Add GNU parallel joblog to training NER models

scripts/03_train_ner_models.sh

@@ -12,9 +12,10 @@ SCHEDULE_NAME=fine-tuning
 
 DOCKER_BUILDKIT=1 docker build --build-arg UID="$(id -u)" --build-arg GID="$(id -g)" --build-arg UNAME="$(id -u -n)" . -f scripts//03_train_ner_models.Dockerfile -t "$IMAGE_NAME"
 
-parallel --halt=soon,fail=100% --jobs=4 --bar --delay 60 \
+parallel --halt=soon,fail=100% --jobs=4 --bar --delay 60 --resume-failed \
+         --joblog scripts/03_train_ner_models_"${SCHEDULE_NAME}".joblog \
          --colsep ' +' \
-         -- '
-           GPU_ID=$(nvidia-smi | grep -F -B 1 -- "0MiB / 15360MiB" | head -n 1 | awk "{ print \$2 }")
-           docker run --rm -u "$(id -u):$(id -g)" --hostname "'"$HOSTNAME"'" --runtime=nvidia -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e NVIDIA_VISIBLE_DEVICES="$GPU_ID" -e TOKENIZERS_PARALLELISM=false -e COMET_API_KEY -v "$PWD"/..:/workdir:rw -w /workdir/"${PWD##*/}" -v "'"$ROOT_PATH"'":"'"$ROOT_PATH"'":rw -v "'"$ANNOTATION_PATH"'":"'"$ANNOTATION_PATH"'":ro -v "'"$OCR_EVAL_PATH"'":"'"$OCR_EVAL_PATH"'":ro "'"$IMAGE_NAME"'" nice -n 19 python scripts/03_train_ner_models.py {1} {2} {3} '"$SCHEDULE_NAME"'
+         -- \
+           'GPU_ID=$(nvidia-smi | grep -F -B 1 -- "0MiB / 15360MiB" | head -n 1 | awk "{ print \$2 }"); '\
+           'docker run --rm -u "$(id -u):$(id -g)" --hostname "'"$HOSTNAME"'" --runtime=nvidia -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e NVIDIA_VISIBLE_DEVICES="$GPU_ID" -e TOKENIZERS_PARALLELISM=false -e COMET_API_KEY -v "$PWD"/..:/workdir:rw -w /workdir/"${PWD##*/}" -v "'"$ROOT_PATH"'":"'"$ROOT_PATH"'":rw -v "'"$ANNOTATION_PATH"'":"'"$ANNOTATION_PATH"'":ro -v "'"$OCR_EVAL_PATH"'":"'"$OCR_EVAL_PATH"'":ro "'"$IMAGE_NAME"'" nice -n 19 python scripts/03_train_ner_models.py {1} {2} {3} '"$SCHEDULE_NAME"'
          ' :::: scripts/03_train_ner_models.tasks