Commit 63d085bd authored by Vít Starý Novotný's avatar Vít Starý Novotný
Browse files

Automatically select GPU in `scripts/03_train_ner_models.sh`

parent 7c0a4044

scripts/03_train_ner_models.sh

+5 −5
Original line number Diff line number Diff line
@@ -11,9 +11,9 @@ OCR_EVAL_PATH=/nlp/projekty/ahisto/ahisto-ocr-eval 


DOCKER_BUILDKIT=1 docker build --build-arg UID="$(id -u)" --build-arg GID="$(id -g)" --build-arg UNAME="$(id -u -n)" . -f scripts//03_train_ner_models.Dockerfile -t "$IMAGE_NAME"



parallel --halt=soon,fail=100% --jobs=100% --resume-failed --bar --delay 30 \

         --joblog=scripts/03_train_ner_models.joblog \

parallel --halt=soon,fail=100% --jobs=100% --bar --delay 60 \

         --colsep ' +' \

         -- \

         docker run --rm -u "$(id -u):$(id -g)" --hostname "$HOSTNAME" --runtime=nvidia -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e NVIDIA_VISIBLE_DEVICES={4} -e TOKENIZERS_PARALLELISM=false -e COMET_API_KEY -v "$PWD"/..:/workdir:rw -w /workdir/"${PWD##*/}" -v "$ROOT_PATH":"$ROOT_PATH":rw -v "$ANNOTATION_PATH":"$ANNOTATION_PATH":ro -v "$OCR_EVAL_PATH":"$OCR_EVAL_PATH":ro "$IMAGE_NAME" nice -n 19 python scripts/03_train_ner_models.py {1} {2} {3} \

         :::: scripts/03_train_ner_models.tasks
 
         -- '

           GPU_ID=$(nvidia-smi | grep -F -B 1 -- "0MiB / 15360MiB" | head -n 1 | awk "{ print \$2 }")

           docker run --rm -u "$(id -u):$(id -g)" --hostname "'"$HOSTNAME"'" --runtime=nvidia -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e NVIDIA_VISIBLE_DEVICES="$GPU_ID" -e TOKENIZERS_PARALLELISM=false -e COMET_API_KEY -v "$PWD"/..:/workdir:rw -w /workdir/"${PWD##*/}" -v "'"$ROOT_PATH"'":"'"$ROOT_PATH"'":rw -v "'"$ANNOTATION_PATH"'":"'"$ANNOTATION_PATH"'":ro -v "'"$OCR_EVAL_PATH"'":"'"$OCR_EVAL_PATH"'":ro "'"$IMAGE_NAME"'" nice -n 19 python scripts/03_train_ner_models.py {1} {2} {3}

         ' :::: scripts/03_train_ner_models.tasks

scripts/03_train_ner_models.tasks

+8 −8
Original line number Diff line number Diff line 
manatee     non-crossing only-relevant  1

manatee     all          only-relevant  2

fuzzy-regex non-crossing only-relevant  3

fuzzy-regex all          only-relevant  4

manatee     non-crossing all            5

manatee     all          all           10

fuzzy-regex non-crossing all           12

fuzzy-regex all          all           13
 
manatee     non-crossing only-relevant

manatee     all          only-relevant

fuzzy-regex non-crossing only-relevant

fuzzy-regex all          only-relevant

manatee     non-crossing all

manatee     all          all

fuzzy-regex non-crossing all

fuzzy-regex all          all