diff --git a/Makefile b/Makefile index 79bd0aca2aa012998befd4254ab0eb4a185611fb..ab682dec4df830e9c15f1bf06b2f8ef424721d69 100644 --- a/Makefile +++ b/Makefile @@ -400,6 +400,42 @@ endef %.accuracy-results-human-without-columns: $(INPUT_HUMAN_JUDGEMENTS_UPSCALED_HIGH_CONFIDENCE_FILENAMES_WITHOUT_COLUMNS) $(OUTPUT_RELEVANT_PAGES) $(INPUT_HUMAN_JUDGEMENTS_WITHOUT_COLUMNS_DIRNAME) % $(PYTHON_RUN) scripts.evaluate_accuracy $^ $@ $@.difficult-pages +output-ocr-google-lowres.language-detection-page-results-lowres-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) output-ocr-google-lowres + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA OLDA + +output-ocr-google-lowres.language-detection-page-results-hires-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) output-ocr-google-lowres + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA OLDA + +output-ocr-google-lowres.language-detection-page-results-human-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) output-ocr-google-lowres + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ annotated OLDA + +output-ocr-google-lowres.language-detection-page-results-lowres-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) output-ocr-google-lowres + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA NLDA + +output-ocr-google-lowres.language-detection-page-results-hires-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) output-ocr-google-lowres + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA NLDA + +output-ocr-google-lowres.language-detection-page-results-human-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) output-ocr-google-lowres + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ annotated NLDA + +output-ocr-google-hires.language-detection-page-results-hires-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) output-ocr-google-hires + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA OLDA + +output-ocr-google-hires.language-detection-page-results-hires-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) output-ocr-google-hires + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA OLDA + +output-ocr-google-hires.language-detection-page-results-human-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) output-ocr-google-hires + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ annotated OLDA + +output-ocr-google-hires.language-detection-page-results-hires-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) output-ocr-google-hires + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA NLDA + +output-ocr-google-hires.language-detection-page-results-hires-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) output-ocr-google-hires + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA NLDA + +output-ocr-google-hires.language-detection-page-results-human-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) output-ocr-google-hires + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ annotated NLDA + %.language-detection-page-results-lowres-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) % $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ OLDA OLDA diff --git a/output-ocr-google-hires.language-detection-page-results-human-nlda b/output-ocr-google-hires.language-detection-page-results-human-nlda index fdbbe895d16c01fb93fbc2b506c31120b0bbd459..c124c3f158463c4242faa0695d047a203997b086 100644 --- a/output-ocr-google-hires.language-detection-page-results-human-nlda +++ b/output-ocr-google-hires.language-detection-page-results-human-nlda @@ -1,8 +1,8 @@ -Mean Mean IOU point estimate: 95.76% -Mean Mean IOU 95.00% CI estimate: [94.86%; 96.67%] +Mean Mean IOU point estimate: 81.63% +Mean Mean IOU 95.00% CI estimate: [79.54%; 83.73%] -Mean Mean Spearman's rho point estimate: 0.96 -Mean Mean Spearman's rho 95.00% CI estimate: [0.91; 1.01] +Mean Mean Spearman's rho point estimate: 0.74 +Mean Mean Spearman's rho 95.00% CI estimate: [0.70; 0.77] Mean Accuracy@1 point estimate: 99.18% Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%] diff --git a/output-ocr-google-hires.language-detection-page-results-human-olda b/output-ocr-google-hires.language-detection-page-results-human-olda index f13500968f290ede6a0b880ee8a26498420d3d34..5e37b6c02e7a3142874a4182e72c3869579cc890 100644 --- a/output-ocr-google-hires.language-detection-page-results-human-olda +++ b/output-ocr-google-hires.language-detection-page-results-human-olda @@ -1,8 +1,16 @@ +<<<<<<< HEAD Mean Mean IOU point estimate: 92.42% Mean Mean IOU 95.00% CI estimate: [91.07%; 93.77%] Mean Mean Spearman's rho point estimate: 0.90 Mean Mean Spearman's rho 95.00% CI estimate: [0.85; 0.95] +======= +Mean Mean IOU point estimate: 81.63% +Mean Mean IOU 95.00% CI estimate: [79.54%; 83.73%] + +Mean Mean Spearman's rho point estimate: 0.74 +Mean Mean Spearman's rho 95.00% CI estimate: [0.70; 0.77] +>>>>>>> e8ca691e6... Evaluate language identification with Google Vision AI without limiting languages Mean Accuracy@1 point estimate: 99.18% Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%] diff --git a/output-ocr-google-hires.language-detection-page-results-lowres-nlda b/output-ocr-google-hires.language-detection-page-results-lowres-nlda new file mode 100644 index 0000000000000000000000000000000000000000..45ebdb38e746c896433a75b36da71c751f6b56d2 --- /dev/null +++ b/output-ocr-google-hires.language-detection-page-results-lowres-nlda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 99.15% +Mean Mean IOU 95.00% CI estimate: [99.13%; 99.17%] + +Mean Mean Spearman's rho point estimate: 0.99 +Mean Mean Spearman's rho 95.00% CI estimate: [0.99; 0.99] + +Mean Accuracy@1 point estimate: 99.72% +Mean Accuracy@1 95.00% CI estimate: [99.68%; 99.76%] + diff --git a/output-ocr-google-lowres.language-detection-page-results-hires-nlda b/output-ocr-google-lowres.language-detection-page-results-hires-nlda new file mode 100644 index 0000000000000000000000000000000000000000..9f62a7881a0d4e82bd7389e84f0fd0541251c326 --- /dev/null +++ b/output-ocr-google-lowres.language-detection-page-results-hires-nlda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 96.62% +Mean Mean IOU 95.00% CI estimate: [96.58%; 96.66%] + +Mean Mean Spearman's rho point estimate: 0.91 +Mean Mean Spearman's rho 95.00% CI estimate: [0.91; 0.91] + +Mean Accuracy@1 point estimate: 99.44% +Mean Accuracy@1 95.00% CI estimate: [99.39%; 99.50%] + diff --git a/output-ocr-google-lowres.language-detection-page-results-hires-olda b/output-ocr-google-lowres.language-detection-page-results-hires-olda index 64b646ffd83297575cc5fac33bf3ef42fc7ce29e..e7483b00e1905613cf26a3631d245483fb46271b 100644 --- a/output-ocr-google-lowres.language-detection-page-results-hires-olda +++ b/output-ocr-google-lowres.language-detection-page-results-hires-olda @@ -1,3 +1,4 @@ +<<<<<<< HEAD Mean Mean IOU point estimate: 98.03% Mean Mean IOU 95.00% CI estimate: [98.00%; 98.06%] @@ -6,4 +7,14 @@ Mean Mean Spearman's rho 95.00% CI estimate: [0.97; 0.98] Mean Accuracy@1 point estimate: 99.48% Mean Accuracy@1 95.00% CI estimate: [99.42%; 99.53%] +======= +Mean Mean IOU point estimate: 96.62% +Mean Mean IOU 95.00% CI estimate: [96.58%; 96.66%] + +Mean Mean Spearman's rho point estimate: 0.91 +Mean Mean Spearman's rho 95.00% CI estimate: [0.91; 0.91] + +Mean Accuracy@1 point estimate: 99.44% +Mean Accuracy@1 95.00% CI estimate: [99.39%; 99.50%] +>>>>>>> e8ca691e6... Evaluate language identification with Google Vision AI without limiting languages diff --git a/output-ocr-google-lowres.language-detection-page-results-human-nlda b/output-ocr-google-lowres.language-detection-page-results-human-nlda index 53832f99d8cd151eea8b5c842c60e54f813b54a1..6fc0625f1f9cb954fb1691f2b784cc79b811cd63 100644 --- a/output-ocr-google-lowres.language-detection-page-results-human-nlda +++ b/output-ocr-google-lowres.language-detection-page-results-human-nlda @@ -1,8 +1,8 @@ -Mean Mean IOU point estimate: 95.79% -Mean Mean IOU 95.00% CI estimate: [94.86%; 96.71%] +Mean Mean IOU point estimate: 81.64% +Mean Mean IOU 95.00% CI estimate: [79.56%; 83.72%] -Mean Mean Spearman's rho point estimate: 0.96 -Mean Mean Spearman's rho 95.00% CI estimate: [0.91; 1.01] +Mean Mean Spearman's rho point estimate: 0.73 +Mean Mean Spearman's rho 95.00% CI estimate: [0.69; 0.77] Mean Accuracy@1 point estimate: 99.18% Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%] diff --git a/output-ocr-google-lowres.language-detection-page-results-human-olda b/output-ocr-google-lowres.language-detection-page-results-human-olda index 524d71fb88c80e4494adb9d244a1cd3d57cb0bd6..86ee4110439278ffc9bf6990f3b6fd15d37122b7 100644 --- a/output-ocr-google-lowres.language-detection-page-results-human-olda +++ b/output-ocr-google-lowres.language-detection-page-results-human-olda @@ -1,8 +1,16 @@ +<<<<<<< HEAD Mean Mean IOU point estimate: 92.39% Mean Mean IOU 95.00% CI estimate: [90.94%; 93.85%] Mean Mean Spearman's rho point estimate: 0.88 Mean Mean Spearman's rho 95.00% CI estimate: [0.82; 0.93] +======= +Mean Mean IOU point estimate: 81.64% +Mean Mean IOU 95.00% CI estimate: [79.56%; 83.72%] + +Mean Mean Spearman's rho point estimate: 0.73 +Mean Mean Spearman's rho 95.00% CI estimate: [0.69; 0.77] +>>>>>>> e8ca691e6... Evaluate language identification with Google Vision AI without limiting languages Mean Accuracy@1 point estimate: 99.18% Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%] diff --git a/scripts/evaluate_language_detection_page.py b/scripts/evaluate_language_detection_page.py index e336d807e26275d3a6663c695ee62174c8026e16..3fa4f79b9c076b3952d00c217f8b93259db93244 100644 --- a/scripts/evaluate_language_detection_page.py +++ b/scripts/evaluate_language_detection_page.py @@ -22,7 +22,7 @@ from .common import read_facts, read_page_languages, get_jaccard_index, get_spea INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES = sys.argv[1] INPUT_OCR_ROOT = Path(sys.argv[2]) OUTPUT_OCR_ROOT = Path(sys.argv[3]) -DETECTED_LANGUAGES = sys.argv[4].split('+') +DETECTED_LANGUAGES = sys.argv[4].split('+') if sys.argv[4] != 'None' else None OUTPUT_LOG_FILENAME = Path(sys.argv[5]) INPUT_ALGORITHM = sys.argv[6] if len(sys.argv) > 6 else 'NLDA' OUTPUT_ALGORITHM = sys.argv[7] if len(sys.argv) > 7 else 'NLDA'