Evaluate language identification with Google Vision AI without limiting languages

Makefile

@@ -400,6 +400,42 @@ endef
 %.accuracy-results-human-without-columns: $(INPUT_HUMAN_JUDGEMENTS_UPSCALED_HIGH_CONFIDENCE_FILENAMES_WITHOUT_COLUMNS) $(OUTPUT_RELEVANT_PAGES) $(INPUT_HUMAN_JUDGEMENTS_WITHOUT_COLUMNS_DIRNAME) %
 	$(PYTHON_RUN) scripts.evaluate_accuracy $^ $@ $@.difficult-pages
 
+output-ocr-google-lowres.language-detection-page-results-lowres-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) output-ocr-google-lowres
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA OLDA
+
+output-ocr-google-lowres.language-detection-page-results-hires-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) output-ocr-google-lowres
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA OLDA
+
+output-ocr-google-lowres.language-detection-page-results-human-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) output-ocr-google-lowres
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ annotated OLDA
+
+output-ocr-google-lowres.language-detection-page-results-lowres-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) output-ocr-google-lowres
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA NLDA
+
+output-ocr-google-lowres.language-detection-page-results-hires-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) output-ocr-google-lowres
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA NLDA
+
+output-ocr-google-lowres.language-detection-page-results-human-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) output-ocr-google-lowres
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ annotated NLDA
+
+output-ocr-google-hires.language-detection-page-results-hires-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) output-ocr-google-hires
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA OLDA
+
+output-ocr-google-hires.language-detection-page-results-hires-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) output-ocr-google-hires
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA OLDA
+
+output-ocr-google-hires.language-detection-page-results-human-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) output-ocr-google-hires
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ annotated OLDA
+
+output-ocr-google-hires.language-detection-page-results-hires-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) output-ocr-google-hires
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA NLDA
+
+output-ocr-google-hires.language-detection-page-results-hires-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) output-ocr-google-hires
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ OLDA NLDA
+
+output-ocr-google-hires.language-detection-page-results-human-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) output-ocr-google-hires
+	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ None $@ annotated NLDA
+
 %.language-detection-page-results-lowres-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) %
 	$(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ OLDA OLDA
 

output-ocr-google-hires.language-detection-page-results-human-nlda

-Mean Mean IOU point estimate: 95.76%
-Mean Mean IOU 95.00% CI estimate: [94.86%; 96.67%]
+Mean Mean IOU point estimate: 81.63%
+Mean Mean IOU 95.00% CI estimate: [79.54%; 83.73%]
 
-Mean Mean Spearman's rho point estimate: 0.96
-Mean Mean Spearman's rho 95.00% CI estimate: [0.91; 1.01]
+Mean Mean Spearman's rho point estimate: 0.74
+Mean Mean Spearman's rho 95.00% CI estimate: [0.70; 0.77]
 
 Mean Accuracy@1 point estimate: 99.18%
 Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%]

output-ocr-google-hires.language-detection-page-results-human-olda

+<<<<<<< HEAD
 Mean Mean IOU point estimate: 92.42%
 Mean Mean IOU 95.00% CI estimate: [91.07%; 93.77%]
 
 Mean Mean Spearman's rho point estimate: 0.90
 Mean Mean Spearman's rho 95.00% CI estimate: [0.85; 0.95]
+=======
+Mean Mean IOU point estimate: 81.63%
+Mean Mean IOU 95.00% CI estimate: [79.54%; 83.73%]
+
+Mean Mean Spearman's rho point estimate: 0.74
+Mean Mean Spearman's rho 95.00% CI estimate: [0.70; 0.77]
+>>>>>>> e8ca691e6... Evaluate language identification with Google Vision AI without limiting languages
 
 Mean Accuracy@1 point estimate: 99.18%
 Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%]

output-ocr-google-hires.language-detection-page-results-lowres-nlda

+Mean Mean IOU point estimate: 99.15%
+Mean Mean IOU 95.00% CI estimate: [99.13%; 99.17%]
+
+Mean Mean Spearman's rho point estimate: 0.99
+Mean Mean Spearman's rho 95.00% CI estimate: [0.99; 0.99]
+
+Mean Accuracy@1 point estimate: 99.72%
+Mean Accuracy@1 95.00% CI estimate: [99.68%; 99.76%]
+

output-ocr-google-lowres.language-detection-page-results-hires-nlda

+Mean Mean IOU point estimate: 96.62%
+Mean Mean IOU 95.00% CI estimate: [96.58%; 96.66%]
+
+Mean Mean Spearman's rho point estimate: 0.91
+Mean Mean Spearman's rho 95.00% CI estimate: [0.91; 0.91]
+
+Mean Accuracy@1 point estimate: 99.44%
+Mean Accuracy@1 95.00% CI estimate: [99.39%; 99.50%]
+

output-ocr-google-lowres.language-detection-page-results-hires-olda

+<<<<<<< HEAD
 Mean Mean IOU point estimate: 98.03%
 Mean Mean IOU 95.00% CI estimate: [98.00%; 98.06%]
 
@@ -6,4 +7,14 @@ Mean Mean Spearman's rho 95.00% CI estimate: [0.97; 0.98]
 
 Mean Accuracy@1 point estimate: 99.48%
 Mean Accuracy@1 95.00% CI estimate: [99.42%; 99.53%]
+=======
+Mean Mean IOU point estimate: 96.62%
+Mean Mean IOU 95.00% CI estimate: [96.58%; 96.66%]
+
+Mean Mean Spearman's rho point estimate: 0.91
+Mean Mean Spearman's rho 95.00% CI estimate: [0.91; 0.91]
+
+Mean Accuracy@1 point estimate: 99.44%
+Mean Accuracy@1 95.00% CI estimate: [99.39%; 99.50%]
+>>>>>>> e8ca691e6... Evaluate language identification with Google Vision AI without limiting languages
 

output-ocr-google-lowres.language-detection-page-results-human-nlda

-Mean Mean IOU point estimate: 95.79%
-Mean Mean IOU 95.00% CI estimate: [94.86%; 96.71%]
+Mean Mean IOU point estimate: 81.64%
+Mean Mean IOU 95.00% CI estimate: [79.56%; 83.72%]
 
-Mean Mean Spearman's rho point estimate: 0.96
-Mean Mean Spearman's rho 95.00% CI estimate: [0.91; 1.01]
+Mean Mean Spearman's rho point estimate: 0.73
+Mean Mean Spearman's rho 95.00% CI estimate: [0.69; 0.77]
 
 Mean Accuracy@1 point estimate: 99.18%
 Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%]

output-ocr-google-lowres.language-detection-page-results-human-olda

+<<<<<<< HEAD
 Mean Mean IOU point estimate: 92.39%
 Mean Mean IOU 95.00% CI estimate: [90.94%; 93.85%]
 
 Mean Mean Spearman's rho point estimate: 0.88
 Mean Mean Spearman's rho 95.00% CI estimate: [0.82; 0.93]
+=======
+Mean Mean IOU point estimate: 81.64%
+Mean Mean IOU 95.00% CI estimate: [79.56%; 83.72%]
+
+Mean Mean Spearman's rho point estimate: 0.73
+Mean Mean Spearman's rho 95.00% CI estimate: [0.69; 0.77]
+>>>>>>> e8ca691e6... Evaluate language identification with Google Vision AI without limiting languages
 
 Mean Accuracy@1 point estimate: 99.18%
 Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%]

scripts/evaluate_language_detection_page.py

@@ -22,7 +22,7 @@ from .common import read_facts, read_page_languages, get_jaccard_index, get_spea
 INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES = sys.argv[1]
 INPUT_OCR_ROOT = Path(sys.argv[2])
 OUTPUT_OCR_ROOT = Path(sys.argv[3])
-DETECTED_LANGUAGES = sys.argv[4].split('+')
+DETECTED_LANGUAGES = sys.argv[4].split('+') if sys.argv[4] != 'None' else None
 OUTPUT_LOG_FILENAME = Path(sys.argv[5])
 INPUT_ALGORITHM = sys.argv[6] if len(sys.argv) > 6 else 'NLDA'
 OUTPUT_ALGORITHM = sys.argv[7] if len(sys.argv) > 7 else 'NLDA'