diff --git a/Makefile b/Makefile index 27e9f8ff09ce2579b9b3d3aed3367e18a254b4fd..79bd0aca2aa012998befd4254ab0eb4a185611fb 100644 --- a/Makefile +++ b/Makefile @@ -394,14 +394,29 @@ endef %.accuracy-results-human: $(INPUT_HUMAN_JUDGEMENTS_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_RELEVANT_PAGES) $(INPUT_HUMAN_JUDGEMENTS_DIRNAME) % $(PYTHON_RUN) scripts.evaluate_accuracy $^ $@ $@.difficult-pages -%.language-detection-page-results-lowres: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) % - $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ +%.accuracy-results-human-with-columns: $(INPUT_HUMAN_JUDGEMENTS_UPSCALED_HIGH_CONFIDENCE_FILENAMES_WITH_COLUMNS) $(OUTPUT_RELEVANT_PAGES) $(INPUT_HUMAN_JUDGEMENTS_WITH_COLUMNS_DIRNAME) % + $(PYTHON_RUN) scripts.evaluate_accuracy $^ $@ $@.difficult-pages + +%.accuracy-results-human-without-columns: $(INPUT_HUMAN_JUDGEMENTS_UPSCALED_HIGH_CONFIDENCE_FILENAMES_WITHOUT_COLUMNS) $(OUTPUT_RELEVANT_PAGES) $(INPUT_HUMAN_JUDGEMENTS_WITHOUT_COLUMNS_DIRNAME) % + $(PYTHON_RUN) scripts.evaluate_accuracy $^ $@ $@.difficult-pages + +%.language-detection-page-results-lowres-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) % + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ OLDA OLDA + +%.language-detection-page-results-hires-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) % + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ OLDA OLDA + +%.language-detection-page-results-human-olda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) % + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ annotated OLDA + +%.language-detection-page-results-lowres-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_LOWRES_DIRNAME) % + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ OLDA NLDA -%.language-detection-page-results-hires: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) % - $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ +%.language-detection-page-results-hires-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(OUTPUT_OCR_GOOGLE_HIRES_DIRNAME) % + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ OLDA NLDA -%.language-detection-page-results-human: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) % - $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ annotated +%.language-detection-page-results-human-nlda: $(INPUT_UPSCALED_HIGH_CONFIDENCE_FILENAMES) $(LANGUAGE_DETECTION_ANNOTATIONS) % + $(PYTHON_RUN) scripts.evaluate_language_detection_page $^ $(DETECTED_LANGUAGES) $@ annotated NLDA $(addprefix %.detected-languages.,$(THRESHOLDS)): $(INPUT_FILENAMES_FILTERED) % $(PYTHON_RUN) scripts.extract_detected_languages $^ $(DETECTED_LANGUAGES) $@ $(subst .,,$(suffix $@)) diff --git a/output-ocr-google-hires.language-detection-page-results-hires-olda b/output-ocr-google-hires.language-detection-page-results-hires-olda new file mode 100644 index 0000000000000000000000000000000000000000..147c2319df21ff088d4c1c9b1db85715d00ee103 --- /dev/null +++ b/output-ocr-google-hires.language-detection-page-results-hires-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 100.00% +Mean Mean IOU 95.00% CI estimate: [nan%; nan%] + +Mean Mean Spearman's rho point estimate: 1.00 +Mean Mean Spearman's rho 95.00% CI estimate: [1.00; 1.00] + +Mean Accuracy@1 point estimate: 100.00% +Mean Accuracy@1 95.00% CI estimate: [nan%; nan%] + diff --git a/output-ocr-google-hires.language-detection-page-results-human b/output-ocr-google-hires.language-detection-page-results-human-nlda similarity index 100% rename from output-ocr-google-hires.language-detection-page-results-human rename to output-ocr-google-hires.language-detection-page-results-human-nlda diff --git a/output-ocr-google-hires.language-detection-page-results-human-olda b/output-ocr-google-hires.language-detection-page-results-human-olda new file mode 100644 index 0000000000000000000000000000000000000000..f13500968f290ede6a0b880ee8a26498420d3d34 --- /dev/null +++ b/output-ocr-google-hires.language-detection-page-results-human-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 92.42% +Mean Mean IOU 95.00% CI estimate: [91.07%; 93.77%] + +Mean Mean Spearman's rho point estimate: 0.90 +Mean Mean Spearman's rho 95.00% CI estimate: [0.85; 0.95] + +Mean Accuracy@1 point estimate: 99.18% +Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%] + diff --git a/output-ocr-google-hires.language-detection-page-results-lowres-olda b/output-ocr-google-hires.language-detection-page-results-lowres-olda new file mode 100644 index 0000000000000000000000000000000000000000..64b646ffd83297575cc5fac33bf3ef42fc7ce29e --- /dev/null +++ b/output-ocr-google-hires.language-detection-page-results-lowres-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 98.03% +Mean Mean IOU 95.00% CI estimate: [98.00%; 98.06%] + +Mean Mean Spearman's rho point estimate: 0.98 +Mean Mean Spearman's rho 95.00% CI estimate: [0.97; 0.98] + +Mean Accuracy@1 point estimate: 99.48% +Mean Accuracy@1 95.00% CI estimate: [99.42%; 99.53%] + diff --git a/output-ocr-google-lowres.language-detection-page-results-hires-olda b/output-ocr-google-lowres.language-detection-page-results-hires-olda new file mode 100644 index 0000000000000000000000000000000000000000..64b646ffd83297575cc5fac33bf3ef42fc7ce29e --- /dev/null +++ b/output-ocr-google-lowres.language-detection-page-results-hires-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 98.03% +Mean Mean IOU 95.00% CI estimate: [98.00%; 98.06%] + +Mean Mean Spearman's rho point estimate: 0.98 +Mean Mean Spearman's rho 95.00% CI estimate: [0.97; 0.98] + +Mean Accuracy@1 point estimate: 99.48% +Mean Accuracy@1 95.00% CI estimate: [99.42%; 99.53%] + diff --git a/output-ocr-google-lowres.language-detection-page-results-human b/output-ocr-google-lowres.language-detection-page-results-human-nlda similarity index 100% rename from output-ocr-google-lowres.language-detection-page-results-human rename to output-ocr-google-lowres.language-detection-page-results-human-nlda diff --git a/output-ocr-google-lowres.language-detection-page-results-human-olda b/output-ocr-google-lowres.language-detection-page-results-human-olda new file mode 100644 index 0000000000000000000000000000000000000000..524d71fb88c80e4494adb9d244a1cd3d57cb0bd6 --- /dev/null +++ b/output-ocr-google-lowres.language-detection-page-results-human-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 92.39% +Mean Mean IOU 95.00% CI estimate: [90.94%; 93.85%] + +Mean Mean Spearman's rho point estimate: 0.88 +Mean Mean Spearman's rho 95.00% CI estimate: [0.82; 0.93] + +Mean Accuracy@1 point estimate: 99.18% +Mean Accuracy@1 95.00% CI estimate: [97.56%; 100.80%] + diff --git a/output-ocr-google-lowres.language-detection-page-results-lowres-olda b/output-ocr-google-lowres.language-detection-page-results-lowres-olda new file mode 100644 index 0000000000000000000000000000000000000000..147c2319df21ff088d4c1c9b1db85715d00ee103 --- /dev/null +++ b/output-ocr-google-lowres.language-detection-page-results-lowres-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 100.00% +Mean Mean IOU 95.00% CI estimate: [nan%; nan%] + +Mean Mean Spearman's rho point estimate: 1.00 +Mean Mean Spearman's rho 95.00% CI estimate: [1.00; 1.00] + +Mean Accuracy@1 point estimate: 100.00% +Mean Accuracy@1 95.00% CI estimate: [nan%; nan%] + diff --git a/output-ocr3+4.language-detection-page-results-hires b/output-ocr3+4.language-detection-page-results-hires-nlda similarity index 100% rename from output-ocr3+4.language-detection-page-results-hires rename to output-ocr3+4.language-detection-page-results-hires-nlda diff --git a/output-ocr3+4.language-detection-page-results-hires-olda b/output-ocr3+4.language-detection-page-results-hires-olda new file mode 100644 index 0000000000000000000000000000000000000000..fa1749f09fe9a75cc56adaef1c3817fda92ec31f --- /dev/null +++ b/output-ocr3+4.language-detection-page-results-hires-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 19.61% +Mean Mean IOU 95.00% CI estimate: [19.43%; 19.79%] + +Mean Mean Spearman's rho point estimate: -0.12 +Mean Mean Spearman's rho 95.00% CI estimate: [-0.12; -0.11] + +Mean Accuracy@1 point estimate: 24.55% +Mean Accuracy@1 95.00% CI estimate: [24.22%; 24.88%] + diff --git a/output-ocr3+4.language-detection-page-results-human b/output-ocr3+4.language-detection-page-results-human-nlda similarity index 100% rename from output-ocr3+4.language-detection-page-results-human rename to output-ocr3+4.language-detection-page-results-human-nlda diff --git a/output-ocr3+4.language-detection-page-results-human-olda b/output-ocr3+4.language-detection-page-results-human-olda new file mode 100644 index 0000000000000000000000000000000000000000..e95d7ecf61d605178e9560bcba7000886a22eb77 --- /dev/null +++ b/output-ocr3+4.language-detection-page-results-human-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 10.36% +Mean Mean IOU 95.00% CI estimate: [7.14%; 13.59%] + +Mean Mean Spearman's rho point estimate: -0.14 +Mean Mean Spearman's rho 95.00% CI estimate: [-0.23; -0.05] + +Mean Accuracy@1 point estimate: 11.48% +Mean Accuracy@1 95.00% CI estimate: [5.74%; 17.21%] + diff --git a/output-ocr3+4.language-detection-page-results-lowres b/output-ocr3+4.language-detection-page-results-lowres-nlda similarity index 100% rename from output-ocr3+4.language-detection-page-results-lowres rename to output-ocr3+4.language-detection-page-results-lowres-nlda diff --git a/output-ocr3+4.language-detection-page-results-lowres-olda b/output-ocr3+4.language-detection-page-results-lowres-olda new file mode 100644 index 0000000000000000000000000000000000000000..1cbe7fcd3d85dbfc9c38e6cab7b7937b7f5a488c --- /dev/null +++ b/output-ocr3+4.language-detection-page-results-lowres-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 19.60% +Mean Mean IOU 95.00% CI estimate: [19.42%; 19.78%] + +Mean Mean Spearman's rho point estimate: -0.12 +Mean Mean Spearman's rho 95.00% CI estimate: [-0.12; -0.12] + +Mean Accuracy@1 point estimate: 24.53% +Mean Accuracy@1 95.00% CI estimate: [24.20%; 24.86%] + diff --git a/output-ocr3.language-detection-page-results-hires b/output-ocr3.language-detection-page-results-hires-nlda similarity index 100% rename from output-ocr3.language-detection-page-results-hires rename to output-ocr3.language-detection-page-results-hires-nlda diff --git a/output-ocr3.language-detection-page-results-hires-olda b/output-ocr3.language-detection-page-results-hires-olda new file mode 100644 index 0000000000000000000000000000000000000000..6b3d1ec86245b2e55f4b0c41b9a40fc0000a5a57 --- /dev/null +++ b/output-ocr3.language-detection-page-results-hires-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 48.26% +Mean Mean IOU 95.00% CI estimate: [48.02%; 48.50%] + +Mean Mean Spearman's rho point estimate: 0.36 +Mean Mean Spearman's rho 95.00% CI estimate: [0.36; 0.36] + +Mean Accuracy@1 point estimate: 65.01% +Mean Accuracy@1 95.00% CI estimate: [64.64%; 65.38%] + diff --git a/output-ocr3.language-detection-page-results-human b/output-ocr3.language-detection-page-results-human-nlda similarity index 100% rename from output-ocr3.language-detection-page-results-human rename to output-ocr3.language-detection-page-results-human-nlda diff --git a/output-ocr3.language-detection-page-results-human-olda b/output-ocr3.language-detection-page-results-human-olda new file mode 100644 index 0000000000000000000000000000000000000000..c00c2faa8f22521d87c1edb3414e4ba458b887ae --- /dev/null +++ b/output-ocr3.language-detection-page-results-human-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 27.97% +Mean Mean IOU 95.00% CI estimate: [23.03%; 32.92%] + +Mean Mean Spearman's rho point estimate: 0.26 +Mean Mean Spearman's rho 95.00% CI estimate: [0.17; 0.35] + +Mean Accuracy@1 point estimate: 42.62% +Mean Accuracy@1 95.00% CI estimate: [33.72%; 51.52%] + diff --git a/output-ocr3.language-detection-page-results-lowres b/output-ocr3.language-detection-page-results-lowres-nlda similarity index 100% rename from output-ocr3.language-detection-page-results-lowres rename to output-ocr3.language-detection-page-results-lowres-nlda diff --git a/output-ocr3.language-detection-page-results-lowres-olda b/output-ocr3.language-detection-page-results-lowres-olda new file mode 100644 index 0000000000000000000000000000000000000000..6add29b1a92115780b74b785cdc17a4010832f72 --- /dev/null +++ b/output-ocr3.language-detection-page-results-lowres-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 48.25% +Mean Mean IOU 95.00% CI estimate: [48.01%; 48.49%] + +Mean Mean Spearman's rho point estimate: 0.36 +Mean Mean Spearman's rho 95.00% CI estimate: [0.36; 0.36] + +Mean Accuracy@1 point estimate: 65.00% +Mean Accuracy@1 95.00% CI estimate: [64.63%; 65.36%] + diff --git a/output-ocr4-onepass.language-detection-page-results-hires b/output-ocr4-onepass.language-detection-page-results-hires-nlda similarity index 100% rename from output-ocr4-onepass.language-detection-page-results-hires rename to output-ocr4-onepass.language-detection-page-results-hires-nlda diff --git a/output-ocr4-onepass.language-detection-page-results-hires-olda b/output-ocr4-onepass.language-detection-page-results-hires-olda new file mode 100644 index 0000000000000000000000000000000000000000..6797db1348eb74470cd87adf1ccf8ca4e4cf1161 --- /dev/null +++ b/output-ocr4-onepass.language-detection-page-results-hires-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 49.84% +Mean Mean IOU 95.00% CI estimate: [49.60%; 50.08%] + +Mean Mean Spearman's rho point estimate: 0.33 +Mean Mean Spearman's rho 95.00% CI estimate: [0.32; 0.33] + +Mean Accuracy@1 point estimate: 66.49% +Mean Accuracy@1 95.00% CI estimate: [66.13%; 66.85%] + diff --git a/output-ocr4-onepass.language-detection-page-results-human b/output-ocr4-onepass.language-detection-page-results-human-nlda similarity index 100% rename from output-ocr4-onepass.language-detection-page-results-human rename to output-ocr4-onepass.language-detection-page-results-human-nlda diff --git a/output-ocr4-onepass.language-detection-page-results-human-olda b/output-ocr4-onepass.language-detection-page-results-human-olda new file mode 100644 index 0000000000000000000000000000000000000000..2dada04fda289519ffc6342fcf9eeee20aea88fb --- /dev/null +++ b/output-ocr4-onepass.language-detection-page-results-human-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 26.18% +Mean Mean IOU 95.00% CI estimate: [21.13%; 31.23%] + +Mean Mean Spearman's rho point estimate: 0.22 +Mean Mean Spearman's rho 95.00% CI estimate: [0.12; 0.32] + +Mean Accuracy@1 point estimate: 36.89% +Mean Accuracy@1 95.00% CI estimate: [28.20%; 45.57%] + diff --git a/output-ocr4-onepass.language-detection-page-results-lowres b/output-ocr4-onepass.language-detection-page-results-lowres-nlda similarity index 100% rename from output-ocr4-onepass.language-detection-page-results-lowres rename to output-ocr4-onepass.language-detection-page-results-lowres-nlda diff --git a/output-ocr4-onepass.language-detection-page-results-lowres-olda b/output-ocr4-onepass.language-detection-page-results-lowres-olda new file mode 100644 index 0000000000000000000000000000000000000000..b05745f1ffd79d0e74c9a20ea9fed4ef6ad2447d --- /dev/null +++ b/output-ocr4-onepass.language-detection-page-results-lowres-olda @@ -0,0 +1,9 @@ +Mean Mean IOU point estimate: 49.84% +Mean Mean IOU 95.00% CI estimate: [49.60%; 50.08%] + +Mean Mean Spearman's rho point estimate: 0.33 +Mean Mean Spearman's rho 95.00% CI estimate: [0.32; 0.33] + +Mean Accuracy@1 point estimate: 66.48% +Mean Accuracy@1 95.00% CI estimate: [66.12%; 66.84%] +