Take word-level language annotations in HOCR output into account

scripts/common.py

 # -*- coding:utf-8 -*-
 
+from collections import defaultdict
 import csv
 from itertools import repeat, chain
 import json
@@ -692,13 +693,20 @@ def _read_page_languages_json(f):
 def _read_page_languages_hocr(f):
     html5_parser = etree.HTMLParser(huge_tree=True)
     xml_document = etree.parse(f, html5_parser)
-    languages = dict()
+    languages = defaultdict(lambda: 0.0)
+
+    def get_confidence(element):
+        return float(len(''.join(element.itertext())))
+
     for paragraph in xml_document.xpath('//p[@lang]'):
-        language_code = paragraph.attrib['lang']
-        confidence = float(len(''.join(paragraph.itertext())))
-        if language_code not in languages:
-            languages[language_code] = 0.0
-        languages[language_code] += confidence
+        paragraph_language_code = paragraph.attrib['lang']
+        paragraph_confidence = get_confidence(paragraph)
+        for word in paragraph.xpath('//span[@class="ocrx_word" and @lang]'):
+            word_language_code = word.attrib['lang']
+            word_confidence = get_confidence(word)
+            languages[word_language_code] += word_confidence
+            paragraph_confidence -= word_confidence
+        languages[paragraph_language_code] += paragraph_confidence
     return languages