Take word-level language annotations in HOCR output into account (cont.)

scripts/common.py

@@ -696,16 +696,17 @@ def _read_page_languages_hocr(f):
     languages = defaultdict(lambda: 0.0)
 
     def get_confidence(element):
-        return float(len(''.join(element.itertext())))
+        return float(len(''.join(element.itertext()).strip()))
 
     for paragraph in xml_document.xpath('//p[@lang]'):
         paragraph_language_code = paragraph.attrib['lang']
         paragraph_confidence = get_confidence(paragraph)
-        for word in paragraph.xpath('//span[@class="ocrx_word" and @lang]'):
+        for word in paragraph.xpath('.//span[@class="ocrx_word" and @lang]'):
             word_language_code = word.attrib['lang']
             word_confidence = get_confidence(word)
             languages[word_language_code] += word_confidence
             paragraph_confidence -= word_confidence
+        assert paragraph_confidence >= 0.0
         languages[paragraph_language_code] += paragraph_confidence
     return languages