Commit fc384e4a authored by Vít Novotný's avatar Vít Novotný
Browse files

Strip punctuation at ends of sentences

parent cd8adc77
......@@ -100,7 +100,7 @@ class BIOTokenPunctuationStrippingClassification(TokenClassification):
label_ids = islice(label_ids, len(token_ids))
if label != 'O':
if label_index + 1 >= len(labels):
should_strip_punctuation = False
should_strip_punctuation = True
else:
label_type = get_label_type(label)
next_label = labels[label_index + 1]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment