Add `NerModel.tokenizer`

ahisto_named_entity_search/recognition/model.py

@@ -11,7 +11,7 @@ from adaptor.evaluators.token_classification import MeanFScore
 from adaptor.objectives.MLM import MaskedLanguageModeling
 from adaptor.lang_module import LangModule
 from adaptor.utils import StoppingStrategy, AdaptationArguments
-from transformers import AutoModelForTokenClassification
+from transformers import AutoModelForTokenClassification, AutoTokenizer
 
 from ..config import CONFIG as _CONFIG
 from ..document import Document, Sentence
@@ -42,6 +42,11 @@ class NerModel:
     def __init__(self, model_name_or_basename: str):
         self.model_name_or_basename = model_name_or_basename
 
+    @property
+    def tokenizer(self) -> AutoTokenizer:
+        tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_basename)
+        return tokenizer
+
     @property
     def model(self) -> AutoModelForTokenClassification:
         model = AutoModelForTokenClassification.from_pretrained(self.model_name_or_basename)