Merge pull request #2 from kkew3/fix-notimplementederror

bug fix: NotImplementedError when constructing CharacterTokenizer

Author: dariush-bahrami

charactertokenizer/core.py

@@ -41,6 +41,18 @@ def __init__(self, characters: Sequence[str], model_max_length: int, **kwargs):
 
         mask_token = AddedToken("[MASK]", lstrip=True, rstrip=False)
 
+        self._vocab_str_to_int = {
+            "[CLS]": 0,
+            "[SEP]": 1,
+            "[BOS]": 2,
+            "[MASK]": 3,
+            "[PAD]": 4,
+            "[RESERVED]": 5,
+            "[UNK]": 6,
+            **{ch: i + 7 for i, ch in enumerate(characters)},
+        }
+        self._vocab_int_to_str = {v: k for k, v in self._vocab_str_to_int.items()}
+
         super().__init__(
             bos_token=bos_token,
             eos_token=eos_token,
@@ -54,22 +66,13 @@ def __init__(self, characters: Sequence[str], model_max_length: int, **kwargs):
             **kwargs,
         )
 
-        self._vocab_str_to_int = {
-            "[CLS]": 0,
-            "[SEP]": 1,
-            "[BOS]": 2,
-            "[MASK]": 3,
-            "[PAD]": 4,
-            "[RESERVED]": 5,
-            "[UNK]": 6,
-            **{ch: i + 7 for i, ch in enumerate(characters)},
-        }
-        self._vocab_int_to_str = {v: k for k, v in self._vocab_str_to_int.items()}
-
     @property
     def vocab_size(self) -> int:
         return len(self._vocab_str_to_int)
 
+    def get_vocab(self):
+        return self._vocab_str_to_int
+
     def _tokenize(self, text: str) -> List[str]:
         return list(text)