UniTok 3.0.3a0__tar.gz → 3.0.4a0__tar.gz

{UniTok-3.0.3a0 → UniTok-3.0.4a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.0.3a0
+Version: 3.0.4a0
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.0.3a0 → UniTok-3.0.4a0}/UniTok/vocab.py

@@ -29,6 +29,7 @@ class Vocab:
 
         self._editable = True  # whether vocab is editable
         self._oov_token = None  # out of vocabulary token
+        self._stable_mode = False
 
         self._count_mode = False  # whether count mode is on
         self._counter = {}  # counter for counting occurrence of each token
@@ -80,8 +81,10 @@ class Vocab:
         if obj in self.o2i:
             return self.o2i[obj]
 
-        if self._count_mode:
-            return self._oov_token or -1
+        if self._stable_mode:
+            if self._oov_token is not None:
+                return self._oov_token
+            return -1
 
         if not self._editable:
             if self._oov_token is not None:
@@ -205,7 +208,7 @@ class Vocab:
             self.reserve(self.reserved_tokens)
         self.extend(vocabs)
 
-        # self.frequency_mode = True
+        self._stable_mode = True
         return self
 
     def summarize(self, base=10):

{UniTok-3.0.3a0 → UniTok-3.0.4a0}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.0.3a0
+Version: 3.0.4a0
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.0.3a0 → UniTok-3.0.4a0}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text()
 
 setup(
     name='UniTok',
-    version='3.0.3.alpha',
+    version='3.0.4.alpha',
     keywords=['token', 'tokenizer', 'bert'],
     description='Unified Tokenizer',
     long_description=long_description,