UniTok 3.1.4__tar.gz → 3.1.6__tar.gz

{UniTok-3.1.4 → UniTok-3.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.1.4
+Version: 3.1.6
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
@@ -138,3 +138,5 @@ news_dep = UniDep('data/news')  # 读取分词结果
 print(len(news_dep))
 print(news_dep[0])
 ```
+
+

{UniTok-3.1.4 → UniTok-3.1.6}/UniTok/column.py

@@ -65,7 +65,7 @@ class Column:
                 pad=kwargs.get('pad', 0),
             )
 
-        self.list = bool(operator) or tok.return_list is True  # whether the column is a list-element column
+        self.list = bool(self.operator) or tok.return_list is True  # whether the column is a list-element column
 
         self.data = []
         self.lengths = Lengths()

{UniTok-3.1.4 → UniTok-3.1.6}/UniTok/tok/bert_tok.py

@@ -30,8 +30,4 @@ class BertTok(BaseTok):
             ids = self.tokenizer.convert_tokens_to_ids(ts)
         else:
             ids = []
-
-        for index in ids:
-            self.vocab.count(index)
-
         return ids

{UniTok-3.1.4 → UniTok-3.1.6}/UniTok/tok/number_tok.py

@@ -40,5 +40,5 @@ class NumberTok(BaseTok):
             if o >= len(self.vocab):
                 if self.vocab_size is not None:
                     raise ValueError('vocab_size is {}, but {} is given'.format(self.vocab_size, o))
-                self.vocab.extend([str(i) for i in range(len(self.vocab), o + 1)])
+                self.vocab.extend([str(i) for i in range(len(self.vocab), o + 1)], count=False)
         return obj

{UniTok-3.1.4 → UniTok-3.1.6}/UniTok/tok/tok.py

@@ -44,6 +44,9 @@ class BaseTok:
         wrapped tokenize method, filter out unknown token
         """
         ids = self.t(obj)
+
+        self.vocab.counts(ids if isinstance(ids, list) else [ids])
+
         if isinstance(ids, list):
             return list(filter(lambda index: index > -1, ids))
         if ids == -1:

{UniTok-3.1.4 → UniTok-3.1.6}/UniTok/vocab.py

@@ -63,12 +63,17 @@ class Vocab:
 
     def append(self, obj):
         index = self._append(obj)
-        self.count(index)
         return index
 
-    def count(self, index):
-        if self._count_mode and index > -1:
-            self._counter[index] = self._counter.get(index, 0) + 1
+    def counts(self, indexes):
+        if self._count_mode:
+            for index in indexes:
+                if index > -1:
+                    self._counter[index] = self._counter.get(index, 0) + 1
+
+    # def count(self, index):
+    #     if self._count_mode and index > -1:
+    #         self._counter[index] = self._counter.get(index, 0) + 1
 
     def _append(self, obj):
         """

{UniTok-3.1.4 → UniTok-3.1.6}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.1.4
+Version: 3.1.6
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
@@ -138,3 +138,5 @@ news_dep = UniDep('data/news')  # 读取分词结果
 print(len(news_dep))
 print(news_dep[0])
 ```
+
+

{UniTok-3.1.4 → UniTok-3.1.6}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text()
 
 setup(
     name='UniTok',
-    version='3.1.4',
+    version='3.1.6',
     keywords=['token', 'tokenizer', 'bert'],
     description='Unified Tokenizer',
     long_description=long_description,