UniTok 3.0.3a0__tar.gz → 3.0.3b0__tar.gz

{UniTok-3.0.3a0 → UniTok-3.0.3b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.0.3a0
+Version: 3.0.3b0
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.0.3a0 → UniTok-3.0.3b0}/UniTok/unidep.py

@@ -1,14 +1,12 @@
 import json
 import os
 import random
+import warnings
 from typing import Dict, List, Callable
 
 import numpy as np
 import tqdm
 
-from .compatible.uni_warnings import MetaDataDeprecationWarning, VocabInfoDeprecationWarning, \
-    ColInfoDeprecationWarning, GetMaxLengthDeprecationWarning, GetVocabDeprecationWarning, \
-    GetVocabSizeDeprecationWarning, IsListColDeprecationWarning, ShuffleDeprecationWarning
 from .meta import Meta, Col, Voc
 from .vocab import Vocab
 from .vocabs import Vocabs
@@ -178,38 +176,41 @@ class UniDep:
 
     @property
     def meta_data(self):
-        MetaDataDeprecationWarning()
+        warnings.warn('meta_data is deprecated, '
+                      'use meta instead (will be removed in 4.x version)', DeprecationWarning)
         return self.meta
 
     @property
     def vocab_info(self):
-        VocabInfoDeprecationWarning()
+        warnings.warn('vocab_info is deprecated, '
+                      'use vocs instead (will be removed in 4.x version)', DeprecationWarning)
         return self.vocs
 
     @property
     def col_info(self):
-        ColInfoDeprecationWarning()
+        warnings.warn('col_info is deprecated, '
+                      'use cols instead (will be removed in 4.x version)', DeprecationWarning)
         return self.cols
 
     def get_vocab_size(self, col_name, as_vocab=False):
-        GetVocabSizeDeprecationWarning()
+        warnings.warn('unidep.get_vocab_size is deprecated (will be removed in 4.x version)', DeprecationWarning)
         vocab_id = col_name if as_vocab else self.get_vocab(col_name)
         return self.vocs[vocab_id].size
 
     def get_vocab(self, col_name):
-        GetVocabDeprecationWarning()
+        warnings.warn('unidep.get_vocab is deprecated (will be removed in 4.x version)', DeprecationWarning)
         return self.cols[col_name].voc.name
 
     def get_max_length(self, col_name):
-        GetMaxLengthDeprecationWarning()
+        warnings.warn('unidep.get_max_length is deprecated (will be removed in 4.x version)', DeprecationWarning)
         return self.cols[col_name].max_length
 
     def is_list_col(self, col_name):
-        IsListColDeprecationWarning()
+        warnings.warn('unidep.is_list_col is deprecated (will be removed in 4.x version)', DeprecationWarning)
         return self.cols[col_name].list
 
     def shuffle(self, shuffle=True):
-        ShuffleDeprecationWarning()
+        warnings.warn('unidep.shuffle is deprecated (will be removed in 4.x version)', DeprecationWarning)
         if shuffle:
             random.shuffle(self._indexes)
         else:

{UniTok-3.0.3a0 → UniTok-3.0.3b0}/UniTok/unitok.py

@@ -1,12 +1,12 @@
 import json
 import os
+import warnings
 from typing import Optional
 
 import numpy as np
 import pandas as pd
 
 from .cols import Cols
-from .compatible.uni_warnings import VocabDepotDeprecationWarning, GetTokPathDeprecationWarning
 from .column import Column, IndexColumn
 from .tok.bert_tok import BertTok
 from .tok.entity_tok import EntTok
@@ -29,7 +29,8 @@ class UniTok:
 
     @property
     def vocab_depots(self):
-        VocabDepotDeprecationWarning()
+        warnings.warn('vocab_depot is deprecated, '
+                      'use vocabs instead (will be removed in 4.x version)', DeprecationWarning)
         return self.vocabs
 
     def add_col(self, col: Column):
@@ -110,6 +111,9 @@ class UniTok:
         if not self.id_col:
             raise ValueError('id column is not set')
 
+        for vocab in self.vocabs.values():
+            vocab.set_count_mode(False)
+
         for col_name in self.cols:
             print('[ COL:', col_name, ']')
             col = self.cols[col_name]  # type: Column
@@ -121,7 +125,7 @@ class UniTok:
         """
         Get the store path of the tokenizer of a column
         """
-        GetTokPathDeprecationWarning()
+        warnings.warn('unitok.get_tok_path is deprecated (will be removed in 4.x version)', DeprecationWarning)
         return self.cols[col_name].tok.vocab.get_store_path(store_dir)
 
     def store_data(self, store_dir):

{UniTok-3.0.3a0 → UniTok-3.0.3b0}/UniTok/vocab.py

@@ -1,12 +1,10 @@
 import math
 import os
+import warnings
 from typing import Union, List
 
 import numpy as np
 
-from UniTok.compatible.uni_warnings import VocabMapDeprecationWarning, OOVDefaultDeprecationWarning, \
-    MinFrequencyDeprecationWarning
-
 
 class VocabMap(dict):
     def __call__(self, *args, **kwargs):
@@ -29,6 +27,7 @@ class Vocab:
 
         self._editable = True  # whether vocab is editable
         self._oov_token = None  # out of vocabulary token
+        self._stable_mode = False
 
         self._count_mode = False  # whether count mode is on
         self._counter = {}  # counter for counting occurrence of each token
@@ -48,7 +47,8 @@ class Vocab:
         """
         Deprecated, use o2i instead
         """
-        VocabMapDeprecationWarning()
+        warnings.warn('vocab.index2obj and vocab.obj2index are deprecated, '
+                      'use vocab.i2o and vocab.o2i instead (will be removed in 4.x version)', DeprecationWarning)
         return self.o2i
 
     @property
@@ -56,7 +56,8 @@ class Vocab:
         """
         Deprecated, use i2o instead
         """
-        VocabMapDeprecationWarning()
+        warnings.warn('vocab.index2obj and vocab.obj2index are deprecated, '
+                      'use vocab.i2o and vocab.o2i instead (will be removed in 4.x version)', DeprecationWarning)
         return self.i2o
 
     def extend(self, objs):
@@ -80,8 +81,10 @@ class Vocab:
         if obj in self.o2i:
             return self.o2i[obj]
 
-        if self._count_mode:
-            return self._oov_token or -1
+        if self._stable_mode:
+            if self._oov_token is not None:
+                return self._oov_token
+            return -1
 
         if not self._editable:
             if self._oov_token is not None:
@@ -127,7 +130,8 @@ class Vocab:
 
     @property
     def oov_default(self):
-        OOVDefaultDeprecationWarning()
+        warnings.warn('vocab.oov_default is deprecated, '
+                      'use vocab.oov_token instead (will be removed in 4.x version)', DeprecationWarning)
         return self._oov_token
 
     def allow_edit(self):
@@ -189,7 +193,8 @@ class Vocab:
         :return:
         """
         if min_count is None:
-            MinFrequencyDeprecationWarning()
+            warnings.warn('vocab.min_frequency is deprecated, '
+                          'use vocab.min_count instead (will be removed in 4.x version)', DeprecationWarning)
             min_count = min_frequency
 
         vocabs = []
@@ -205,7 +210,7 @@ class Vocab:
             self.reserve(self.reserved_tokens)
         self.extend(vocabs)
 
-        # self.frequency_mode = True
+        self._stable_mode = True
         return self
 
     def summarize(self, base=10):
@@ -258,6 +263,12 @@ class Vocab:
 
     def __getattr__(self, item):
         if item in ['frequency_mode', 'frequency', 'max_frequency', 'frequent_vocab']:
-            return AttributeError(f'{item} is deprecated after UniTok 3.0, '
-                                  f'degrade to 2.4.3.2 or lower to use it, '
-                                  f'or check new features of Vocab class')
+            raise AttributeError(f'{item} is deprecated after UniTok 3.0, '
+                                 f'degrade to 2.4.3.2 or lower to use it, '
+                                 f'or check new features of Vocab class')
+
+    @property
+    def trim_vocab(self):
+        warnings.warn('vocab.trim_vocab is deprecated, '
+                      'use vocab.trim instead (will be removed in 4.x version)', DeprecationWarning)
+        return self.trim

{UniTok-3.0.3a0 → UniTok-3.0.3b0}/UniTok/vocabs.py

@@ -1,4 +1,5 @@
-from .compatible.uni_warnings import ColMapDeprecationWarning, DepotsDeprecationWarning
+import warnings
+
 from .vocab import Vocab
 
 
@@ -9,12 +10,14 @@ class Vocabs(dict):
 
     @property
     def col_map(self):
-        ColMapDeprecationWarning()
+        warnings.warn('vocab_depot.col_map is deprecated, '
+                      'use vocabs.cols instead (will be removed in 4.x version)', DeprecationWarning)
         return self.cols
 
     @property
     def depots(self):
-        DepotsDeprecationWarning()
+        warnings.warn('vocab_depot.depots is deprecated, '
+                      'use vocabs instead (will be removed in 4.x version)', DeprecationWarning)
         return self
 
     def append(self, col_or_vocab):

{UniTok-3.0.3a0 → UniTok-3.0.3b0}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.0.3a0
+Version: 3.0.3b0
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.0.3a0 → UniTok-3.0.3b0}/UniTok.egg-info/SOURCES.txt

@@ -17,8 +17,6 @@ UniTok.egg-info/top_level.txt
 UniTok/analysis/__init__.py
 UniTok/analysis/lengths.py
 UniTok/analysis/plot.py
-UniTok/compatible/__init__.py
-UniTok/compatible/uni_warnings.py
 UniTok/tok/__init__.py
 UniTok/tok/bert_tok.py
 UniTok/tok/entity_tok.py

{UniTok-3.0.3a0 → UniTok-3.0.3b0}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text()
 
 setup(
     name='UniTok',
-    version='3.0.3.alpha',
+    version='3.0.3.beta',
     keywords=['token', 'tokenizer', 'bert'],
     description='Unified Tokenizer',
     long_description=long_description,

UniTok-3.0.3a0/UniTok/compatible/__init__.py

@@ -1,21 +0,0 @@
-from .uni_warnings import VocabDepotDeprecationWarning, GetTokPathDeprecationWarning, \
-    VocabMapDeprecationWarning, OOVDefaultDeprecationWarning, MinFrequencyDeprecationWarning, \
-    ColMapDeprecationWarning, DepotsDeprecationWarning, MetaDataDeprecationWarning, \
-    VocabInfoDeprecationWarning, ColInfoDeprecationWarning, GetMaxLengthDeprecationWarning, \
-    GetVocabDeprecationWarning
-
-
-__all__ = [
-    'VocabDepotDeprecationWarning',
-    'GetTokPathDeprecationWarning',
-    'VocabMapDeprecationWarning',
-    'OOVDefaultDeprecationWarning',
-    'MinFrequencyDeprecationWarning',
-    'ColMapDeprecationWarning',
-    'DepotsDeprecationWarning',
-    'MetaDataDeprecationWarning',
-    'VocabInfoDeprecationWarning',
-    'ColInfoDeprecationWarning',
-    'GetMaxLengthDeprecationWarning',
-    'GetVocabDeprecationWarning',
-]

UniTok-3.0.3a0/UniTok/compatible/uni_warnings.py

@@ -1,70 +0,0 @@
-import warnings
-from typing import Callable
-
-warned_flags = set()
-
-
-class UniWarning:
-    def __init__(self, msg, type_: Callable = warnings.warn):
-        self.msg = msg
-        self.type = type_
-
-    def __call__(self, *args, **kwargs):
-        if self not in warned_flags:
-            warned_flags.add(self)
-            self.type(self.msg.format(*args, **kwargs))
-
-
-VocabMapDeprecationWarning = UniWarning(
-    'vocab.index2obj and vocab.obj2index are deprecated, '
-    'use vocab.i2o and vocab.o2i instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-OOVDefaultDeprecationWarning = UniWarning(
-    'vocab.oov_default is deprecated, '
-    'use vocab.oov_token instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-MinFrequencyDeprecationWarning = UniWarning(
-    'vocab.min_frequency is deprecated, '
-    'use vocab.min_count instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-VocabDepotDeprecationWarning = UniWarning(
-    'vocab_depot is deprecated, '
-    'use vocabs instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-ColMapDeprecationWarning = UniWarning(
-    'vocab_depot.col_map is deprecated, '
-    'use vocabs.cols instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-GetTokPathDeprecationWarning = UniWarning(
-    'unitok.get_tok_path is deprecated (will be removed in 4.x version)', type_=DeprecationWarning)
-
-DepotsDeprecationWarning = UniWarning(
-    'vocab_depot.depots is deprecated, '
-    'use vocabs instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-MetaDataDeprecationWarning = UniWarning(
-    'meta_data is deprecated, '
-    'use meta instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-VocabInfoDeprecationWarning = UniWarning(
-    'vocab_info is deprecated, '
-    'use vocs instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-ColInfoDeprecationWarning = UniWarning(
-    'col_info is deprecated, '
-    'use cols instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-GetMaxLengthDeprecationWarning = UniWarning(
-    'unidep.get_max_length is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)
-
-GetVocabDeprecationWarning = UniWarning(
-    'unidep.get_vocab is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)
-
-GetVocabSizeDeprecationWarning = UniWarning(
-    'unidep.get_vocab_size is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)
-
-IsListColDeprecationWarning = UniWarning(
-    'unidep.is_list_col is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)
-
-ShuffleDeprecationWarning = UniWarning(
-    'unidep.shuffle is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)