UniTok 4.4.0__tar.gz → 4.4.2__tar.gz

{UniTok-4.4.0 → UniTok-4.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.4.0
+Version: 4.4.2
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
@@ -18,6 +18,7 @@ Requires-Dist: transformers
 Requires-Dist: oba
 Requires-Dist: prettytable
 Requires-Dist: rich
+Requires-Dist: fastparquet
 
 # UniTok V4
 

{UniTok-4.4.0 → UniTok-4.4.2}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.4.0
+Version: 4.4.2
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
@@ -18,6 +18,7 @@ Requires-Dist: transformers
 Requires-Dist: oba
 Requires-Dist: prettytable
 Requires-Dist: rich
+Requires-Dist: fastparquet
 
 # UniTok V4
 

{UniTok-4.4.0 → UniTok-4.4.2}/UniTok.egg-info/requires.txt

@@ -6,3 +6,4 @@ transformers
 oba
 prettytable
 rich
+fastparquet

{UniTok-4.4.0 → UniTok-4.4.2}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text(encoding='utf8')
 
 setup(
     name='UniTok',
-    version='4.4.0',
+    version='4.4.2',
     keywords=['token', 'tokenizer', 'NLP', 'transformers', 'glove', 'bert', 'llama'],
     description='Unified Tokenizer',
     long_description=long_description,
@@ -25,7 +25,8 @@ setup(
         'transformers',
         'oba',
         'prettytable',
-        'rich'
+        'rich',
+        'fastparquet'
     ],
     entry_points={
         'console_scripts': [

{UniTok-4.4.0 → UniTok-4.4.2}/unitok/job.py

@@ -4,7 +4,7 @@ from unitok.feature import Feature, FeatureHub
 
 class Job(Feature):
     def __init__(self, **kwargs):
-        warnings.deprecated(f'Job is deprecated, use Feature instead.')
+        warnings.warn(f'`Job` class is deprecated, use `Feature`.', DeprecationWarning, stacklevel=2)
         super().__init__(**kwargs)
 
 

{UniTok-4.4.0 → UniTok-4.4.2}/unitok/meta.py

@@ -27,6 +27,11 @@ class Meta:
         self.tokenizers = TokenizerSet()
         self.features = FeatureSet()
 
+    @property
+    def jobs(self):
+        warnings.warn('`jobs` is deprecated, use `features` instead.', DeprecationWarning, stacklevel=2)
+        return self.features
+
     @staticmethod
     def parse_vocabulary(name: str, **kwargs):
         return Vocab(name)
@@ -66,7 +71,7 @@ class Meta:
 
     @staticmethod
     def parse_job(name: str, column: str, tokenizer: str, truncate: int, order: int, key: bool, max_len: int):
-        warnings.deprecated('`parse_job` is deprecated, use `parse_feature` instead.', stacklevel=2)
+        warnings.warn('`parse_job` is deprecated, use `parse_feature` instead.', DeprecationWarning, stacklevel=2)
         return Meta.parse_feature(name, column, tokenizer, truncate, order, key, max_len)
 
     @staticmethod

{UniTok-4.4.0 → UniTok-4.4.2}/unitok/unitok.py

@@ -41,12 +41,12 @@ class UniTok(Status):
 
     @property
     def key_job(self):
-        warnings.deprecated('key_job is deprecated, use key_feat instead', stacklevel=2)
+        warnings.warn('key_job is deprecated, use key_feat instead', DeprecationWarning, stacklevel=2)
         return self.key_feature
 
     @key_job.setter
     def key_job(self, value):
-        warnings.deprecated('key_job is deprecated, use key_feat instead', stacklevel=2)
+        warnings.warn('key_job is deprecated, use key_feat instead', DeprecationWarning, stacklevel=2)
         self.key_feature = value
 
     @property
@@ -130,7 +130,7 @@ class UniTok(Status):
         )
 
     def add_index_job(self, name: str = 'index', tokenizer: DigitTokenizer = None):
-        warnings.deprecated('`add_index_job` is deprecated, use `add_job` instead', stacklevel=2)
+        warnings.warn('`add_index_job` is deprecated, use `add_job` instead', DeprecationWarning, stacklevel=2)
         return self.add_index_feature(name=name, tokenizer=tokenizer)
 
     def add_job(
@@ -141,7 +141,7 @@ class UniTok(Status):
             truncate: int = None,
             key: bool = False,
     ):
-        warnings.deprecated('`add_job` is deprecated, use `add_feature` instead', stacklevel=2)
+        warnings.warn('`add_job` is deprecated, use `add_feature` instead', DeprecationWarning, stacklevel=2)
         return self.add_feature(
             tokenizer=tokenizer,
             column=column,
@@ -543,5 +543,5 @@ class UniTok(Status):
             self.data.pop(feature.name)
 
     def remove_job(self, feature: Union[Feature, str]):
-        warnings.deprecated(f'`remove_job` is deprecated, use `remove_feature` instead.', stacklevel=2)
+        warnings.warn(f'`remove_job` is deprecated, use `remove_feature` instead.', DeprecationWarning, stacklevel=2)
         self.remove_feature(feature)