UniTok 4.3.5__tar.gz → 4.3.7__tar.gz

{UniTok-4.3.5 → UniTok-4.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.3.5
+Version: 4.3.7
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.3.5 → UniTok-4.3.7}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.3.5
+Version: 4.3.7
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.3.5 → UniTok-4.3.7}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text(encoding='utf8')
 
 setup(
     name='UniTok',
-    version='4.3.5',
+    version='4.3.7',
     keywords=['token', 'tokenizer', 'NLP', 'transformers', 'glove', 'bert', 'llama'],
     description='Unified Tokenizer',
     long_description=long_description,

{UniTok-4.3.5 → UniTok-4.3.7}/unitok/tokenizer/transformers_tokenizer.py

@@ -56,6 +56,15 @@ class TransformersTokenizer(BaseTokenizer):
             self.vocab.counter(token)
         return tokens
 
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state['tokenizer'] = None
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        self.tokenizer = AutoTokenizer.from_pretrained(self.key, **self.kwargs)
+
 
 class BertTokenizer(TransformersTokenizer):
     param_list = []

{UniTok-4.3.5 → UniTok-4.3.7}/unitok/unitok.py

@@ -289,6 +289,7 @@ class UniTok(Status):
             raise ValueError(f'job {job.name} is from a soft union, please use hard union or save-and-load the unitok.')
 
         new_job = job.clone(name=new_name)
+        self.meta.jobs.add(new_job)
 
         if lazy or not job.return_list:
             self.data[new_job.name] = self.data[job.name]