UniTok 4.3.4__tar.gz → 4.3.5__tar.gz

{UniTok-4.3.4 → UniTok-4.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.3.4
+Version: 4.3.5
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.3.4 → UniTok-4.3.5}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.3.4
+Version: 4.3.5
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.3.4 → UniTok-4.3.5}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text(encoding='utf8')
 
 setup(
     name='UniTok',
-    version='4.3.4',
+    version='4.3.5',
     keywords=['token', 'tokenizer', 'NLP', 'transformers', 'glove', 'bert', 'llama'],
     description='Unified Tokenizer',
     long_description=long_description,

{UniTok-4.3.4 → UniTok-4.3.5}/unitok/unitok.py

@@ -278,7 +278,7 @@ class UniTok(Status):
 
     @Status.require_not_initialized
     @Status.to_organized
-    def replicate(self, job: Union[Job, str], new_name: str):
+    def replicate(self, job: Union[Job, str], new_name: str, lazy=False):
         if isinstance(job, str):
             job = self.meta.jobs[job]
 
@@ -289,7 +289,12 @@ class UniTok(Status):
             raise ValueError(f'job {job.name} is from a soft union, please use hard union or save-and-load the unitok.')
 
         new_job = job.clone(name=new_name)
-        self.data[new_job.name] = self.data[job.name]
+
+        if lazy or not job.return_list:
+            self.data[new_job.name] = self.data[job.name]
+        else:
+            # deep copy the data
+            self.data[new_job.name] = [line.copy() for line in self.data[job.name]]
 
     @Status.require_not_initialized
     def summarize(self):