UniTok 4.3.3__tar.gz → 4.3.5__tar.gz

{UniTok-4.3.3 → UniTok-4.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.3.3
+Version: 4.3.5
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.3.3 → UniTok-4.3.5}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.3.3
+Version: 4.3.5
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.3.3 → UniTok-4.3.5}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text(encoding='utf8')
 
 setup(
     name='UniTok',
-    version='4.3.3',
+    version='4.3.5',
     keywords=['token', 'tokenizer', 'NLP', 'transformers', 'glove', 'bert', 'llama'],
     description='Unified Tokenizer',
     long_description=long_description,

{UniTok-4.3.3 → UniTok-4.3.5}/unitok/unitok.py

@@ -276,6 +276,26 @@ class UniTok(Status):
             if job is not other.key_job:
                 self.data[job.name] = union_data[job.name]
 
+    @Status.require_not_initialized
+    @Status.to_organized
+    def replicate(self, job: Union[Job, str], new_name: str, lazy=False):
+        if isinstance(job, str):
+            job = self.meta.jobs[job]
+
+        if not job.is_processed:
+            raise ValueError(f'job {job.name} is not processed')
+
+        if job.from_union and self.is_soft_union:
+            raise ValueError(f'job {job.name} is from a soft union, please use hard union or save-and-load the unitok.')
+
+        new_job = job.clone(name=new_name)
+
+        if lazy or not job.return_list:
+            self.data[new_job.name] = self.data[job.name]
+        else:
+            # deep copy the data
+            self.data[new_job.name] = [line.copy() for line in self.data[job.name]]
+
     @Status.require_not_initialized
     def summarize(self):
         console = Console()