UniTok 4.0.3__tar.gz → 4.3.0__tar.gz

{UniTok-4.0.3 → UniTok-4.3.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.0.3
+Version: 4.3.0
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
 Author-email: liu@qijiong.work
 License: MIT Licence
-Keywords: token,tokenizer
+Keywords: token,tokenizer,NLP,transformers,glove,bert,llama
 Platform: any
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -29,17 +29,18 @@ UniTok is designed to simplify preprocessing by offering reusable components suc
 
 ### Changes and Comparisons
 
-| Feature           | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
-|-------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
-| `UniTok` class    | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
-| `UniDep` class    | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
-| `Column` class    | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
-| `Job` class       | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
-| `Tokenizer` class | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
-| `Tokenizer` class | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
-| `analyse` method  | Supported                                                   | Not supported Currently                             |                                                                               |
-| `Meta` class      | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
-| `unitok` command  | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               | 
+| Feature                         | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
+|---------------------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
+| `UniTok` class                  | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
+| `UniDep` class                  | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
+| `Column` class                  | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
+| `Job` class                     | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
+| `Tokenizer` class               | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
+| `Tokenizer` class               | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
+| `analyse` method                | Supported                                                   | Not supported Currently                             |                                                                               |
+| `Meta` class                    | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
+| `unitok` command                | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               |
+| `Vocab` class (unitok >= 4.1.0) | Save and load vocabulary using text files                   | Save and load vocabulary using pickle files         | Avoids issues with special characters in text files                           |
 
 ### How to Migrate the Processed Data
 

{UniTok-4.0.3 → UniTok-4.3.0}/README.md

@@ -16,17 +16,18 @@ UniTok is designed to simplify preprocessing by offering reusable components suc
 
 ### Changes and Comparisons
 
-| Feature           | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
-|-------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
-| `UniTok` class    | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
-| `UniDep` class    | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
-| `Column` class    | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
-| `Job` class       | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
-| `Tokenizer` class | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
-| `Tokenizer` class | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
-| `analyse` method  | Supported                                                   | Not supported Currently                             |                                                                               |
-| `Meta` class      | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
-| `unitok` command  | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               | 
+| Feature                         | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
+|---------------------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
+| `UniTok` class                  | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
+| `UniDep` class                  | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
+| `Column` class                  | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
+| `Job` class                     | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
+| `Tokenizer` class               | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
+| `Tokenizer` class               | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
+| `analyse` method                | Supported                                                   | Not supported Currently                             |                                                                               |
+| `Meta` class                    | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
+| `unitok` command                | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               |
+| `Vocab` class (unitok >= 4.1.0) | Save and load vocabulary using text files                   | Save and load vocabulary using pickle files         | Avoids issues with special characters in text files                           |
 
 ### How to Migrate the Processed Data
 

{UniTok-4.0.3 → UniTok-4.3.0}/UniTok.egg-info/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.0.3
+Version: 4.3.0
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
 Author-email: liu@qijiong.work
 License: MIT Licence
-Keywords: token,tokenizer
+Keywords: token,tokenizer,NLP,transformers,glove,bert,llama
 Platform: any
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -29,17 +29,18 @@ UniTok is designed to simplify preprocessing by offering reusable components suc
 
 ### Changes and Comparisons
 
-| Feature           | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
-|-------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
-| `UniTok` class    | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
-| `UniDep` class    | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
-| `Column` class    | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
-| `Job` class       | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
-| `Tokenizer` class | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
-| `Tokenizer` class | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
-| `analyse` method  | Supported                                                   | Not supported Currently                             |                                                                               |
-| `Meta` class      | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
-| `unitok` command  | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               | 
+| Feature                         | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
+|---------------------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
+| `UniTok` class                  | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
+| `UniDep` class                  | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
+| `Column` class                  | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
+| `Job` class                     | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
+| `Tokenizer` class               | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
+| `Tokenizer` class               | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
+| `analyse` method                | Supported                                                   | Not supported Currently                             |                                                                               |
+| `Meta` class                    | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
+| `unitok` command                | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               |
+| `Vocab` class (unitok >= 4.1.0) | Save and load vocabulary using text files                   | Save and load vocabulary using pickle files         | Avoids issues with special characters in text files                           |
 
 ### How to Migrate the Processed Data
 

{UniTok-4.0.3 → UniTok-4.3.0}/UniTok.egg-info/SOURCES.txt

@@ -38,9 +38,9 @@ unitok/status.py
 unitok/unitok.py
 unitok/tokenizer/__init__.py
 unitok/tokenizer/base_tokenizer.py
-unitok/tokenizer/cachable_tokenizer.py
 unitok/tokenizer/digit_tokenizer.py
 unitok/tokenizer/entity_tokenizer.py
+unitok/tokenizer/glove_tokenizer.py
 unitok/tokenizer/split_tokenizer.py
 unitok/tokenizer/transformers_tokenizer.py
 unitok/tokenizer/union_tokenizer.py

{UniTok-4.0.3 → UniTok-4.3.0}/setup.py

@@ -6,8 +6,8 @@ long_description = (this_directory / "README.md").read_text(encoding='utf8')
 
 setup(
     name='UniTok',
-    version='4.0.3',
-    keywords=['token', 'tokenizer'],
+    version='4.3.0',
+    keywords=['token', 'tokenizer', 'NLP', 'transformers', 'glove', 'bert', 'llama'],
     description='Unified Tokenizer',
     long_description=long_description,
     long_description_content_type='text/markdown',

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/__init__.py

@@ -9,6 +9,7 @@ from unitok.tokenizer import BaseTokenizer, TokenizerHub
 from unitok.tokenizer import EntityTokenizer, EntitiesTokenizer
 from unitok.tokenizer import TransformersTokenizer, BertTokenizer
 from unitok.tokenizer import SplitTokenizer, DigitTokenizer, DigitsTokenizer
+from unitok.tokenizer import GloVeTokenizer
 from unitok.job import Job, JobHub
 
 from unitok.utils.index_set import IndexSet, VocabSet, TokenizerSet, JobSet
@@ -29,6 +30,7 @@ __all__ = [
     'EntityTokenizer', 'EntitiesTokenizer',
     'TransformersTokenizer', 'BertTokenizer',
     'SplitTokenizer', 'DigitTokenizer', 'DigitsTokenizer',
+    'GloVeTokenizer',
     'Job', 'JobHub',
     'IndexSet', 'VocabSet', 'TokenizerSet', 'JobSet',
     'Meta',

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/__main__.py

@@ -1,7 +1,9 @@
 import argparse
 
 import pandas as pd
+from pigmento import pnt
 
+from unitok import Vocab
 from unitok.tokenizer import BaseTokenizer
 from unitok.unitok import UniTok
 from unitok.utils.class_pool import ClassPool
@@ -29,7 +31,7 @@ def integrate():
         if arg.startswith('--t.'):
             current_param = arg[4:]
         elif arg.startswith('--tokenizer.'):
-            current_param = arg[11:]
+            current_param = arg[12:]
 
     if args.file.endswith('.csv') or args.file.endswith('.tsv'):
         df = pd.read_csv(args.file, sep='\t')
@@ -39,17 +41,32 @@ def integrate():
         raise ValueError(f'Unsupported file format: {args.file}')
 
     with UniTok.load(args.path, tokenizer_lib=args.lib) as ut:
+        tokenizer = None
+
         if args.tokenizer_id:
             for t in ut.meta.tokenizers:  # type: BaseTokenizer
                 if t.get_tokenizer_id() == args.tokenizer_id:
                     tokenizer = t
                     break
             else:
-                raise ValueError(f'Unknown tokenizer id: {args.tokenizer_id}')
-        else:
-            assert args.tokenizer is not None and args.vocab is not None, 'Tokenizer classname and vocabulary must be specified'
+                pnt(f'Unknown tokenizer id: {args.tokenizer_id}, will create a new tokenizer')
+                tokenizer_params['tokenizer_id'] = args.tokenizer_id
+
+        if not tokenizer:
+            if args.tokenizer is None and args.vocab is None:
+                raise ValueError('Tokenizer classname and vocabulary must be specified')
+
+            if args.vocab.endswith('.vocab'):
+                if '/' in args.vocab:
+                    vocab_path, vocab_name = args.vocab.rsplit('/', 1)
+                else:
+                    vocab_path, vocab_name = '.', args.vocab
+                vocab_name = vocab_name[:-6]
+                args.vocab = Vocab(vocab_name).load(vocab_path)
+
             tokenizers = ClassPool.tokenizers(args.lib)
-            assert args.tokenizer in tokenizers, f'Unknown tokenizer: {args.tokenizer}. Available tokenizers: {tokenizers.keys()}'
+            if args.tokenizer not in tokenizers:
+                raise ValueError(f'Unknown tokenizer: {args.tokenizer}. Available tokenizers: {tokenizers.keys()}')
             tokenizer = tokenizers[args.tokenizer](vocab=args.vocab, **tokenizer_params)
 
         ut.add_job(tokenizer=tokenizer, column=args.column, name=args.name, truncate=args.truncate)
@@ -65,14 +82,27 @@ def summarize():
         ut.summarize()
 
 
+def remove():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('path', type=str, default='.', help='path to a unitok data directory')
+    parser.add_argument('--name', type=str, help='job name to remove')
+    args, _ = parser.parse_known_args()
+
+    with UniTok.load(args.path) as ut:
+        ut.remove_job(args.name)
+        ut.save(args.path)
+
+
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('--action', '-a', type=str, default='summarize', choices=['summarize', 'integrate'])
+    parser.add_argument('--action', '-a', type=str, default='summarize', choices=['summarize', 'integrate', 'remove'])
 
     args, _ = parser.parse_known_args()
     action = args.action
 
     if action == 'integrate':
         integrate()
+    elif action == 'remove':
+        remove()
     else:
         summarize()

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/job.py

@@ -1,3 +1,6 @@
+from typing import Union
+
+from unitok import TokenizerHub, VocabHub
 from unitok.tokenizer.union_tokenizer import UnionTokenizer
 
 from unitok.tokenizer import BaseTokenizer
@@ -8,7 +11,7 @@ from unitok.utils.hub import Hub
 class Job:
     def __init__(
             self,
-            tokenizer: BaseTokenizer,
+            tokenizer: Union[BaseTokenizer, str],
             column: str,
             name: str = None,
             truncate: int = None,
@@ -16,7 +19,13 @@ class Job:
             key: bool = False,
             max_len: int = 0,
     ):
+        if isinstance(tokenizer, str):
+            if TokenizerHub.has(tokenizer):
+                tokenizer = TokenizerHub.get(tokenizer)
+            else:
+                raise ValueError(f"Tokenizer {tokenizer} not found in the tokenizer hub.")
         self.tokenizer: BaseTokenizer = tokenizer
+
         self.column: str = column
         self.name: str = name
         self.truncate: int = truncate
@@ -26,7 +35,8 @@ class Job:
         self.max_len = max_len
         self.from_union = isinstance(self.tokenizer, UnionTokenizer)
 
-        JobHub.add(self.name, self)
+        JobHub.add(self)
+        VocabHub.add(self.tokenizer.vocab)
 
     @property
     def return_list(self):
@@ -77,3 +87,8 @@ class Job:
 
 class JobHub(Hub[Job]):
     _instance = Instance(compulsory_space=True)
+
+    @classmethod
+    def add(cls, key, obj: Job = None):
+        key, obj = key.name, key
+        return super().add(key, obj)

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/meta.py

@@ -15,7 +15,7 @@ from unitok.vocabulary import Vocab, VocabHub
 
 
 class Meta:
-    version = 'unidep-v4beta'
+    version = 'unidep-v4'
 
     def __init__(self):
         self.note = ('Not compatible with unitok-v3 or lower version, '
@@ -116,6 +116,7 @@ class Meta:
         meta.vocabularies = VocabSet({cls.parse_vocabulary(**v).load(save_dir) for v in kwargs.get('vocabularies')})
         meta.tokenizers = TokenizerSet({cls.parse_tokenizer(**t) for t in kwargs.get('tokenizers')})
         meta.jobs = JobSet({cls.parse_job(**j) for j in kwargs.get('jobs')})
+        meta.version = kwargs.get('version')
 
         return meta
 

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/tokenizer/__init__.py

@@ -1,6 +1,6 @@
 from unitok.tokenizer.base_tokenizer import BaseTokenizer, TokenizerHub
-from unitok.tokenizer.cachable_tokenizer import CachableTokenizer
 from unitok.tokenizer.entity_tokenizer import EntityTokenizer, EntitiesTokenizer
+from unitok.tokenizer.glove_tokenizer import GloVeTokenizer
 from unitok.tokenizer.transformers_tokenizer import TransformersTokenizer, BertTokenizer
 from unitok.tokenizer.split_tokenizer import SplitTokenizer
 from unitok.tokenizer.digit_tokenizer import DigitTokenizer, DigitsTokenizer
@@ -8,7 +8,6 @@ from unitok.tokenizer.digit_tokenizer import DigitTokenizer, DigitsTokenizer
 
 __all__ = [
     BaseTokenizer,
-    CachableTokenizer,
     EntityTokenizer,
     EntitiesTokenizer,
     TransformersTokenizer,
@@ -16,5 +15,6 @@ __all__ = [
     SplitTokenizer,
     DigitTokenizer,
     DigitsTokenizer,
-    TokenizerHub
+    GloVeTokenizer,
+    TokenizerHub,
 ]

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/tokenizer/base_tokenizer.py

@@ -28,7 +28,8 @@ class BaseTokenizer(abc.ABC):
 
         self._tokenizer_id = tokenizer_id
 
-        TokenizerHub.add(self.get_tokenizer_id(), self)
+        TokenizerHub.add(self)
+        VocabHub.add(self.vocab)
 
     def get_tokenizer_id(self):
         if self._tokenizer_id is None:
@@ -81,3 +82,8 @@ class BaseTokenizer(abc.ABC):
 
 class TokenizerHub(Hub[BaseTokenizer]):
     _instance = Instance()
+
+    @classmethod
+    def add(cls, key, obj: BaseTokenizer = None):
+        key, obj = key.get_tokenizer_id(), key
+        return super().add(key, obj)

UniTok-4.3.0/unitok/tokenizer/glove_tokenizer.py

@@ -0,0 +1,21 @@
+import nltk
+
+from unitok.vocabulary import VocabHub
+from unitok.tokenizer import BaseTokenizer
+
+
+class GloVeTokenizer(BaseTokenizer):
+    return_list = True
+    param_list = ['language']
+
+    def __init__(self, vocab, language='english', **kwargs):
+        if isinstance(vocab, str) and not VocabHub.has(vocab):
+            raise ValueError('GloVeTokenizer requires a pre-filled Vocab object that stores valid tokens')
+
+        super().__init__(vocab=vocab, **kwargs)
+
+        self.language = language
+
+    def __call__(self, obj):
+        objs = nltk.tokenize.word_tokenize(obj.lower(), language=self.language)
+        return [self.vocab[o] for o in objs if o in self.vocab]

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/tokenizer/transformers_tokenizer.py

@@ -1,31 +1,37 @@
 from typing import Union
 
+from pigmento import pnt
 from transformers import AutoTokenizer
 
-from UniTokv3.vocab import Vocab
-from unitok.tokenizer import CachableTokenizer
+from unitok.vocabulary import Vocab
+from unitok.tokenizer import BaseTokenizer
 
 
-class TransformersTokenizer(CachableTokenizer):
+class TransformersTokenizer(BaseTokenizer):
     return_list = True
-    param_list = ['key']
 
-    def __init__(self, vocab: Union[str, Vocab], tokenizer_id: str = None, use_cache=False, key: str = None, **kwargs):
-        super().__init__(vocab=vocab, tokenizer_id=tokenizer_id, use_cache=use_cache)
+    def __init__(self, vocab: Union[str, Vocab], tokenizer_id: str = None, key: str = None, **kwargs):
+        super().__init__(vocab=vocab, tokenizer_id=tokenizer_id)
         self.key = key
 
         self.kwargs = kwargs
+        self.param_list = ['key']
         self.param_list.extend(list(kwargs.keys()))
 
         self.tokenizer = AutoTokenizer.from_pretrained(self.key, **self.kwargs)
         self.vocab.extend(self._generate_token_list())
 
     def _generate_token_list(self):
+        if not hasattr(self.tokenizer, 'vocab'):
+            pnt(f'transformer({self.key}): does not provide vocabulary, generating placeholders instead')
+            return list(range(self.tokenizer.vocab_size))
+
         tokens = self.tokenizer.vocab
         if isinstance(tokens, list):
             return tokens
         if not isinstance(tokens, dict):
-            raise ValueError(f'transformer({self.key}): unsupported type of vocabulary')
+            pnt(f'transformer({self.key}): unsupported type of vocabulary, generating placeholders instead')
+            return list(range(self.tokenizer.vocab_size))
 
         num_tokens = len(tokens)
         token_ids = list(tokens.values())
@@ -45,11 +51,15 @@ class TransformersTokenizer(CachableTokenizer):
 
     def __call__(self, obj):
         tokens = self.tokenizer.tokenize(obj)
-        return super().__call__(tokens)
+        tokens = self.tokenizer.convert_tokens_to_ids(tokens)
+        for token in tokens:
+            self.vocab.counter(token)
+        return tokens
 
 
 class BertTokenizer(TransformersTokenizer):
     param_list = []
 
     def __init__(self, **kwargs):
+        kwargs.pop('key', None)
         super().__init__(key='bert-base-uncased', **kwargs)

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/unitok.py

@@ -173,7 +173,7 @@ class UniTok(Status):
             if tokenizer.return_list:
                 raise AttributeError('Column content of the key job should be tokenized into atomic value')
             if self.key_job:
-                raise ValueError(f'key key already exists: {self.key_job.name}')
+                raise ValueError(f'Key column already exists: {self.key_job.name}')
             self.key_job = job
 
     @Status.require_not_organized
@@ -282,7 +282,10 @@ class UniTok(Status):
 
         # Prepare introduction header
         introduction_header = Text.assemble(
-            (f"UniTok ({self.meta.parse_version(self.meta.version)})\n", "bold cyan"),
+            (
+                f"UniTok (v{self.meta.parse_version(Meta.version)}), "
+                f"Data (v{self.meta.parse_version(self.meta.version)})\n",
+                "bold cyan"),
             (f"Sample Size: {self._sample_size}\n", "green"),
             (f"ID Column: {self.key_job.name}\n", "magenta"),
             style="dim"
@@ -462,3 +465,28 @@ class UniTok(Status):
 
         job.max_len = max_len
         self.data[job.name] = series
+
+    def remove_job(self, job: Union[Job, str]):
+        if isinstance(job, str):
+            job = self.meta.jobs[job]
+
+        if job.key:
+            raise ValueError('key job cannot be removed')
+
+        self.meta.jobs.remove(job)
+
+        tokenizer = job.tokenizer
+        for j in self.meta.jobs:
+            if j.tokenizer == tokenizer:
+                break
+        else:
+            self.meta.tokenizers.remove(tokenizer)
+            vocab = tokenizer.vocab
+            for t in self.meta.tokenizers:
+                if t.vocab == vocab:
+                    break
+            else:
+                self.meta.vocabularies.remove(vocab)
+
+        if job.is_processed:
+            self.data.pop(job.name)

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/utils/handler/pkl_handler.py

@@ -14,6 +14,6 @@ class PickleHandler:
         return pickle.load(open(path, "rb"))
 
     @staticmethod
-    def save(data: dict, path: str):
+    def save(data: any, path: str):
         with open(path, "wb") as f:
             pickle.dump(data, cast(SupportsWrite, f))

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/utils/hub/hub.py

@@ -11,7 +11,7 @@ class Hub(abc.ABC, Generic[T]):
     _instance: Instance
 
     @classmethod
-    def add(cls, key, obj: T):
+    def add(cls, key, obj: T = None):
         instance = cls._instance.current()
         if key in instance and instance[key] is not obj:
             raise ValueError(f'Conflict object declaration: {obj} and {instance[key]}')

{UniTok-4.0.3 → UniTok-4.3.0}/unitok/vocabulary/vocabulary.py

@@ -1,5 +1,7 @@
 import os
+from typing import Optional, Union
 
+from unitok import PickleHandler
 from unitok.utils import Map, Instance
 from unitok.utils.hub import Hub
 from unitok.vocabulary.counter import Counter
@@ -17,7 +19,7 @@ class Vocabulary:
         self._editable = True  # whether vocab is editable
         self.counter = Counter()
 
-        VocabularyHub.add(self.name, self)
+        VocabularyHub.add(self)
 
     def equals(self, other: 'Vocabulary'):
         return self.name == other.name and len(self) == len(other)
@@ -42,7 +44,7 @@ class Vocabulary:
         """
         return [self.append(obj) for obj in objs]
 
-    def append(self, obj, oov_token=None):
+    def append(self, obj, oov_token: Optional[Union[int, str]] = None):
         obj = str(obj)
         if obj not in self.o2i:
             if '\n' in obj:
@@ -51,7 +53,11 @@ class Vocabulary:
             if not self._editable:
                 if oov_token is None:
                     raise ValueError(f'the fixed vocab {self.name} is not allowed to add new token ({obj})')
-                return oov_token
+                if isinstance(oov_token, str):
+                    return self[oov_token]
+                if len(self) > oov_token >= 0:
+                    return oov_token
+                raise ValueError(f'oov_token ({oov_token}) is not in the vocab')
 
             index = len(self)
             self.o2i[obj] = index
@@ -80,6 +86,9 @@ class Vocabulary:
             return self.i2o[item]
         return self.o2i[item]
 
+    def __contains__(self, item: str):
+        return item in self.o2i
+
     def __str__(self):
         return f'Vocabulary({self.name}, vocab_size={len(self)})'
 
@@ -87,6 +96,10 @@ class Vocabulary:
     Editable Methods
     """
 
+    @property
+    def editable(self):
+        return self._editable
+
     def allow_edit(self):
         self._editable = True
         return self
@@ -112,8 +125,8 @@ class Vocabulary:
     Save & Load Methods
     """
 
-    def filepath(self, store_dir):
-        return os.path.join(store_dir, self.filename)
+    def filepath(self, save_dir):
+        return os.path.join(save_dir, self.filename)
 
     @property
     def filename(self):
@@ -124,8 +137,7 @@ class Vocabulary:
             save_dir = self.filepath(save_dir)
 
         self.o2i, self.i2o = {}, {}
-        with open(save_dir, 'r') as f:
-            objs = f.read().strip().split('\n')
+        objs = PickleHandler.load(save_dir)
         for index, obj in enumerate(objs):
             self.o2i[obj] = index
             self.i2o[index] = obj
@@ -134,8 +146,7 @@ class Vocabulary:
 
     def save(self, save_dir):
         store_path = self.filepath(save_dir)
-        with open(store_path, 'w') as f:
-            f.write('\n'.join(self))
+        PickleHandler.save(list(self), store_path)
 
         return self
 
@@ -148,3 +159,8 @@ class Vocabulary:
 
 class VocabularyHub(Hub[Vocabulary]):
     _instance = Instance()
+
+    @classmethod
+    def add(cls, key, obj: Vocabulary = None):
+        key, obj = key.name, key
+        return super().add(key, obj)

UniTok-4.0.3/unitok/tokenizer/cachable_tokenizer.py

@@ -1,25 +0,0 @@
-from typing import Hashable
-
-from unitok import warning
-from unitok.tokenizer import BaseTokenizer
-
-
-class CachableTokenizer(BaseTokenizer):
-    def __init__(self, use_cache=False, **kwargs):
-        super().__init__(**kwargs)
-
-        if not self.return_list and use_cache:
-            warning(f'Only the tokenizer that return_list=True may need cache, use_cache of {self.get_classname()} will be set to False')
-            use_cache = False
-        self._use_cache = use_cache
-        self._cache = dict()
-
-    def __call__(self, objs):
-        if self._use_cache and isinstance(objs, Hashable):
-            if objs in self._cache:
-                return self._cache[objs]
-            value = super().__call__(objs)
-            self._cache[objs] = value
-            return value
-
-        return super().__call__(objs)