UniTok 4.0.3__tar.gz → 4.2.5__tar.gz

{UniTok-4.0.3 → UniTok-4.2.5}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.0.3
+Version: 4.2.5
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
 Author-email: liu@qijiong.work
 License: MIT Licence
-Keywords: token,tokenizer
+Keywords: token,tokenizer,NLP,transformers,glove,bert,llama
 Platform: any
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -29,17 +29,18 @@ UniTok is designed to simplify preprocessing by offering reusable components suc
 
 ### Changes and Comparisons
 
-| Feature           | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
-|-------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
-| `UniTok` class    | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
-| `UniDep` class    | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
-| `Column` class    | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
-| `Job` class       | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
-| `Tokenizer` class | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
-| `Tokenizer` class | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
-| `analyse` method  | Supported                                                   | Not supported Currently                             |                                                                               |
-| `Meta` class      | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
-| `unitok` command  | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               | 
+| Feature                         | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
+|---------------------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
+| `UniTok` class                  | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
+| `UniDep` class                  | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
+| `Column` class                  | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
+| `Job` class                     | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
+| `Tokenizer` class               | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
+| `Tokenizer` class               | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
+| `analyse` method                | Supported                                                   | Not supported Currently                             |                                                                               |
+| `Meta` class                    | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
+| `unitok` command                | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               |
+| `Vocab` class (unitok >= 4.1.0) | Save and load vocabulary using text files                   | Save and load vocabulary using pickle files         | Avoids issues with special characters in text files                           |
 
 ### How to Migrate the Processed Data
 

{UniTok-4.0.3 → UniTok-4.2.5}/README.md

@@ -16,17 +16,18 @@ UniTok is designed to simplify preprocessing by offering reusable components suc
 
 ### Changes and Comparisons
 
-| Feature           | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
-|-------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
-| `UniTok` class    | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
-| `UniDep` class    | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
-| `Column` class    | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
-| `Job` class       | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
-| `Tokenizer` class | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
-| `Tokenizer` class | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
-| `analyse` method  | Supported                                                   | Not supported Currently                             |                                                                               |
-| `Meta` class      | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
-| `unitok` command  | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               | 
+| Feature                         | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
+|---------------------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
+| `UniTok` class                  | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
+| `UniDep` class                  | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
+| `Column` class                  | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
+| `Job` class                     | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
+| `Tokenizer` class               | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
+| `Tokenizer` class               | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
+| `analyse` method                | Supported                                                   | Not supported Currently                             |                                                                               |
+| `Meta` class                    | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
+| `unitok` command                | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               |
+| `Vocab` class (unitok >= 4.1.0) | Save and load vocabulary using text files                   | Save and load vocabulary using pickle files         | Avoids issues with special characters in text files                           |
 
 ### How to Migrate the Processed Data
 

{UniTok-4.0.3 → UniTok-4.2.5}/UniTok.egg-info/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.0.3
+Version: 4.2.5
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
 Author-email: liu@qijiong.work
 License: MIT Licence
-Keywords: token,tokenizer
+Keywords: token,tokenizer,NLP,transformers,glove,bert,llama
 Platform: any
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -29,17 +29,18 @@ UniTok is designed to simplify preprocessing by offering reusable components suc
 
 ### Changes and Comparisons
 
-| Feature           | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
-|-------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
-| `UniTok` class    | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
-| `UniDep` class    | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
-| `Column` class    | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
-| `Job` class       | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
-| `Tokenizer` class | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
-| `Tokenizer` class | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
-| `analyse` method  | Supported                                                   | Not supported Currently                             |                                                                               |
-| `Meta` class      | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
-| `unitok` command  | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               | 
+| Feature                         | UniTok v3                                                   | UniTok v4                                           | Comments                                                                      |
+|---------------------------------|-------------------------------------------------------------|-----------------------------------------------------|-------------------------------------------------------------------------------|
+| `UniTok` class                  | Solely for tokenization                                     | Manages the entire preprocessing lifecycle          |                                                                               |
+| `UniDep` class                  | Data loading and combining                                  | Removed                                             | V4 combines the functionalities of `UniTok` and `UniDep` into a single class. |
+| `Column` class                  | Column name is for both the original and tokenized datasets | N/A                                                 | V4 introduces a `Job` class.                                                  |
+| `Job` class                     | N/A                                                         | Defines how a specific column should be tokenized   |                                                                               |
+| `Tokenizer` class               | Ambiguous return type definition                            | `return_list` parameter must be of type `bool`      |                                                                               |
+| `Tokenizer` class               | Only supports `BertTokenizer` for text processing           | Supports all Tokenizers in the transformers library | New `TransformersTokenizer` class                                             |
+| `analyse` method                | Supported                                                   | Not supported Currently                             |                                                                               |
+| `Meta` class                    | Only for human-friendly displaying                          | Manager for `Job`, `Tokenizer`, and `Vocab`         |                                                                               |
+| `unitok` command                | Visualization in the terminal                               | More colorful and detailed output                   |                                                                               |
+| `Vocab` class (unitok >= 4.1.0) | Save and load vocabulary using text files                   | Save and load vocabulary using pickle files         | Avoids issues with special characters in text files                           |
 
 ### How to Migrate the Processed Data
 

{UniTok-4.0.3 → UniTok-4.2.5}/UniTok.egg-info/SOURCES.txt

@@ -38,9 +38,9 @@ unitok/status.py
 unitok/unitok.py
 unitok/tokenizer/__init__.py
 unitok/tokenizer/base_tokenizer.py
-unitok/tokenizer/cachable_tokenizer.py
 unitok/tokenizer/digit_tokenizer.py
 unitok/tokenizer/entity_tokenizer.py
+unitok/tokenizer/glove_tokenizer.py
 unitok/tokenizer/split_tokenizer.py
 unitok/tokenizer/transformers_tokenizer.py
 unitok/tokenizer/union_tokenizer.py

{UniTok-4.0.3 → UniTok-4.2.5}/setup.py

@@ -6,8 +6,8 @@ long_description = (this_directory / "README.md").read_text(encoding='utf8')
 
 setup(
     name='UniTok',
-    version='4.0.3',
-    keywords=['token', 'tokenizer'],
+    version='4.2.5',
+    keywords=['token', 'tokenizer', 'NLP', 'transformers', 'glove', 'bert', 'llama'],
     description='Unified Tokenizer',
     long_description=long_description,
     long_description_content_type='text/markdown',

{UniTok-4.0.3 → UniTok-4.2.5}/unitok/__main__.py

@@ -1,7 +1,9 @@
 import argparse
 
 import pandas as pd
+from pigmento import pnt
 
+from unitok import Vocab
 from unitok.tokenizer import BaseTokenizer
 from unitok.unitok import UniTok
 from unitok.utils.class_pool import ClassPool
@@ -29,7 +31,7 @@ def integrate():
         if arg.startswith('--t.'):
             current_param = arg[4:]
         elif arg.startswith('--tokenizer.'):
-            current_param = arg[11:]
+            current_param = arg[12:]
 
     if args.file.endswith('.csv') or args.file.endswith('.tsv'):
         df = pd.read_csv(args.file, sep='\t')
@@ -39,17 +41,32 @@ def integrate():
         raise ValueError(f'Unsupported file format: {args.file}')
 
     with UniTok.load(args.path, tokenizer_lib=args.lib) as ut:
+        tokenizer = None
+
         if args.tokenizer_id:
             for t in ut.meta.tokenizers:  # type: BaseTokenizer
                 if t.get_tokenizer_id() == args.tokenizer_id:
                     tokenizer = t
                     break
             else:
-                raise ValueError(f'Unknown tokenizer id: {args.tokenizer_id}')
-        else:
-            assert args.tokenizer is not None and args.vocab is not None, 'Tokenizer classname and vocabulary must be specified'
+                pnt(f'Unknown tokenizer id: {args.tokenizer_id}, will create a new tokenizer')
+                tokenizer_params['tokenizer_id'] = args.tokenizer_id
+
+        if not tokenizer:
+            if args.tokenizer is None and args.vocab is None:
+                raise ValueError('Tokenizer classname and vocabulary must be specified')
+
+            if args.vocab.endswith('.vocab'):
+                if '/' in args.vocab:
+                    vocab_path, vocab_name = args.vocab.rsplit('/', 1)
+                else:
+                    vocab_path, vocab_name = '.', args.vocab
+                vocab_name = vocab_name[:-6]
+                args.vocab = Vocab(vocab_name).load(vocab_path)
+
             tokenizers = ClassPool.tokenizers(args.lib)
-            assert args.tokenizer in tokenizers, f'Unknown tokenizer: {args.tokenizer}. Available tokenizers: {tokenizers.keys()}'
+            if args.tokenizer not in tokenizers:
+                raise ValueError(f'Unknown tokenizer: {args.tokenizer}. Available tokenizers: {tokenizers.keys()}')
             tokenizer = tokenizers[args.tokenizer](vocab=args.vocab, **tokenizer_params)
 
         ut.add_job(tokenizer=tokenizer, column=args.column, name=args.name, truncate=args.truncate)
@@ -65,14 +82,27 @@ def summarize():
         ut.summarize()
 
 
+def remove():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('path', type=str, default='.', help='path to a unitok data directory')
+    parser.add_argument('--name', type=str, help='job name to remove')
+    args, _ = parser.parse_known_args()
+
+    with UniTok.load(args.path) as ut:
+        ut.remove_job(args.name)
+        ut.save(args.path)
+
+
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('--action', '-a', type=str, default='summarize', choices=['summarize', 'integrate'])
+    parser.add_argument('--action', '-a', type=str, default='summarize', choices=['summarize', 'integrate', 'remove'])
 
     args, _ = parser.parse_known_args()
     action = args.action
 
     if action == 'integrate':
         integrate()
+    elif action == 'remove':
+        remove()
     else:
         summarize()

{UniTok-4.0.3 → UniTok-4.2.5}/unitok/tokenizer/__init__.py

@@ -1,5 +1,4 @@
 from unitok.tokenizer.base_tokenizer import BaseTokenizer, TokenizerHub
-from unitok.tokenizer.cachable_tokenizer import CachableTokenizer
 from unitok.tokenizer.entity_tokenizer import EntityTokenizer, EntitiesTokenizer
 from unitok.tokenizer.transformers_tokenizer import TransformersTokenizer, BertTokenizer
 from unitok.tokenizer.split_tokenizer import SplitTokenizer
@@ -8,7 +7,6 @@ from unitok.tokenizer.digit_tokenizer import DigitTokenizer, DigitsTokenizer
 
 __all__ = [
     BaseTokenizer,
-    CachableTokenizer,
     EntityTokenizer,
     EntitiesTokenizer,
     TransformersTokenizer,

UniTok-4.2.5/unitok/tokenizer/glove_tokenizer.py

@@ -0,0 +1,21 @@
+import nltk
+
+from unitok.vocabulary import VocabHub
+from unitok.tokenizer import BaseTokenizer
+
+
+class GloVeTokenizer(BaseTokenizer):
+    return_list = True
+    param_list = ['language']
+
+    def __init__(self, vocab, language='english', **kwargs):
+        if isinstance(vocab, str) and not VocabHub.has(vocab):
+            raise ValueError('GloVeTokenizer requires a pre-filled Vocab object that stores valid tokens')
+
+        super().__init__(vocab=vocab, **kwargs)
+
+        self.language = language
+
+    def __call__(self, obj):
+        objs = nltk.tokenize.word_tokenize(obj.lower())
+        return [self.vocab[o] for o in objs if o in self.vocab]

{UniTok-4.0.3 → UniTok-4.2.5}/unitok/tokenizer/transformers_tokenizer.py

@@ -1,31 +1,37 @@
 from typing import Union
 
+from pigmento import pnt
 from transformers import AutoTokenizer
 
-from UniTokv3.vocab import Vocab
-from unitok.tokenizer import CachableTokenizer
+from unitok.vocabulary import Vocab
+from unitok.tokenizer import BaseTokenizer
 
 
-class TransformersTokenizer(CachableTokenizer):
+class TransformersTokenizer(BaseTokenizer):
     return_list = True
-    param_list = ['key']
 
-    def __init__(self, vocab: Union[str, Vocab], tokenizer_id: str = None, use_cache=False, key: str = None, **kwargs):
-        super().__init__(vocab=vocab, tokenizer_id=tokenizer_id, use_cache=use_cache)
+    def __init__(self, vocab: Union[str, Vocab], tokenizer_id: str = None, key: str = None, **kwargs):
+        super().__init__(vocab=vocab, tokenizer_id=tokenizer_id)
         self.key = key
 
         self.kwargs = kwargs
+        self.param_list = ['key']
         self.param_list.extend(list(kwargs.keys()))
 
         self.tokenizer = AutoTokenizer.from_pretrained(self.key, **self.kwargs)
         self.vocab.extend(self._generate_token_list())
 
     def _generate_token_list(self):
+        if not hasattr(self.tokenizer, 'vocab'):
+            pnt(f'transformer({self.key}): does not provide vocabulary, generating placeholders instead')
+            return list(range(self.tokenizer.vocab_size))
+
         tokens = self.tokenizer.vocab
         if isinstance(tokens, list):
             return tokens
         if not isinstance(tokens, dict):
-            raise ValueError(f'transformer({self.key}): unsupported type of vocabulary')
+            pnt(f'transformer({self.key}): unsupported type of vocabulary, generating placeholders instead')
+            return list(range(self.tokenizer.vocab_size))
 
         num_tokens = len(tokens)
         token_ids = list(tokens.values())
@@ -45,11 +51,15 @@ class TransformersTokenizer(CachableTokenizer):
 
     def __call__(self, obj):
         tokens = self.tokenizer.tokenize(obj)
-        return super().__call__(tokens)
+        tokens = self.tokenizer.convert_tokens_to_ids(tokens)
+        for token in tokens:
+            self.vocab.counter(token)
+        return tokens
 
 
 class BertTokenizer(TransformersTokenizer):
     param_list = []
 
     def __init__(self, **kwargs):
+        kwargs.pop('key', None)
         super().__init__(key='bert-base-uncased', **kwargs)

{UniTok-4.0.3 → UniTok-4.2.5}/unitok/unitok.py

@@ -462,3 +462,28 @@ class UniTok(Status):
 
         job.max_len = max_len
         self.data[job.name] = series
+
+    def remove_job(self, job: Union[Job, str]):
+        if isinstance(job, str):
+            job = self.meta.jobs[job]
+
+        if job.key:
+            raise ValueError('key job cannot be removed')
+
+        self.meta.jobs.remove(job)
+
+        tokenizer = job.tokenizer
+        for j in self.meta.jobs:
+            if j.tokenizer == tokenizer:
+                break
+        else:
+            self.meta.tokenizers.remove(tokenizer)
+            vocab = tokenizer.vocab
+            for t in self.meta.tokenizers:
+                if t.vocab == vocab:
+                    break
+            else:
+                self.meta.vocabularies.remove(vocab)
+
+        if job.is_processed:
+            self.data.pop(job.name)

{UniTok-4.0.3 → UniTok-4.2.5}/unitok/utils/handler/pkl_handler.py

@@ -14,6 +14,6 @@ class PickleHandler:
         return pickle.load(open(path, "rb"))
 
     @staticmethod
-    def save(data: dict, path: str):
+    def save(data: any, path: str):
         with open(path, "wb") as f:
             pickle.dump(data, cast(SupportsWrite, f))

{UniTok-4.0.3 → UniTok-4.2.5}/unitok/vocabulary/vocabulary.py

@@ -1,5 +1,6 @@
 import os
 
+from unitok import PickleHandler
 from unitok.utils import Map, Instance
 from unitok.utils.hub import Hub
 from unitok.vocabulary.counter import Counter
@@ -124,8 +125,7 @@ class Vocabulary:
             save_dir = self.filepath(save_dir)
 
         self.o2i, self.i2o = {}, {}
-        with open(save_dir, 'r') as f:
-            objs = f.read().strip().split('\n')
+        objs = PickleHandler.load(save_dir)
         for index, obj in enumerate(objs):
             self.o2i[obj] = index
             self.i2o[index] = obj
@@ -134,8 +134,7 @@ class Vocabulary:
 
     def save(self, save_dir):
         store_path = self.filepath(save_dir)
-        with open(store_path, 'w') as f:
-            f.write('\n'.join(self))
+        PickleHandler.save(list(self), store_path)
 
         return self
 

UniTok-4.0.3/unitok/tokenizer/cachable_tokenizer.py

@@ -1,25 +0,0 @@
-from typing import Hashable
-
-from unitok import warning
-from unitok.tokenizer import BaseTokenizer
-
-
-class CachableTokenizer(BaseTokenizer):
-    def __init__(self, use_cache=False, **kwargs):
-        super().__init__(**kwargs)
-
-        if not self.return_list and use_cache:
-            warning(f'Only the tokenizer that return_list=True may need cache, use_cache of {self.get_classname()} will be set to False')
-            use_cache = False
-        self._use_cache = use_cache
-        self._cache = dict()
-
-    def __call__(self, objs):
-        if self._use_cache and isinstance(objs, Hashable):
-            if objs in self._cache:
-                return self._cache[objs]
-            value = super().__call__(objs)
-            self._cache[objs] = value
-            return value
-
-        return super().__call__(objs)