UniTok 4.2.5__tar.gz → 4.3.0__tar.gz

{UniTok-4.2.5 → UniTok-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.2.5
+Version: 4.3.0
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.2.5 → UniTok-4.3.0}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.2.5
+Version: 4.3.0
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.2.5 → UniTok-4.3.0}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text(encoding='utf8')
 
 setup(
     name='UniTok',
-    version='4.2.5',
+    version='4.3.0',
     keywords=['token', 'tokenizer', 'NLP', 'transformers', 'glove', 'bert', 'llama'],
     description='Unified Tokenizer',
     long_description=long_description,

{UniTok-4.2.5 → UniTok-4.3.0}/unitok/__init__.py

@@ -9,6 +9,7 @@ from unitok.tokenizer import BaseTokenizer, TokenizerHub
 from unitok.tokenizer import EntityTokenizer, EntitiesTokenizer
 from unitok.tokenizer import TransformersTokenizer, BertTokenizer
 from unitok.tokenizer import SplitTokenizer, DigitTokenizer, DigitsTokenizer
+from unitok.tokenizer import GloVeTokenizer
 from unitok.job import Job, JobHub
 
 from unitok.utils.index_set import IndexSet, VocabSet, TokenizerSet, JobSet
@@ -29,6 +30,7 @@ __all__ = [
     'EntityTokenizer', 'EntitiesTokenizer',
     'TransformersTokenizer', 'BertTokenizer',
     'SplitTokenizer', 'DigitTokenizer', 'DigitsTokenizer',
+    'GloVeTokenizer',
     'Job', 'JobHub',
     'IndexSet', 'VocabSet', 'TokenizerSet', 'JobSet',
     'Meta',

{UniTok-4.2.5 → UniTok-4.3.0}/unitok/job.py

@@ -1,3 +1,6 @@
+from typing import Union
+
+from unitok import TokenizerHub, VocabHub
 from unitok.tokenizer.union_tokenizer import UnionTokenizer
 
 from unitok.tokenizer import BaseTokenizer
@@ -8,7 +11,7 @@ from unitok.utils.hub import Hub
 class Job:
     def __init__(
             self,
-            tokenizer: BaseTokenizer,
+            tokenizer: Union[BaseTokenizer, str],
             column: str,
             name: str = None,
             truncate: int = None,
@@ -16,7 +19,13 @@ class Job:
             key: bool = False,
             max_len: int = 0,
     ):
+        if isinstance(tokenizer, str):
+            if TokenizerHub.has(tokenizer):
+                tokenizer = TokenizerHub.get(tokenizer)
+            else:
+                raise ValueError(f"Tokenizer {tokenizer} not found in the tokenizer hub.")
         self.tokenizer: BaseTokenizer = tokenizer
+
         self.column: str = column
         self.name: str = name
         self.truncate: int = truncate
@@ -26,7 +35,8 @@ class Job:
         self.max_len = max_len
         self.from_union = isinstance(self.tokenizer, UnionTokenizer)
 
-        JobHub.add(self.name, self)
+        JobHub.add(self)
+        VocabHub.add(self.tokenizer.vocab)
 
     @property
     def return_list(self):
@@ -77,3 +87,8 @@ class Job:
 
 class JobHub(Hub[Job]):
     _instance = Instance(compulsory_space=True)
+
+    @classmethod
+    def add(cls, key, obj: Job = None):
+        key, obj = key.name, key
+        return super().add(key, obj)

{UniTok-4.2.5 → UniTok-4.3.0}/unitok/meta.py

@@ -15,7 +15,7 @@ from unitok.vocabulary import Vocab, VocabHub
 
 
 class Meta:
-    version = 'unidep-v4beta'
+    version = 'unidep-v4'
 
     def __init__(self):
         self.note = ('Not compatible with unitok-v3 or lower version, '
@@ -116,6 +116,7 @@ class Meta:
         meta.vocabularies = VocabSet({cls.parse_vocabulary(**v).load(save_dir) for v in kwargs.get('vocabularies')})
         meta.tokenizers = TokenizerSet({cls.parse_tokenizer(**t) for t in kwargs.get('tokenizers')})
         meta.jobs = JobSet({cls.parse_job(**j) for j in kwargs.get('jobs')})
+        meta.version = kwargs.get('version')
 
         return meta
 

{UniTok-4.2.5 → UniTok-4.3.0}/unitok/tokenizer/__init__.py

@@ -1,5 +1,6 @@
 from unitok.tokenizer.base_tokenizer import BaseTokenizer, TokenizerHub
 from unitok.tokenizer.entity_tokenizer import EntityTokenizer, EntitiesTokenizer
+from unitok.tokenizer.glove_tokenizer import GloVeTokenizer
 from unitok.tokenizer.transformers_tokenizer import TransformersTokenizer, BertTokenizer
 from unitok.tokenizer.split_tokenizer import SplitTokenizer
 from unitok.tokenizer.digit_tokenizer import DigitTokenizer, DigitsTokenizer
@@ -14,5 +15,6 @@ __all__ = [
     SplitTokenizer,
     DigitTokenizer,
     DigitsTokenizer,
-    TokenizerHub
+    GloVeTokenizer,
+    TokenizerHub,
 ]

{UniTok-4.2.5 → UniTok-4.3.0}/unitok/tokenizer/base_tokenizer.py

@@ -28,7 +28,8 @@ class BaseTokenizer(abc.ABC):
 
         self._tokenizer_id = tokenizer_id
 
-        TokenizerHub.add(self.get_tokenizer_id(), self)
+        TokenizerHub.add(self)
+        VocabHub.add(self.vocab)
 
     def get_tokenizer_id(self):
         if self._tokenizer_id is None:
@@ -81,3 +82,8 @@ class BaseTokenizer(abc.ABC):
 
 class TokenizerHub(Hub[BaseTokenizer]):
     _instance = Instance()
+
+    @classmethod
+    def add(cls, key, obj: BaseTokenizer = None):
+        key, obj = key.get_tokenizer_id(), key
+        return super().add(key, obj)

{UniTok-4.2.5 → UniTok-4.3.0}/unitok/tokenizer/glove_tokenizer.py

@@ -17,5 +17,5 @@ class GloVeTokenizer(BaseTokenizer):
         self.language = language
 
     def __call__(self, obj):
-        objs = nltk.tokenize.word_tokenize(obj.lower())
+        objs = nltk.tokenize.word_tokenize(obj.lower(), language=self.language)
         return [self.vocab[o] for o in objs if o in self.vocab]

{UniTok-4.2.5 → UniTok-4.3.0}/unitok/unitok.py

@@ -173,7 +173,7 @@ class UniTok(Status):
             if tokenizer.return_list:
                 raise AttributeError('Column content of the key job should be tokenized into atomic value')
             if self.key_job:
-                raise ValueError(f'key key already exists: {self.key_job.name}')
+                raise ValueError(f'Key column already exists: {self.key_job.name}')
             self.key_job = job
 
     @Status.require_not_organized
@@ -282,7 +282,10 @@ class UniTok(Status):
 
         # Prepare introduction header
         introduction_header = Text.assemble(
-            (f"UniTok ({self.meta.parse_version(self.meta.version)})\n", "bold cyan"),
+            (
+                f"UniTok (v{self.meta.parse_version(Meta.version)}), "
+                f"Data (v{self.meta.parse_version(self.meta.version)})\n",
+                "bold cyan"),
             (f"Sample Size: {self._sample_size}\n", "green"),
             (f"ID Column: {self.key_job.name}\n", "magenta"),
             style="dim"

{UniTok-4.2.5 → UniTok-4.3.0}/unitok/utils/hub/hub.py

@@ -11,7 +11,7 @@ class Hub(abc.ABC, Generic[T]):
     _instance: Instance
 
     @classmethod
-    def add(cls, key, obj: T):
+    def add(cls, key, obj: T = None):
         instance = cls._instance.current()
         if key in instance and instance[key] is not obj:
             raise ValueError(f'Conflict object declaration: {obj} and {instance[key]}')

{UniTok-4.2.5 → UniTok-4.3.0}/unitok/vocabulary/vocabulary.py

@@ -1,4 +1,5 @@
 import os
+from typing import Optional, Union
 
 from unitok import PickleHandler
 from unitok.utils import Map, Instance
@@ -18,7 +19,7 @@ class Vocabulary:
         self._editable = True  # whether vocab is editable
         self.counter = Counter()
 
-        VocabularyHub.add(self.name, self)
+        VocabularyHub.add(self)
 
     def equals(self, other: 'Vocabulary'):
         return self.name == other.name and len(self) == len(other)
@@ -43,7 +44,7 @@ class Vocabulary:
         """
         return [self.append(obj) for obj in objs]
 
-    def append(self, obj, oov_token=None):
+    def append(self, obj, oov_token: Optional[Union[int, str]] = None):
         obj = str(obj)
         if obj not in self.o2i:
             if '\n' in obj:
@@ -52,7 +53,11 @@ class Vocabulary:
             if not self._editable:
                 if oov_token is None:
                     raise ValueError(f'the fixed vocab {self.name} is not allowed to add new token ({obj})')
-                return oov_token
+                if isinstance(oov_token, str):
+                    return self[oov_token]
+                if len(self) > oov_token >= 0:
+                    return oov_token
+                raise ValueError(f'oov_token ({oov_token}) is not in the vocab')
 
             index = len(self)
             self.o2i[obj] = index
@@ -81,6 +86,9 @@ class Vocabulary:
             return self.i2o[item]
         return self.o2i[item]
 
+    def __contains__(self, item: str):
+        return item in self.o2i
+
     def __str__(self):
         return f'Vocabulary({self.name}, vocab_size={len(self)})'
 
@@ -88,6 +96,10 @@ class Vocabulary:
     Editable Methods
     """
 
+    @property
+    def editable(self):
+        return self._editable
+
     def allow_edit(self):
         self._editable = True
         return self
@@ -113,8 +125,8 @@ class Vocabulary:
     Save & Load Methods
     """
 
-    def filepath(self, store_dir):
-        return os.path.join(store_dir, self.filename)
+    def filepath(self, save_dir):
+        return os.path.join(save_dir, self.filename)
 
     @property
     def filename(self):
@@ -147,3 +159,8 @@ class Vocabulary:
 
 class VocabularyHub(Hub[Vocabulary]):
     _instance = Instance()
+
+    @classmethod
+    def add(cls, key, obj: Vocabulary = None):
+        key, obj = key.name, key
+        return super().add(key, obj)