UniTok 4.3.1__tar.gz → 4.3.3__tar.gz

{UniTok-4.3.1 → UniTok-4.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.3.1
+Version: 4.3.3
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.3.1 → UniTok-4.3.3}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 4.3.1
+Version: 4.3.3
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-4.3.1 → UniTok-4.3.3}/UniTok.egg-info/SOURCES.txt

@@ -36,6 +36,12 @@ unitok/meta.py
 unitok/selector.py
 unitok/status.py
 unitok/unitok.py
+unitok.egg-info/PKG-INFO
+unitok.egg-info/SOURCES.txt
+unitok.egg-info/dependency_links.txt
+unitok.egg-info/entry_points.txt
+unitok.egg-info/requires.txt
+unitok.egg-info/top_level.txt
 unitok/tokenizer/__init__.py
 unitok/tokenizer/base_tokenizer.py
 unitok/tokenizer/digit_tokenizer.py

{UniTok-4.3.1 → UniTok-4.3.3}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text(encoding='utf8')
 
 setup(
     name='UniTok',
-    version='4.3.1',
+    version='4.3.3',
     keywords=['token', 'tokenizer', 'NLP', 'transformers', 'glove', 'bert', 'llama'],
     description='Unified Tokenizer',
     long_description=long_description,

{UniTok-4.3.1 → UniTok-4.3.3}/unitok/unitok.py

@@ -98,12 +98,12 @@ class UniTok(Status):
 
     def __enter__(self):
         from unitok.utils import Space
-        Space.set(self)
+        Space.push(self)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         from unitok.utils import Space
-        Space.unset()
+        Space.pop(self)
 
     @Status.require_initialized
     def add_index_job(self, name: str = 'index', tokenizer: DigitTokenizer = None):

{UniTok-4.3.1 → UniTok-4.3.3}/unitok/utils/hub/hub.py

@@ -13,10 +13,14 @@ class Hub(abc.ABC, Generic[T]):
     @classmethod
     def add(cls, key, obj: T = None):
         instance = cls._instance.current()
-        if key in instance and instance[key] is not obj:
+        if key in instance and cls.notequal(instance[key], obj):
             raise ValueError(f'Conflict object declaration: {obj} and {instance[key]}')
         instance[key] = obj
 
+    @classmethod
+    def notequal(cls, a: T, b: T) -> bool:
+        return a is not b
+
     @classmethod
     def get(cls, name: str, **kwargs) -> T:
         """

UniTok-4.3.3/unitok/utils/space.py

@@ -0,0 +1,35 @@
+class Space:
+    """
+    UniTok allows multiple instances to be created, but the "with" statement can only be used with one instance.
+    """
+
+    _stack = []
+
+    @classmethod
+    def push(cls, obj):
+        """
+        Lock the unitok instance as the current active instance
+        """
+        cls._stack.append(obj)
+        # if cls._active_instance is not None:
+        #     raise ValueError(f'Space is already locked to {cls._active_instance}')
+
+    @classmethod
+    def pop(cls, obj):
+        """
+        Unlock the current active instance
+        """
+        # cls._active_instance = None
+        if not cls._stack:
+            raise ValueError('Space stack is empty')
+        if cls._stack[-1] != obj:
+            raise ValueError('Space stack is not in order')
+        cls._stack.pop()
+
+    @classmethod
+    def get_space(cls):
+        """
+        Get the current active instance
+        """
+        # return cls._active_space
+        return cls._stack[-1] if cls._stack else None

{UniTok-4.3.1 → UniTok-4.3.3}/unitok/vocabulary/vocabulary.py

@@ -164,3 +164,7 @@ class VocabularyHub(Hub[Vocabulary]):
     def add(cls, key, obj: Vocabulary = None):
         key, obj = key.name, key
         return super().add(key, obj)
+
+    @classmethod
+    def notequal(cls, a: Vocabulary, b: Vocabulary) -> bool:
+        return a.name != b.name or a.size != b.size

UniTok-4.3.1/unitok/utils/space.py

@@ -1,29 +0,0 @@
-class Space:
-    """
-    UniTok allows multiple instances to be created, but the "with" statement can only be used with one instance.
-    """
-
-    _active_instance = None
-
-    @classmethod
-    def set(cls, obj):
-        """
-        Lock the unitok instance as the current active instance
-        """
-        if cls._active_instance is not None:
-            raise ValueError(f'Space is already locked to {cls._active_instance}')
-        cls._active_instance = obj
-
-    @classmethod
-    def unset(cls):
-        """
-        Unlock the current active instance
-        """
-        cls._active_instance = None
-
-    @classmethod
-    def get_space(cls):
-        """
-        Get the current active instance
-        """
-        return cls._active_instance