UniTok 3.1.0__tar.gz → 3.1.1__tar.gz

{UniTok-3.1.0 → UniTok-3.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.1.0
+Version: 3.1.1
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.1.0 → UniTok-3.1.1}/UniTok/column.py

@@ -1,5 +1,4 @@
-from typing import Type
-
+from typing import Type, Union
 
 from tqdm import tqdm
 
@@ -45,7 +44,7 @@ class Column:
         tok (BaseTok): The tokenizer of the column.
         operator (SeqOperator): The operator of the column.
     """
-    def __init__(self, tok: BaseTok | Type[BaseTok], name=None, operator: SeqOperator = None, **kwargs):
+    def __init__(self, tok: Union[BaseTok, Type[BaseTok]], name=None, operator: SeqOperator = None, **kwargs):
         self.tok = tok
         self.name = name or tok.vocab.name
         self.operator = operator

{UniTok-3.1.0 → UniTok-3.1.1}/UniTok/meta.py

@@ -1,13 +1,13 @@
 import json
 import os
 import warnings
-from typing import List
+from typing import List, Union
 
 
 class Col:
     def __init__(self, name, voc=None, max_length=None, padding=None, vocab=None):
         self.name: str = name
-        self.voc: Voc | str = voc or vocab
+        self.voc: Union[Voc, str] = voc or vocab
         self.max_length = max_length
         self.padding = padding
         self.list = max_length is not None
@@ -29,7 +29,7 @@ class Voc:
     def __init__(self, name, size, cols, store_dir):
         self.name: str = name
         self.size: int = size
-        self.cols: List[Col | str] = cols
+        self.cols: List[Union[Col, str]] = cols
         self.store_dir = store_dir
 
     def __eq__(self, other):

{UniTok-3.1.0 → UniTok-3.1.1}/UniTok/unidep.py

@@ -46,7 +46,7 @@ class UniDep:
             self.print('resize sample_size to', self._sample_size)
             self.sample_size = self._sample_size
 
-        self.vocabs = Vocabs()
+        self.vocabs = Vocabs()  # type: Union[Dict[str, Vocab], Vocabs]
         for vocab_name in self.vocs:
             self.vocabs.append(Vocab(name=vocab_name).load(self.store_dir))
         self.id2index = self.vocabs[self.id_voc.name].o2i

{UniTok-3.1.0 → UniTok-3.1.1}/UniTok/unitok.py

@@ -1,7 +1,7 @@
 import json
 import os
 import warnings
-from typing import Optional, Type
+from typing import Optional, Type, Dict, Union
 
 import numpy as np
 import pandas as pd
@@ -58,7 +58,7 @@ class UniTok:
 
     def __init__(self):
         self.cols = Cols()
-        self.vocabs = Vocabs()
+        self.vocabs = Vocabs()  # type: Union[Dict[str, Vocab], Vocabs]
         self.id_col = None  # type: Optional[Column]
         self.data = None
 
@@ -68,7 +68,7 @@ class UniTok:
                       'use vocabs instead (will be removed in 4.x version)', DeprecationWarning)
         return self.vocabs
 
-    def add_col(self, col: Column | str, tok: BaseTok | Type[BaseTok] = None):
+    def add_col(self, col: Union[Column, str], tok: Union[BaseTok, Type[BaseTok]] = None):
         """
         Declare a column in the DataFrame to be tokenized.
         """

{UniTok-3.1.0 → UniTok-3.1.1}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.1.0
+Version: 3.1.1
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.1.0 → UniTok-3.1.1}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text()
 
 setup(
     name='UniTok',
-    version='3.1.0',
+    version='3.1.1',
     keywords=['token', 'tokenizer', 'bert'],
     description='Unified Tokenizer',
     long_description=long_description,