UniTok 3.0.4a0__tar.gz → 3.0.6__tar.gz

{UniTok-3.0.4a0 → UniTok-3.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.0.4a0
+Version: 3.0.6
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.0.4a0 → UniTok-3.0.6}/UniTok/column.py

@@ -42,10 +42,22 @@ class Column:
         tok (BaseTok): The tokenizer of the column.
         operator (SeqOperator): The operator of the column.
     """
-    def __init__(self, name, tok: BaseTok, operator: SeqOperator = None):
+    def __init__(self, name, tok: BaseTok, operator: SeqOperator = None, **kwargs):
         self.name = name
         self.tok = tok
         self.operator = operator
+
+        if kwargs:
+            if operator:
+                raise ValueError('operator and kwargs cannot be set at the same time')
+            self.operator = SeqOperator(
+                max_length=kwargs.get('max_length', 0),
+                padding=kwargs.get('padding', False),
+                slice_post=kwargs.get('slice_post', False),
+                pad_pre=kwargs.get('pad_pre', False),
+                pad=kwargs.get('pad', 0),
+            )
+
         self.list = bool(operator) or tok.return_list is True  # whether the column is a list-element column
 
         self.data = []

UniTok-3.0.6/UniTok/meta.py

@@ -0,0 +1,108 @@
+import json
+import os
+import warnings
+from typing import List
+
+
+class Col:
+    def __init__(self, name, voc=None, max_length=None, padding=None, vocab=None):
+        self.name: str = name
+        self.voc: Voc = voc or vocab
+        self.max_length = max_length
+        self.padding = padding
+        self.list = max_length is not None
+
+    def __eq__(self, other):
+        return self.name == other.name and self.voc.name == other.voc.name and self.max_length == other.max_length
+
+    def get_info(self):
+        info = {
+            'voc': self.voc.name,
+        }
+        if self.list:
+            info['max_length'] = self.max_length
+            info['padding'] = self.padding
+        return info
+
+
+class Voc:
+    def __init__(self, name, size, cols):
+        self.name: str = name
+        self.size: int = size
+        self.cols: List[Col] = cols
+
+    def __eq__(self, other):
+        return self.name == other.name and self.size == other.size
+
+    def get_info(self):
+        return {
+            'size': self.size,
+            'cols': [col.name for col in self.cols]
+        }
+
+
+class Meta:
+    VER = 'UniDep-2.0'
+
+    def __init__(self, store_dir):
+        self.store_dir = store_dir
+        self.path = os.path.join(self.store_dir, 'meta.data.json')
+
+        data = self.load()
+        self.version = data['version']
+        self.cols = data.get('cols') or data['col_info']
+        self.vocs = data.get('vocs') or data['vocab_info']
+        self.id_col = data['id_col']
+
+        # build col-voc graph
+        self.cols = {col: Col(**self.cols[col], name=col) for col in self.cols}
+        self.vocs = {voc: Voc(**self.vocs[voc], name=voc) for voc in self.vocs}
+
+        # connect class objects
+        for col in self.cols.values():
+            col.voc = self.vocs[col.voc]
+        for voc in self.vocs.values():
+            voc.cols = [self.cols[col] for col in voc.cols]
+
+        self.version_check()
+
+    @staticmethod
+    def parse_version(version):
+        if version.startswith('UniDep-'):
+            return version[7:]
+        return f'0.{version}'
+
+    def get_info(self):
+        return {
+            'version': Meta.VER,
+            'id_col': self.id_col,
+            'cols': {col.name: col.get_info() for col in self.cols.values()},
+            'vocs': {voc.name: voc.get_info() for voc in self.vocs.values()}
+        }
+
+    def load(self) -> dict:
+        return json.load(open(self.path))
+
+    def save(self):
+        json.dump(self.get_info(), open(os.path.join(self.store_dir, 'meta.data.json'), 'w'), indent=2)
+
+    def version_check(self):
+        current_version = self.parse_version(Meta.VER)
+        depot_version = self.parse_version(self.version)
+
+        if current_version != depot_version:
+            warnings.warn(
+                f'meta version of depot ({self.store_dir}) mismatch, '
+                f'current version: {current_version}, '
+                f'depot version: {depot_version}. '
+                f'It may cause unexpected error.')
+
+        if current_version <= depot_version:
+            return
+
+        command = input('Press Y to upgrade meta data for future use (Y/n): ')
+        if command.lower() == 'y':
+            os.rename(self.path, self.path + '.bak')
+            print('Old meta data backed up to {}.'.format(self.path + '.bak'))
+            self.save()
+            print('Meta data upgraded.')

{UniTok-3.0.4a0 → UniTok-3.0.6}/UniTok/tok/number_tok.py

@@ -1,3 +1,5 @@
+from typing import Iterable
+
 from .tok import BaseTok
 
 
@@ -28,9 +30,15 @@ class NumberTok(BaseTok):
             self.vocab.extend([str(i) for i in range(vocab_size)])
 
     def t(self, obj):
-        obj = int(obj)
-        if obj >= len(self.vocab):
-            if self.vocab_size is not None:
-                raise ValueError('vocab_size is {}, but {} is given'.format(self.vocab_size, obj))
-            self.vocab.extend([str(i) for i in range(len(self.vocab), obj + 1)])
+        # check is iterable
+        if isinstance(obj, Iterable) and not isinstance(obj, str):
+            obj = [int(o) for o in obj]
+        else:
+            obj = int(obj)
+        objs = [obj] if isinstance(obj, int) else obj
+        for o in objs:
+            if o >= len(self.vocab):
+                if self.vocab_size is not None:
+                    raise ValueError('vocab_size is {}, but {} is given'.format(self.vocab_size, o))
+                self.vocab.extend([str(i) for i in range(len(self.vocab), o + 1)])
         return obj

{UniTok-3.0.4a0 → UniTok-3.0.6}/UniTok/unidep.py

@@ -1,14 +1,11 @@
-import json
 import os
 import random
+import warnings
 from typing import Dict, List, Callable
 
 import numpy as np
 import tqdm
 
-from .compatible.uni_warnings import MetaDataDeprecationWarning, VocabInfoDeprecationWarning, \
-    ColInfoDeprecationWarning, GetMaxLengthDeprecationWarning, GetVocabDeprecationWarning, \
-    GetVocabSizeDeprecationWarning, IsListColDeprecationWarning, ShuffleDeprecationWarning
 from .meta import Meta, Col, Voc
 from .vocab import Vocab
 from .vocabs import Vocabs
@@ -19,14 +16,13 @@ class UniDep:
 
     def __init__(self, store_dir, silent=False):
         self.store_dir = os.path.expanduser(store_dir)
+        self.meta = Meta(self.store_dir)
+
         self.silent = silent
 
         self.cached = False
         self.cached_samples = []
 
-        self.meta_path = os.path.join(self.store_dir, 'meta.data.json')
-        self.meta = Meta(**json.load(open(self.meta_path)))
-
         self.data_path = os.path.join(self.store_dir, 'data.npy')
         self.data = np.load(self.data_path, allow_pickle=True)
         try:
@@ -178,38 +174,41 @@ class UniDep:
 
     @property
     def meta_data(self):
-        MetaDataDeprecationWarning()
+        warnings.warn('meta_data is deprecated, '
+                      'use meta instead (will be removed in 4.x version)', DeprecationWarning)
         return self.meta
 
     @property
     def vocab_info(self):
-        VocabInfoDeprecationWarning()
+        warnings.warn('vocab_info is deprecated, '
+                      'use vocs instead (will be removed in 4.x version)', DeprecationWarning)
         return self.vocs
 
     @property
     def col_info(self):
-        ColInfoDeprecationWarning()
+        warnings.warn('col_info is deprecated, '
+                      'use cols instead (will be removed in 4.x version)', DeprecationWarning)
         return self.cols
 
     def get_vocab_size(self, col_name, as_vocab=False):
-        GetVocabSizeDeprecationWarning()
+        warnings.warn('unidep.get_vocab_size is deprecated (will be removed in 4.x version)', DeprecationWarning)
         vocab_id = col_name if as_vocab else self.get_vocab(col_name)
         return self.vocs[vocab_id].size
 
     def get_vocab(self, col_name):
-        GetVocabDeprecationWarning()
+        warnings.warn('unidep.get_vocab is deprecated (will be removed in 4.x version)', DeprecationWarning)
         return self.cols[col_name].voc.name
 
     def get_max_length(self, col_name):
-        GetMaxLengthDeprecationWarning()
+        warnings.warn('unidep.get_max_length is deprecated (will be removed in 4.x version)', DeprecationWarning)
         return self.cols[col_name].max_length
 
     def is_list_col(self, col_name):
-        IsListColDeprecationWarning()
+        warnings.warn('unidep.is_list_col is deprecated (will be removed in 4.x version)', DeprecationWarning)
         return self.cols[col_name].list
 
     def shuffle(self, shuffle=True):
-        ShuffleDeprecationWarning()
+        warnings.warn('unidep.shuffle is deprecated (will be removed in 4.x version)', DeprecationWarning)
         if shuffle:
             random.shuffle(self._indexes)
         else:

{UniTok-3.0.4a0 → UniTok-3.0.6}/UniTok/unitok.py

@@ -1,12 +1,12 @@
 import json
 import os
+import warnings
 from typing import Optional
 
 import numpy as np
 import pandas as pd
 
 from .cols import Cols
-from .compatible.uni_warnings import VocabDepotDeprecationWarning, GetTokPathDeprecationWarning
 from .column import Column, IndexColumn
 from .tok.bert_tok import BertTok
 from .tok.entity_tok import EntTok
@@ -29,7 +29,8 @@ class UniTok:
 
     @property
     def vocab_depots(self):
-        VocabDepotDeprecationWarning()
+        warnings.warn('vocab_depot is deprecated, '
+                      'use vocabs instead (will be removed in 4.x version)', DeprecationWarning)
         return self.vocabs
 
     def add_col(self, col: Column):
@@ -110,6 +111,9 @@ class UniTok:
         if not self.id_col:
             raise ValueError('id column is not set')
 
+        for vocab in self.vocabs.values():
+            vocab.set_count_mode(False)
+
         for col_name in self.cols:
             print('[ COL:', col_name, ']')
             col = self.cols[col_name]  # type: Column
@@ -121,7 +125,7 @@ class UniTok:
         """
         Get the store path of the tokenizer of a column
         """
-        GetTokPathDeprecationWarning()
+        warnings.warn('unitok.get_tok_path is deprecated (will be removed in 4.x version)', DeprecationWarning)
         return self.cols[col_name].tok.vocab.get_store_path(store_dir)
 
     def store_data(self, store_dir):
@@ -145,7 +149,7 @@ class UniTok:
             cols=self.cols.get_info(),
             id_col=self.id_col.name,
         )
-        json.dump(meta_data, open(os.path.join(store_dir, 'meta.data.json'), 'w'))
+        json.dump(meta_data, open(os.path.join(store_dir, 'meta.data.json'), 'w'), indent=2)
         return self
 
 

{UniTok-3.0.4a0 → UniTok-3.0.6}/UniTok/vocab.py

@@ -1,12 +1,10 @@
 import math
 import os
+import warnings
 from typing import Union, List
 
 import numpy as np
 
-from UniTok.compatible.uni_warnings import VocabMapDeprecationWarning, OOVDefaultDeprecationWarning, \
-    MinFrequencyDeprecationWarning
-
 
 class VocabMap(dict):
     def __call__(self, *args, **kwargs):
@@ -34,12 +32,6 @@ class Vocab:
         self._count_mode = False  # whether count mode is on
         self._counter = {}  # counter for counting occurrence of each token
 
-        # self.frequency_mode = False
-        # self.frequency = {}
-        # self.max_frequency = 0
-
-        # self.frequent_vocab = []
-
     """
     Basic Methods
     """
@@ -49,7 +41,8 @@ class Vocab:
         """
         Deprecated, use o2i instead
         """
-        VocabMapDeprecationWarning()
+        warnings.warn('vocab.index2obj and vocab.obj2index are deprecated, '
+                      'use vocab.i2o and vocab.o2i instead (will be removed in 4.x version)', DeprecationWarning)
         return self.o2i
 
     @property
@@ -57,7 +50,8 @@ class Vocab:
         """
         Deprecated, use i2o instead
         """
-        VocabMapDeprecationWarning()
+        warnings.warn('vocab.index2obj and vocab.obj2index are deprecated, '
+                      'use vocab.i2o and vocab.o2i instead (will be removed in 4.x version)', DeprecationWarning)
         return self.i2o
 
     def extend(self, objs):
@@ -130,7 +124,8 @@ class Vocab:
 
     @property
     def oov_default(self):
-        OOVDefaultDeprecationWarning()
+        warnings.warn('vocab.oov_default is deprecated, '
+                      'use vocab.oov_token instead (will be removed in 4.x version)', DeprecationWarning)
         return self._oov_token
 
     def allow_edit(self):
@@ -192,7 +187,8 @@ class Vocab:
         :return:
         """
         if min_count is None:
-            MinFrequencyDeprecationWarning()
+            warnings.warn('vocab.min_frequency is deprecated, '
+                          'use vocab.min_count instead (will be removed in 4.x version)', DeprecationWarning)
             min_count = min_frequency
 
         vocabs = []
@@ -261,6 +257,12 @@ class Vocab:
 
     def __getattr__(self, item):
         if item in ['frequency_mode', 'frequency', 'max_frequency', 'frequent_vocab']:
-            return AttributeError(f'{item} is deprecated after UniTok 3.0, '
-                                  f'degrade to 2.4.3.2 or lower to use it, '
-                                  f'or check new features of Vocab class')
+            raise AttributeError(f'{item} is deprecated after UniTok 3.0, '
+                                 f'degrade to 2.4.3.2 or lower to use it, '
+                                 f'or check new features of Vocab class')
+
+    @property
+    def trim_vocab(self):
+        warnings.warn('vocab.trim_vocab is deprecated, '
+                      'use vocab.trim instead (will be removed in 4.x version)', DeprecationWarning)
+        return self.trim

{UniTok-3.0.4a0 → UniTok-3.0.6}/UniTok/vocabs.py

@@ -1,4 +1,5 @@
-from .compatible.uni_warnings import ColMapDeprecationWarning, DepotsDeprecationWarning
+import warnings
+
 from .vocab import Vocab
 
 
@@ -9,12 +10,14 @@ class Vocabs(dict):
 
     @property
     def col_map(self):
-        ColMapDeprecationWarning()
+        warnings.warn('vocab_depot.col_map is deprecated, '
+                      'use vocabs.cols instead (will be removed in 4.x version)', DeprecationWarning)
         return self.cols
 
     @property
     def depots(self):
-        DepotsDeprecationWarning()
+        warnings.warn('vocab_depot.depots is deprecated, '
+                      'use vocabs instead (will be removed in 4.x version)', DeprecationWarning)
         return self
 
     def append(self, col_or_vocab):
@@ -33,11 +36,14 @@ class Vocabs(dict):
             raise ValueError(f'vocab {vocab.name} already exists')
         self[vocab.name] = vocab
 
-    def get_info(self):
+    def get_info(self) -> dict:
+        """
+        Get the information of all vocabs
+        """
         return {vocab.name: dict(
             size=vocab.get_size(),
             cols=self.cols[vocab.name],
-        ) for vocab in self}
+        ) for vocab in self.values()}
 
     def __call__(self, name) -> Vocab:
         return self[name]

{UniTok-3.0.4a0 → UniTok-3.0.6}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.0.4a0
+Version: 3.0.6
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.0.4a0 → UniTok-3.0.6}/UniTok.egg-info/SOURCES.txt

@@ -17,8 +17,6 @@ UniTok.egg-info/top_level.txt
 UniTok/analysis/__init__.py
 UniTok/analysis/lengths.py
 UniTok/analysis/plot.py
-UniTok/compatible/__init__.py
-UniTok/compatible/uni_warnings.py
 UniTok/tok/__init__.py
 UniTok/tok/bert_tok.py
 UniTok/tok/entity_tok.py

{UniTok-3.0.4a0 → UniTok-3.0.6}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text()
 
 setup(
     name='UniTok',
-    version='3.0.4.alpha',
+    version='3.0.6',
     keywords=['token', 'tokenizer', 'bert'],
     description='Unified Tokenizer',
     long_description=long_description,

UniTok-3.0.4a0/UniTok/compatible/__init__.py

@@ -1,21 +0,0 @@
-from .uni_warnings import VocabDepotDeprecationWarning, GetTokPathDeprecationWarning, \
-    VocabMapDeprecationWarning, OOVDefaultDeprecationWarning, MinFrequencyDeprecationWarning, \
-    ColMapDeprecationWarning, DepotsDeprecationWarning, MetaDataDeprecationWarning, \
-    VocabInfoDeprecationWarning, ColInfoDeprecationWarning, GetMaxLengthDeprecationWarning, \
-    GetVocabDeprecationWarning
-
-
-__all__ = [
-    'VocabDepotDeprecationWarning',
-    'GetTokPathDeprecationWarning',
-    'VocabMapDeprecationWarning',
-    'OOVDefaultDeprecationWarning',
-    'MinFrequencyDeprecationWarning',
-    'ColMapDeprecationWarning',
-    'DepotsDeprecationWarning',
-    'MetaDataDeprecationWarning',
-    'VocabInfoDeprecationWarning',
-    'ColInfoDeprecationWarning',
-    'GetMaxLengthDeprecationWarning',
-    'GetVocabDeprecationWarning',
-]

UniTok-3.0.4a0/UniTok/compatible/uni_warnings.py

@@ -1,70 +0,0 @@
-import warnings
-from typing import Callable
-
-warned_flags = set()
-
-
-class UniWarning:
-    def __init__(self, msg, type_: Callable = warnings.warn):
-        self.msg = msg
-        self.type = type_
-
-    def __call__(self, *args, **kwargs):
-        if self not in warned_flags:
-            warned_flags.add(self)
-            self.type(self.msg.format(*args, **kwargs))
-
-
-VocabMapDeprecationWarning = UniWarning(
-    'vocab.index2obj and vocab.obj2index are deprecated, '
-    'use vocab.i2o and vocab.o2i instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-OOVDefaultDeprecationWarning = UniWarning(
-    'vocab.oov_default is deprecated, '
-    'use vocab.oov_token instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-MinFrequencyDeprecationWarning = UniWarning(
-    'vocab.min_frequency is deprecated, '
-    'use vocab.min_count instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-VocabDepotDeprecationWarning = UniWarning(
-    'vocab_depot is deprecated, '
-    'use vocabs instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-ColMapDeprecationWarning = UniWarning(
-    'vocab_depot.col_map is deprecated, '
-    'use vocabs.cols instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-GetTokPathDeprecationWarning = UniWarning(
-    'unitok.get_tok_path is deprecated (will be removed in 4.x version)', type_=DeprecationWarning)
-
-DepotsDeprecationWarning = UniWarning(
-    'vocab_depot.depots is deprecated, '
-    'use vocabs instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-MetaDataDeprecationWarning = UniWarning(
-    'meta_data is deprecated, '
-    'use meta instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-VocabInfoDeprecationWarning = UniWarning(
-    'vocab_info is deprecated, '
-    'use vocs instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-ColInfoDeprecationWarning = UniWarning(
-    'col_info is deprecated, '
-    'use cols instead (will be removed in 4.x version)', type_=DeprecationWarning)
-
-GetMaxLengthDeprecationWarning = UniWarning(
-    'unidep.get_max_length is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)
-
-GetVocabDeprecationWarning = UniWarning(
-    'unidep.get_vocab is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)
-
-GetVocabSizeDeprecationWarning = UniWarning(
-    'unidep.get_vocab_size is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)
-
-IsListColDeprecationWarning = UniWarning(
-    'unidep.is_list_col is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)
-
-ShuffleDeprecationWarning = UniWarning(
-    'unidep.shuffle is deprecated, (will be removed in 4.x version)', type_=DeprecationWarning)

UniTok-3.0.4a0/UniTok/meta.py

@@ -1,57 +0,0 @@
-import warnings
-from typing import List
-
-
-class Col:
-    def __init__(self, name, voc, max_length=None, padding=None):
-        self.name: str = name
-        self.voc: Voc = voc
-        self.max_length = max_length
-        self.padding = padding
-        self.list = max_length is not None
-
-    def __eq__(self, other):
-        return self.name == other.name and self.voc.name == other.voc.name and self.max_length == other.max_length
-
-
-class Voc:
-    def __init__(self, name, size, cols):
-        self.name: str = name
-        self.size: int = size
-        self.cols: List[Col] = cols
-
-    def __eq__(self, other):
-        return self.name == other.name and self.size == other.size
-
-
-class Meta:
-    VER = 'UniDep-2.0'
-
-    def __init__(self, version, id_col, col_info=None, vocab_info=None, cols=None, vocs=None):
-        self.version = version
-
-        self.cols = cols or col_info
-        self.vocs = vocs or vocab_info
-        self.id_col = id_col
-
-        # build col-voc graph
-        self.cols = {col: Col(**self.cols[col], name=col) for col in self.cols}
-        self.vocs = {voc: Voc(**self.vocs[voc], name=voc) for voc in self.vocs}
-
-        # connect class objects
-        for col in self.cols.values():
-            col.voc = self.vocs[col.voc]
-        for voc in self.vocs.values():
-            voc.cols = [self.cols[col] for col in voc.cols]
-
-        self.version_check()
-
-    def version_check(self):
-        if self.version != Meta.VER:
-            warnings.warn(
-                'Meta version mismatch, '
-                'current version: {}, '
-                'depot version: {}. '
-                'It may cause unexpected error.'.format(
-                    Meta.VER, self.version
-                ))