UniTok 3.0.4__tar.gz → 3.0.5__tar.gz

{UniTok-3.0.4 → UniTok-3.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.0.4
+Version: 3.0.5
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.0.4 → UniTok-3.0.5}/UniTok/meta.py

@@ -44,12 +44,15 @@ class Voc:
 class Meta:
     VER = 'UniDep-2.0'
 
-    def __init__(self, version, id_col, col_info=None, vocab_info=None, cols=None, vocs=None):
-        self.version = version
+    def __init__(self, store_dir):
+        self.store_dir = store_dir
+        self.path = os.path.join(self.store_dir, 'meta.data.json')
 
-        self.cols = cols or col_info
-        self.vocs = vocs or vocab_info
-        self.id_col = id_col
+        data = self.load()
+        self.version = data['version']
+        self.cols = data.get('cols') or data['col_info']
+        self.vocs = data.get('vocs') or data['vocab_info']
+        self.id_col = data['id_col']
 
         # build col-voc graph
         self.cols = {col: Col(**self.cols[col], name=col) for col in self.cols}
@@ -61,7 +64,7 @@ class Meta:
         for voc in self.vocs.values():
             voc.cols = [self.cols[col] for col in voc.cols]
 
-        self.upgrade = self.version_check()
+        self.version_check()
 
     @staticmethod
     def parse_version(version):
@@ -77,8 +80,11 @@ class Meta:
             'vocs': {voc.name: voc.get_info() for voc in self.vocs.values()}
         }
 
-    def save(self, store_dir):
-        json.dump(self.get_info(), open(os.path.join(store_dir, 'meta.data.json'), 'w'), indent=2)
+    def load(self) -> dict:
+        return json.load(open(self.path))
+
+    def save(self):
+        json.dump(self.get_info(), open(os.path.join(self.store_dir, 'meta.data.json'), 'w'), indent=2)
 
     def version_check(self):
         current_version = self.parse_version(Meta.VER)
@@ -86,15 +92,17 @@ class Meta:
 
         if current_version != depot_version:
             warnings.warn(
-                'Meta version mismatch, '
-                'current version: {}, '
-                'depot version: {}. '
-                'It may cause unexpected error.'.format(
-                    current_version, depot_version
-                ))
+                f'meta version of depot ({self.store_dir}) mismatch, '
+                f'current version: {current_version}, '
+                f'depot version: {depot_version}. '
+                f'It may cause unexpected error.')
 
         if current_version <= depot_version:
-            return False
+            return
 
         command = input('Press Y to upgrade meta data for future use (Y/n): ')
-        return command.lower() == 'y'
+        if command.lower() == 'y':
+            os.rename(self.path, self.path + '.bak')
+            print('Old meta data backed up to {}.'.format(self.path + '.bak'))
+            self.save()
+            print('Meta data upgraded.')

{UniTok-3.0.4 → UniTok-3.0.5}/UniTok/unidep.py

@@ -23,13 +23,7 @@ class UniDep:
         self.cached_samples = []
 
         self.meta_path = os.path.join(self.store_dir, 'meta.data.json')
-        self.meta = Meta(**json.load(open(self.meta_path)))
-
-        if self.meta.upgrade:
-            # backup old meta
-            os.rename(self.meta_path, self.meta_path + '.bak')
-            # save new meta
-            self.meta.save(self.store_dir)
+        self.meta = Meta(**json.load(open(self.meta_path)), store_dir=self.store_dir)
 
         self.data_path = os.path.join(self.store_dir, 'data.npy')
         self.data = np.load(self.data_path, allow_pickle=True)

{UniTok-3.0.4 → UniTok-3.0.5}/UniTok.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.0.4
+Version: 3.0.5
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu

{UniTok-3.0.4 → UniTok-3.0.5}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text()
 
 setup(
     name='UniTok',
-    version='3.0.4',
+    version='3.0.5',
     keywords=['token', 'tokenizer', 'bert'],
     description='Unified Tokenizer',
     long_description=long_description,