UniTok 3.5.1__tar.gz → 3.5.3__tar.gz

{UniTok-3.5.1 → UniTok-3.5.3}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.5.1
+Version: 3.5.3
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
-Author-email: i@6-79.cn
+Author-email: liu@qijiong.work
 License: MIT Licence
 Keywords: token,tokenizer
 Platform: any

UniTok-3.5.3/UniTok/__main__.py

@@ -0,0 +1,42 @@
+import argparse
+import random
+
+from prettytable import PrettyTable
+
+from UniTok import UniDep
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('path', nargs='?', type=str, default='.', help='Path to the UniDep dataset')
+    parser.add_argument('--head', type=int, default=0, help='Number of rows to display')
+    parser.add_argument('--tail', type=int, default=0, help='Number of rows to display')
+    parser.add_argument('--rand', type=int, default=0, help='Number of rows to display')
+    parser.add_argument('--cols', type=str, nargs='+', help='Columns to display')
+
+    args = parser.parse_args()
+    path = args.path
+
+    depot = UniDep(path, silent=True)
+
+    if not args.head and not args.tail and not args.rand:
+        print(depot)
+        exit(0)
+
+    indices = []
+    sample_size = depot.sample_size
+    if args.head:
+        indices += list(range(args.head))
+    if args.rand:
+        for _ in range(args.rand):
+            indices.append(random.randint(0, sample_size - 1))
+    if args.tail:
+        indices += list(range(sample_size - args.tail, sample_size))
+
+    cols = args.cols if args.cols else list(depot.cols.keys())
+
+    table = PrettyTable(cols)
+    for i in indices:
+        row = [depot[i][col] for col in cols]
+        table.add_row(row)
+    print(table)

{UniTok-3.5.1 → UniTok-3.5.3}/UniTok/unidep.py

@@ -357,8 +357,8 @@ class UniDep:
                 data[col_name].append(sample[col_name])
 
         for col_name in data:
-            data[col_name] = np.array(data[col_name])
-        np.save(os.path.join(store_dir, 'data.npy'), cast(np.ndarray, data), allow_pickle=True)
+            data[col_name] = np.array(data[col_name], dtype=object)
+        np.save(os.path.join(store_dir, 'data.npy'), data, allow_pickle=True)
 
         meta_data = self.meta.get_info()
         json.dump(meta_data, open(os.path.join(store_dir, 'meta.data.json'), 'w'), indent=2)

{UniTok-3.5.1 → UniTok-3.5.3}/UniTok.egg-info/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: UniTok
-Version: 3.5.1
+Version: 3.5.3
 Summary: Unified Tokenizer
 Home-page: https://github.com/Jyonn/UnifiedTokenizer
 Author: Jyonn Liu
-Author-email: i@6-79.cn
+Author-email: liu@qijiong.work
 License: MIT Licence
 Keywords: token,tokenizer
 Platform: any

{UniTok-3.5.1 → UniTok-3.5.3}/UniTok.egg-info/requires.txt

@@ -4,3 +4,4 @@ numpy
 pandas
 transformers
 oba
+prettytable

{UniTok-3.5.1 → UniTok-3.5.3}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text(encoding='utf8')
 
 setup(
     name='UniTok',
-    version='3.5.1',
+    version='3.5.3',
     keywords=['token', 'tokenizer'],
     description='Unified Tokenizer',
     long_description=long_description,
@@ -14,7 +14,7 @@ setup(
     license='MIT Licence',
     url='https://github.com/Jyonn/UnifiedTokenizer',
     author='Jyonn Liu',
-    author_email='i@6-79.cn',
+    author_email='liu@qijiong.work',
     platforms='any',
     packages=find_packages(),
     install_requires=[
@@ -23,7 +23,8 @@ setup(
         'numpy',
         'pandas',
         'transformers',
-        'oba'
+        'oba',
+        'prettytable',
     ],
     entry_points={
         'console_scripts': [

UniTok-3.5.1/UniTok/__main__.py

@@ -1,14 +0,0 @@
-import argparse
-
-from UniTok import UniDep
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('path', nargs='?', type=str, default='.', help='Path to the UniDep dataset')
-
-    args = parser.parse_args()
-    path = args.path
-
-    depot = UniDep(path)
-    print(depot)