xlin 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

xlin/jsonl.py

@@ -47,6 +47,28 @@ def jsonlist_to_dataframe(json_list: List[Dict[str, str]]):
     return pd.DataFrame(json_list)
 
 
+def is_jsonl(filepath: str):
+    with open(filepath) as f:
+        try:
+            l = next(f)  # 读取一行，用来判断文件是json还是jsonl格式
+            f.seek(0)
+        except:
+            return False
+
+        try:
+            _ = json.loads(l)
+        except ValueError:
+            return False  # 第一行不是json，所以是json格式
+        else:
+            return True  # 第一行是json，所以是jsonl格式
+
+
+def load_json_or_jsonl(filepath: str):
+    if is_jsonl(filepath):
+        return load_json_list(filepath)
+    return load_json(filepath)
+
+
 def load_json(filename: str):
     with open(filename, "r", encoding="utf-8") as f:
         return json.load(f)
@@ -66,6 +88,8 @@ def load_json_list(filename: str):
             try:
                 obj = json.loads(i.strip())
             except:
+                print("格式损坏数据，无法加载")
+                print(i)
                 continue
             json_list.append(obj)
         return json_list

xlin/multiprocess_mapping.py

@@ -9,7 +9,7 @@ from pathlib import Path
 from tqdm import tqdm
 from loguru import logger
 
-from jsonl import dataframe_to_json_list, load_json_list, save_json_list, load_json, save_json
+from xlin.jsonl import dataframe_to_json_list, load_json_list, save_json_list, load_json, save_json
 
 
 cpu_count = multiprocessing.cpu_count()

xlin/read_as_dataframe.py

@@ -7,9 +7,9 @@ from loguru import logger
 import pandas as pd
 import pyexcel
 
-from util import ls
-from jsonl import dataframe_to_json_list, load_json, load_json_list, save_json_list
-from xls2xlsx import is_xslx
+from xlin.util import ls
+from xlin.jsonl import dataframe_to_json_list, load_json, load_json_list, save_json_list
+from xlin.xls2xlsx import is_xslx
 
 
 def valid_to_read_as_dataframe(filename: str) -> bool:

{xlin-0.1.0.dist-info → xlin-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xlin
-Version: 0.1.0
+Version: 0.1.2
 Summary: toolbox for LinXueyuan
 License: MIT
 Author: XiChen

xlin-0.1.2.dist-info/RECORD

@@ -0,0 +1,15 @@
+xlin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xlin/ischinese.py,sha256=Ia9IMQ6q-UHkdLwqS70L1fTnfSPbluFrv_I1UqsKquo,293
+xlin/jsonl.py,sha256=oE8w8IFVEnBQdWUCMGYF9BlE3wtEhFsmjaLpZPKwSXg,6605
+xlin/multiprocess_mapping.py,sha256=DE-liX94UlWJUuDAgIb_FpM6oaiNZ5Hq-HapdbU-DAg,6971
+xlin/read_as_dataframe.py,sha256=ir3HUT6dt3crqa3xnlcNn8j3wqjSIGJgiIVLP3KkBaQ,8678
+xlin/statistic.py,sha256=BLj8hszlbBT5xDIfd70_YtOb8QgZEvYXiFJDGXBwCfw,881
+xlin/terminal_color.py,sha256=nfE-CY2BzjY2eZbm9yk8r-AuyJ-hchmLXhASCb4HAIA,191
+xlin/util.py,sha256=hme7Zl4Sa_-FTA9TEVzr1qTdaKW1eq5dTWZgd4owcDc,11303
+xlin/uuid.py,sha256=gouvm7_DL22sIhXl-g4e6S2qzIZtmE3SEp00xy1upyg,271
+xlin/xls2xlsx.py,sha256=5zfcM0gmunFQOcOj9nYd9Dj0HMhU7-cPKnPIy6Ot9iU,930
+xlin/yaml.py,sha256=kICi7G3Td5q2MaSXXt85qNTWoHMgjzt7pvn7r3C4dME,183
+xlin-0.1.2.dist-info/LICENSE,sha256=KX0dDCYlO4DskqMZY8qeY94EZMrDRNnNqlGLkXVlKyM,1063
+xlin-0.1.2.dist-info/METADATA,sha256=PKad7J7_9fQ2148ciupAUuSS8Js_3_1C2ZPmM2hnt44,772
+xlin-0.1.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+xlin-0.1.2.dist-info/RECORD,,

xlin-0.1.0.dist-info/RECORD

@@ -1,14 +0,0 @@
-xlin/ischinese.py,sha256=Ia9IMQ6q-UHkdLwqS70L1fTnfSPbluFrv_I1UqsKquo,293
-xlin/jsonl.py,sha256=Fg9JqyzV9gmBW7R4P8aqdURyM3pD_pS2c2wVeFK2MVY,5938
-xlin/multiprocess_mapping.py,sha256=WfMCV9g15HufvE8Y9a8ken983rZV0_I7sQV3H6-5zx0,6966
-xlin/read_as_dataframe.py,sha256=32u5AkUjdAtvGjBg0BEAxKsO5qpRAtWR38_JTYfJyn8,8663
-xlin/statistic.py,sha256=BLj8hszlbBT5xDIfd70_YtOb8QgZEvYXiFJDGXBwCfw,881
-xlin/terminal_color.py,sha256=nfE-CY2BzjY2eZbm9yk8r-AuyJ-hchmLXhASCb4HAIA,191
-xlin/util.py,sha256=hme7Zl4Sa_-FTA9TEVzr1qTdaKW1eq5dTWZgd4owcDc,11303
-xlin/uuid.py,sha256=gouvm7_DL22sIhXl-g4e6S2qzIZtmE3SEp00xy1upyg,271
-xlin/xls2xlsx.py,sha256=5zfcM0gmunFQOcOj9nYd9Dj0HMhU7-cPKnPIy6Ot9iU,930
-xlin/yaml.py,sha256=kICi7G3Td5q2MaSXXt85qNTWoHMgjzt7pvn7r3C4dME,183
-xlin-0.1.0.dist-info/LICENSE,sha256=KX0dDCYlO4DskqMZY8qeY94EZMrDRNnNqlGLkXVlKyM,1063
-xlin-0.1.0.dist-info/METADATA,sha256=b4hLo7rTVP9S_87OT3ljqFz4KAZsEAgbJnB_yWdOtdQ,772
-xlin-0.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-xlin-0.1.0.dist-info/RECORD,,