xlin 0.1.5__tar.gz → 0.1.7__tar.gz

{xlin-0.1.5 → xlin-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xlin
-Version: 0.1.5
+Version: 0.1.7
 Summary: toolbox for LinXueyuan
 License: MIT
 Author: XiChen

{xlin-0.1.5 → xlin-0.1.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "xlin"
-version = "0.1.5"
+version = "0.1.7"
 description = "toolbox for LinXueyuan"
 authors = ["XiChen <23211526+LinXueyuanStdio@users.noreply.github.com>"]
 license = "MIT"

{xlin-0.1.5 → xlin-0.1.7}/xlin/jsonl.py

@@ -62,6 +62,10 @@ def is_jsonl(filepath: str):
         else:
             return True  # 第一行是json，所以是jsonl格式
 
+def load_text(filename):
+    with open(filename, 'r') as f:
+        return f.read()
+
 
 def load_json_or_jsonl(filepath: str):
     if is_jsonl(filepath):

{xlin-0.1.5 → xlin-0.1.7}/xlin/multiprocess_mapping.py

@@ -23,6 +23,7 @@ def element_mapping(
     if use_multiprocessing:
         pool = ThreadPool(thread_pool_size)
         results = pool.map(mapping_func, iterator)
+        pool.close()
         for ok, row in results:
             if ok:
                 rows.append(row)
@@ -116,6 +117,7 @@ def multiprocessing_mapping_jsonlist(
     if len(tmp_list) > 0:
         results = pool.map(partial_func, tmp_list)
         output_list.extend([x for x in results])
+    pool.close()
     if need_caching:
         save_json_list(output_list, output_path)
     return output_list
@@ -176,6 +178,7 @@ def multiprocessing_mapping(
     if len(tmp_list) > 0:
         results = pool.map(partial_func, tmp_list)
         output_list.extend([x for x in results])
+    pool.close()
     output_df = pd.DataFrame(output_list)
     if need_caching:
         output_df.to_excel(output_path, index=False)