xlin 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

xlin/multiprocess_mapping.py

@@ -12,18 +12,11 @@ from loguru import logger
 from xlin.jsonl import dataframe_to_json_list, load_json_list, save_json_list, load_json, save_json
 
 
-cpu_count = multiprocessing.cpu_count()
-# pool = ThreadPool(cpu_count)  # 大模型接口辣鸡，太快会截断答案
-thread_pool_size = int(os.getenv("THREAD_POOL_SIZE", 5))
-pool = ThreadPool(thread_pool_size)
-logger.debug(f"pool size: {thread_pool_size}, cpu count: {cpu_count}")
-
-
 def multiprocessing_mapping_jsonlist(
     jsonlist: List[Any],
     output_path: Optional[Union[str, Path]],
     partial_func,
-    batch_size=cpu_count * 2,
+    batch_size=multiprocessing.cpu_count(),
     cache_batch_num=1,
     thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
 ):
@@ -47,7 +40,7 @@ def multiprocessing_mapping_jsonlist(
         else:
             output_path.parent.mkdir(parents=True, exist_ok=True)
     pool = ThreadPool(thread_pool_size)
-    logger.debug(f"pool size: {thread_pool_size}, cpu count: {cpu_count}")
+    logger.debug(f"pool size: {thread_pool_size}, cpu count: {multiprocessing.cpu_count()}")
     start_time = time.time()
     last_save_time = start_time
     for i, line in tqdm(list(enumerate(jsonlist))):
@@ -78,7 +71,7 @@ def multiprocessing_mapping(
     df: pd.DataFrame,
     output_path: Optional[Union[str, Path]],
     partial_func,
-    batch_size=cpu_count * 2,
+    batch_size=multiprocessing.cpu_count(),
     cache_batch_num=1,
     thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
 ):
@@ -104,7 +97,7 @@ def multiprocessing_mapping(
         else:
             output_path.parent.mkdir(parents=True, exist_ok=True)
     pool = ThreadPool(thread_pool_size)
-    logger.debug(f"pool size: {thread_pool_size}, cpu count: {cpu_count}")
+    logger.debug(f"pool size: {thread_pool_size}, cpu count: {multiprocessing.cpu_count()}")
     start_time = time.time()
     last_save_time = start_time
     for i, line in tqdm(list(df.iterrows())):
@@ -135,29 +128,51 @@ def multiprocessing_mapping(
     return output_df, output_list
 
 
-def dataframe_by_row_mapping(
+def dataframe_with_row_mapping(
     df: pd.DataFrame,
-    mapping_func: Callable[[dict], Tuple[bool, dict]],
+    mapping_func: Callable[[Tuple[int, dict]], Tuple[bool, dict]],
     use_multiprocessing=True,
     thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
 ):
     rows = []
     if use_multiprocessing:
         pool = ThreadPool(thread_pool_size)
-        logger.debug(f"pool size: {thread_pool_size}, cpu count: {cpu_count}")
-        results = pool.map(mapping_func, dataframe_to_json_list(df))
+        logger.debug(f"pool size: {thread_pool_size}, cpu count: {multiprocessing.cpu_count()}")
+        results = pool.map(mapping_func, enumerate(dataframe_to_json_list(df)))
         for ok, row in results:
             if ok:
                 rows.append(row)
     else:
-        for i, row in df.iterrows():
-            ok, row = mapping_func(row)
+        for i, row in tqdm(df.iterrows()):
+            ok, row = mapping_func(i, row)
             if ok:
                 rows.append(row)
     df = pd.DataFrame(rows)
     return df
 
 
+def list_with_element_mapping(
+    iterator: List[Any],
+    mapping_func: Callable[[Tuple[int, Any]], Tuple[bool, Any]],
+    use_multiprocessing=True,
+    thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
+):
+    rows = []
+    if use_multiprocessing:
+        pool = ThreadPool(thread_pool_size)
+        logger.debug(f"pool size: {thread_pool_size}, cpu count: {multiprocessing.cpu_count()}")
+        results = pool.map(mapping_func, enumerate(iterator))
+        for ok, row in results:
+            if ok:
+                rows.append(row)
+    else:
+        for i, row in tqdm(enumerate(iterator)):
+            ok, row = mapping_func(i, row)
+            if ok:
+                rows.append(row)
+    return rows
+
+
 def continue_run(
     jsonfiles: List[str],
     save_dir: str,

{xlin-0.1.2.dist-info → xlin-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xlin
-Version: 0.1.2
+Version: 0.1.4
 Summary: toolbox for LinXueyuan
 License: MIT
 Author: XiChen

{xlin-0.1.2.dist-info → xlin-0.1.4.dist-info}/RECORD

@@ -1,7 +1,7 @@
 xlin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 xlin/ischinese.py,sha256=Ia9IMQ6q-UHkdLwqS70L1fTnfSPbluFrv_I1UqsKquo,293
 xlin/jsonl.py,sha256=oE8w8IFVEnBQdWUCMGYF9BlE3wtEhFsmjaLpZPKwSXg,6605
-xlin/multiprocess_mapping.py,sha256=DE-liX94UlWJUuDAgIb_FpM6oaiNZ5Hq-HapdbU-DAg,6971
+xlin/multiprocess_mapping.py,sha256=bJKYAvhbeRICn7p3iXYJTh-NRloxgvCa4rpGJSRLh6Y,7516
 xlin/read_as_dataframe.py,sha256=ir3HUT6dt3crqa3xnlcNn8j3wqjSIGJgiIVLP3KkBaQ,8678
 xlin/statistic.py,sha256=BLj8hszlbBT5xDIfd70_YtOb8QgZEvYXiFJDGXBwCfw,881
 xlin/terminal_color.py,sha256=nfE-CY2BzjY2eZbm9yk8r-AuyJ-hchmLXhASCb4HAIA,191
@@ -9,7 +9,7 @@ xlin/util.py,sha256=hme7Zl4Sa_-FTA9TEVzr1qTdaKW1eq5dTWZgd4owcDc,11303
 xlin/uuid.py,sha256=gouvm7_DL22sIhXl-g4e6S2qzIZtmE3SEp00xy1upyg,271
 xlin/xls2xlsx.py,sha256=5zfcM0gmunFQOcOj9nYd9Dj0HMhU7-cPKnPIy6Ot9iU,930
 xlin/yaml.py,sha256=kICi7G3Td5q2MaSXXt85qNTWoHMgjzt7pvn7r3C4dME,183
-xlin-0.1.2.dist-info/LICENSE,sha256=KX0dDCYlO4DskqMZY8qeY94EZMrDRNnNqlGLkXVlKyM,1063
-xlin-0.1.2.dist-info/METADATA,sha256=PKad7J7_9fQ2148ciupAUuSS8Js_3_1C2ZPmM2hnt44,772
-xlin-0.1.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-xlin-0.1.2.dist-info/RECORD,,
+xlin-0.1.4.dist-info/LICENSE,sha256=KX0dDCYlO4DskqMZY8qeY94EZMrDRNnNqlGLkXVlKyM,1063
+xlin-0.1.4.dist-info/METADATA,sha256=ryDOmmS9Tt8A_ig_5zx6Gcme0Wwg1Wb4V0HPo_h2PNs,772
+xlin-0.1.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+xlin-0.1.4.dist-info/RECORD,,