xlin 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

xlin/multiprocess_mapping.py

@@ -9,7 +9,61 @@ from pathlib import Path
 from tqdm import tqdm
 from loguru import logger
 
-from xlin.jsonl import dataframe_to_json_list, load_json_list, save_json_list, load_json, save_json
+from xlin.jsonl import load_json_list, save_json_list, load_json, save_json
+from xlin.util import ls
+
+
+def element_mapping(
+    iterator: List[Any],
+    mapping_func: Callable[[Any], Tuple[bool, Any]],
+    use_multiprocessing=True,
+    thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
+):
+    rows = []
+    if use_multiprocessing:
+        pool = ThreadPool(thread_pool_size)
+        results = pool.map(mapping_func, iterator)
+        for ok, row in results:
+            if ok:
+                rows.append(row)
+    else:
+        for row in tqdm(iterator):
+            ok, row = mapping_func(row)
+            if ok:
+                rows.append(row)
+    return rows
+
+
+def batch_mapping(
+    iterator: List[Any],
+    mapping_func: Callable[[List[Any]], Tuple[bool, List[Any]]],
+    use_multiprocessing=True,
+    thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
+    batch_size=4,
+):
+    batch_iterator = []
+    batch = []
+    for i, item in enumerate(iterator):
+        batch.append(item)
+        if len(batch) == batch_size:
+            batch_iterator.append(batch)
+            batch = []
+    if len(batch) > 0:
+        batch_iterator.append(batch)
+    rows = element_mapping(batch_iterator, mapping_func, use_multiprocessing, thread_pool_size)
+    rows = [row for batch in rows for row in batch]
+    return rows
+
+
+def dataframe_with_row_mapping(
+    df: pd.DataFrame,
+    mapping_func: Callable[[dict], Tuple[bool, dict]],
+    use_multiprocessing=True,
+    thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
+):
+    rows = element_mapping(df.iterrows(), lambda x: mapping_func(x[1]), use_multiprocessing, thread_pool_size)
+    df = pd.DataFrame(rows)
+    return df
 
 
 def multiprocessing_mapping_jsonlist(
@@ -128,51 +182,6 @@ def multiprocessing_mapping(
     return output_df, output_list
 
 
-def dataframe_with_row_mapping(
-    df: pd.DataFrame,
-    mapping_func: Callable[[int, dict], Tuple[bool, dict]],
-    use_multiprocessing=True,
-    thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
-):
-    rows = []
-    if use_multiprocessing:
-        pool = ThreadPool(thread_pool_size)
-        logger.debug(f"pool size: {thread_pool_size}, cpu count: {multiprocessing.cpu_count()}")
-        results = pool.map(mapping_func, enumerate(dataframe_to_json_list(df)))
-        for ok, row in results:
-            if ok:
-                rows.append(row)
-    else:
-        for i, row in tqdm(df.iterrows()):
-            ok, row = mapping_func(i, row)
-            if ok:
-                rows.append(row)
-    df = pd.DataFrame(rows)
-    return df
-
-
-def list_with_element_mapping(
-    iterator: List[Any],
-    mapping_func: Callable[[int, Any], Tuple[bool, Any]],
-    use_multiprocessing=True,
-    thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
-):
-    rows = []
-    if use_multiprocessing:
-        pool = ThreadPool(thread_pool_size)
-        logger.debug(f"pool size: {thread_pool_size}, cpu count: {multiprocessing.cpu_count()}")
-        results = pool.map(mapping_func, enumerate(iterator))
-        for ok, row in results:
-            if ok:
-                rows.append(row)
-    else:
-        for i, row in tqdm(enumerate(iterator)):
-            ok, row = mapping_func(i, row)
-            if ok:
-                rows.append(row)
-    return rows
-
-
 def continue_run(
     jsonfiles: List[str],
     save_dir: str,
@@ -185,8 +194,7 @@ def continue_run(
     save_dir: Path = Path(save_dir)
     save_dir.mkdir(parents=True, exist_ok=True)
     new_jsonfiles = []
-    for jsonfile in jsonfiles:
-        jsonfile = Path(jsonfile)
+    for jsonfile in ls(jsonfiles):
         jsonlist = load_func(jsonfile)
         output_filepath = save_dir / jsonfile.name
         for row in jsonlist:

xlin/util.py

@@ -1,5 +1,5 @@
-from collections import defaultdict
 from typing import *
+from collections import defaultdict
 from pathlib import Path
 import pandas as pd
 import os

{xlin-0.1.3.dist-info → xlin-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xlin
-Version: 0.1.3
+Version: 0.1.5
 Summary: toolbox for LinXueyuan
 License: MIT
 Author: XiChen

{xlin-0.1.3.dist-info → xlin-0.1.5.dist-info}/RECORD

@@ -1,15 +1,15 @@
 xlin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 xlin/ischinese.py,sha256=Ia9IMQ6q-UHkdLwqS70L1fTnfSPbluFrv_I1UqsKquo,293
 xlin/jsonl.py,sha256=oE8w8IFVEnBQdWUCMGYF9BlE3wtEhFsmjaLpZPKwSXg,6605
-xlin/multiprocess_mapping.py,sha256=mvkxra4uPvHfwHfI38FMEZU4dVYf6mu3gMVMAC1-P4o,7502
+xlin/multiprocess_mapping.py,sha256=lLw0sBFDymk38DdvNV65IPV3_3Vqw2SiYtRpwTgCbr0,7597
 xlin/read_as_dataframe.py,sha256=ir3HUT6dt3crqa3xnlcNn8j3wqjSIGJgiIVLP3KkBaQ,8678
 xlin/statistic.py,sha256=BLj8hszlbBT5xDIfd70_YtOb8QgZEvYXiFJDGXBwCfw,881
 xlin/terminal_color.py,sha256=nfE-CY2BzjY2eZbm9yk8r-AuyJ-hchmLXhASCb4HAIA,191
-xlin/util.py,sha256=hme7Zl4Sa_-FTA9TEVzr1qTdaKW1eq5dTWZgd4owcDc,11303
+xlin/util.py,sha256=SOQUh506GQlljJYLYuI6nScSTOrgRQnMq2xfxSvKIlI,11303
 xlin/uuid.py,sha256=gouvm7_DL22sIhXl-g4e6S2qzIZtmE3SEp00xy1upyg,271
 xlin/xls2xlsx.py,sha256=5zfcM0gmunFQOcOj9nYd9Dj0HMhU7-cPKnPIy6Ot9iU,930
 xlin/yaml.py,sha256=kICi7G3Td5q2MaSXXt85qNTWoHMgjzt7pvn7r3C4dME,183
-xlin-0.1.3.dist-info/LICENSE,sha256=KX0dDCYlO4DskqMZY8qeY94EZMrDRNnNqlGLkXVlKyM,1063
-xlin-0.1.3.dist-info/METADATA,sha256=Z0r6K1HueSI2T-U8I3xxV5Vl-92NZuwnQVuztiXClIk,772
-xlin-0.1.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-xlin-0.1.3.dist-info/RECORD,,
+xlin-0.1.5.dist-info/LICENSE,sha256=KX0dDCYlO4DskqMZY8qeY94EZMrDRNnNqlGLkXVlKyM,1063
+xlin-0.1.5.dist-info/METADATA,sha256=SpRGumS27m1HewBbF_pPDuT8SlGnIuKwD9C2nNvhGaQ,772
+xlin-0.1.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+xlin-0.1.5.dist-info/RECORD,,