xlin 0.1.37__py2.py3-none-any.whl → 0.1.39__py2.py3-none-any.whl

xlin/__init__.py

@@ -1,10 +1,12 @@
-from .ischinese import *
-from .jsonl import *
+from .dataframe_util import *
+from .datetime_util import *
+from .file_util import *
+from .image_util import *
+from .jsonlist_util import *
 from .metric import *
-from .multiprocess_mapping import *
-from .read_as_dataframe import *
+from .multiprocess_util import *
 from .statistic import *
-from .timing import *
-from .util import *
-from .xls2xlsx import *
-from .yaml import *
+from .text_util import *
+from .timing_util import *
+from .xlsx_util import *
+from .yaml_util import *

xlin/{read_as_dataframe.py → dataframe_util.py}

@@ -1,3 +1,4 @@
+import asyncio
 from collections import defaultdict
 import os
 from typing import Callable, Dict, List, Optional, Tuple, Union
@@ -7,9 +8,9 @@ from loguru import logger
 import pandas as pd
 import pyexcel
 
-from xlin.util import ls
-from xlin.jsonl import dataframe_to_json_list, load_json, load_json_list, save_json_list
-from xlin.xls2xlsx import is_xslx
+from xlin.file_util import ls
+from xlin.jsonlist_util import dataframe_to_json_list, load_json, load_json_list, save_json_list
+from xlin.xlsx_util import is_xslx
 
 
 def read_as_dataframe(
@@ -77,7 +78,9 @@ def read_as_dataframe(
     elif filename.endswith(".html"):
         df = pd.read_html(filepath)[0]
     elif filename.endswith(".db"):
-        df = pd.read_sql_table(sheet_name, filepath)
+        if sheet_name is None:
+            raise ValueError("读取 .db 文件需要提供 sheet_name 作为表名")
+        df = pd.read_sql_table(sheet_name, f"sqlite:///{filepath}")
     else:
         raise ValueError(
             (
@@ -267,3 +270,39 @@ def split_dataframe(
         df_i.to_excel(filepath, index=False)
         df_list.append(df_i)
     return df_list
+
+def append_column(df: pd.DataFrame, query_column: str, output_column: str, transform):
+    query = df[query_column].tolist()
+    loop = asyncio.get_event_loop()
+    result = loop.run_until_complete(transform(query))
+    df[output_column] = [str(r) for r in result]
+    return df
+
+def grouped_col_list(df: pd.DataFrame, key_col="query", value_col="output"):
+    grouped = defaultdict(list)
+    if key_col not in df.columns:
+        logger.warning(f"`{key_col}` not in columns: {list(df.columns)}")
+        return grouped
+    for i, row in df.iterrows():
+        grouped[row[key_col]].append(row[value_col])
+    return grouped
+
+
+def grouped_col(df: pd.DataFrame, key_col="query", value_col="output"):
+    grouped = {}
+    if key_col not in df.columns:
+        logger.warning(f"`{key_col}` not in columns: {list(df.columns)}")
+        return grouped
+    for i, row in df.iterrows():
+        grouped[row[key_col]] = row[value_col]
+    return grouped
+
+
+def grouped_row(df: pd.DataFrame, key_col="query"):
+    grouped = defaultdict(list)
+    if key_col not in df.columns:
+        logger.warning(f"`{key_col}` not in columns: {list(df.columns)}")
+        return grouped
+    for i, row in df.iterrows():
+        grouped[row[key_col]].append(row)
+    return grouped

xlin/datetime_util.py

@@ -0,0 +1,21 @@
+
+
+import datetime
+import random
+
+
+date_str = datetime.datetime.now().strftime("%Y%m%d")
+datetime_str = datetime.datetime.now().strftime("%Y%m%d_%Hh%Mm%Ss")
+
+
+def random_timestamp(start_timestamp=None, end_timestamp=None):
+    if start_timestamp is None:
+        start_timestamp = datetime.datetime(2024, 1, 1).timestamp()
+    if end_timestamp is None:
+        end_timestamp = datetime.datetime.now().timestamp()
+    return random.uniform(start_timestamp, end_timestamp)
+
+
+def random_timestamp_str(start_timestamp=None, end_timestamp=None, format="%Y年%m月%d日%H时%M分"):
+    return datetime.datetime.fromtimestamp(random_timestamp(start_timestamp, end_timestamp)).strftime(format)
+

xlin/{util.py → file_util.py}

@@ -1,30 +1,12 @@
 from typing import *
 from collections import defaultdict
 from pathlib import Path
-import pandas as pd
 import os
 import asyncio
-import datetime
-from loguru import logger
 import shutil
-import random
-
-
-date_str = datetime.datetime.now().strftime("%Y%m%d")
-datetime_str = datetime.datetime.now().strftime("%Y%m%d_%Hh%Mm%Ss")
-
-
-def random_timestamp(start_timestamp=None, end_timestamp=None):
-    if start_timestamp is None:
-        start_timestamp = datetime.datetime(2024, 1, 1).timestamp()
-    if end_timestamp is None:
-        end_timestamp = datetime.datetime.now().timestamp()
-    return random.uniform(start_timestamp, end_timestamp)
-
-
-def random_timestamp_str(start_timestamp=None, end_timestamp=None, format="%Y年%m月%d日%H时%M分"):
-    return datetime.datetime.fromtimestamp(random_timestamp(start_timestamp, end_timestamp)).strftime(format)
 
+import pandas as pd
+from loguru import logger
 
 def auto_retry_to_get_data(retry_times, request, data_key="data", *args, **kwargs):
     if retry_times == 0:
@@ -39,13 +21,6 @@ def auto_retry_to_get_data(retry_times, request, data_key="data", *args, **kwarg
     return auto_retry_to_get_data(retry_times - 1, request, data_key, *args, **kwargs)
 
 
-def append_column(df: pd.DataFrame, query_column: str, output_column: str, transform):
-    query = df[query_column].tolist()
-    loop = asyncio.get_event_loop()
-    result = loop.run_until_complete(transform(query))
-    df[output_column] = [str(r) for r in result]
-    return df
-
 
 def request_wrapper(request_num=10):
     def request_wrapper_body(func):
@@ -201,47 +176,6 @@ def clean_empty_folder(dir_path):
                 clean_empty_folder(path)
 
 
-def grouped_col_list(df: pd.DataFrame, key_col="query", value_col="output"):
-    grouped = defaultdict(list)
-    if key_col not in df.columns:
-        logger.warning(f"`{key_col}` not in columns: {list(df.columns)}")
-        return grouped
-    for i, row in df.iterrows():
-        grouped[row[key_col]].append(row[value_col])
-    return grouped
-
-
-def grouped_col(df: pd.DataFrame, key_col="query", value_col="output"):
-    grouped = {}
-    if key_col not in df.columns:
-        logger.warning(f"`{key_col}` not in columns: {list(df.columns)}")
-        return grouped
-    for i, row in df.iterrows():
-        grouped[row[key_col]] = row[value_col]
-    return grouped
-
-
-def grouped_row(df: pd.DataFrame, key_col="query"):
-    grouped = defaultdict(list)
-    if key_col not in df.columns:
-        logger.warning(f"`{key_col}` not in columns: {list(df.columns)}")
-        return grouped
-    for i, row in df.iterrows():
-        grouped[row[key_col]].append(row)
-    return grouped
-
-
-def grouped_row_in_jsonlist(jsonlist: List[Dict[str, Any]], key_col="query"):
-    grouped = defaultdict(list)
-    for i, row in enumerate(jsonlist):
-        if key_col not in row:
-            logger.warning(f"`{key_col}` not in row: {row}")
-            notfound_key = f"NotFound:{key_col}"
-            grouped[notfound_key].append(row)
-            continue
-        grouped[row[key_col]].append(row)
-    return grouped
-
 
 def submit_file(path: Union[str, Path], target_dir: Union[str, Path]):
     p = Path(path).absolute()
@@ -266,44 +200,3 @@ def submit_file(path: Union[str, Path], target_dir: Union[str, Path]):
     logger.info("现在目标文件夹下的文件有：\n" + "\n".join(filenames))
 
 
-def pretty_limited_text(text: str, limited_length: int = 300, language="zh"):
-    text = str(text).strip()
-    if len(text) > limited_length:
-        # if language == "zh":
-        #     tail = f"...(共{len(text)}字)"
-        # else:
-        #     tail = f"...({len(text)} words in total)"
-        # return text[: limited_length - len(tail)] + tail
-        return text[: limited_length // 2] + text[-limited_length // 2 :]
-    return text
-
-
-def bucket_count(length):
-    grouped_count = []
-    j = 0
-    for i in range(0, max(length), 50):
-        grouped_count.append(0)
-        while length[j] < i:
-            grouped_count[i // 50] += 1
-            j += 1
-    for i, j in enumerate(grouped_count):
-        if i == 0 or j == 0:
-            continue
-        print(f"[{(i-1)*50}, {i*50})  {j}   {sum(grouped_count[:i+1])/len(length)*100:.2f}%")
-
-
-def sortedCounter(obj, by="key", reverse=False, return_list=False):
-    c = Counter(obj)
-    c_list = [(k, c[k]) for k in c]
-    if by == "key":
-        c_list = sorted(c_list, key=lambda x: x[0], reverse=reverse)
-    elif by in ["value", "count"]:
-        c_list = sorted(c_list, key=lambda x: x[1], reverse=reverse)
-    else:
-        raise Exception(f"unsupported by: {by}")
-    c = Counter()
-    for k, v in c_list:
-        c[k] = v
-    if return_list:
-        return c, c_list
-    return c

xlin/image_util.py

@@ -0,0 +1,237 @@
+
+import base64
+from io import BytesIO
+from loguru import logger
+from PIL import Image, ImageDraw, ImageFont
+import uuid
+import os
+
+
+def image_to_base64(image: Image.Image) -> str:
+    buffered = BytesIO()
+    image.save(buffered, format="PNG")
+    b64 = base64.b64encode(buffered.getvalue()).decode()
+    return f"data:image/png;base64,{b64}"
+
+
+def base64_to_image(base64_str: str) -> Image.Image:
+    """
+    Convert a base64 string to an image.
+    """
+    prefix_list = [
+        "data:image/png;base64,",
+        "data:image/jpeg;base64,",
+        "data:image/gif;base64,",
+        "data:image/webp;base64,",
+    ]
+    for prefix in prefix_list:
+        if base64_str.startswith(prefix):
+            base64_str = base64_str[len(prefix):]
+            break
+    image_data = base64.b64decode(base64_str)
+    image = Image.open(BytesIO(image_data))
+    return image
+
+
+def generate_short_uuid(length=8):
+    # 生成标准 UUID
+    uuid_value = uuid.uuid4().bytes
+
+    # 使用 Base64 编码并转换为 URL 安全格式
+    encoded = base64.urlsafe_b64encode(uuid_value).decode("ascii")
+
+    # 移除可能的填充字符 '='
+    encoded = encoded.rstrip("=")
+
+    # 截取指定长度的字符串
+    return encoded[:length]
+
+
+
+def scale_to_fit(image: Image.Image, target_size: tuple[int, int]=(512, 512)) -> Image.Image:
+    """
+    将图像缩放到适合目标大小的尺寸，同时保持原始宽高比。
+
+    args:
+        image: PIL.Image.Image
+            要缩放的图像。
+        target_size: tuple[int, int]
+            目标大小，格式为 (width, height)。
+
+    return: PIL.Image.Image
+        缩放后的图像。
+    """
+    original_width, original_height = image.size
+    target_width, target_height = target_size
+
+    # 计算缩放比例
+    width_ratio = target_width / original_width
+    height_ratio = target_height / original_height
+    scale_ratio = min(width_ratio, height_ratio)
+    if scale_ratio >= 1:
+        # 如果图像已经小于或等于目标大小，则不需要缩放
+        return image
+
+    # 计算新的尺寸
+    new_width = round(original_width * scale_ratio)
+    new_height = round(original_height * scale_ratio)
+
+    # 缩放图像
+    resized_image = image.resize((new_width, new_height), Image.Resampling.LANCZOS)
+    return resized_image
+
+
+def add_scale_bar(
+    image: Image.Image,
+    spacing=64,
+    color=(0, 0, 0),
+    font_size=12,
+    left_margin=50,
+    top_margin=50,
+    tick_length=8,
+    tick_width=2,
+    text_offset=2,
+    origin_size: tuple[int, int] = None,
+):
+    """
+    为图像添加顶部和左侧标尺，并将文字标签放在空白边距中，不与原图重叠。
+
+    args:
+        image: PIL.Image.Image
+            要添加标尺的图像。
+        spacing: int
+            刻度之间的间隔，单位为像素。
+        color: tuple
+            刻度线和文字的颜色，RGB格式。
+        font_size: int
+            文字的字体大小。
+        left_margin: int
+            左侧边距的宽度，单位为像素。
+        top_margin: int
+            顶部边距的高度，单位为像素。
+        tick_length: int
+            刻度线的长度，单位为像素。
+        tick_width: int
+            刻度线的宽度，单位为像素。
+        text_offset: int
+            文字与刻度线之间的距离，单位为像素。
+        origin_size: tuple[int, int]
+            原图的尺寸，格式为 (width, height)。如果未提供，则使用图像的实际尺寸。
+    return: PIL.Image.Image
+
+    示例用法
+    ```
+    img = Image.open("/Pictures/example.png")
+    out = add_scale_bar(
+        img,
+        spacing=100,
+        color=(0, 0, 0),
+        font_size=12,
+        left_margin=50,
+        top_margin=50,
+        tick_length=8,
+        text_offset=4,
+        origin_size=(img.width, img.height)  # 可选，指定原图尺寸
+    )
+    out
+    ```
+    """
+    # 加载字体
+    try:
+        font_path = "C:/Windows/Fonts/arial.ttf"
+        if not os.path.exists(font_path):
+            font_path = "/System/Library/Fonts/Supplemental/Arial.ttf"
+        if not os.path.exists(font_path):
+            font_path = "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"
+        if not os.path.exists(font_path):
+            font_path = "/usr/share/fonts/truetype/freefont/FreeMono.ttf"
+        if not os.path.exists(font_path):
+            font_path = "/usr/share/fonts/truetype/liberation/LiberationMono-Regular.ttf"
+        if not os.path.exists(font_path):
+            font_path = "/usr/share/fonts/truetype/noto/NotoSansMono-Regular.ttf"
+        if not os.path.exists(font_path):
+            font_path = "/usr/share/fonts/truetype/ubuntu/Ubuntu-C.ttf"
+        font = ImageFont.truetype(font_path, font_size)
+    except:
+        font = ImageFont.load_default()
+
+    w, h = image.size
+    new_w, new_h = w + left_margin, h + top_margin
+
+    # 创建背景画布并粘贴原图
+    mode = image.mode
+    bg = (255, 255, 255) if mode == "RGB" else (255,)
+    canvas = Image.new(mode, (new_w, new_h), bg)
+    canvas.paste(image, (left_margin, top_margin))
+
+    draw = ImageDraw.Draw(canvas)
+
+    # 计算文字宽高的 helper
+    def text_dimensions(txt):
+        bbox = draw.textbbox((0, 0), txt, font=font)
+        return bbox[2] - bbox[0], bbox[3] - bbox[1]
+
+    origin_width, origin_height = origin_size if origin_size else (w, h)
+
+    # 顶部刻度和文字
+    x_ticks = range(0, w + 1, spacing)
+    for i, x in enumerate(x_ticks):
+        # 计算刻度线的 x 坐标
+        px = left_margin + x
+        if i == len(x_ticks) - 1:
+            # 最后一个刻度线在右侧边界
+            px = new_w - tick_width
+        # 刻度线
+        draw.line([(px, top_margin), (px, top_margin - tick_length)], width=tick_width, fill=color)
+        # 文字
+        origin_x = x * origin_width // w  # 将刻度值映射到原图尺寸
+        if i == len(x_ticks) - 1:
+            origin_x = origin_width  # 确保最后一个刻度值是原图宽度
+        txt = str(origin_x)
+        tw, th = text_dimensions(txt)
+        tx = px - tw / 2
+        if i == len(x_ticks) - 1:
+            # 最后一个刻度的文字放在刻度线的左边
+            tx = tx - tw / 2
+        ty = top_margin - tick_length - th - text_offset
+        draw.text((tx, ty), txt, fill=color, font=font)
+
+    # 左侧刻度和文字
+    y_ticks = range(0, h + 1, spacing)
+    for i, y in enumerate(y_ticks):
+        # 计算刻度线的 y 坐标
+        py = top_margin + y
+        if i == len(y_ticks) - 1:
+            # 最后一个刻度线在底部边界
+            py = new_h - tick_width
+        # 刻度线
+        draw.line([(left_margin, py), (left_margin - tick_length, py)], width=tick_width, fill=color)
+        # 文字
+        origin_y = y * origin_height // h  # 将刻度值映射到原图尺寸
+        if i == len(y_ticks) - 1:
+            origin_y = origin_height
+        txt = str(origin_y)
+        tw, th = text_dimensions(txt)
+        tx = left_margin - tick_length - tw - text_offset
+        ty = py - th / 2
+        if i == len(y_ticks) - 1:
+            # 最后一个刻度的文字放在刻度线的上边
+            ty = ty - th / 3 * 2
+        draw.text((tx, ty), txt, fill=color, font=font)
+
+    return canvas
+
+
+
+def scale_to_fit_and_add_scale_bar(image: Image.Image, debug=False) -> Image.Image:
+    origin_width, origin_height = image.size
+    target_width, target_height = 512, 512
+    if debug:
+        logger.debug(f"原图尺寸: {origin_width}x{origin_height}, 目标尺寸: {target_width}x{target_height}")
+    image = scale_to_fit(image, target_size=(target_width, target_height))  # 缩放图片到目标大小，为了省 image tokens
+    if debug:
+        logger.debug(f"缩放后图片尺寸: {image.size[0]}x{image.size[1]}")
+    image = add_scale_bar(image, origin_size=(origin_width, origin_height))  # 保持缩放后的比例尺为原图的比例尺，方便模型在原图上定位坐标和长宽用于裁剪
+    if debug:
+        logger.debug(f"添加比例尺后图片尺寸: {image.size[0]}x{image.size[1]}")
+    return image

xlin/{jsonl.py → jsonlist_util.py}

@@ -1,9 +1,14 @@
+from collections import defaultdict
 import json
 from typing import *
 
 from pathlib import Path
 from loguru import logger
 import pandas as pd
+import pyexcel
+
+from xlin.file_util import ls
+from xlin.xlsx_util import is_xslx
 
 
 def dataframe_to_json_list(df: pd.DataFrame):
@@ -68,11 +73,91 @@ def load_text(filename):
 
 
 def load_json_or_jsonl(filepath: str):
+    """
+    read_as_json_list 更好用，可以无缝切换到：read_as_json_list(filepath)
+    """
     if is_jsonl(filepath):
         return load_json_list(filepath)
     return load_json(filepath)
 
 
+def read_as_json_list(
+    filepath: Union[str, Path, List[str], List[Path]],
+    sheet_name: Optional[str] = None,
+    skip_None: bool = True,
+    skip_blank: bool = True,
+    filter: Callable[[Path], bool] = lambda x: True,
+) -> List[Dict]:
+    """
+    读取文件或递归读取文件夹里的文件为 JSON list（List[Dict]）。
+    支持格式：json, jsonl, xlsx, xls, csv, parquet, feather, pkl, h5, txt, tsv, xml, html, db
+    """
+    if isinstance(filepath, list):
+        json_list = []
+        for path in filepath:
+            try:
+                sub_list = read_as_json_list(path, sheet_name, skip_None, skip_blank, filter)
+                for obj in sub_list:
+                    if isinstance(obj, dict):
+                        obj["数据来源"] = Path(path).name
+                json_list.extend(sub_list)
+            except Exception as e:
+                print(f"读取失败 {path}: {e}")
+        return json_list
+
+    filepath = Path(filepath)
+    if filepath.is_dir():
+        paths = ls(filepath, filter=filter, expand_all_subdir=True)
+        return read_as_json_list(paths, sheet_name, skip_None, skip_blank, filter)
+
+    filename = filepath.name
+    if filename.endswith(".json") or filename.endswith(".jsonl"):
+        if is_jsonl(filepath):
+            return load_json_list(filepath)
+        else:
+            return [load_json(filepath)]
+
+    elif filename.endswith(".xlsx"):
+        if sheet_name is None:
+            df = pd.read_excel(filepath)
+        else:
+            df = pd.read_excel(filepath, sheet_name)
+    elif filename.endswith(".xls"):
+        if is_xslx(filepath):
+            if sheet_name is None:
+                df = pd.read_excel(filepath)
+            else:
+                df = pd.read_excel(filepath, sheet_name)
+        else:
+            df = pyexcel.get_sheet(file_name=filepath)
+    elif filename.endswith(".csv"):
+        df = pd.read_csv(filepath)
+    elif filename.endswith(".parquet"):
+        df = pd.read_parquet(filepath)
+    elif filename.endswith(".feather"):
+        df = pd.read_feather(filepath)
+    elif filename.endswith(".pkl"):
+        df = pd.read_pickle(filepath)
+    elif filename.endswith(".h5"):
+        df = pd.read_hdf(filepath)
+    elif filename.endswith(".txt"):
+        df = pd.read_csv(filepath, delimiter="\t")
+    elif filename.endswith(".tsv"):
+        df = pd.read_csv(filepath, delimiter="\t")
+    elif filename.endswith(".xml"):
+        df = pd.read_xml(filepath)
+    elif filename.endswith(".html"):
+        df = pd.read_html(filepath)[0]
+    elif filename.endswith(".db"):
+        if sheet_name is None:
+            raise ValueError("读取 .db 文件需要提供 sheet_name 作为表名")
+        df = pd.read_sql_table(sheet_name, f"sqlite:///{filepath}")
+    else:
+        raise ValueError(f"Unsupported file type: {filepath}")
+
+    return df.to_dict(orient="records")
+
+
 def load_json(filename: str):
     with open(filename, "r", encoding="utf-8") as f:
         return json.load(f)
@@ -84,16 +169,24 @@ def save_json(json_list: Union[Dict[str, str], List[Dict[str, str]]], filename:
         return json.dump(json_list, f, ensure_ascii=False, separators=(",", ":"), indent=2)
 
 
-def load_json_list(filename: str):
+def load_json_list(filename: str, skip_None=True, skip_blank=True) -> List[Dict[str, str]]:
     with open(filename, "r", encoding="utf-8") as f:
         lines = f.readlines()
         json_list = []
-        for i in lines:
+        for i, line in enumerate(lines):
+            line = line.strip()
+            if line == "":
+                if not skip_blank:
+                    json_list.append("")
+                continue
+            if line == "None":
+                if not skip_None:
+                    json_list.append(None)
+                continue
             try:
-                obj = json.loads(i.strip())
+                obj = json.loads(line)
             except:
-                print("格式损坏数据，无法加载")
-                print(i)
+                print(f"格式损坏，跳过第 {i} 行: {repr(line)}")
                 continue
             json_list.append(obj)
         return json_list
@@ -176,7 +269,7 @@ def apply_changes_to_paths(
 ):
     total_updated = 0
     total_deleted = 0
-    for path in paths:
+    for path in ls(paths):
         if verbose:
             print("checking", path)
         jsonlist = load_json(path)
@@ -199,25 +292,8 @@ def apply_changes_to_paths(
     print(f"total: updated {total_updated}, deleted {total_deleted}")
 
 
-def backup_current_output(row: Dict[str, str], output_key="output"):
-    if "old_output" in row:
-        for i in range(1, 10):
-            if f"old_output{i}" not in row:
-                row[f"old_output{i}"] = row[output_key]
-                break
-    else:
-        row["old_output"] = row[output_key]
-    return row
-
-
-def backup_and_set_output(row: Dict[str, str], output: str):
-    backup_current_output(row)
-    row["output"] = output
-    return row
-
-
 def generator_from_paths(paths: List[Path], load_data: Callable[[Path], List[Dict[str, Any]]] = load_json):
-    for path in paths:
+    for path in ls(paths):
         jsonlist: List[Dict[str, Any]] = load_data(path)
         for row in jsonlist:
             yield path, row
@@ -262,3 +338,15 @@ def generator_from_jsonl(path):
     jsonlist = load_json_list(path)
     for line in jsonlist:
         yield line
+
+def grouped_row_in_jsonlist(jsonlist: List[Dict[str, Any]], key_col="query"):
+    grouped = defaultdict(list)
+    for i, row in enumerate(jsonlist):
+        if key_col not in row:
+            logger.warning(f"`{key_col}` not in row: {row}")
+            notfound_key = f"NotFound:{key_col}"
+            grouped[notfound_key].append(row)
+            continue
+        grouped[row[key_col]].append(row)
+    return grouped
+

xlin/{multiprocess_mapping.py → multiprocess_util.py}

@@ -9,9 +9,9 @@ from pathlib import Path
 from tqdm import tqdm
 from loguru import logger
 
-from xlin.jsonl import append_to_json_list, dataframe_to_json_list, load_json_list, row_to_json, save_json_list, load_json, save_json
-from xlin.read_as_dataframe import read_as_dataframe
-from xlin.util import ls
+from xlin.jsonlist_util import append_to_json_list, dataframe_to_json_list, load_json_list, row_to_json, save_json_list, load_json, save_json
+from xlin.dataframe_util import read_as_dataframe
+from xlin.file_util import ls
 
 
 def element_mapping(
@@ -100,6 +100,16 @@ def xmap(
         preserve_order (bool): 是否保持结果顺序
         chunksize (Optional[int]): 单个任务分块大小，None为自动计算
         retry_count (int): 任务失败重试次数
+
+    Example:
+        >>> from xlin.multiprocess_mapping import xmap
+        >>> jsonlist = [{"id": 1, "text": "Hello"}, {"id": 2, "text": "World"}]
+        >>> def work_func(item):
+        ...     item["text"] = item["text"].upper()
+        ...     return item
+        >>> results = xmap(jsonlist, work_func, output_path="output.jsonl", batch_size=2)
+        >>> print(results)
+        [{'id': 1, 'text': 'HELLO'}, {'id': 2, 'text': 'WORLD'}]
     """
     need_caching = output_path is not None
     output_list = []
@@ -204,7 +214,7 @@ def xmap(
 def multiprocessing_mapping(
     df: pd.DataFrame,
     output_path: Optional[Union[str, Path]],
-    partial_func,
+    partial_func: Callable[[Dict[str, str]], Dict[str, str]],
     batch_size=multiprocessing.cpu_count(),
     cache_batch_num=1,
     thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
@@ -215,6 +225,9 @@ def multiprocessing_mapping(
         df (DataFrame): [description]
         output_path (Path): 数据量大的时候需要缓存
         partial_func (function): (Dict[str, str]) -> Dict[str, str]
+        batch_size (int): batch size
+        cache_batch_num (int): cache batch num
+        thread_pool_size (int): thread pool size
     """
     need_caching = output_path is not None
     tmp_list, output_list = list(), list()

xlin/statistic.py

@@ -1,10 +1,26 @@
 import sys
 from typing import List, Optional
-from collections import defaultdict
+from collections import Counter, defaultdict
 
 import pandas as pd
 
 
+def sortedCounter(obj, by="key", reverse=False, return_list=False):
+    c = Counter(obj)
+    c_list = [(k, c[k]) for k in c]
+    if by == "key":
+        c_list = sorted(c_list, key=lambda x: x[0], reverse=reverse)
+    elif by in ["value", "count"]:
+        c_list = sorted(c_list, key=lambda x: x[1], reverse=reverse)
+    else:
+        raise Exception(f"unsupported by: {by}")
+    c = Counter()
+    for k, v in c_list:
+        c[k] = v
+    if return_list:
+        return c, c_list
+    return c
+
 
 def bucket_count(length: List[int], step=50, skip_zero_count=False):
     grouped_count = []

xlin/text_util.py

@@ -0,0 +1,24 @@
+def text_is_all_chinese(test: str):
+    for ch in test:
+        if '\u4e00' <= ch <= '\u9fff':
+            continue
+        return False
+    return True
+
+
+def text_contains_chinese(test: str):
+    for ch in test:
+        if '\u4e00' <= ch <= '\u9fff':
+            return True
+    return False
+
+def pretty_limited_text(text: str, limited_length: int = 300, language="zh"):
+    text = str(text).strip()
+    if len(text) > limited_length:
+        # if language == "zh":
+        #     tail = f"...(共{len(text)}字)"
+        # else:
+        #     tail = f"...({len(text)} words in total)"
+        # return text[: limited_length - len(tail)] + tail
+        return text[: limited_length // 2] + text[-limited_length // 2 :]
+    return text

xlin/{timing.py → timing_util.py}

@@ -41,3 +41,17 @@ class Timer:
     def __exit__(self, *args):
         self.end = time.perf_counter()
         self.interval = self.end - self.start
+
+if __name__ == "__main__":
+    with Timer() as t:
+        time.sleep(1)
+    print(t.interval)
+    with Benchmark("Test Benchmark") as b:
+        time.sleep(1)
+    print(b.time)
+    @timing
+    def test_function(x, y):
+        time.sleep(1)
+        return x + y
+    result = test_function(1, 2)
+    print(f"Result of test_function: {result}")

xlin-0.1.39.dist-info/METADATA

@@ -0,0 +1,282 @@
+Metadata-Version: 2.1
+Name: xlin
+Version: 0.1.39
+Summary: toolbox for LinXueyuan
+License: MIT
+Author: LinXueyuanStdio
+Author-email: 23211526+LinXueyuanStdio@users.noreply.github.com
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: loguru
+Requires-Dist: pandas
+Requires-Dist: pyexcel
+Requires-Dist: pyexcel-xls
+Requires-Dist: pyexcel-xlsx
+Requires-Dist: pyyaml
+Requires-Dist: tqdm
+Requires-Dist: xlsxwriter
+Description-Content-Type: text/markdown
+
+# xlin
+
+Python 工具代码集合，提供了丰富的工具函数，涵盖文件操作、数据处理、多进程处理等多个方面，旨在提高开发效率。
+
+## 安装
+
+```bash
+pip install xlin --upgrade
+```
+
+## 使用方法
+
+```python
+from xlin import *
+```
+
+### 文件操作类：`ls`，`rm` 和 `cp`
+- `ls`: 列出文件或文件夹下的所有文件。
+- `rm`: 删除文件或文件夹。
+- `cp`: 复制文件或文件夹。
+
+```python
+from xlin import ls, rm, cp
+
+dir_path = "./data"
+dir_path = "/mnt/data.json"
+dir_path = "./data,/mnt/data.json"
+dir_path = ["./data", "/mnt/data.json", "./data,/mnt/data.json"]
+def filter_func(path: Path) -> bool:
+    return path.name.endswith('.json')
+
+filepaths: list[Path] = ls(dir_path, filter=filter_func)
+rm(dir_path)
+cp(dir_path, "./backup_data")  # 会根据最大公共前缀保持文件夹结构
+```
+
+### 读取类
+
+- `read_as_json_list`：读取 JSON 文件为列表。
+- `read_as_dataframe`：读取文件为表格。如果是文件夹，则读取文件夹下的所有文件为表格并拼接。
+- `read_as_dataframe_dict`：读取文件为字典，键为表头，值为列数据。
+- `load_text`：加载文本文件。
+- `load_yaml`：加载 YAML 文件。
+- `load_json`：加载 JSON 文件。
+- `load_json_list`：加载 JSON 列表文件。
+
+
+> `read_as_**` 函数支持文件夹或者文件，支持多种文件格式，包括 Excel、CSV、JSON、Parquet 等。
+>
+> `load_**` 函数主要用于加载单个文件，支持文本、YAML 和 JSON 格式。
+
+```python
+from xlin import *
+import pandas as pd
+
+dir_path = "./data"
+dir_path = "./data,data.xlsx,data.csv,data.json,data.jsonl,data.parquet,data.feather,data.pkl,data.h5,data.txt,data.tsv,data.xml,data.html,data.db"
+dir_path = "./data,/mnt/data.json"
+dir_path = ["./data", "/mnt/data.json", "./data,/mnt/data.json"]
+df_single = read_as_dataframe(dir_path)
+jsonlist = read_as_json_list(dir_path)
+df_dict = read_as_dataframe_dict(dir_path)  # xlsx or dirs
+for sheet_name, df in df_dict.items():
+    print(f"Sheet: {sheet_name}")
+    print(df)
+
+text = load_text("example.txt")
+yaml_data = load_yaml("example.yaml")
+json_data = load_json("example.json")
+json_list_data = load_json_list("example.jsonl")
+```
+
+### 保存类
+
+```python
+save_json(data, 'output.json')
+save_json_list(jsonlist, 'output.jsonl')
+save_df(df, 'output.xlsx')
+save_df_dict(df_dict, 'output.xlsx')  # 将 read_as_dataframe_dict 返回的字典保存为 Excel 文件。
+save_df_from_jsonlist(jsonlist, 'output_from_jsonlist.xlsx')
+append_to_json_list(data, 'output.jsonl')
+```
+
+### 并行处理类：`xmap`
+高效处理 JSON 列表，支持多进程/多线程。
+
+```python
+from xlin import xmap
+
+jsonlist = [{"id": 1, "text": "Hello"}, {"id": 2, "text": "World"}]
+
+def work_func(item):
+    item["text"] = item["text"].upper()
+    return item
+
+results = xmap(jsonlist, work_func, output_path="output.jsonl", batch_size=2)
+print(results)
+```
+
+### 合并多个文件：`merge_json_list`，`merge_multiple_df_dict`
+合并多个 JSONL 文件。
+
+```python
+from xlin import merge_json_list
+
+filenames = ['example1.jsonl', 'example2.jsonl']
+output_filename = 'merged.jsonl'
+merge_json_list(filenames, output_filename)
+```
+
+合并多个 `read_as_dataframe_dict` 返回的字典。
+
+```python
+from xlin import read_as_dataframe_dict, merge_multiple_df_dict
+
+df_dict1 = read_as_dataframe_dict('example1.xlsx')
+df_dict2 = read_as_dataframe_dict('example2.xlsx')
+merged_df_dict = merge_multiple_df_dict([df_dict1, df_dict2])
+for sheet_name, df in merged_df_dict.items():
+    print(f"Sheet: {sheet_name}")
+    print(df)
+```
+
+### 对 json 文件批量操作
+- 对 JSON 列表应用更改：`apply_changes_to_paths`，`apply_changes_to_jsonlist`
+
+```python
+from xlin import *
+
+paths = [Path('example1.jsonl'), Path('example2.jsonl')]
+jsonlist = [{"id": 1, "text": "Hello"}, {"id": 2, "text": "World"}]
+
+def change_func(row):
+    if row["id"] == 1:
+        row["text"] = "New Hello"
+        return "updated", row
+    return "unchanged", row
+
+changes = {"update_text": change_func}
+
+# 1. 对文件路径应用更改
+apply_changes_to_paths(paths, changes, save=True)
+# 2. 对 JSON 列表应用更改
+new_jsonlist, updated, deleted = apply_changes_to_jsonlist(jsonlist, changes)
+print(new_jsonlist)
+```
+
+### 生成器
+- 从多个文件中生成 JSON 列表的生成器：`generator_from_paths`
+
+```python
+from xlin import generator_from_paths
+from pathlib import Path
+
+paths = [Path('example1.jsonl'), Path('example2.jsonl')]
+
+for path, row in generator_from_paths(paths):
+    print(f"Path: {path}, Row: {row}")
+```
+
+### 数据转换
+- DataFrame 和 JSON 列表之间的转换：`dataframe_to_json_list` 和 `jsonlist_to_dataframe`
+
+```python
+from xlin import dataframe_to_json_list, jsonlist_to_dataframe
+import pandas as pd
+
+data = {'col1': [1, 2], 'col2': [3, 4]}
+df = pd.DataFrame(data)
+
+json_list = dataframe_to_json_list(df)
+print(json_list)
+
+new_df = jsonlist_to_dataframe(json_list)
+print(new_df)
+```
+
+### 分组
+- 对 DataFrame 进行分组：`grouped_col_list`、`grouped_col` 和 `grouped_row`
+
+```python
+from xlin import grouped_col_list, grouped_col, grouped_row
+import pandas as pd
+
+data = {'query': ['a', 'a', 'b'], 'output': [1, 2, 3]}
+df = pd.DataFrame(data)
+
+grouped_col_list_result = grouped_col_list(df)
+print(grouped_col_list_result)
+
+grouped_col_result = grouped_col(df)
+print(grouped_col_result)
+
+grouped_row_result = grouped_row(df)
+print(grouped_row_result)
+```
+
+- 对 JSON 列表进行分组：`grouped_row_in_jsonlist`
+
+```python
+from xlin import grouped_row_in_jsonlist
+
+jsonlist = [{"query": "a", "output": 1}, {"query": "a", "output": 2}, {"query": "b", "output": 3}]
+grouped_row_in_jsonlist_result = grouped_row_in_jsonlist(jsonlist)
+print(grouped_row_in_jsonlist_result)
+```
+
+### 工具类
+
+- `random_timestamp` 和 `random_timestamp_str`：生成随机时间戳和格式化的随机时间字符串。
+
+```python
+from xlin import random_timestamp, random_timestamp_str
+
+timestamp = random_timestamp()
+print(timestamp)
+
+timestamp_str = random_timestamp_str()
+print(timestamp_str)
+```
+
+
+- `df_dict_summary`: 对 `read_as_dataframe_dict` 返回的字典进行总结，返回一个 DataFrame 包含每个表的基本信息。
+
+```python
+from xlin import read_as_dataframe_dict, df_dict_summary
+
+df_dict = read_as_dataframe_dict('example.xlsx')
+summary = df_dict_summary(df_dict)
+print(summary)
+```
+
+- `text_is_all_chinese` 和 `text_contains_chinese`：判断文本是否全为中文或是否包含中文。
+
+```python
+from xlin import text_is_all_chinese, text_contains_chinese
+
+text1 = "你好"
+text2 = "Hello 你好"
+
+print(text_is_all_chinese(text1))  # True
+print(text_is_all_chinese(text2))  # False
+print(text_contains_chinese(text2))  # True
+```
+
+## 许可证
+
+本项目采用 MIT 许可证，详情请参阅 [LICENSE](LICENSE) 文件。
+
+## 作者
+
+- LinXueyuanStdio <23211526+LinXueyuanStdio@users.noreply.github.com>

xlin-0.1.39.dist-info/RECORD

@@ -0,0 +1,17 @@
+xlin/__init__.py,sha256=CIhMAGhFgqwC6w16MzKcwo2mDjmaRUAcrlZFR3Am--I,321
+xlin/dataframe_util.py,sha256=Z8k3_XLMP5B13IMov2dQJhe-7dPh_YUJGokibWSx8II,10460
+xlin/datetime_util.py,sha256=jzdF-58PTb_ofBy6F-LBDnEmsTQ9jvoCgqKZJmyDtqE,687
+xlin/file_util.py,sha256=mYTABNywdYoSfh1RLJcH7l1FzgKTFWN2-JZMFzv-ehw,7270
+xlin/image_util.py,sha256=j1QlVXS-aikTDFDINbTmxjZi6CokPDTVlQ6_ABctMWQ,8109
+xlin/jsonlist_util.py,sha256=dLgrgrSTvg_1plVRCEnilajPM_s3vYdVx2bCTqrZAN8,11316
+xlin/metric.py,sha256=N7wJ35y-C-IaBr1I1CJ_37lTG7gA69zmn9Xg6xSwKoI,1690
+xlin/multiprocess_util.py,sha256=-tskCWQlBBCOPycXLj9Y2MugYg-tHF_QYYWW7c1ixOk,17300
+xlin/statistic.py,sha256=ioJJjL4qwHiwNPeBFBB67keoAIbB-uZM51zkDYviar0,17037
+xlin/text_util.py,sha256=ejFD8-j8tLCbPlCPFg0Tu3MEMPEpF7R5_IpXXjl6qzA,735
+xlin/timing_util.py,sha256=nNVKtSXel-Cc8SF_BqPRNkyNDOjGqOMxTol-L1vpON4,1340
+xlin/xlsx_util.py,sha256=uSmXcDvIhi5Sq0LGidMXy0wErNBXdjaoa6EftYVjTXs,947
+xlin/yaml_util.py,sha256=kICi7G3Td5q2MaSXXt85qNTWoHMgjzt7pvn7r3C4dME,183
+xlin-0.1.39.dist-info/LICENSE,sha256=60ys6rRtc1dZOP8UjSUr9fAqhZudT3WpKe5WbMCralM,1066
+xlin-0.1.39.dist-info/METADATA,sha256=d9F6S7LBxur58Jm817Kww54kVOBeWTRvc4R0AgOWYfw,7992
+xlin-0.1.39.dist-info/WHEEL,sha256=IrRNNNJ-uuL1ggO5qMvT1GGhQVdQU54d6ZpYqEZfEWo,92
+xlin-0.1.39.dist-info/RECORD,,

xlin/ischinese.py

@@ -1,13 +0,0 @@
-def text_is_all_chinese(test: str):
-    for ch in test:
-        if '\u4e00' <= ch <= '\u9fff':
-            continue
-        return False
-    return True
-
-
-def text_contains_chinese(test: str):
-    for ch in test:
-        if '\u4e00' <= ch <= '\u9fff':
-            return True
-    return False

xlin-0.1.37.dist-info/METADATA

@@ -1,33 +0,0 @@
-Metadata-Version: 2.1
-Name: xlin
-Version: 0.1.37
-Summary: toolbox for LinXueyuan
-License: MIT
-Author: LinXueyuanStdio
-Author-email: 23211526+LinXueyuanStdio@users.noreply.github.com
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 2
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.4
-Classifier: Programming Language :: Python :: 3.5
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: loguru
-Requires-Dist: pandas
-Requires-Dist: pyexcel
-Requires-Dist: pyexcel-xls
-Requires-Dist: pyexcel-xlsx
-Requires-Dist: pyyaml
-Requires-Dist: tqdm
-Requires-Dist: xlsxwriter
-Description-Content-Type: text/markdown
-
-# xlin
-个人 python 工具代码
-

xlin-0.1.37.dist-info/RECORD

@@ -1,15 +0,0 @@
-xlin/__init__.py,sha256=MWWCNPgJFS_oV2US52ULa4yg4Ku61qjn40NVKqcp9-c,248
-xlin/ischinese.py,sha256=Ia9IMQ6q-UHkdLwqS70L1fTnfSPbluFrv_I1UqsKquo,293
-xlin/jsonl.py,sha256=IDRydHh2x-8iAGCxt9ScK2wfNLNA40PxNxR5hhr4v6k,7903
-xlin/metric.py,sha256=N7wJ35y-C-IaBr1I1CJ_37lTG7gA69zmn9Xg6xSwKoI,1690
-xlin/multiprocess_mapping.py,sha256=q4EVU8JPLcRAbNf9NUEzmn8rDLDfIQ3jaW0yxVKPECk,16669
-xlin/read_as_dataframe.py,sha256=MqY57L7Wp9UoWTRlZLSBKQNaZa-dKw51-ufrKvHKf8s,9041
-xlin/statistic.py,sha256=nwFSN8BWfTQRimI-zfp6RwfA-I9aFDbemtV2cyh6Hq8,16533
-xlin/timing.py,sha256=XMT8dMcMolOMohDvAZOIM_BAiPMREhGQKnO1kc5s6PU,998
-xlin/util.py,sha256=TTWJaqF5D_r-gAZ_fj0kyHomvCagjwHXQZ2OPSgwd54,10976
-xlin/xls2xlsx.py,sha256=uSmXcDvIhi5Sq0LGidMXy0wErNBXdjaoa6EftYVjTXs,947
-xlin/yaml.py,sha256=kICi7G3Td5q2MaSXXt85qNTWoHMgjzt7pvn7r3C4dME,183
-xlin-0.1.37.dist-info/LICENSE,sha256=60ys6rRtc1dZOP8UjSUr9fAqhZudT3WpKe5WbMCralM,1066
-xlin-0.1.37.dist-info/METADATA,sha256=yMXp-Qqsf8Ch_xEzx4M9y3b21v0zRVsiDAGi4kbKEIo,1098
-xlin-0.1.37.dist-info/WHEEL,sha256=IrRNNNJ-uuL1ggO5qMvT1GGhQVdQU54d6ZpYqEZfEWo,92
-xlin-0.1.37.dist-info/RECORD,,