xlin 0.1.0__tar.gz

xlin-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 兮尘
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

xlin-0.1.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: xlin
+Version: 0.1.0
+Summary: toolbox for LinXueyuan
+License: MIT
+Author: XiChen
+Author-email: 23211526+LinXueyuanStdio@users.noreply.github.com
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: loguru (>=0.7.2,<0.8.0)
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: pyexcel (>=0.7.1,<0.8.0)
+Requires-Dist: pyexcel-xls (>=0.7.0,<0.8.0)
+Requires-Dist: pyexcel-xlsx (>=0.6.0,<0.7.0)
+Requires-Dist: xlsxwriter (==3.1.2)
+Description-Content-Type: text/markdown
+
+# xlin
+个人 python 工具代码
+

xlin-0.1.0/README.md

@@ -0,0 +1,2 @@
+# xlin
+个人 python 工具代码

xlin-0.1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "xlin"
+version = "0.1.0"
+description = "toolbox for LinXueyuan"
+authors = ["XiChen <23211526+LinXueyuanStdio@users.noreply.github.com>"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+
+loguru = "^0.7.2"
+pandas = "^2.2.3"
+pyexcel = "^0.7.1"
+pyexcel-xls = "^0.7.0"
+pyexcel-xlsx = "^0.6.0"
+xlsxwriter = "3.1.2"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

xlin-0.1.0/xlin/ischinese.py

@@ -0,0 +1,13 @@
+def text_is_all_chinese(test: str):
+    for ch in test:
+        if '\u4e00' <= ch <= '\u9fff':
+            continue
+        return False
+    return True
+
+
+def text_contains_chinese(test: str):
+    for ch in test:
+        if '\u4e00' <= ch <= '\u9fff':
+            return True
+    return False

xlin-0.1.0/xlin/jsonl.py

@@ -0,0 +1,195 @@
+import json
+from typing import *
+
+from pathlib import Path
+from loguru import logger
+import pandas as pd
+
+
+def dataframe_to_json_list(df: pd.DataFrame):
+    """
+    Args:
+        df (pd.DataFrame): df
+
+    Returns:
+        List[Dict[str, str]]: json list: [{"col1": "xxx", "col2": "xxx", ...}, ...]
+    """
+    json_list = []
+    for i, line in df.iterrows():
+        json_list.append(dict(line))
+    return json_list
+
+
+def transform_dataframe_to_json_list(df: pd.DataFrame, row_transform):
+    """
+    Args:
+        df (pd.DataFrame): df
+        row_transform : lambda row: prompt_template.format(row['query']), "", row['label']
+
+    Returns:
+        List[Dict[str, str]]: json list: [{"instruction": "xxx", "input": "xxx", "output": "xxx"}, ...]
+    """
+    out_list = list()
+    for _, row in df.iterrows():
+        instruction, input, output = row_transform(row)
+        out_list.append({"instruction": instruction, "input": input, "output": output})
+    return out_list
+
+
+def jsonlist_to_dataframe(json_list: List[Dict[str, str]]):
+    """
+    Args:
+        json_list (List[Dict[str, str]]): json list: [{"col1": "xxx", "col2": "xxx", ...}, ...]
+
+    Returns:
+        pd.DataFrame: df
+    """
+    return pd.DataFrame(json_list)
+
+
+def load_json(filename: str):
+    with open(filename, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def save_json(json_list: Union[Dict[str, str], List[Dict[str, str]]], filename: str):
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
+    with open(filename, "w", encoding="utf-8") as f:
+        return json.dump(json_list, f, ensure_ascii=False, separators=(",", ":"), indent=2)
+
+
+def load_json_list(filename: str):
+    with open(filename, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+        json_list = []
+        for i in lines:
+            try:
+                obj = json.loads(i.strip())
+            except:
+                continue
+            json_list.append(obj)
+        return json_list
+
+
+def save_json_list(json_list: List[Dict[str, str]], filename: str):
+    Path(filename).parent.mkdir(parents=True, exist_ok=True)
+    with open(filename, "w", encoding="utf-8") as f:
+        f.write("\n".join([json.dumps(line, ensure_ascii=False, separators=(",", ":")) for line in json_list]))
+
+
+def merge_json_list(filenames: List[str], output_filename: str):
+    json_list = []
+    for filename in filenames:
+        json_list.extend(load_json_list(filename))
+    save_json_list(json_list, output_filename)
+
+
+def jsonlist_dict_summary(jsonlist_dict: Dict[str, List[dict]]):
+    rows = []
+    for k, jsonlist in jsonlist_dict.items():
+        if len(jsonlist) == 0:
+            continue
+        row = {
+            "sheet_name": k,
+            "length": len(jsonlist),
+            "columns": str(list(jsonlist[0].keys())),
+        }
+        rows.append(row)
+    df = pd.DataFrame(rows)
+    return df
+
+
+def print_in_json(text: str):
+    print(json.dumps({"text": text}, indent=2, ensure_ascii=False))
+
+
+def apply_changes_to_jsonlist(
+    jsonlist: List[Dict[str, str]],
+    changes: Dict[str, Callable[[Dict[str, str]], Tuple[Literal["deleted", "updated", "unchanged"], Dict[str, str]]]],
+    verbose=False,
+    **kwargs,
+):
+    rows = jsonlist
+    total_updated = 0
+    total_deleted = 0
+    for name, change in changes.items():
+        new_rows = []
+        updated = 0
+        deleted = 0
+        for row in rows:
+            status, new_row = change(row, **kwargs)
+            if status == "deleted":
+                deleted += 1
+                continue
+            if status == "updated":
+                updated += 1
+            new_rows.append(new_row)
+        rows = new_rows
+        msgs = []
+        if updated > 0:
+            total_updated += updated
+            msgs += [f"updated {updated}"]
+        if deleted > 0:
+            total_deleted += deleted
+            msgs += [f"deleted {deleted}"]
+        if verbose and updated > 0 or deleted > 0:
+            logger.info(f"{name}: {', '.join(msgs)}, remained {len(new_rows)} rows.")
+    return rows, total_updated, total_deleted
+
+
+def apply_changes_to_paths(
+    paths: List[Path],
+    changes: Dict[str, Callable[[Dict[str, str]], Tuple[Literal["deleted", "updated", "unchanged"], Dict[str, str]]]],
+    verbose=False,
+    save=False,
+    load_json=load_json,
+    save_json=save_json,
+    **kwargs,
+):
+    total_updated = 0
+    total_deleted = 0
+    for path in paths:
+        if verbose:
+            print("checking", path)
+        jsonlist = load_json(path)
+        kwargs["path"] = path
+        new_jsonlist, updated, deleted = apply_changes_to_jsonlist(jsonlist, changes, verbose, **kwargs)
+        msgs = [f"total {len(jsonlist)} -> {len(new_jsonlist)}"]
+        if updated > 0:
+            total_updated += updated
+            msgs += [f"updated {updated}"]
+        if deleted > 0:
+            msgs += [f"deleted {deleted}"]
+            total_deleted += deleted
+        if updated > 0 or deleted > 0:
+            print(f"{path}: {', '.join(msgs)}")
+            if save:
+                if len(new_jsonlist) > 0:
+                    save_json(new_jsonlist, path)
+                else:
+                    path.unlink()
+    print(f"total: updated {total_updated}, deleted {total_deleted}")
+
+
+def backup_current_output(row: Dict[str, str], output_key="output"):
+    if "old_output" in row:
+        for i in range(1, 10):
+            if f"old_output{i}" not in row:
+                row[f"old_output{i}"] = row[output_key]
+                break
+    else:
+        row["old_output"] = row[output_key]
+    return row
+
+
+def backup_and_set_output(row: Dict[str, str], output: str):
+    backup_current_output(row)
+    row["output"] = output
+    return row
+
+
+def generator_from_paths(paths: List[Path], load_data: Callable[[Path], List[Dict[str, Any]]] = load_json):
+    for path in paths:
+        jsonlist: List[Dict[str, Any]] = load_data(path)
+        for row in jsonlist:
+            yield path, row

xlin-0.1.0/xlin/multiprocess_mapping.py

@@ -0,0 +1,188 @@
+import time
+import os
+import multiprocessing
+from multiprocessing.pool import ThreadPool
+from typing import *
+
+import pandas as pd
+from pathlib import Path
+from tqdm import tqdm
+from loguru import logger
+
+from jsonl import dataframe_to_json_list, load_json_list, save_json_list, load_json, save_json
+
+
+cpu_count = multiprocessing.cpu_count()
+# pool = ThreadPool(cpu_count)  # 大模型接口辣鸡，太快会截断答案
+thread_pool_size = int(os.getenv("THREAD_POOL_SIZE", 5))
+pool = ThreadPool(thread_pool_size)
+logger.debug(f"pool size: {thread_pool_size}, cpu count: {cpu_count}")
+
+
+def multiprocessing_mapping_jsonlist(
+    jsonlist: List[Any],
+    output_path: Optional[Union[str, Path]],
+    partial_func,
+    batch_size=cpu_count * 2,
+    cache_batch_num=1,
+    thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
+):
+    """mapping a column to another column
+
+    Args:
+        df (DataFrame): [description]
+        output_path (Path): 数据量大的时候需要缓存
+        partial_func (function): (Dict[str, str]) -> Dict[str, str]
+    """
+    need_caching = output_path is not None
+    tmp_list, output_list = list(), list()
+    start_idx = 0
+    if need_caching:
+        output_path = Path(output_path)
+        if output_path.exists():
+            output_list = load_json_list(output_path)
+            start_idx = len(output_list)
+            logger.warning(f"Cache found {output_path} has {start_idx} rows. This process will continue at row index {start_idx}.")
+            logger.warning(f"缓存 {output_path} 存在 {start_idx} 行. 本次处理将从第 {start_idx} 行开始.")
+        else:
+            output_path.parent.mkdir(parents=True, exist_ok=True)
+    pool = ThreadPool(thread_pool_size)
+    logger.debug(f"pool size: {thread_pool_size}, cpu count: {cpu_count}")
+    start_time = time.time()
+    last_save_time = start_time
+    for i, line in tqdm(list(enumerate(jsonlist))):
+        if i < start_idx:
+            continue
+        tmp_list.append(line)
+        if len(tmp_list) == batch_size:
+            results = pool.map(partial_func, tmp_list)
+            output_list.extend([x for x in results])
+            tmp_list = list()
+        if need_caching and (i // batch_size) % cache_batch_num == 0:
+            current_time = time.time()
+            if current_time - last_save_time < 3:
+                # 如果多进程处理太快，为了不让 IO 成为瓶颈拉慢进度，不足 3 秒的批次都忽略，也不缓存中间结果
+                last_save_time = current_time
+                continue
+            save_json_list(output_list, output_path)
+            last_save_time = time.time()
+    if len(tmp_list) > 0:
+        results = pool.map(partial_func, tmp_list)
+        output_list.extend([x for x in results])
+    if need_caching:
+        save_json_list(output_list, output_path)
+    return output_list
+
+
+def multiprocessing_mapping(
+    df: pd.DataFrame,
+    output_path: Optional[Union[str, Path]],
+    partial_func,
+    batch_size=cpu_count * 2,
+    cache_batch_num=1,
+    thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
+):
+    """mapping a column to another column
+
+    Args:
+        df (DataFrame): [description]
+        output_path (Path): 数据量大的时候需要缓存
+        partial_func (function): (Dict[str, str]) -> Dict[str, str]
+    """
+    need_caching = output_path is not None
+    tmp_list, output_list = list(), list()
+    start_idx = 0
+    if need_caching:
+        output_path = Path(output_path)
+        if output_path.exists():
+            # existed_df = read_as_dataframe(output_path)
+            # start_idx = len(existed_df)
+            # output_list = dataframe_to_json_list(existed_df)
+            # logger.warning(f"Cache found {output_path} has {start_idx} rows. This process will continue at row index {start_idx}.")
+            # logger.warning(f"缓存 {output_path} 存在 {start_idx} 行. 本次处理将从第 {start_idx} 行开始.")
+            pass
+        else:
+            output_path.parent.mkdir(parents=True, exist_ok=True)
+    pool = ThreadPool(thread_pool_size)
+    logger.debug(f"pool size: {thread_pool_size}, cpu count: {cpu_count}")
+    start_time = time.time()
+    last_save_time = start_time
+    for i, line in tqdm(list(df.iterrows())):
+        if i < start_idx:
+            continue
+        line_info: dict = line.to_dict()
+        line_info: Dict[str, str] = {str(k): str(v) for k, v in line_info.items()}
+        tmp_list.append(line_info)
+        if len(tmp_list) == batch_size:
+            results = pool.map(partial_func, tmp_list)
+            output_list.extend([x for x in results])
+            tmp_list = list()
+        if need_caching and (i // batch_size) % cache_batch_num == 0:
+            current_time = time.time()
+            if current_time - last_save_time < 3:
+                # 如果多进程处理太快，为了不让 IO 成为瓶颈拉慢进度，不足 3 秒的批次都忽略，也不缓存中间结果
+                last_save_time = current_time
+                continue
+            output_df = pd.DataFrame(output_list)
+            output_df.to_excel(output_path, index=False)
+            last_save_time = time.time()
+    if len(tmp_list) > 0:
+        results = pool.map(partial_func, tmp_list)
+        output_list.extend([x for x in results])
+    output_df = pd.DataFrame(output_list)
+    if need_caching:
+        output_df.to_excel(output_path, index=False)
+    return output_df, output_list
+
+
+def dataframe_by_row_mapping(
+    df: pd.DataFrame,
+    mapping_func: Callable[[dict], Tuple[bool, dict]],
+    use_multiprocessing=True,
+    thread_pool_size=int(os.getenv("THREAD_POOL_SIZE", 5)),
+):
+    rows = []
+    if use_multiprocessing:
+        pool = ThreadPool(thread_pool_size)
+        logger.debug(f"pool size: {thread_pool_size}, cpu count: {cpu_count}")
+        results = pool.map(mapping_func, dataframe_to_json_list(df))
+        for ok, row in results:
+            if ok:
+                rows.append(row)
+    else:
+        for i, row in df.iterrows():
+            ok, row = mapping_func(row)
+            if ok:
+                rows.append(row)
+    df = pd.DataFrame(rows)
+    return df
+
+
+def continue_run(
+    jsonfiles: List[str],
+    save_dir: str,
+    mapping_func,
+    load_func=load_json,
+    save_func=save_json,
+    batch_size=1024,
+    cache_size=8,
+):
+    save_dir: Path = Path(save_dir)
+    save_dir.mkdir(parents=True, exist_ok=True)
+    new_jsonfiles = []
+    for jsonfile in jsonfiles:
+        jsonfile = Path(jsonfile)
+        jsonlist = load_func(jsonfile)
+        output_filepath = save_dir / jsonfile.name
+        for row in jsonlist:
+            row["来源"] = jsonfile.name
+        new_jsonlist = multiprocessing_mapping_jsonlist(
+            jsonlist,
+            output_filepath,
+            mapping_func,
+            batch_size,
+            cache_size,
+        )
+        save_func(new_jsonlist, output_filepath)
+        new_jsonfiles.append(output_filepath)
+    return new_jsonfiles

xlin-0.1.0/xlin/read_as_dataframe.py

@@ -0,0 +1,233 @@
+from collections import defaultdict
+import os
+from typing import Callable, Dict, List, Optional, Tuple, Union
+from pathlib import Path
+from loguru import logger
+
+import pandas as pd
+import pyexcel
+
+from util import ls
+from jsonl import dataframe_to_json_list, load_json, load_json_list, save_json_list
+from xls2xlsx import is_xslx
+
+
+def valid_to_read_as_dataframe(filename: str) -> bool:
+    suffix_list = [".json", ".jsonl", ".xlsx", "xls", ".csv"]
+    return any([filename.endswith(suffix) for suffix in suffix_list])
+
+
+def read_as_dataframe(filepath: Union[str, Path], sheet_name: Optional[str] = None, fill_empty_str_to_na=True) -> pd.DataFrame:
+    """
+    读取文件为表格
+    """
+    filepath = Path(filepath)
+    if filepath.is_dir():
+        paths = ls(filepath, expand_all_subdir=True)
+        df_list = []
+        for path in paths:
+            try:
+                df = read_as_dataframe(path, sheet_name, fill_empty_str_to_na)
+                df['数据来源'] = path.name
+            except:
+                df = pd.DataFrame()
+            df_list.append(df)
+        df = pd.concat(df_list)
+        if fill_empty_str_to_na:
+            df.fillna("", inplace=True)
+        return df
+    filename = filepath.name
+    if filename.endswith(".json") or filename.endswith(".jsonl"):
+        try:
+            json_list = load_json(filepath)
+        except:
+            json_list = load_json_list(filepath)
+        df = pd.DataFrame(json_list)
+    elif filename.endswith(".xlsx"):
+        df = pd.read_excel(filepath) if sheet_name is None else pd.read_excel(filepath, sheet_name)
+    elif filename.endswith(".xls"):
+        if is_xslx(filepath):
+            df = pd.read_excel(filepath) if sheet_name is None else pd.read_excel(filepath, sheet_name)
+        else:
+            df = pyexcel.get_sheet(file_name=filepath)
+    elif filename.endswith(".csv"):
+        df = pd.read_csv(filepath)
+    else:
+        raise ValueError(f"Unsupported filetype {filepath}. filetype not in [json, jsonl, xlsx, xls, csv]")
+    if fill_empty_str_to_na:
+        df.fillna("", inplace=True)
+    return df
+
+
+def read_maybe_dir_as_dataframe(filepath: Union[str, Path], sheet_name: Optional[str] = None) -> pd.DataFrame:
+    """
+    input path 可能是文件夹，此时将文件夹下的所有表格拼接到一起返回，要求所有表头一致
+    如果不是文件夹，则为文件，尝试直接读取为表格返回
+    """
+    out_list = list()
+    if not isinstance(filepath, Path):
+        filepath = Path(filepath)
+    if not filepath.exists():
+        raise ValueError(f"Path Not Exist: {filepath}")
+    if not filepath.is_dir():
+        return read_as_dataframe(filepath, sheet_name)
+
+    files = os.listdir(filepath)
+    for file_name in files:
+        if not valid_to_read_as_dataframe(file_name):
+            continue
+        input_file = filepath / file_name
+        df = read_as_dataframe(input_file, sheet_name)
+        df.fillna("", inplace=True)
+        for _, line in df.iterrows():
+            line = line.to_dict()
+            out_list.append(line)
+    df = pd.DataFrame(out_list)
+    return df
+
+
+def read_as_dataframe_dict(filepath: Union[str, Path], fill_empty_str_to_na=True):
+    filepath = Path(filepath)
+    if filepath.is_dir():
+        paths = ls(filepath, expand_all_subdir=True)
+        df_dict_list = []
+        for path in paths:
+            try:
+                df_dict = read_as_dataframe_dict(path, fill_empty_str_to_na)
+            except:
+                df_dict = {}
+            df_dict_list.append(df_dict)
+        df_dict = merge_multiple_df_dict(df_dict_list)
+        return df_dict
+    df_dict: Dict[str, pd.DataFrame] = pd.read_excel(filepath, sheet_name=None)
+    if isinstance(df_dict, dict):
+        for name, df in df_dict.items():
+            if fill_empty_str_to_na:
+                df.fillna("", inplace=True)
+            df['数据来源'] = filepath.name
+    elif isinstance(df_dict, pd.DataFrame):
+        if fill_empty_str_to_na:
+            df_dict.fillna("", inplace=True)
+        df_dict['数据来源'] = filepath.name
+    return df_dict
+
+
+def df_dict_summary(df_dict: Dict[str, pd.DataFrame]):
+    rows = []
+    for k, df in df_dict.items():
+        row = {
+            "sheet_name": k,
+            "length": len(df),
+            "columns": str(list(df.columns)),
+        }
+        rows.append(row)
+    df = pd.DataFrame(rows)
+    return df
+
+
+def save_df_dict(df_dict: Dict[str, pd.DataFrame], output_filepath: Union[str, Path]):
+    if not isinstance(output_filepath, Path):
+        output_filepath = Path(output_filepath)
+        output_filepath.parent.mkdir(parents=True, exist_ok=True)
+    with pd.ExcelWriter(output_filepath, engine="xlsxwriter") as writer:
+        for k, df in df_dict.items():
+            if len(k) > 31:
+                logger.warning(f"表名太长，自动截断了：[{k}]的长度为{len(k)}")
+            df.to_excel(writer, sheet_name=k[:31], index=False)
+    return output_filepath
+
+
+def save_df_from_jsonlist(jsonlist: List[Dict[str, str]], output_filepath: Union[str, Path]):
+    df = pd.DataFrame(jsonlist)
+    return save_df(df, output_filepath)
+
+
+def save_df(df: pd.DataFrame, output_filepath: Union[str, Path]):
+    if not isinstance(output_filepath, Path):
+        output_filepath = Path(output_filepath)
+        output_filepath.parent.mkdir(parents=True, exist_ok=True)
+    df.to_excel(output_filepath, index=False)
+    return output_filepath
+
+
+def lazy_build_dataframe(name: str, output_filepath: Path, func, filetype: str = "xlsx"):
+    logger.info(name)
+    output_filepath.parent.mkdir(parents=True, exist_ok=True)
+    if output_filepath.exists():
+        df = read_as_dataframe(output_filepath)
+    else:
+        df: pd.DataFrame = func()
+        filename = output_filepath.name.split(".")[0]
+        if filetype == "xlsx":
+            df.to_excel(output_filepath.parent / f"{filename}.xlsx", index=False)
+        elif filetype == "json":
+            save_json_list(dataframe_to_json_list(df), output_filepath.parent / f"{filename}.json")
+        elif filetype == "jsonl":
+            save_json_list(dataframe_to_json_list(df), output_filepath.parent / f"{filename}.jsonl")
+        else:
+            logger.warning(f"不认识的 {filetype}，默认保存为 xlsx")
+            df.to_excel(output_filepath.parent / f"{filename}.xlsx", index=False)
+    logger.info(f"{name}结果保存在 {output_filepath}")
+    return df
+
+
+def lazy_build_dataframe_dict(name: str, output_filepath: Path, df_dict: Dict[str, pd.DataFrame], func, skip_sheets: List[str] = list()):
+    logger.info(name)
+    output_filepath.parent.mkdir(parents=True, exist_ok=True)
+    if output_filepath.exists():
+        new_df_dict = read_as_dataframe_dict(output_filepath)
+    else:
+        new_df_dict = {}
+        for sheet_name, df in df_dict.items():
+            if sheet_name in skip_sheets:
+                continue
+            df = func(sheet_name, df)
+            new_df_dict[sheet_name] = df
+        save_df_dict(new_df_dict, output_filepath)
+    logger.info(f"{name}结果保存在 {output_filepath}")
+    return new_df_dict
+
+
+def merge_multiple_df_dict(list_of_df_dict: List[Dict[str, pd.DataFrame]], sort=True):
+    df_dict_merged = defaultdict(list)
+    for df_dict in list_of_df_dict:
+        for k, df in df_dict.items():
+            df_dict_merged[k].append(df)
+    df_dict_merged: Dict[str, pd.DataFrame] = {k: pd.concat(v) for k, v in df_dict_merged.items()}
+    if sort:
+        df_dict_merged: Dict[str, pd.DataFrame] = {k: df_dict_merged[k] for k in sorted(df_dict_merged)}
+    return df_dict_merged
+
+
+def remove_duplicate_and_sort(df: pd.DataFrame, key_col="query", sort_by='label'):
+    query_to_rows = {}
+    for i, row in df.iterrows():
+        query_to_rows[row[key_col]] = row
+    rows = sorted(list(query_to_rows.values()), key=lambda row: row[sort_by])
+    df_filtered = pd.DataFrame(rows)
+    return df_filtered
+
+
+def color_negative_red(x):
+    color = "red" if x < 0 else ""
+    return f"color: {color}"
+
+
+def highlight_max(x):
+    is_max = x == x.max()
+    return [("background-color: yellow" if m else "") for m in is_max]
+
+
+def split_dataframe(df: pd.DataFrame, output_dir: Union[str, Path], tag: str, split_count=6):
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    rows = dataframe_to_json_list(df)
+    split_step = len(rows) // split_count + 1
+    df_list = []
+    for i in range(0, len(rows), split_step):
+        filepath = output_dir / f"{tag}_{i // split_step}.xlsx"
+        df_i = pd.DataFrame(rows[i:i+split_step])
+        df_i.to_excel(filepath, index=False)
+        df_list.append(df_i)
+    return df_list
+

xlin-0.1.0/xlin/statistic.py

@@ -0,0 +1,33 @@
+from typing import List
+
+import pandas as pd
+
+
+
+def bucket_count(length: List[int], step=50, skip_zero_count=False):
+    grouped_count = []
+    j = 0
+    for i in range(0, max(length) + step, step):
+        grouped_count.append(0)
+        while j < len(length) and length[j] < i:
+            grouped_count[i // step] += 1
+            j += 1
+    x, y = [], []
+    for i, j in enumerate(grouped_count):
+        if i == 0:
+            continue
+        if skip_zero_count and j == 0:
+            continue
+        print(f"[{(i-1)*step}, {i*step})  {j}   {sum(grouped_count[:i+1])/len(length)*100:.2f}%")
+        x.append((i - 1) * step)
+        y.append(j)
+    return x, y
+
+
+def statistic_char_length(df: pd.DataFrame, instruction_key="instruction"):
+    length = []
+    for i, row in df.iterrows():
+        length.append(len(row[instruction_key]))
+    length.sort()
+    return length
+

xlin-0.1.0/xlin/terminal_color.py

@@ -0,0 +1,10 @@
+
+blue = "\x1b[34m"
+cyan = "\x1b[36;21m"
+green = "\x1b[32;21m"
+orange = "\x1b[33;21m"
+grey = "\x1b[38;21m"
+yellow = "\x1b[33;21m"
+red = "\x1b[31;21m"
+bold_red = "\x1b[31;1m"
+reset = "\x1b[0m"

xlin-0.1.0/xlin/util.py

@@ -0,0 +1,321 @@
+from collections import defaultdict
+from typing import *
+from pathlib import Path
+import pandas as pd
+import os
+import asyncio
+import datetime
+from loguru import logger
+import shutil
+import random
+
+
+date_str = datetime.datetime.now().strftime("%Y%m%d")
+datetime_str = datetime.datetime.now().strftime("%Y%m%d_%Hh%Mm%Ss")
+
+
+def random_timestamp(start_timestamp=None, end_timestamp=None):
+    if start_timestamp is None:
+        start_timestamp = datetime.datetime(2024, 1, 1).timestamp()
+    if end_timestamp is None:
+        end_timestamp = datetime.datetime.now().timestamp()
+    return random.uniform(start_timestamp, end_timestamp)
+
+
+def random_timestamp_str(start_timestamp=None, end_timestamp=None, format="%Y年%m月%d日%H时%M分"):
+    return datetime.datetime.fromtimestamp(random_timestamp(start_timestamp, end_timestamp)).strftime(format)
+
+
+def auto_retry_to_get_data(retry_times, request, data_key="data", *args, **kwargs):
+    if retry_times == 0:
+        return {}
+    resp = request(*args, **kwargs)
+    if resp is not None:
+        if data_key is None:
+            return resp
+        elif data_key in resp and resp[data_key] is not None:
+            return resp[data_key]
+    logger.debug("[error! retrying...]", resp)
+    return auto_retry_to_get_data(retry_times - 1, request, data_key, *args, **kwargs)
+
+
+def append_column(df: pd.DataFrame, query_column: str, output_column: str, transform):
+    query = df[query_column].tolist()
+    loop = asyncio.get_event_loop()
+    result = loop.run_until_complete(transform(query))
+    df[output_column] = [str(r) for r in result]
+    return df
+
+
+def request_wrapper(request_num=10):
+    def request_wrapper_body(func):
+        def wrapper(*args, **kwargs):
+            c = request_num
+            excute_num = 0
+            while c > 0:
+                c -= 1
+                res = func(*args, **kwargs)
+                excute_num += 1
+                if res != "-1":
+                    logger.debug("{} excute_num: {}".format(func.__name__, excute_num))
+                    return res
+            logger.debug("{} excute_num: {}".format(func.__name__, excute_num))
+            return ""
+
+        return wrapper
+
+    return request_wrapper_body
+
+
+def copy_file(input_filepath, output_filepath, force_overwrite=False, verbose=False):
+    if verbose:
+        logger.info(f"正在复制 {input_filepath} 到 {output_filepath}")
+    if not isinstance(output_filepath, Path):
+        output_filepath = Path(output_filepath)
+    if output_filepath.exists() and not force_overwrite:
+        if verbose:
+            logger.warning(f"文件已存在，跳过复制：{output_filepath}")
+        return output_filepath
+    shutil.copy(input_filepath, output_filepath, follow_symlinks=True)
+    return output_filepath
+
+
+def rm(dir_path: Union[str, Path, List[str], List[Path]], filter: Callable[[Path], bool] = lambda filepath: True, expand_all_subdir=True, debug=False):
+    if isinstance(dir_path, str) and "," in dir_path:
+        for path in dir_path.split(","):
+            rm(path, filter, expand_all_subdir)
+        return
+    if isinstance(dir_path, list):
+        for path in dir_path:
+            rm(path, filter, expand_all_subdir)
+        return
+    dir_path = Path(dir_path)
+    if not dir_path.exists():
+        if debug:
+            print(f"路径不存在 {dir_path}")
+        return
+    if not dir_path.is_dir():
+        if filter(dir_path):
+            dir_path.unlink()
+            if debug:
+                print(f"删除文件 {dir_path}")
+        return
+    filenames = os.listdir(dir_path)
+    for filename in sorted(filenames):
+        filepath = dir_path / filename
+        if debug:
+            print("checking", filepath)
+        if filepath.is_dir():
+            paths = ls(filepath, filter, expand_all_subdir)
+            if len(paths) > 0:
+                rm(paths, filter, expand_all_subdir)
+            child = filepath
+            while child.exists() and len(os.listdir(child)) > 0:
+                child = child / os.listdir(child)[0]
+            while child != filepath:
+                if child.exists() and len(os.listdir(child)) == 0:
+                    child.rmdir()
+                    if debug:
+                        print(f"删除空文件夹 {child}")
+                else:
+                    break
+            if filepath.exists() and len(os.listdir(filepath)) == 0:
+                filepath.rmdir()
+                if debug:
+                    print(f"删除空文件夹 {filepath}")
+        elif filter(filepath):
+            rm(filepath, filter, expand_all_subdir)
+    if dir_path.exists() and len(os.listdir(dir_path)) == 0:
+        dir_path.rmdir()
+        if debug:
+            print(f"删除空文件夹 {dir_path}")
+
+
+def cp(
+    input_dir_path: Union[str, Path, List[str], List[Path]],
+    output_dir_path: Union[str, Path],
+    base_input_dir: Optional[Union[Path, str]] = None,
+    force_overwrite: bool = False,
+    filter: Callable[[Path], bool] = lambda filepath: True,
+    expand_all_subdir=True,
+    verbose=False,
+):
+    input_paths = ls(input_dir_path, filter, expand_all_subdir)
+    if len(input_paths) == 0:
+        if verbose:
+            logger.warning(f"no files in {input_dir_path}")
+        return
+    if base_input_dir is None:
+        # 计算最大公共路径
+        if len(input_paths) > 1:
+            base_input_dir = os.path.commonpath([str(p) for p in input_paths])
+        else:
+            base_input_dir = input_paths[0].parent
+    base_input_dir = Path(base_input_dir)
+    output_dir_path = Path(output_dir_path)
+    for input_path in input_paths:
+        relative_path = input_path.relative_to(base_input_dir)
+        output_path = output_dir_path / relative_path
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        copy_file(input_path, output_path, force_overwrite, verbose)
+
+
+def ls(dir_path: Union[str, Path, List[str], List[Path]], filter: Callable[[Path], bool] = lambda filepath: True, expand_all_subdir=True):
+    """list all files, return a list of filepaths
+
+    Args:
+        dir_path (Union[str, Path]): dir
+        filter ((Path) -> bool, optional): filter. Defaults to lambda filepath:True.
+        expand_all_subdir (bool, optional): _description_. Defaults to True.
+
+    Returns:
+        List[Path]: not null, may be empty list []
+    """
+    filepaths: List[Path] = []
+    if isinstance(dir_path, str) and "," in dir_path:
+        for path in dir_path.split(","):
+            filepaths.extend(ls(path, filter, expand_all_subdir))
+        return filepaths
+    if isinstance(dir_path, list):
+        for path in dir_path:
+            filepaths.extend(ls(path, filter, expand_all_subdir))
+        return filepaths
+    dir_path = Path(dir_path)
+    if not dir_path.exists():
+        return filepaths
+    if not dir_path.is_dir():
+        if filter(dir_path):
+            return [dir_path]
+        else:
+            return filepaths
+    filenames = os.listdir(dir_path)
+    for filename in sorted(filenames):
+        filepath = dir_path / filename
+        if filepath.is_dir():
+            if expand_all_subdir:
+                filepaths.extend(ls(filepath, filter, expand_all_subdir))
+        elif filter(filepath):
+            filepaths.append(filepath)
+    return filepaths
+
+
+def clean_empty_folder(dir_path):
+    dir_path = Path(dir_path)
+    sub_names = os.listdir(dir_path)
+    if not sub_names or len(sub_names) == 0:
+        print(f"clean empty folder: {dir_path}")
+        dir_path.rmdir()
+        clean_empty_folder(dir_path.parent)
+    else:
+        for sub_name in sub_names:
+            path = dir_path / sub_name
+            if path.is_dir():
+                clean_empty_folder(path)
+
+
+def grouped_col_list(df: pd.DataFrame, key_col="query", value_col="output"):
+    grouped = defaultdict(list)
+    if key_col not in df.columns:
+        logger.warning(f"`{key_col}` not in columns: {list(df.columns)}")
+        return grouped
+    for i, row in df.iterrows():
+        grouped[row[key_col]].append(row[value_col])
+    return grouped
+
+
+def grouped_col(df: pd.DataFrame, key_col="query", value_col="output"):
+    grouped = {}
+    if key_col not in df.columns:
+        logger.warning(f"`{key_col}` not in columns: {list(df.columns)}")
+        return grouped
+    for i, row in df.iterrows():
+        grouped[row[key_col]] = row[value_col]
+    return grouped
+
+
+def grouped_row(df: pd.DataFrame, key_col="query"):
+    grouped = defaultdict(list)
+    if key_col not in df.columns:
+        logger.warning(f"`{key_col}` not in columns: {list(df.columns)}")
+        return grouped
+    for i, row in df.iterrows():
+        grouped[row[key_col]].append(row)
+    return grouped
+
+
+def grouped_row_in_jsonlist(jsonlist: List[Dict[str, Any]], key_col="query"):
+    grouped = defaultdict(list)
+    for i, row in enumerate(jsonlist):
+        if key_col not in row:
+            logger.warning(f"`{key_col}` not in row: {row}")
+            notfound_key = f"NotFound:{key_col}"
+            grouped[notfound_key].append(row)
+            continue
+        grouped[row[key_col]].append(row)
+    return grouped
+
+
+def submit_file(path: Union[str, Path], target_dir: Union[str, Path]):
+    p = Path(path).absolute()
+    target_dir = Path(target_dir).absolute()
+    logger.info(f"正在复制到目标文件夹 {target_dir}")
+    if p.is_dir():
+        logger.info(f"文件夹 {p}")
+        filenames = os.listdir(path)
+        for filename in filenames:
+            src_file = p / filename
+            tgt_file = target_dir / filename
+            copy_file(src_file, tgt_file)
+            logger.info(f"已复制 {filename} 到 {tgt_file}")
+    else:
+        filename = p.name
+        logger.info(f"文件 {filename}")
+        src_file = p
+        tgt_file = target_dir / filename
+        copy_file(src_file, tgt_file)
+        logger.info(f"已复制 {filename} 到 {tgt_file}")
+    filenames = os.listdir(target_dir)
+    logger.info("现在目标文件夹下的文件有：\n" + "\n".join(filenames))
+
+
+def pretty_limited_text(text: str, limited_length: int = 300, language="zh"):
+    text = str(text).strip()
+    if len(text) > limited_length:
+        # if language == "zh":
+        #     tail = f"...(共{len(text)}字)"
+        # else:
+        #     tail = f"...({len(text)} words in total)"
+        # return text[: limited_length - len(tail)] + tail
+        return text[: limited_length // 2] + text[-limited_length // 2 :]
+    return text
+
+
+def bucket_count(length):
+    grouped_count = []
+    j = 0
+    for i in range(0, max(length), 50):
+        grouped_count.append(0)
+        while length[j] < i:
+            grouped_count[i // 50] += 1
+            j += 1
+    for i, j in enumerate(grouped_count):
+        if i == 0 or j == 0:
+            continue
+        print(f"[{(i-1)*50}, {i*50})  {j}   {sum(grouped_count[:i+1])/len(length)*100:.2f}%")
+
+
+def sortedCounter(obj, by="key", reverse=False, return_list=False):
+    c = Counter(obj)
+    c_list = [(k, c[k]) for k in c]
+    if by == "key":
+        c_list = sorted(c_list, key=lambda x: x[0], reverse=reverse)
+    elif by in ["value", "count"]:
+        c_list = sorted(c_list, key=lambda x: x[1], reverse=reverse)
+    else:
+        raise Exception(f"unsupported by: {by}")
+    c = Counter()
+    for k, v in c_list:
+        c[k] = v
+    if return_list:
+        return c, c_list
+    return c

xlin-0.1.0/xlin/uuid.py

@@ -0,0 +1,13 @@
+from typing import *
+import uuid
+
+import pandas as pd
+
+
+def append_uuid_column(df: pd.DataFrame, uuid_key="uuid"):
+    rows = []
+    for i, row in df.iterrows():
+        row[uuid_key] = str(uuid.uuid4())
+        rows.append(row)
+    df = pd.DataFrame(rows)
+    return df

xlin-0.1.0/xlin/xls2xlsx.py

@@ -0,0 +1,33 @@
+# pip install pyexcel pyexcel-xls pyexcel-xlsx
+import os
+
+import pyexcel as p
+
+
+def convert_xls_dir_to_xlsx(data_dir):
+    filenames = os.listdir(data_dir)
+    for filename in filenames:
+        if filename.endswith(".xls"):
+            convert_xls_to_xlsx(os.path.join(data_dir, filename))
+
+def convert_xls_to_xlsx(file_name):
+    converted_filename = file_name + 'x'
+    if is_xslx(file_name):
+        # rename to .xlsx
+        with open(file_name, 'rb') as f:
+            with open(converted_filename, 'wb') as f2:
+                f2.write(f.read())
+        return converted_filename
+    sheet = p.get_sheet(file_name=file_name)
+    sheet.save_as(converted_filename)
+    return converted_filename
+
+
+def is_xslx(filename):
+    with open(filename, 'rb') as f:
+        first_four_bytes = f.read()[:4]
+        return first_four_bytes == b'PK\x03\x04'
+
+if __name__ == "__main__":
+    import sys
+    convert_xls_to_xlsx(sys.argv[1])

xlin-0.1.0/xlin/yaml.py

@@ -0,0 +1,8 @@
+import yaml
+
+
+def load_yaml(yaml_file: str):
+    with open(yaml_file, "r", encoding="utf-8") as f:
+        file_str = f.read()
+    schema = yaml.safe_load(file_str)
+    return schema