xlin 0.1.11__py2.py3-none-any.whl → 0.1.13__py2.py3-none-any.whl

xlin/read_as_dataframe.py

@@ -12,23 +12,23 @@ from xlin.jsonl import dataframe_to_json_list, load_json, load_json_list, save_j
 from xlin.xls2xlsx import is_xslx
 
 
-def valid_to_read_as_dataframe(filename: str) -> bool:
-    suffix_list = [".json", ".jsonl", ".xlsx", "xls", ".csv"]
-    return any([filename.endswith(suffix) for suffix in suffix_list])
-
-
-def read_as_dataframe(filepath: Union[str, Path], sheet_name: Optional[str] = None, fill_empty_str_to_na=True) -> pd.DataFrame:
+def read_as_dataframe(
+    filepath: Union[str, Path],
+    sheet_name: Optional[str] = None,
+    fill_empty_str_to_na=True,
+    filter=lambda x: True,
+) -> pd.DataFrame:
     """
-    读取文件为表格
+    读取文件为表格。如果是文件夹，则读取文件夹下的所有文件为表格并拼接
     """
     filepath = Path(filepath)
     if filepath.is_dir():
-        paths = ls(filepath, expand_all_subdir=True)
+        paths = ls(filepath, filter=filter, expand_all_subdir=True)
         df_list = []
         for path in paths:
             try:
-                df = read_as_dataframe(path, sheet_name, fill_empty_str_to_na)
-                df['数据来源'] = path.name
+                df = read_as_dataframe(path, sheet_name, fill_empty_str_to_na, filter)
+                df["数据来源"] = path.name
             except:
                 df = pd.DataFrame()
             df_list.append(df)
@@ -44,56 +44,64 @@ def read_as_dataframe(filepath: Union[str, Path], sheet_name: Optional[str] = No
             json_list = load_json_list(filepath)
         df = pd.DataFrame(json_list)
     elif filename.endswith(".xlsx"):
-        df = pd.read_excel(filepath) if sheet_name is None else pd.read_excel(filepath, sheet_name)
+        if sheet_name is None:
+            df = pd.read_excel(filepath)
+        else:
+            df = pd.read_excel(filepath, sheet_name)
     elif filename.endswith(".xls"):
         if is_xslx(filepath):
-            df = pd.read_excel(filepath) if sheet_name is None else pd.read_excel(filepath, sheet_name)
+            if sheet_name is None:
+                df = pd.read_excel(filepath)
+            else:
+                df = pd.read_excel(filepath, sheet_name)
         else:
             df = pyexcel.get_sheet(file_name=filepath)
     elif filename.endswith(".csv"):
         df = pd.read_csv(filepath)
+    elif filename.endswith(".parquet"):
+        df = pd.read_parquet(filepath)
+    elif filename.endswith(".feather"):
+        df = pd.read_feather(filepath)
+    elif filename.endswith(".pkl"):
+        df = pd.read_pickle(filepath)
+    elif filename.endswith(".h5"):
+        df = pd.read_hdf(filepath)
+    elif filename.endswith(".txt"):
+        df = pd.read_csv(filepath, delimiter="\t")
+    elif filename.endswith(".tsv"):
+        df = pd.read_csv(filepath, delimiter="\t")
+    elif filename.endswith(".xml"):
+        df = pd.read_xml(filepath)
+    elif filename.endswith(".html"):
+        df = pd.read_html(filepath)[0]
+    elif filename.endswith(".db"):
+        df = pd.read_sql_table(sheet_name, filepath)
     else:
-        raise ValueError(f"Unsupported filetype {filepath}. filetype not in [json, jsonl, xlsx, xls, csv]")
+        raise ValueError(
+            (
+                f"Unsupported filetype {filepath}. filetype not in \n"
+                "[json, jsonl, xlsx, xls, csv, "
+                "parquet, feather, pkl, h5, txt, "
+                "tsv, xml, html, db]"
+            )
+        )
     if fill_empty_str_to_na:
         df.fillna("", inplace=True)
     return df
 
 
-def read_maybe_dir_as_dataframe(filepath: Union[str, Path], sheet_name: Optional[str] = None) -> pd.DataFrame:
-    """
-    input path 可能是文件夹，此时将文件夹下的所有表格拼接到一起返回，要求所有表头一致
-    如果不是文件夹，则为文件，尝试直接读取为表格返回
-    """
-    out_list = list()
-    if not isinstance(filepath, Path):
-        filepath = Path(filepath)
-    if not filepath.exists():
-        raise ValueError(f"Path Not Exist: {filepath}")
-    if not filepath.is_dir():
-        return read_as_dataframe(filepath, sheet_name)
-
-    files = os.listdir(filepath)
-    for file_name in files:
-        if not valid_to_read_as_dataframe(file_name):
-            continue
-        input_file = filepath / file_name
-        df = read_as_dataframe(input_file, sheet_name)
-        df.fillna("", inplace=True)
-        for _, line in df.iterrows():
-            line = line.to_dict()
-            out_list.append(line)
-    df = pd.DataFrame(out_list)
-    return df
-
-
-def read_as_dataframe_dict(filepath: Union[str, Path], fill_empty_str_to_na=True):
+def read_as_dataframe_dict(
+    filepath: Union[str, Path],
+    fill_empty_str_to_na=True,
+    filter=lambda x: True,
+):
     filepath = Path(filepath)
     if filepath.is_dir():
-        paths = ls(filepath, expand_all_subdir=True)
+        paths = ls(filepath, filter=filter, expand_all_subdir=True)
         df_dict_list = []
         for path in paths:
             try:
-                df_dict = read_as_dataframe_dict(path, fill_empty_str_to_na)
+                df_dict = read_as_dataframe_dict(path, fill_empty_str_to_na, filter)
             except:
                 df_dict = {}
             df_dict_list.append(df_dict)
@@ -104,11 +112,11 @@ def read_as_dataframe_dict(filepath: Union[str, Path], fill_empty_str_to_na=True
         for name, df in df_dict.items():
             if fill_empty_str_to_na:
                 df.fillna("", inplace=True)
-            df['数据来源'] = filepath.name
+            df["数据来源"] = filepath.name
     elif isinstance(df_dict, pd.DataFrame):
         if fill_empty_str_to_na:
             df_dict.fillna("", inplace=True)
-        df_dict['数据来源'] = filepath.name
+        df_dict["数据来源"] = filepath.name
     return df_dict
 
 
@@ -137,7 +145,10 @@ def save_df_dict(df_dict: Dict[str, pd.DataFrame], output_filepath: Union[str, P
     return output_filepath
 
 
-def save_df_from_jsonlist(jsonlist: List[Dict[str, str]], output_filepath: Union[str, Path]):
+def save_df_from_jsonlist(
+    jsonlist: List[Dict[str, str]],
+    output_filepath: Union[str, Path],
+):
     df = pd.DataFrame(jsonlist)
     return save_df(df, output_filepath)
 
@@ -150,7 +161,9 @@ def save_df(df: pd.DataFrame, output_filepath: Union[str, Path]):
     return output_filepath
 
 
-def lazy_build_dataframe(name: str, output_filepath: Path, func, filetype: str = "xlsx"):
+def lazy_build_dataframe(
+    name: str, output_filepath: Path, func, filetype: str = "xlsx"
+):
     logger.info(name)
     output_filepath.parent.mkdir(parents=True, exist_ok=True)
     if output_filepath.exists():
@@ -161,9 +174,13 @@ def lazy_build_dataframe(name: str, output_filepath: Path, func, filetype: str =
         if filetype == "xlsx":
             df.to_excel(output_filepath.parent / f"{filename}.xlsx", index=False)
         elif filetype == "json":
-            save_json_list(dataframe_to_json_list(df), output_filepath.parent / f"{filename}.json")
+            save_json_list(
+                dataframe_to_json_list(df), output_filepath.parent / f"{filename}.json"
+            )
         elif filetype == "jsonl":
-            save_json_list(dataframe_to_json_list(df), output_filepath.parent / f"{filename}.jsonl")
+            save_json_list(
+                dataframe_to_json_list(df), output_filepath.parent / f"{filename}.jsonl"
+            )
         else:
             logger.warning(f"不认识的 {filetype}，默认保存为 xlsx")
             df.to_excel(output_filepath.parent / f"{filename}.xlsx", index=False)
@@ -171,7 +188,13 @@ def lazy_build_dataframe(name: str, output_filepath: Path, func, filetype: str =
     return df
 
 
-def lazy_build_dataframe_dict(name: str, output_filepath: Path, df_dict: Dict[str, pd.DataFrame], func, skip_sheets: List[str] = list()):
+def lazy_build_dataframe_dict(
+    name: str,
+    output_filepath: Path,
+    df_dict: Dict[str, pd.DataFrame],
+    func: Callable[[str, pd.DataFrame], pd.DataFrame],
+    skip_sheets: List[str] = list(),
+):
     logger.info(name)
     output_filepath.parent.mkdir(parents=True, exist_ok=True)
     if output_filepath.exists():
@@ -193,13 +216,17 @@ def merge_multiple_df_dict(list_of_df_dict: List[Dict[str, pd.DataFrame]], sort=
     for df_dict in list_of_df_dict:
         for k, df in df_dict.items():
             df_dict_merged[k].append(df)
-    df_dict_merged: Dict[str, pd.DataFrame] = {k: pd.concat(v) for k, v in df_dict_merged.items()}
+    df_dict_merged: Dict[str, pd.DataFrame] = {
+        k: pd.concat(v) for k, v in df_dict_merged.items()
+    }
     if sort:
-        df_dict_merged: Dict[str, pd.DataFrame] = {k: df_dict_merged[k] for k in sorted(df_dict_merged)}
+        df_dict_merged: Dict[str, pd.DataFrame] = {
+            k: df_dict_merged[k] for k in sorted(df_dict_merged)
+        }
     return df_dict_merged
 
 
-def remove_duplicate_and_sort(df: pd.DataFrame, key_col="query", sort_by='label'):
+def remove_duplicate_and_sort(df: pd.DataFrame, key_col="query", sort_by="label"):
     query_to_rows = {}
     for i, row in df.iterrows():
         query_to_rows[row[key_col]] = row
@@ -218,16 +245,20 @@ def highlight_max(x):
     return [("background-color: yellow" if m else "") for m in is_max]
 
 
-def split_dataframe(df: pd.DataFrame, output_dir: Union[str, Path], tag: str, split_count=6):
+def split_dataframe(
+    df: pd.DataFrame,
+    output_dir: Union[str, Path],
+    output_filename_prefix: str,
+    split_count=6,
+):
     output_dir = Path(output_dir)
     output_dir.mkdir(parents=True, exist_ok=True)
     rows = dataframe_to_json_list(df)
     split_step = len(rows) // split_count + 1
     df_list = []
     for i in range(0, len(rows), split_step):
-        filepath = output_dir / f"{tag}_{i // split_step}.xlsx"
-        df_i = pd.DataFrame(rows[i:i+split_step])
+        filepath = output_dir / f"{output_filename_prefix}_{i // split_step}.xlsx"
+        df_i = pd.DataFrame(rows[i : i + split_step])
         df_i.to_excel(filepath, index=False)
         df_list.append(df_i)
     return df_list
-

xlin/statistic.py

@@ -31,3 +31,86 @@ def statistic_char_length(df: pd.DataFrame, instruction_key="instruction"):
     length.sort()
     return length
 
+
+def statistic_token_length(df: pd.DataFrame, model_path: str, row_to_prompt: lambda row: row["prompt"]):
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
+    lengths = []
+    for i, row in df.iterrows():
+        prompt = row_to_prompt(row)
+        inputs = tokenizer(prompt, return_tensors="pt")
+        length = inputs["input_ids"].shape[1]
+        lengths.append(length)
+    lengths.sort()
+    return lengths
+
+
+def draw_histogram(data: list[int], bins=30, title="Data Distribution Analysis"):
+    import numpy as np
+    import matplotlib.pyplot as plt
+    from scipy.stats import gaussian_kde
+
+    data = np.array(data)
+
+    # 计算统计指标
+    mean = np.mean(data)
+    median = np.median(data)
+    std = np.std(data)
+    q25, q75 = np.percentile(data, [25, 75])
+    data_range = (np.min(data), np.max(data))
+
+    # 创建图形和坐标轴
+    plt.figure(figsize=(12, 7), dpi=100)
+
+    # 绘制直方图
+    plt.hist(data, bins=bins, density=True, alpha=0.5, color="skyblue", edgecolor="white", label="Distribution")
+
+    # 绘制核密度估计（KDE）
+    kde = gaussian_kde(data)
+    x_vals = np.linspace(data_range[0] - 1, data_range[1] + 1, 1000)
+    plt.plot(x_vals, kde(x_vals), color="navy", linewidth=2, label="KDE Curve")
+
+    # 添加统计线
+    plt.axvline(mean, color="red", linestyle="--", linewidth=2, label=f"Mean ({mean:.2f})")
+    plt.axvline(median, color="green", linestyle="-.", linewidth=2, label=f"Median ({median:.2f})")
+    plt.axvspan(mean - std, mean + std, color="orange", alpha=0.1, label=f"±1 Std.Dev ({std:.2f})")
+
+    # 添加四分位线
+    plt.axvline(q25, color="purple", linestyle=":", alpha=0.8, label=f"25th Percentile ({q25:.2f})")
+    plt.axvline(q75, color="purple", linestyle=":", alpha=0.8, label=f"75th Percentile ({q75:.2f})")
+
+    # 添加统计摘要
+    stats_text = f"""\
+Data Range: [{data_range[0]:.2f}, {data_range[1]:.2f}]
+Observations: {len(data):,}
+Standard Deviation: {std:.2f}
+IQR: {q75 - q25:.2f}
+Skewness: {float((data - mean).mean()**3 / std**3):.4f}
+Kurtosis: {float((data - mean).mean()**4 / std**4):.4f}\
+"""
+# 文字左对齐 align
+    plt.annotate(stats_text, xy=(0.99, 0.98), xycoords="axes fraction", ha="right", va="top", fontfamily="monospace", bbox=dict(boxstyle="round", facecolor="white", alpha=0.8),)
+
+    # 设置图形属性
+    plt.title(title, fontsize=14, pad=20)
+    plt.xlabel("Value", fontsize=12)
+    plt.ylabel("Density", fontsize=12)
+    plt.grid(True, linestyle="--", alpha=0.4)
+    plt.legend(loc="upper left", frameon=True, framealpha=0.9, shadow=True)
+
+    # 调整坐标轴范围
+    buffer = (data_range[1] - data_range[0]) * 0.1
+    plt.xlim(data_range[0] - buffer, data_range[1] + buffer)
+
+    # 显示图形
+    plt.tight_layout()
+    plt.show()
+
+
+def draw_pie(numbers: List[int], title="Pie Chart of Numbers"):
+    import matplotlib.pyplot as plt
+
+    plt.pie(numbers, labels=[str(i) for i in range(len(numbers))], autopct='%1.1f%%')
+    plt.title(title)
+    plt.show()

{xlin-0.1.11.dist-info → xlin-0.1.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: xlin
-Version: 0.1.11
+Version: 0.1.13
 Summary: toolbox for LinXueyuan
 License: MIT
 Author: XiChen
@@ -18,11 +18,13 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: loguru
 Requires-Dist: pandas
 Requires-Dist: pyexcel
 Requires-Dist: pyexcel-xls
 Requires-Dist: pyexcel-xlsx
+Requires-Dist: pyyaml
 Requires-Dist: tqdm
 Requires-Dist: xlsxwriter
 Description-Content-Type: text/markdown

{xlin-0.1.11.dist-info → xlin-0.1.13.dist-info}/RECORD

@@ -3,12 +3,12 @@ xlin/ischinese.py,sha256=Ia9IMQ6q-UHkdLwqS70L1fTnfSPbluFrv_I1UqsKquo,293
 xlin/jsonl.py,sha256=DvVM241a9VgQlp5WIMPRv-JIolT0RdSxw47IG_fc7xE,6690
 xlin/metric.py,sha256=N7wJ35y-C-IaBr1I1CJ_37lTG7gA69zmn9Xg6xSwKoI,1690
 xlin/multiprocess_mapping.py,sha256=pmzyEUYpbpIZ_ezyvWWWRpr7D7n4t3E3jW1nGXBbVck,7652
-xlin/read_as_dataframe.py,sha256=ir3HUT6dt3crqa3xnlcNn8j3wqjSIGJgiIVLP3KkBaQ,8678
-xlin/statistic.py,sha256=BLj8hszlbBT5xDIfd70_YtOb8QgZEvYXiFJDGXBwCfw,881
+xlin/read_as_dataframe.py,sha256=P8bOYW-zm8uGhehCldZI9ZQhHHLGqDPDbSMNWI2li6g,8885
+xlin/statistic.py,sha256=ldTSClpPiPJeA2Yo2rcy14dDtbX8waRL3rZxkxJqTMo,3911
 xlin/util.py,sha256=SOQUh506GQlljJYLYuI6nScSTOrgRQnMq2xfxSvKIlI,11303
 xlin/xls2xlsx.py,sha256=5zfcM0gmunFQOcOj9nYd9Dj0HMhU7-cPKnPIy6Ot9iU,930
 xlin/yaml.py,sha256=kICi7G3Td5q2MaSXXt85qNTWoHMgjzt7pvn7r3C4dME,183
-xlin-0.1.11.dist-info/LICENSE,sha256=KX0dDCYlO4DskqMZY8qeY94EZMrDRNnNqlGLkXVlKyM,1063
-xlin-0.1.11.dist-info/METADATA,sha256=uX2_qdpKUG3TEUnG5iT19lOOim7YAIk5UuagU-6Vi8Y,1067
-xlin-0.1.11.dist-info/WHEEL,sha256=IrRNNNJ-uuL1ggO5qMvT1GGhQVdQU54d6ZpYqEZfEWo,92
-xlin-0.1.11.dist-info/RECORD,,
+xlin-0.1.13.dist-info/LICENSE,sha256=KX0dDCYlO4DskqMZY8qeY94EZMrDRNnNqlGLkXVlKyM,1063
+xlin-0.1.13.dist-info/METADATA,sha256=21YKyI-upWi6HFq2zkBoruE_8GS7WtPbkt_Pl6SaixE,1140
+xlin-0.1.13.dist-info/WHEEL,sha256=aiTauIPAnqOMBuVimVqk4bevIILHLWXNdkzTocSR-tg,92
+xlin-0.1.13.dist-info/RECORD,,

{xlin-0.1.11.dist-info → xlin-0.1.13.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 2.1.1
 Root-Is-Purelib: true
 Tag: py2.py3-none-any