kevin-toolbox-dev 1.3.5__py3-none-any.whl → 1.3.6__py3-none-any.whl

kevin_toolbox/data_flow/file/markdown/parse_table.py

@@ -0,0 +1,135 @@
+import re
+from typing import Union
+from kevin_toolbox.data_flow.file.markdown.variable import Table_Format
+
+
+def parse_table(raw_table, output_format: Union[Table_Format, str] = Table_Format.COMPLETE_DICT, orientation="vertical",
+                chunk_size=None, chunk_nums=None, b_remove_empty_lines=False, f_gen_order_of_values=None):
+    """
+        将二维数组形式的表格（比如find_tables()的返回列表的元素），解析成指定的格式
+
+        参数：
+            raw_table:                  <list of list> 二维数组形式的表格
+            output_format:              <Table_Format or str> 目标格式
+                                            具体可以参考 Table_Format 的介绍
+            orientation:                <str> 解释表格时取哪个方向
+                                            支持以下值：
+                                            "vertical" / "v":       将第一行作为标题
+                                            "horizontal" / "h":     将第一列作为标题
+            chunk_nums:                 <int> 表格被平均分割为多少份进行并列显示。
+            chunk_size:                 <int> 表格被按照最大长度进行分割，然后并列显示。
+                以上两个参数是用于解释 generate_table() 中使用对应参数生成的表格，其中 chunk_size 仅作检验行数是否符合要求，
+                对解释表格无作用。但是当指定该参数时，将视为表格有可能是多个表格并列的情况，因此将尝试根据标题的重复规律，
+                推断出对应的 chunk_nums，并最终将其拆分成多个表格。
+            b_remove_empty_lines:       <boolean> 移除空的行、列
+            f_gen_order_of_values:      <callable> 生成values排序顺序的函数
+                                            具体参考 generate_table() 中的对应参数
+    """
+    assert isinstance(raw_table, (list, tuple,))
+
+    # 转换为字典形式
+    if orientation not in ["vertical", "v"]:
+        # 需要转为垂直方向
+        raw_table = list(zip(*raw_table))
+    r_nums, c_nums = len(raw_table), len(raw_table[0])
+    if chunk_size is not None:
+        assert chunk_size == r_nums - 1, \
+            (f'The number of values {r_nums - 1} actually contained in the table '
+             f'does not match the specified chunk_size {chunk_size}')
+        chunk_nums = c_nums // _find_shortest_repeating_pattern_size(arr=raw_table[0])
+    chunk_nums = 1 if chunk_nums is None else chunk_nums
+    assert c_nums % chunk_nums == 0, \
+        f'The number of headers actually contained in the table does not match the specified chunk_nums, ' \
+        f'Expected n*{chunk_nums}, but got {c_nums}'
+    # 解释出标题
+    keys = raw_table[0][0:c_nums // chunk_nums]
+    # 解释出值
+    if chunk_nums == 1:
+        values = raw_table[1:]
+    else:
+        values = []
+        for i in range(chunk_nums):
+            for j in range(1, r_nums):
+                values.append(raw_table[j][i * len(keys):(i + 1) * len(keys)])
+    # 去除空行
+    if b_remove_empty_lines:
+        values = [line for line in values if any(i != '' for i in line)]
+    table_s = {i: {"title": k, "values": list(v)} for i, (k, v) in enumerate(zip(keys, list(zip(*values))))}
+    # 去除空列
+    if b_remove_empty_lines:
+        table_s = {k: v_s for k, v_s in table_s.items() if v_s["title"] != '' and any(i != '' for i in v_s["values"])}
+    # 对值进行排序
+    if callable(f_gen_order_of_values):
+        breakpoint()
+        # 检查是否有重复的 title
+        temp = [v["title"] for v in table_s.values()]
+        assert len(set(temp)) == len(temp), \
+            f'table has duplicate titles, thus cannot be sorted using f_gen_order_of_values'
+        idx_ls = list(range(len(values)))
+        idx_ls.sort(key=lambda x: f_gen_order_of_values({v["title"]: v["values"][x] for v in table_s.values()}))
+        for v in table_s.values():
+            v["values"] = [v["values"][i] for i in idx_ls]
+
+    #
+    if output_format is Table_Format.SIMPLE_DICT:
+        temp = {v_s["title"] for v_s in table_s.values()}
+        if len(temp) != len(set(temp)):
+            raise AssertionError(
+                f'There are columns with the same title in the table, '
+                f'please check the orientation of the table or use output_format="complete_dict"')
+        table_s = {v_s["title"]: v_s["values"] for v_s in table_s.values()}
+
+    return table_s
+
+
+def _find_shortest_repeating_pattern_size(arr):
+    n = len(arr)
+
+    # 部分匹配表
+    pi = [0] * n
+    k = 0
+    for i in range(1, n):
+        if k > 0 and arr[k] != arr[i]:
+            k = 0
+        if arr[k] == arr[i]:
+            k += 1
+        pi[i] = k
+
+    # 最短重复模式的长度
+    pattern_length = n - pi[n - 1]
+    # 是否是完整的重复模式
+    if n % pattern_length != 0:
+        pattern_length = n
+    return pattern_length
+
+
+if __name__ == '__main__':
+    from kevin_toolbox.data_flow.file.markdown import find_tables
+    # # 示例Markdown表格文本
+    # file_path = ""
+    # with open(file_path, 'r') as f:
+    #     markdown_text = f.read()
+
+    # markdown_text = """
+    # | Name | Age | Occupation |
+    # |------|-----|------------|
+    # | Alice | 28  | Engineer   |
+    # | Bob   | 23  | Teacher    |
+    # | Name | Age | Occupation |
+    # | Carol | 32  | Hacker   |
+    # | David | 18  | Student   |
+    # """
+
+    markdown_text = """
+|  | a | b |  | a | b |  | a | b |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+|  | 0 | 2 |  | 4 | 6 |  | 7 | 9 |
+|  | 1 | 3 |  | 5 | 7 |  | 8 | : |
+|  | 2 | 4 |  | 6 | 8 |  | 9 | ; |
+|  | 3 | 5 |  |  |  |  |  |  |
+"""
+    table_ls = find_tables(text=markdown_text)
+
+    # 调用函数并打印结果
+    tables = parse_table(raw_table=table_ls[0], output_format="complete_dict", chunk_nums=3, b_remove_empty_lines=True)
+    print(tables)

kevin_toolbox/data_flow/file/markdown/save_images_in_ndl.py

@@ -0,0 +1,81 @@
+import os
+import warnings
+import torch
+import cv2
+import numpy as np
+from PIL import Image
+from collections import defaultdict
+from kevin_toolbox.data_flow.file import markdown
+from kevin_toolbox.patches.for_os.path import replace_illegal_chars, find_illegal_chars
+import kevin_toolbox.nested_dict_list as ndl
+
+
+def save_images_in_ndl(var, plot_dir, doc_dir=None, setting_s=None):
+    """
+        将ndl结构叶节点下的图片对象保存到 plot_dir 中，并替换为该图片的markdown链接
+
+        参数：
+            var:                <dict> 待处理的 ndl 结构
+            plot_dir:           <path> 图片保存的目录
+            doc_dir:            <path> 输出的markdown文档保存的目录
+                                    当有指定时，图片链接将以相对于 doc_dir 的相对路径的形式保存
+                                    默认为 None，此时保存的markdown图片链接使用的是绝对路径
+            setting_s:          <dict> 配置
+                                    指定要在哪些节点下去寻找图片对象，以及转换图片对象时使用的参数
+                                    形式为 {<node name>: {"b_is_rgb":<boolean>, ...}, ...}
+                                    其中配置项支持：
+                                        - b_is_rgb:             待保存的图片是RGB顺序还是BGR顺序
+                                        - saved_image_format:   保存图片时使用的格式
+                                    默认为 None，此时等效于 {"": {"b_is_rgb": False, "saved_image_format": ".jpg"}}
+    """
+    if len(find_illegal_chars(file_name=plot_dir, b_is_path=True)) > 0:
+        warnings.warn(f'plot_dir {plot_dir} contains illegal symbols, '
+                      f'which may cause compatibility issues on certain systems.', UserWarning)
+    setting_s = setting_s or {"": {"b_is_rgb": False, "saved_image_format": ".jpg"}}
+
+    # 将配置解释到各个叶节点
+    #   从最浅的路径开始，若更深的路径有另外的设置，则以更新的为准
+    root_ls = list(setting_s.keys())
+    root_ls.sort(key=lambda x: len(ndl.name_handler.parse_name(name=x)[-1]))
+    root_to_leaf_s = defaultdict(set)
+    leaf_to_root_s = dict()
+    leaf_to_value_s = dict()
+    for root in root_ls:
+        for leaf, v in ndl.get_nodes(var=ndl.get_value(var=var, name=root, b_pop=False), level=-1, b_strict=True):
+            leaf = root + leaf
+            if leaf in leaf_to_root_s:
+                root_to_leaf_s[leaf_to_root_s[leaf]].remove(leaf)
+            root_to_leaf_s[root].add(leaf)
+            leaf_to_root_s[leaf] = root
+            leaf_to_value_s[leaf] = v
+
+    for root, leaf_ls in root_to_leaf_s.items():
+        setting_ = setting_s[root]
+        for leaf in leaf_ls:
+            v = leaf_to_value_s[leaf]
+            if isinstance(v, Image.Image):
+                v = np.asarray(v)
+            elif torch.is_tensor(v):
+                v = v.detach().cpu().numpy()
+            #
+            if isinstance(v, np.ndarray):
+                image_path = os.path.join(
+                    plot_dir, replace_illegal_chars(
+                        file_name=f'{leaf}_{setting_["saved_image_format"]}', b_is_path=False)
+                )
+                os.makedirs(os.path.dirname(image_path), exist_ok=True)
+                if setting_["b_is_rgb"]:
+                    v = cv2.cvtColor(v, cv2.COLOR_RGB2BGR)
+                cv2.imwrite(image_path, v)
+                v_new = markdown.generate_link(
+                    name=os.path.basename(image_path),
+                    target=os.path.relpath(image_path, doc_dir) if doc_dir is not None else image_path, type_="image")
+            elif v is None:
+                v_new = "/"
+            else:
+                v_new = v
+            ndl.set_value(var=var, name=leaf, b_force=False, value=v_new)
+    return var
+
+
+

kevin_toolbox/data_flow/file/markdown/variable.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+
+class Table_Format(Enum):
+    """
+        表格的几种模式
+            1.simple_dict 简易字典模式：
+                content_s = {<title>: <list of value>, ...}
+                此时键作为标题，值作为标题下的一系列值。
+                由于字典的无序性，此时标题的顺序是不能保证的，若要额外指定顺序，请使用下面的 完整模式。
+            2. complete_dict 完整字典模式:
+                content_s = {<index>: {"title": <title>,"values":<list of value>}, ...}
+                此时将取第 <index> 个 "title" 的值来作为第 <index> 个标题的值。values 同理。
+                该模式允许缺省某些 <index>，此时这些 <index> 对应的行/列将全部置空。
+    """
+    SIMPLE_DICT = "simple_dict"
+    COMPLETE_DICT = "complete_dict"

kevin_toolbox/nested_dict_list/serializer/read.py

@@ -18,7 +18,7 @@ def read(input_path, **kwargs):
     assert os.path.exists(input_path)
 
     with tempfile.TemporaryDirectory(dir=os.path.dirname(input_path)) as temp_dir:
-        if os.path.isfile(input_path) and input_path.endswith(".tar"): # 解压
+        if os.path.isfile(input_path) and input_path.endswith(".tar"):  # 解压
             for_os.unpack(source=input_path, target=temp_dir)
             input_path = os.path.join(temp_dir, os.listdir(temp_dir)[0])
         var = _read_unpacked_ndl(input_path, **kwargs)

kevin_toolbox/nested_dict_list/serializer/write.py

@@ -13,7 +13,7 @@ from .saved_node_name_builder import Saved_Node_Name_Builder
 
 def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_pack_into_tar=True,
           strictness_level=Strictness_Level.COMPATIBLE, saved_node_name_format='{count}_{hash_name}',
-          b_keep_identical_relations=False, **kwargs):
+          b_keep_identical_relations=False, b_allow_overwrite=False, **kwargs):
     """
         将输入的嵌套字典列表 var 的结构和节点值保存到文件中
             遍历 var，匹配并使用 settings 中设置的保存方式来对各部分结构/节点进行序列化
@@ -106,6 +106,8 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
                                             替换为单个节点和其多个引用的形式。
                                         对于 ndl 中存在大量具有相同 id 的重复节点的情况，使用该操作可以额外达到压缩的效果。
                                         默认为 False
+            b_allow_overwrite:          <boolean> 是否允许强制覆盖已有文件
+                                        默认为 False，此时若目标文件已存在则报错
     """
     from kevin_toolbox.nested_dict_list.serializer.variable import SERIALIZER_BACKEND
 
@@ -113,7 +115,12 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
     traversal_mode = Traversal_Mode(traversal_mode)
     strictness_level = Strictness_Level(strictness_level)
     #
-    assert not os.path.exists(output_dir + ".tar" if b_pack_into_tar else output_dir), f'target already exists'
+    tgt_path = output_dir + ".tar" if b_pack_into_tar else output_dir
+    if os.path.exists(tgt_path):
+        if b_allow_overwrite:
+            for_os.remove(path=tgt_path, ignore_errors=True)
+        else:
+            raise FileExistsError(f"target {tgt_path} already exists")
     os.makedirs(os.path.dirname(output_dir), exist_ok=True)
     temp_dir = tempfile.TemporaryDirectory(dir=os.path.dirname(output_dir))
     temp_output_dir = os.path.join(temp_dir.name, os.path.basename(output_dir))

kevin_toolbox/patches/for_matplotlib/common_charts/__init__.py

@@ -0,0 +1,6 @@
+from .plot_lines import plot_lines
+from .plot_scatters import plot_scatters
+from .plot_distribution import plot_distribution
+from .plot_bars import plot_bars
+from .plot_scatters_matrix import plot_scatters_matrix
+from .plot_confusion_matrix import plot_confusion_matrix

kevin_toolbox/patches/for_matplotlib/common_charts/plot_bars.py

@@ -0,0 +1,54 @@
+import os
+import copy
+from kevin_toolbox.computer_science.algorithm import for_seq
+import matplotlib.pyplot as plt
+from kevin_toolbox.patches.for_os.path import replace_illegal_chars
+
+# TODO 在 linux 系统下遇到中文时，尝试自动下载中文字体，并尝试自动设置字体
+# font_path = os.path.join(root_dir, "utils/SimHei.ttf")
+# font_name = FontProperties(fname=font_path)
+
+
+def plot_bars(data_s, title, x_name, y_label=None, output_dir=None, **kwargs):
+    data_s = copy.deepcopy(data_s)
+    paras = {
+        "dpi": 200
+    }
+    paras.update(kwargs)
+
+    plt.clf()
+    #
+    x_all_ls = data_s.pop(x_name)
+    #
+    for i, (k, y_ls) in enumerate(data_s.items()):
+        if i == 0:
+            plt.bar([j - 0.1 for j in range(len(x_all_ls))], y_ls, width=0.2, align='center', label=k)
+        else:
+            plt.bar([j + 0.1 for j in range(len(x_all_ls))], y_ls, width=0.2, align='center', label=k)
+
+    plt.xlabel(f'{x_name}')
+    plt.ylabel(f'{y_label if y_label else "value"}')
+    temp = for_seq.flatten_list([list(i) for i in data_s.values()])
+    y_min, y_max = min(temp), max(temp)
+    plt.ylim(max(min(y_min, 0), y_min - (y_max - y_min) * 0.2), y_max + (y_max - y_min) * 0.1)
+    plt.xticks(list(range(len(x_all_ls))), labels=x_all_ls)  # , fontproperties=font_name
+    plt.title(f'{title}')
+    # 显示图例
+    plt.legend()
+
+    if output_dir is None:
+        plt.show()
+        return None
+    else:
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f'{replace_illegal_chars(title)}.png')
+        plt.savefig(output_path, dpi=paras["dpi"])
+        return output_path
+
+
+if __name__ == '__main__':
+    plot_bars(data_s={
+        'a': [1.5, 2, 3, 4, 5],
+        'b': [5, 4, 3, 2, 1],
+        'c': [1, 2, 3, 4, 5]},
+        title='test', x_name='a', output_dir=os.path.join(os.path.dirname(__file__), "temp"))

kevin_toolbox/patches/for_matplotlib/common_charts/plot_confusion_matrix.py

@@ -0,0 +1,60 @@
+import os
+from sklearn.metrics import confusion_matrix
+import matplotlib.pyplot as plt
+import seaborn as sns
+from kevin_toolbox.patches.for_os.path import replace_illegal_chars
+
+
+def plot_confusion_matrix(data_s, title, gt_name, pd_name, label_to_value_s=None, output_dir=None, **kwargs):
+    paras = {
+        "dpi": 200,
+        "normalize": None,  # "true", "pred", "all",
+        "b_return_cfm": False,  # 是否输出混淆矩阵
+    }
+    paras.update(kwargs)
+
+    value_set = set(data_s[gt_name]).union(set(data_s[pd_name]))
+    if label_to_value_s is None:
+        label_to_value_s = {f'{i}': i for i in value_set}
+    else:
+        assert all(i in value_set for i in label_to_value_s.values())
+    # 计算混淆矩阵
+    cfm = confusion_matrix(y_true=data_s[gt_name], y_pred=data_s[pd_name], labels=list(label_to_value_s.values()),
+                           normalize=paras["normalize"])
+    # 绘制混淆矩阵热力图
+    plt.clf()
+    plt.figure(figsize=(8, 6))
+    sns.heatmap(cfm, annot=True, fmt='.2%' if paras["normalize"] is not None else 'd',
+                xticklabels=list(label_to_value_s.keys()), yticklabels=list(label_to_value_s.keys()),
+                cmap='viridis')
+
+    plt.xlabel(f'{pd_name}')
+    plt.ylabel(f'{gt_name}')
+    plt.title(f'{title}')
+
+    if output_dir is None:
+        plt.show()
+        output_path = None
+    else:
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f'{replace_illegal_chars(title)}.png')
+        plt.savefig(output_path, dpi=paras["dpi"])
+
+    if paras["b_return_cfm"]:
+        return output_path, cfm
+    else:
+        return output_path
+
+
+if __name__ == '__main__':
+    import numpy as np
+
+    # 示例真实标签和预测标签
+    y_true = np.array([0, 1, 2, 0, 1, 2, 0, 1, 2, 5])
+    y_pred = np.array([0, 2, 1, 0, 2, 1, 0, 1, 1, 5])
+
+    plot_confusion_matrix(data_s={'a': y_true, 'b': y_pred},
+                          title='test', gt_name='a', pd_name='b',
+                          label_to_value_s={"A": 5, "B": 0, "C": 1, "D": 2},
+                          # output_dir=os.path.join(os.path.dirname(__file__), "temp"),
+                          normalize="true")

kevin_toolbox/patches/for_matplotlib/common_charts/plot_distribution.py

@@ -0,0 +1,65 @@
+import os
+import math
+import matplotlib.pyplot as plt
+import numpy as np
+from kevin_toolbox.patches.for_os.path import replace_illegal_chars
+
+
+def plot_distribution(data_s, title, x_name=None, x_name_ls=None, type_="hist", output_dir=None, **kwargs):
+    paras = {
+        "dpi": 200
+    }
+    paras.update(kwargs)
+    if x_name is not None:
+        x_name_ls = [x_name, ]
+    assert isinstance(x_name_ls, (list, tuple)) and len(x_name_ls) > 0
+
+    plt.clf()
+
+    alpha = max(1 / len(x_name_ls), 0.3)
+    # 检查数据类型
+    if type_ in ["histogram", "hist"]:
+        # 数字数据，绘制概率分布图
+        for x_name in x_name_ls:
+            data = data_s[x_name]
+            assert all(isinstance(x, (int, float)) for x in data), \
+                f'输入数组中的元素类型不一致'
+            if "steps" in paras:
+                min_ = math.floor(min(data) / paras["steps"]) * paras["steps"]
+                max_ = math.ceil(max(data) / paras["steps"]) * paras["steps"]
+                bins = np.arange(min_, max_ + paras["steps"], paras["steps"])
+            else:
+                bins = np.linspace(paras.get("min", min(data)), paras.get("max", max(data)), paras["bin_nums"] + 1)
+            plt.hist(data, density=True, bins=bins, alpha=alpha, label=x_name)
+    elif type_ in ["category", "cate"]:
+        # 字符串数据，绘制概率直方图
+        for x_name in x_name_ls:
+            data = data_s[x_name]
+            unique_values, counts = np.unique(data, return_counts=True)
+            probabilities = counts / len(data)
+            plt.bar([f'{i}' for i in unique_values], probabilities, label=x_name, alpha=alpha)
+    else:
+        raise ValueError(f'unsupported plot type {type_}')
+
+    plt.xlabel(f'value')
+    plt.ylabel('prob')
+    plt.title(f'{title}')
+    # 显示图例
+    plt.legend()
+
+    if output_dir is None:
+        plt.show()
+        return None
+    else:
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f'{replace_illegal_chars(title)}.png')
+        plt.savefig(output_path, dpi=paras["dpi"])
+        return output_path
+
+
+if __name__ == '__main__':
+    plot_distribution(data_s={
+        'a': [1, 2, 3, 4, 5, 3, 2, 1],
+        'c': [1, 2, 3, 4, 5, 0, 0, 0]},
+        title='test', x_name_ls=['a', 'c'], type_="category",
+        output_dir=os.path.join(os.path.dirname(__file__), "temp"))

kevin_toolbox/patches/for_matplotlib/common_charts/plot_lines.py

@@ -0,0 +1,61 @@
+import os
+import matplotlib.pyplot as plt
+from kevin_toolbox.patches.for_os.path import replace_illegal_chars
+from kevin_toolbox.patches.for_matplotlib import generate_color_list
+
+
+def plot_lines(data_s, title, x_name, output_dir=None, **kwargs):
+    line_nums = len(data_s) - 1
+    paras = {
+        "dpi": 200,
+        "color_ls": generate_color_list(nums=line_nums),
+        "marker_ls": None,
+        "linestyle_ls": '-',
+    }
+    paras.update(kwargs)
+    for k, v in paras.items():
+        if k.endswith("_ls") and not isinstance(v, (list, tuple)):
+            paras[k] = [v] * line_nums
+    assert line_nums == len(paras["color_ls"]) == len(paras["marker_ls"]) == len(paras["linestyle_ls"])
+
+    plt.clf()
+    #
+    x_all_ls = data_s.pop(x_name)
+    data_s, temp = dict(), data_s
+    for k, v_ls in temp.items():
+        y_ls, x_ls = [], []
+        for x, v in zip(x_all_ls, v_ls):
+            if x is None or v is None:
+                continue
+            x_ls.append(x)
+            y_ls.append(v)
+        if len(x_ls) == 0:
+            continue
+        data_s[k] = (x_ls, y_ls)
+    #
+    for i, (k, (x_ls, y_ls)) in enumerate(data_s.items()):
+        plt.plot(x_ls, y_ls, label=f'{k}', color=paras["color_ls"][i], marker=paras["marker_ls"][i],
+                 linestyle=paras["linestyle_ls"][i])
+    plt.xlabel(f'{x_name}')
+    plt.ylabel('value')
+    plt.title(f'{title}')
+    # 显示图例
+    plt.legend()
+
+    if output_dir is None:
+        plt.show()
+        return None
+    else:
+        # 对非法字符进行替换
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f'{replace_illegal_chars(title)}.png')
+        plt.savefig(output_path, dpi=paras["dpi"])
+        return output_path
+
+
+if __name__ == '__main__':
+    plot_lines(data_s={
+        'a': [1, 2, 3, 4, 5],
+        'b': [5, 4, 3, 2, 1],
+        'c': [1, 2, 3, 4, 5]},
+        title='test', x_name='a', output_dir=os.path.join(os.path.dirname(__file__), "temp"))

kevin_toolbox/patches/for_matplotlib/common_charts/plot_scatters.py

@@ -0,0 +1,53 @@
+import os
+import matplotlib.pyplot as plt
+from kevin_toolbox.patches.for_matplotlib import generate_color_list
+from kevin_toolbox.patches.for_os.path import replace_illegal_chars
+
+
+def plot_scatters(data_s, title, x_name, y_name, cate_name=None, output_dir=None, **kwargs):
+    paras = {
+        "dpi": 200,
+        "scatter_size": 5
+    }
+    paras.update(kwargs)
+
+    plt.clf()
+    #
+    color_s = None
+    if cate_name is not None:
+        cates = list(set(data_s[cate_name]))
+        color_s = {i: j for i, j in zip(cates, generate_color_list(nums=len(cates)))}
+        c = [color_s[i] for i in data_s[cate_name]]
+    else:
+        c = "blue"
+    # 创建散点图
+    plt.scatter(data_s[x_name], data_s[y_name], s=paras["scatter_size"], c=c, alpha=0.8)
+    #
+    plt.xlabel(f'{x_name}')
+    plt.ylabel(f'{y_name}')
+    plt.title(f'{title}')
+    # 添加图例
+    if cate_name is not None:
+        plt.legend(handles=[
+            plt.Line2D([0], [0], marker='o', color='w', label=i, markerfacecolor=j,
+                       markersize=min(paras["scatter_size"], 5)) for i, j in color_s.items()
+        ])
+
+    if output_dir is None:
+        plt.show()
+        return None
+    else:
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f'{replace_illegal_chars(title)}.png')
+        plt.savefig(output_path, dpi=paras["dpi"])
+        return output_path
+
+
+if __name__ == '__main__':
+    data_s_ = dict(
+        x=[1, 2, 3, 4, 5],
+        y=[2, 4, 6, 8, 10],
+        categories=['A', 'B', 'A', 'B', 'A']
+    )
+
+    plot_scatters(data_s=data_s_, title='test', x_name='x', y_name='y', cate_name='categories')

kevin_toolbox/patches/for_matplotlib/common_charts/plot_scatters_matrix.py

@@ -0,0 +1,54 @@
+import os
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+from kevin_toolbox.patches.for_matplotlib import generate_color_list
+from kevin_toolbox.patches.for_os.path import replace_illegal_chars
+
+
+def plot_scatters_matrix(data_s, title, x_name_ls, cate_name=None, output_dir=None, cate_color_s=None, **kwargs):
+    paras = {
+        "dpi": 200,
+        "diag_kind": "kde"  # 设置对角线图直方图/密度图 {‘hist’, ‘kde’}
+    }
+    assert cate_name in data_s and len(set(x_name_ls).difference(set(data_s.keys()))) == 0
+    if cate_color_s is None:
+        temp = set(data_s[cate_name])
+        cate_color_s = {k: v for k, v in zip(temp, generate_color_list(len(temp)))}
+    assert set(cate_color_s.keys()) == set(data_s[cate_name])
+    paras.update(kwargs)
+
+    plt.clf()
+    # 使用seaborn绘制散点图矩阵
+    sns.pairplot(
+        pd.DataFrame(data_s),
+        diag_kind=paras["diag_kind"],  # 设置对角线图直方图/密度图 {‘hist’, ‘kde’}
+        hue=cate_name,  # hue 表示根据该列的值进行分类
+        palette=cate_color_s, x_vars=x_name_ls, y_vars=x_name_ls,  # x_vars，y_vars 指定子图的排列顺序
+    )
+    #
+    plt.subplots_adjust(top=0.95)
+    plt.suptitle(f'{title}', y=0.98, x=0.47)
+    # g.fig.suptitle(f'{title}', y=1.05)
+
+    if output_dir is None:
+        plt.show()
+        return None
+    else:
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f'{replace_illegal_chars(title)}.png')
+        plt.savefig(output_path, dpi=paras["dpi"])
+        return output_path
+
+
+if __name__ == '__main__':
+    data_s_ = dict(
+        x=[1, 2, 3, 4, 5],
+        y=[2, 4, 6, 8, 10],
+        z=[2, 4, 6, 8, 10],
+        categories=['A', 'B', 'A', 'B', 'A'],
+        title='test',
+    )
+
+    plot_scatters_matrix(data_s=data_s_, title='test', x_name_ls=['y', 'x', 'z'], cate_name='categories',
+                         cate_color_s={'A': 'red', 'B': 'blue'})

kevin_toolbox/patches/for_os/__init__.py

@@ -3,3 +3,5 @@ from .copy import copy
 from .pack import pack
 from .unpack import unpack
 from .walk import walk, Path_Ignorer, Ignore_Scope
+from .find_files_in_dir import find_files_in_dir
+from . import path

kevin_toolbox/patches/for_os/find_files_in_dir.py

@@ -0,0 +1,30 @@
+import os
+from kevin_toolbox.patches.for_os import walk
+
+
+def find_files_in_dir(input_dir, suffix_ls, b_relative_path=True, b_ignore_case=True):
+    """
+        找出目录下带有给定后缀的所有文件的生成器
+            主要利用了 for_os.walk 中的过滤规则进行实现
+
+        参数：
+            suffix_ls:          <list/tuple of str> 可选的后缀
+            b_relative_path:    <bool> 是否返回相对路径
+            b_ignore_case:      <bool> 是否忽略大小写
+    """
+    suffix_ls = tuple(set(suffix_ls))
+    suffix_ls = tuple(map(lambda x: x.lower(), suffix_ls)) if b_ignore_case else suffix_ls
+    for root, dirs, files in walk(top=input_dir, topdown=True,
+                                  ignore_s=[{
+                                      "func": lambda _, b_is_symlink, path: b_is_symlink or not (
+                                              path.lower() if b_ignore_case else path).endswith(suffix_ls),
+                                      "scope": ["files", ]
+                                  }]):
+        for file in files:
+            file_path = os.path.join(root, file)
+            if b_relative_path:
+                file_path = os.path.relpath(file_path, start=input_dir)
+            yield file_path
+
+
+