kevin-toolbox-dev 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl

kevin_toolbox/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.4.4"
+__version__ = "1.4.6"
 
 
 import os
@@ -12,5 +12,5 @@ os.system(
 os.system(
     f'python {os.path.split(__file__)[0]}/env_info/check_validity_and_uninstall.py '
     f'--package_name kevin-toolbox-dev '
-    f'--expiration_timestamp 1749817728 --verbose 0'
+    f'--expiration_timestamp 1753280251 --verbose 0'
 )

kevin_toolbox/computer_science/algorithm/cache_manager/cache_manager.py

@@ -44,6 +44,12 @@ class Cache_Manager:
                                             当设置为 float 时表示占 upper_bound 的比例
                                             默认为 0.5
                 strategy:               <str/dict/Strategy_Base> 管理策略
+                                            目前支持以下策略：
+                                                - ":by_initial_time:FIFO"     删除最后一次访问时间最久远的部分
+                                                - ":by_counts:LFU"            删除访问频率最低的部分
+                                                - ":by_last_time:LRU"          删除最后一次访问时间最久远的部分
+                                                - ":by_survival_time:LST"      删除访问频率最低的部分
+                                            默认使用 LRU 策略
                 cache:                  <str/dict/Cache_Base> 缓存种类
         """
         # 默认参数

kevin_toolbox/data_flow/file/excel/__init__.py

@@ -0,0 +1 @@
+from .write_excel_with_matrix import write_excel_with_matrix as write_with_matrix

kevin_toolbox/data_flow/file/excel/write_excel_with_matrix.py

@@ -0,0 +1,105 @@
+import os
+import openpyxl
+from openpyxl.styles import Alignment, Font, PatternFill
+from kevin_toolbox.patches.for_os import remove
+
+
+# excel
+def write_excel_with_matrix(matrix, file_path=None, file_obj=None, sheet_name="matrix",
+                            column_label_ls=None, row_label_ls=None, column_title="", row_title="", main_title=""):
+    """
+        将矩阵写入到 excel 文件中
+
+        参数:
+            file_path:                      要写入到哪个文件
+            file_obj:                       <openpyxl.Workbook> 文件对象
+                    注意！！以上两个参数指定其一即可，同时指定时候，以后者为准。
+            sheet_name:                     要写入到哪个sheet页面
+            matrix:                         <np.array or np.matrix> 矩阵
+            column_label_ls, row_label_ls:  行列标签
+            column_title, row_title:        行列标题
+            main_title:                     总标题
+    """
+    assert file_path is not None or file_obj is not None
+
+    if file_obj is None:
+        file_path = os.path.abspath(os.path.expanduser(file_path))
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        # 判断文件是否存在，不存在则新建，否则读取文件
+        if not os.path.isfile(file_path):
+            wb = openpyxl.Workbook()  # 创建文件对象
+            # wb对象创建后，默认含有一个默认的名为 Sheet 的 页面,将其删除
+            ws_ = wb.active
+            wb.remove(ws_)
+        else:
+            wb = openpyxl.load_workbook(file_path)
+    else:
+        assert isinstance(file_obj, openpyxl.Workbook)
+        wb = file_obj
+    # 判断sheet是否存在，不存在则建立，否则先删除再建立
+    if sheet_name in wb.sheetnames:
+        ws = wb[sheet_name]
+        wb.remove(ws)
+    ws = wb.create_sheet(sheet_name)
+
+    # 开始写
+    matrix_r_offset, matrix_c_offset = 1, 1  # 矩阵的起始位置
+    r_offset, c_offset = 1, 1  # 目前的写入位置
+    for i in [main_title, column_title, column_label_ls]:
+        if i:
+            matrix_r_offset += 1
+    for j in [row_title, row_label_ls]:
+        if j:
+            matrix_c_offset += 1
+    # print(matrix)
+    matrix_row_num, matrix_column_num = matrix.shape[0], matrix.shape[1]
+    # 标题
+    alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
+    if main_title:
+        ws.merge_cells(start_row=r_offset, start_column=1, end_row=r_offset,
+                       end_column=matrix_c_offset + matrix_column_num - 1)
+        ws.cell(row=r_offset, column=1).value = main_title
+        ws.cell(row=r_offset, column=1).alignment = alignment
+        ws.cell(row=r_offset, column=1).font = Font(size=10, bold=True, name='微软雅黑', color="000000")
+        r_offset += 1
+    if column_title:
+        ws.merge_cells(start_row=r_offset, start_column=matrix_c_offset, end_row=r_offset,
+                       end_column=matrix_c_offset + matrix_column_num - 1)
+        ws.cell(row=r_offset, column=matrix_c_offset).value = column_title
+        ws.cell(row=r_offset, column=matrix_c_offset).alignment = alignment
+        ws.cell(row=r_offset, column=matrix_c_offset).font = Font(size=10, bold=True, name='微软雅黑', color="000000")
+        r_offset += 1
+    if row_title:
+        ws.merge_cells(start_row=matrix_r_offset, start_column=1, end_row=matrix_r_offset + matrix_row_num - 1,
+                       end_column=1)
+        ws.cell(row=matrix_r_offset, column=1).value = row_title
+        ws.cell(row=matrix_r_offset, column=1).alignment = alignment
+        ws.cell(row=matrix_r_offset, column=1).font = Font(size=10, bold=True, name='微软雅黑', color="000000")
+        c_offset += 1
+    # 标签
+    if column_label_ls:
+        for i in range(matrix_column_num):
+            ws.cell(row=r_offset, column=matrix_c_offset + i).value = column_label_ls[i]
+            ws.cell(row=r_offset, column=matrix_c_offset + i).alignment = alignment
+            ws.cell(row=r_offset, column=matrix_c_offset + i).fill = PatternFill(patternType="solid",
+                                                                                 start_color="33CCFF")
+        r_offset += 1
+    if row_label_ls:
+        for i in range(matrix_row_num):
+            ws.cell(row=matrix_r_offset + i, column=c_offset).value = row_label_ls[i]
+            ws.cell(row=matrix_r_offset + i, column=c_offset).alignment = alignment
+            ws.cell(row=matrix_r_offset + i, column=c_offset).fill = PatternFill(patternType="solid",
+                                                                                 start_color="33CCFF")
+        c_offset += 1
+    # 校验，可省略
+    if not (c_offset == matrix_c_offset and r_offset == matrix_r_offset):
+        print("wrong here")
+    for r_ in range(matrix_row_num):
+        for c_ in range(matrix_column_num):
+            ws.cell(row=matrix_r_offset + r_, column=matrix_c_offset + c_).value = matrix[r_][c_]
+
+    if file_path is not None:
+        remove(file_path, ignore_errors=True)
+        wb.save(file_path)
+
+    return wb

kevin_toolbox/data_flow/file/json_/read_json.py

@@ -23,6 +23,7 @@ def read_json(file_path=None, file_obj=None, converters=None, b_use_suggested_co
     """
     assert file_path is not None or file_obj is not None
     if file_path is not None:
+        file_path = os.path.abspath(os.path.expanduser(file_path))
         assert os.path.isfile(file_path), f'file {file_path} not found'
         file_obj = open(file_path, 'r')
     elif isinstance(file_obj, (BytesIO,)):

kevin_toolbox/data_flow/file/json_/write_json.py

@@ -38,9 +38,9 @@ def write_json(content, file_path, sort_keys=False, converters=None, b_use_sugge
     content = json.dumps(content, indent=4, ensure_ascii=False, sort_keys=sort_keys)
 
     if file_path is not None:
-        file_path = os.path.abspath(file_path)
+        file_path = os.path.abspath(os.path.expanduser(file_path))
         os.makedirs(os.path.dirname(file_path), exist_ok=True)
-        with open(file_path, 'w') as f:
+        with open(file_path, 'w', encoding="utf-8") as f:
             f.write(content)
     else:
         return content

kevin_toolbox/env_info/__init__.py

@@ -1 +1,2 @@
-
+from .check_version_and_update import check_version_and_update
+from .check_validity_and_uninstall import check_validity_and_uninstall

kevin_toolbox/env_info/check_validity_and_uninstall.py

@@ -1,6 +1,4 @@
-import os
 import subprocess
-import argparse
 import time
 
 """
@@ -8,22 +6,44 @@ import time
 - 若超过指定的有效期，则卸载。
 """
 
-out_parser = argparse.ArgumentParser(description='check_validity_and_uninstall')
-out_parser.add_argument('--package_name', type=str, required=True)
-out_parser.add_argument('--expiration_timestamp', type=int, required=False, default=1e10)
-out_parser.add_argument('--verbose', type=int, required=False, default=1)
-args = out_parser.parse_args().__dict__
-
-cur_timestamp = time.time()
-
-if cur_timestamp > args["expiration_timestamp"]:
-    ex = subprocess.Popen(f'pip uninstall {args["package_name"]} --yes', shell=True, stdout=subprocess.PIPE,
-                          stderr=subprocess.STDOUT)
-    out, _ = ex.communicate()
-    res = out.decode().strip()
-else:
-    res = "still within the validity period"
-
-if args["verbose"]:
-    # print(args)
-    print(res)
+DEFAULT_EXPIRATION_TIMESTAMP = 1e10
+
+
+def check_validity_and_uninstall(package_name, expiration_timestamp=DEFAULT_EXPIRATION_TIMESTAMP):
+    """
+        检查当前机器时间是否超过 expiration_timestamp 指定的有效期，若超过则卸载 package_name 对应的库
+    """
+    cur_timestamp = time.time()
+
+    b_success_uninstalled = False
+    if cur_timestamp > expiration_timestamp:
+        ex = subprocess.Popen(f'pip uninstall {package_name} --yes', shell=True, stdout=subprocess.PIPE,
+                              stderr=subprocess.STDOUT)
+        out, _ = ex.communicate()
+        msg = out.decode().strip()
+        if ex.returncode == 0:
+            b_success_uninstalled = True
+    else:
+        msg = "still within the validity period"
+
+    res_s = dict(cur_timestamp=cur_timestamp, expiration_timestamp=expiration_timestamp,
+                 b_success_uninstalled=b_success_uninstalled, msg=msg)
+    return res_s
+
+
+if __name__ == '__main__':
+    import argparse
+
+    out_parser = argparse.ArgumentParser(description='check_validity_and_uninstall')
+    out_parser.add_argument('--package_name', type=str, required=True)
+    out_parser.add_argument('--expiration_timestamp', type=float, required=False, default=DEFAULT_EXPIRATION_TIMESTAMP)
+    out_parser.add_argument('--verbose', type=lambda x: bool(eval(x)), required=False, default=True)
+    args = out_parser.parse_args().__dict__
+
+    b_version = args.pop("verbose")
+
+    res_s_ = check_validity_and_uninstall(**args)
+
+    if b_version:
+        for k, v in res_s_.items():
+            print(f"{k}: {v}")

kevin_toolbox/env_info/check_version_and_update.py

@@ -1,51 +1,72 @@
-import os
 import subprocess
-import argparse
-import version
-
-"""
-检查当前版本
-- 若在可用版本中，有比当前版本更高的版本，则更新到可以获取到的最新版本。
-"""
-
-out_parser = argparse.ArgumentParser(description='check_version_and_update')
-out_parser.add_argument('--package_name', type=str, required=True)
-out_parser.add_argument('--cur_version', type=str, required=False)
-out_parser.add_argument('--available_versions', nargs='+', type=str, required=False)
-out_parser.add_argument('--verbose', type=int, required=False, default=1)
-args = out_parser.parse_args().__dict__
-
-# try to read cur_version
-if args["cur_version"] is None:
-    ex = subprocess.Popen(f'pip list | grep "{args["package_name"]} "', shell=True, stdout=subprocess.PIPE)
-    out, _ = ex.communicate()
-    out = out.decode().strip()
-    # breakpoint()
-    args["cur_version"] = out.split(args["package_name"])[-1].strip()
-
-# try to read available versions
-if args["available_versions"] is None:
-    ex = subprocess.Popen(f'pip install {args["package_name"]}==?', shell=True, stdout=subprocess.PIPE,
-                          stderr=subprocess.STDOUT)
-    out, _ = ex.communicate()
-    out = out.decode().strip()
-    if "(from versions:" in out:
-        v_ls = out.split("(from versions:")[-1].rsplit(")", 1)[0].split(",", -1)
-        v_ls = [i.strip() for i in v_ls]
+
+try:
+    from kevin_toolbox.env_info import version
+except:
+    import version
+
+
+def check_version_and_update(package_name, cur_version=None, available_versions=None):
+    """
+        检查当前版本，并尝试更新
+            - 若在 pip 的可用版本中，有比当前版本更高的版本，则更新到可以获取到的最新版本。
+    """
+    # try to read cur_version
+    if cur_version is None:
+        ex = subprocess.Popen(f'pip list | grep "{package_name} "', shell=True, stdout=subprocess.PIPE)
+        out, _ = ex.communicate()
+        out = out.decode().strip()
+        # breakpoint()
+        cur_version = out.split(package_name)[-1].strip()
+
+    # try to read available versions
+    if available_versions is None:
+        ex = subprocess.Popen(f'pip install {package_name}==?', shell=True, stdout=subprocess.PIPE,
+                              stderr=subprocess.STDOUT)
+        out, _ = ex.communicate()
+        out = out.decode().strip()
+        if "(from versions:" in out:
+            v_ls = out.split("(from versions:")[-1].rsplit(")", 1)[0].split(",", -1)
+            v_ls = [i.strip() for i in v_ls]
+        else:
+            v_ls = ["none"]
+        available_versions = version.sort_ls(version_ls=v_ls, reverse=True)
+
+    b_success_updated = False
+    new_version = None
+    if len(available_versions) > 0 and version.compare(available_versions[0], ">", cur_version):
+        ex = subprocess.Popen(
+            f'pip install {package_name}=={available_versions[0]} --no-dependencies',
+            shell=True, stdout=subprocess.PIPE
+        )
+        out, _ = ex.communicate()
+        msg = out.decode().strip()
+        if ex.returncode == 0:
+            b_success_updated = True
+            new_version = available_versions[0]
     else:
-        v_ls = ["none"]
-    args["available_versions"] = version.sort_ls(version_ls=v_ls, reverse=True)
-
-if len(args["available_versions"]) > 0 and version.compare(args["available_versions"][0], ">", args["cur_version"]):
-    ex = subprocess.Popen(
-        f'pip install {args["package_name"]}=={args["available_versions"][0]} --no-dependencies',
-        shell=True, stdout=subprocess.PIPE
-    )
-    out, _ = ex.communicate()
-    res = out.decode().strip()
-else:
-    res = "Already the latest version, no need to update"
-
-if args["verbose"]:
-    # print(args)
-    print(res)
+        msg = "Already the latest version, no need to update"
+
+    res_s = dict(version_before_updated=cur_version, version_after_updated=new_version,
+                 available_versions=available_versions, b_success_updated=b_success_updated, msg=msg)
+
+    return res_s
+
+
+if __name__ == '__main__':
+    import argparse
+
+    out_parser = argparse.ArgumentParser(description='check_version_and_update')
+    out_parser.add_argument('--package_name', type=str, required=True)
+    out_parser.add_argument('--cur_version', type=str, required=False)
+    out_parser.add_argument('--available_versions', nargs='+', type=str, required=False)
+    out_parser.add_argument('--verbose', type=lambda x: bool(eval(x)), required=False, default=True)
+    args = out_parser.parse_args().__dict__
+
+    b_version = args.pop("verbose")
+
+    res_s_ = check_version_and_update(**args)
+
+    if b_version:
+        for k, v in res_s_.items():
+            print(f"{k}: {v}")

kevin_toolbox/env_info/test/test_check_.py

@@ -0,0 +1,52 @@
+import os
+import pytest
+from kevin_toolbox.env_info import check_validity_and_uninstall, check_version_and_update
+from kevin_toolbox.patches.for_test import check_consistency
+
+
+@pytest.mark.parametrize(
+    "package_name, expiration_timestamp, expected_s",
+    [
+        ("tqdm", 1, {"b_success_uninstalled": True}),
+        ("tqdm", 1e10, {"b_success_uninstalled": False}),
+    ]
+)
+def test_check_validity_and_uninstall(package_name, expiration_timestamp, expected_s):
+    # call by func
+    res_s = check_validity_and_uninstall(package_name=package_name, expiration_timestamp=expiration_timestamp)
+    print(res_s)
+    for k, v in expected_s.items():
+        check_consistency(res_s[k], v)
+
+    # call by script
+    os.system(
+        f'python {os.path.dirname(os.path.split(__file__)[0])}/check_validity_and_uninstall.py ' +
+        (f'--package_name {package_name} ' if package_name is not None else '') +
+        (f'--expiration_timestamp {expiration_timestamp} ' if expiration_timestamp is not None else '') +
+        f'--verbose 1'
+    )
+
+
+@pytest.mark.parametrize(
+    "package_name, cur_version, available_versions, expected_s",
+    [
+        ("tqdm", None, None, {}),
+        ("tqdm", "1.3.1", ["1.1.2", "1.1.3"], {"b_success_updated": False}),
+    ]
+)
+def test_check_version_and_update(package_name, cur_version, available_versions, expected_s):
+    # call by func
+    res_s = check_version_and_update(package_name=package_name, cur_version=cur_version,
+                                     available_versions=available_versions)
+    print(res_s)
+    for k, v in expected_s.items():
+        check_consistency(res_s[k], v)
+
+    # call by script
+    os.system(
+        f'python {os.path.dirname(os.path.split(__file__)[0])}/check_version_and_update.py ' +
+        (f'--package_name {package_name} ' if package_name is not None else '') +
+        (f'--cur_version {cur_version} ' if cur_version is not None else '') +
+        (f'--available_versions {available_versions} ' if available_versions is not None else '') +
+        f'--verbose 1'
+    )

kevin_toolbox/env_info/test/test_variable_.py

@@ -0,0 +1,50 @@
+import os
+import pytest
+from kevin_toolbox.env_info.variable_ import Env_Vars_Parser
+from kevin_toolbox.patches.for_test import check_consistency
+
+
+@pytest.mark.parametrize(
+    "input_text, expected",
+    [
+        # 验证 ${<cfg_name>:<var_name>} 的情况
+        (
+                "666/123${SYS:HOME}/afasf/${/xxx.../xxx.json:111:222}336",
+                ["666/123", ("SYS", [':'], ['HOME']), "/afasf/", ("/xxx.../xxx.json", [':', ':'], ['111', '222']),
+                 "336"]
+        ),
+        # 验证 ${<cfg_name>} 和 ${:<var_name>} 的混合情况
+        (
+                "start${CFG}middle${:VAR}end",
+                ["start", ("CFG", [], []), "middle", ('', [':'], ['VAR']), "end"]
+        ),
+        (
+                "${:VAR}",
+                [('', [':'], ['VAR'])]
+        ),
+        (
+                "${CFG}",
+                [("CFG", [], [])]
+        ),
+        (
+                "{:VAR}",
+                ["{:VAR}"]
+        ),
+    ]
+)
+def test_split_string_in_env_vars_parser(input_text, expected):
+    result = Env_Vars_Parser.split_string(input_text)
+    check_consistency(result, expected)
+
+
+def test_env_vars_parser_0():
+    env_cfg_file = os.path.expanduser("~/.kvt_cfg/.temp.json")
+    from kevin_toolbox.data_flow.file import json_
+    json_.write(content={"dataset_dir": ["~/data", "~/dataset"], "version": "001"}, file_path=env_cfg_file)
+    input_text = "/root/${KVT_TEMP:dataset_dir@1}/${KVT_TEMP:version}/${HOME}/${SYS:HOME}"
+    expected = "/".join(["/root/~/dataset/001", ] + [os.path.expanduser("~")] * 2)
+
+    #
+    parser = Env_Vars_Parser()
+    result = parser(input_text)
+    check_consistency(expected, result)

kevin_toolbox/env_info/variable_/__init__.py

@@ -0,0 +1,2 @@
+from .env_vars_parser import Env_Vars_Parser
+env_vars_parser = Env_Vars_Parser()

kevin_toolbox/env_info/variable_/env_vars_parser.py

@@ -0,0 +1,88 @@
+import os
+import re
+from kevin_toolbox.data_flow.file import json_
+
+
+class Env_Vars_Parser:
+    """
+        解释并替换字符串中${}形式指定的环境变量
+            支持以下几种方式：
+            - "${HOME}"                 家目录
+            - "${SYS:<var_name>}"       其他系统环境变量
+                                            在 linux 系统可以通过 env 命令来打印当前的环境变量，比如家目录也可以使用 ${SYS:HOME} 来表示
+            - "${KVT_XXX<ndl_name>}"   读取配置文件 ~/.kvt_cfg/.xxx.json 中的变量（xxx将被自动转为小写）
+                                            配置文件要求是 ndl 结构，比如当配置文件 ~/.kvt_cfg/.ndl.json 中保存的值为：
+                                                {"dataset_dir":["~/data", "~/dataset"], ...}
+                                            时，如果要指定使用 dataset_dir 下第二个路径，那么可以使用 ${KVT_NDL:dataset_dir@1} 来表示
+            - "${/xxx.../xxx.json<ndl_name>}"  读取指定路径下的配置文件 /xxx.../xxx.json 中的变量
+    """
+
+    def __init__(self, home_dir=None):
+        self.cfg_s = dict(
+            SYS=dict(os.environ),
+            HOME=home_dir if home_dir is not None else os.path.expanduser("~")
+        )
+
+    def __call__(self, *args, **kwargs):
+        return self.parse(*args, **kwargs)
+
+    def parse(self, text):
+        """
+            解释并替换
+        """
+        temp_ls = []
+        for it in self.split_string(text=text):
+            if isinstance(it, str):
+                temp_ls.append(it)
+                continue
+            root_node, method_ls, node_ls = it
+            if root_node not in self.cfg_s:
+                try:
+                    if root_node.startswith("KVT_"):
+                        t0, t1 = root_node.lower().split("_", 1)
+                        root_node = os.path.expanduser(f'~/.{t0}_cfg/.{t1}.json')
+                    assert os.path.isfile(root_node), f'file not exist: {root_node}'
+                    cfg = json_.read(file_path=root_node, b_use_suggested_converter=True)
+                    self.cfg_s[root_node] = cfg
+                except Exception as e:
+                    raise ValueError(f"invalid cfg_name: {root_node}, because: {e}")
+            cfg = self.cfg_s.get(root_node, None)
+            if cfg is None:
+                raise ValueError(f"invalid cfg_name: {root_node}")
+            #
+            from kevin_toolbox.nested_dict_list import get_value
+            temp_ls.append(get_value(var=cfg, name=it))
+
+        return "".join([f'{i}' for i in temp_ls])
+
+    @staticmethod
+    def split_string(text):
+        """
+            将字符串中 ${<cfg_name>} 部分的内容分割出来
+                比如对于 "666/123${SYS:HOME}/afasf/${/xxx.../xxx.json:111:222}336"
+                应该分割为 ["666/123", ("SYS:HOME", ), "/afasf/", ("/xxx.../xxx.json:111:222", ), "336"]
+                然后再对其中 tuple 部分使用 ndl.name_handler.parse_name 进行解释
+        """
+        from kevin_toolbox.nested_dict_list.name_handler import parse_name
+        pattern = r'\$\{([^}]+)\}'
+        matches = re.finditer(pattern, text)
+
+        result = []
+        last_end = 0
+
+        for match in matches:
+            start = match.start()
+            if start > last_end:
+                result.append(text[last_end:start])
+            result.append(parse_name(name=match.group(1)))
+            last_end = match.end()
+
+        if last_end < len(text):
+            result.append(text[last_end:])
+
+        return result
+
+
+if __name__ == '__main__':
+    env_vars_parser = Env_Vars_Parser()
+    print(env_vars_parser.split_string("666/123${:VAR}/afasf/${/xxx.../xxx.json:111:222}336"))

kevin_toolbox/nested_dict_list/get_value.py

@@ -7,7 +7,7 @@ def get_value(var, name, b_pop=False, **kwargs):
 
         参数：
             var:            任意支持索引取值的变量
-            name:           <str> 名字
+            name:           <str/parsed_name> 名字
                                 名字 name 的具体介绍参见函数 name_handler.parse_name()
                                 假设 var=dict(acc=[0.66,0.78,0.99])，如果你想读取 var["acc"][1] => 0.78，那么可以将 name 写成：
                                     ":acc@1" 或者 "|acc|1" 等。
@@ -19,7 +19,11 @@ def get_value(var, name, b_pop=False, **kwargs):
                                 - 不设置（默认）。当取值失败时将报错。
                                 - 设置为任意值。取值失败时将返回该值。
     """
-    _, method_ls, node_ls = parse_name(name=name, b_de_escape_node=True)
+    if isinstance(name, (tuple, list,)):
+        assert len(name) == 3, f'invalid parsed name {name}'
+        _, method_ls, node_ls = name
+    else:
+        _, method_ls, node_ls = parse_name(name=name, b_de_escape_node=True)
 
     try:
         pre, cur = None, var
@@ -46,3 +50,9 @@ def get_value(var, name, b_pop=False, **kwargs):
             raise IndexError(f'invalid name {name}')
 
     return cur
+
+
+if __name__ == "__main__":
+    var_ = dict(acc=[0.66, 0.78, 0.99])
+    print(get_value(var_, ''))
+    print(get_value(var_, ['', [], []]))

kevin_toolbox/nested_dict_list/serializer/read.py

@@ -1,9 +1,9 @@
 import os
-import time
+import tempfile
 from kevin_toolbox.patches import for_os
 from kevin_toolbox.data_flow.file import json_
 import kevin_toolbox.nested_dict_list as ndl
-import tempfile
+from kevin_toolbox.env_info.variable_ import env_vars_parser
 
 
 def read(input_path, **kwargs):
@@ -13,8 +13,6 @@ def read(input_path, **kwargs):
         参数：
             input_path:             <path> 文件夹或者 .tar 文件，具体结构参考 write()
     """
-    from kevin_toolbox.nested_dict_list.serializer.variable import SERIALIZER_BACKEND
-
     assert os.path.exists(input_path)
 
     with tempfile.TemporaryDirectory(dir=os.path.dirname(input_path)) as temp_dir:
@@ -63,7 +61,10 @@ def _read_unpacked_ndl(input_path, **kwargs):
     for name in processed_nodes:
         value = ndl.get_value(var=var, name=name)
         if isinstance(value, (dict,)) and "backend" in value and "name" in value:
-            bk = SERIALIZER_BACKEND.get(name=value.pop("backend"))(folder=os.path.join(input_path, "nodes"))
+            nodes_dir = env_vars_parser(value.pop("nodes_dir")) if "nodes_dir" in value else os.path.join(input_path,
+                                                                                                          "nodes")
+            assert os.path.exists(nodes_dir), f"nodes_dir {nodes_dir} does not exist"
+            bk = SERIALIZER_BACKEND.get(name=value.pop("backend"))(folder=nodes_dir)
             ndl.set_value(var=var, name=name, value=bk.read(**value))
 
     #

kevin_toolbox/nested_dict_list/serializer/write.py

@@ -7,6 +7,7 @@ from kevin_toolbox.data_flow.file import json_
 from kevin_toolbox.patches import for_os
 import kevin_toolbox.nested_dict_list as ndl
 from kevin_toolbox.nested_dict_list.traverse import Traversal_Mode
+from kevin_toolbox.env_info.variable_ import env_vars_parser
 from .enum_variable import Strictness_Level
 from .saved_node_name_builder import Saved_Node_Name_Builder
 
@@ -32,7 +33,17 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
             var:                    <nested dict list>
             settings:               <list of dict> 指定对于不同节点or部分的处理模式
                                         其结构为：
-                                            [{"match_cond": <匹配模式>, "backend": <序列化方式>, "traversal_mode": <遍历方式>}, ...]
+                                            [
+                                                {
+                                                    "match_cond": <匹配模式>,
+                                                    "backend": <序列化方式>,
+                                                    "traversal_mode": <遍历方式>,
+                                                    ("nodes_dir": <节点保存目录>,
+                                                     "saved_node_name_format": <nodes目录下节点文件/文件夹的命名方式>)
+                                                },
+                                                ...
+                                            ]
+                                        允许专门指定某个处理模式下所使用的 nodes_dir 和 saved_node_name_format，若不指定，则使用后面的默认值。
                                         <匹配模式>支持以下4种：
                                             - "<level>..."  匹配指定层的节点，比如"<level>0"表示根节点，"<level>-1"表示所有叶节点
                                             - "<node>name"  匹配指定name的节点
@@ -85,7 +96,9 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
                                             - "low" / Strictness_Level.IGNORE_FAILURE   匹配不完整，或者某些节点尝试过所有匹配到
                                                                                             的 backend 之后仍然无法写入
                                         默认是 "normal"
-            saved_node_name_format: <str> nodes/目录下节点文件/文件夹的命名方式。
+            nodes_dir:              <path> 节点内容保存目录
+                                        默认为保存在 <output_dir>/nodes 下
+            saved_node_name_format: <str> nodes目录下节点文件/文件夹的命名方式。
                                         基本结构为：  '{<part_0>}...{<part_1>}...'
                                         其中 {} 内将根据 part 指定的类型进行自动填充。目前支持以下几种选项：
                                             - "raw_name"        该节点对应位置的 name。
@@ -132,7 +145,8 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
         var = value_parser.replace_identical_with_reference(var=var, flag="same", b_reverse=False)
     if settings is None:
         settings = [{"match_cond": "<level>-1", "backend": (":skip:simple", ":numpy:npy", ":torch:tensor", ":pickle")}]
-    snn_builder = Saved_Node_Name_Builder(format_=saved_node_name_format)
+    default_snn_builder = Saved_Node_Name_Builder(format_=saved_node_name_format)
+    default_nodes_dir = os.path.join(temp_output_dir, "nodes")
 
     # 构建 processed_s
     #     为了避免重复处理节点/结构，首先构建与 var 具有相似结构的 processed_s 来记录处理处理进度。
@@ -154,13 +168,20 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
         if isinstance(setting["match_cond"], str) and setting["match_cond"].startswith("<eval>"):
             setting["match_cond"] = eval(setting["match_cond"][6:])
         assert callable(setting["match_cond"]) or isinstance(setting["match_cond"], str)
+        # nodes_dir = env_vars_parser(value.pop("nodes_dir")) if "nodes_dir" in value else os.path.join(input_path,
+        #                                                                                               "nodes")
+        # assert os.path.exists(nodes_dir), f"nodes_dir {nodes_dir} does not exist"
         # backend
         backend_name_ls = setting["backend"] if isinstance(setting["backend"], (list, tuple)) else [setting["backend"]]
+        nodes_dir = env_vars_parser(setting["nodes_dir"]) if "nodes_dir" in setting else default_nodes_dir
         for i in backend_name_ls:
             if i not in backend_s:
-                backend_s[i] = SERIALIZER_BACKEND.get(name=i)(folder=os.path.join(temp_output_dir, "nodes"))
+                backend_s[i] = SERIALIZER_BACKEND.get(name=i)(folder=nodes_dir)
         #
         t_mode = Traversal_Mode(setting.get("traversal_mode", traversal_mode))
+        # snn_builder
+        snn_builder = Saved_Node_Name_Builder(
+            format_=setting["saved_node_name_format"]) if "saved_node_name_format" in setting else default_snn_builder
         # _process and  paras
         if callable(setting["match_cond"]):
             if t_mode in (Traversal_Mode.DFS_PRE_ORDER, Traversal_Mode.BFS):
@@ -181,7 +202,7 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
             # print(processed_s)
             # print(f'backend: {i}')
             _process(backend=backend_s[i], strictness_level=strictness_level, processed_s=processed_s,
-                     snn_builder=snn_builder, **paras)
+                     snn_builder=snn_builder, b_record_nodes_dir=nodes_dir != default_nodes_dir, **paras)
             if "_hook_for_debug" in kwargs:
                 kwargs["_hook_for_debug"]["processed"].append([i, ndl.copy_(var=processed_s, b_deepcopy=True)])
 
@@ -246,13 +267,29 @@ def _judge_processed_or_not(processed_s, name):
     return b_processed
 
 
-def _process_for_level(var, processed_s, processed_s_bak, level, backend, strictness_level, snn_builder):
+def _process_for_level(var, processed_s, processed_s_bak, level, backend, strictness_level, snn_builder,
+                       b_record_nodes_dir):
     for name, _ in ndl.get_nodes(var=processed_s_bak, level=level, b_strict=True):
         _process_for_name(var=var, processed_s=processed_s, name=name, backend=backend,
-                          strictness_level=strictness_level, snn_builder=snn_builder)
+                          strictness_level=strictness_level, snn_builder=snn_builder,
+                          b_record_nodes_dir=b_record_nodes_dir)
 
 
-def _process_for_name(var, processed_s, name, backend, strictness_level, snn_builder):
+def _write_by_backend(backend, snn_builder, raw_name, value, strictness_level, b_record_nodes_dir):
+    snn_name = snn_builder(name=raw_name, value=value)
+    try:
+        res = backend.write(name=snn_name, var=value)
+    except:
+        assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
+            f'An error occurred when node {snn_name} was saved using the first matched backend {backend}'
+        return False, None  # b_success, res
+    if b_record_nodes_dir and isinstance(res, (dict,)) and res is not value:
+        # 记录节点位置
+        res["nodes_dir"] = backend.paras["folder"]
+    return True, res
+
+
+def _process_for_name(var, processed_s, name, backend, strictness_level, snn_builder, b_record_nodes_dir):
     if _judge_processed_or_not(processed_s=processed_s, name=name) is True:
         # has been processed
         return
@@ -262,18 +299,15 @@ def _process_for_name(var, processed_s, name, backend, strictness_level, snn_bui
         return
 
     # write by backend
-    snn_name = snn_builder(name=name, value=value)
-    try:
-        res = backend.write(name=snn_name, var=value)
-    except:
-        assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
-            f'An error occurred when node {name} was saved using the first matched backend {backend}'
+    b_success, res = _write_by_backend(backend, snn_builder, name, value, strictness_level, b_record_nodes_dir)
+    if not b_success:
         return
     ndl.set_value(var=processed_s, name=name, value=True, b_force=False)
     ndl.set_value(var=var, name=name, value=res, b_force=False)
 
 
-def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_mode, strictness_level, snn_builder):
+def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_mode, strictness_level, snn_builder,
+                              b_record_nodes_dir):
     def match_cond_(parent_type, idx, value):
         nonlocal match_cond, processed_s
 
@@ -288,12 +322,8 @@ def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_m
         nonlocal processed_s, backend, strictness_level
 
         # write by backend
-        snn_name = snn_builder(name=idx, value=value)
-        try:
-            res = backend.write(name=snn_name, var=value)
-        except:
-            assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
-                f'An error occurred when node {name} was saved using the first matched backend {backend}'
+        b_success, res = _write_by_backend(backend, snn_builder, idx, value, strictness_level, b_record_nodes_dir)
+        if not b_success:
             return value
         ndl.set_value(var=processed_s, name=idx, value=True, b_force=True)
         return res
@@ -302,7 +332,8 @@ def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_m
                  b_use_name_as_idx=True, traversal_mode=traversal_mode, b_traverse_matched_element=False)
 
 
-def _process_from_down_to_top(var, processed_s, match_cond, backend, traversal_mode, strictness_level, snn_builder):
+def _process_from_down_to_top(var, processed_s, match_cond, backend, traversal_mode, strictness_level, snn_builder,
+                              b_record_nodes_dir):
     processed_s_raw, processed_s = processed_s, ndl.copy_(var=processed_s, b_deepcopy=True)
 
     def match_cond_(parent_type, idx, value):
@@ -320,12 +351,8 @@ def _process_from_down_to_top(var, processed_s, match_cond, backend, traversal_m
         nonlocal processed_s, backend, processed_s_raw, strictness_level
 
         # write by backend
-        snn_name = snn_builder(name=idx, value=value)
-        try:
-            res = backend.write(name=snn_name, var=value)
-        except:
-            assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
-                f'An error occurred when node {name} was saved using the first matched backend {backend}'
+        b_success, res = _write_by_backend(backend, snn_builder, idx, value, strictness_level, b_record_nodes_dir)
+        if not b_success:
             return value
         ndl.set_value(var=processed_s, name=idx, value=True, b_force=True)
         ndl.set_value(var=processed_s_raw, name=idx, value=True, b_force=True)

kevin_toolbox/nested_dict_list/set_value.py

@@ -1,5 +1,5 @@
 from kevin_toolbox.nested_dict_list import get_value
-from kevin_toolbox.nested_dict_list.name_handler import parse_name, escape_node
+from kevin_toolbox.nested_dict_list.name_handler import parse_name
 
 
 def set_value(var, name, value, b_force=False):
@@ -8,7 +8,7 @@ def set_value(var, name, value, b_force=False):
 
         参数：
             var:            任意支持索引赋值的变量
-            name:           <string> 名字
+            name:           <string/parsed_name> 名字
                                 名字 name 的具体介绍参见函数 name_handler.parse_name()
                                 假设 var=dict(acc=[0.66,0.78,0.99])，如果你想将 var["acc"][1] 设置为 100，那么可以将 name 写成：
                                     ":acc@1" 或者 "|acc|1" 等。
@@ -25,14 +25,18 @@ def set_value(var, name, value, b_force=False):
                     若 b_force 为 True 有可能不会在 var 的基础上进行改变，而是返回一个新的ndl结构，
                     因此建议使用赋值 var = ndl.set_value(var) 来避免可能的错误。
     """
-    _, method_ls, node_ls = parse_name(name=name, b_de_escape_node=False)
+    if isinstance(name, (tuple, list,)):
+        assert len(name) == 3, f'invalid parsed name {name}'
+        _, method_ls, node_ls = name
+    else:
+        _, method_ls, node_ls = parse_name(name=name, b_de_escape_node=True)
     if len(node_ls) == 0:
         return value
 
-    raw_key = escape_node(node=node_ls[-1], b_reversed=True, times=1)
+    raw_key = node_ls[-1]
 
     try:
-        item = get_value(var=var, name=name[:-1 - len(node_ls[-1])])
+        item = get_value(var=var, name=('', method_ls[:-1], node_ls[:-1]))
         if method_ls[-1] == "@":
             key = eval(raw_key)
         elif method_ls[-1] == "|":
@@ -63,6 +67,12 @@ def set_value(var, name, value, b_force=False):
                 else:
                     # 其他，比如当 key 为元组、浮点数等等时，则使用 dict 构建
                     value = {key: value}
-            var = set_value(var=var, name=name[:-1 - len(node_ls[-1])], value=value, b_force=b_force)
+            var = set_value(var=var, name=('', method_ls[:-1], node_ls[:-1]), value=value, b_force=b_force)
 
     return var
+
+
+if __name__ == "__main__":
+    var_ = []
+    set_value(var=var_, name="@2:data", value=1, b_force=True)
+    print(var_)

kevin_toolbox/patches/for_os/find_files_in_dir.py

@@ -2,7 +2,7 @@ import os
 from kevin_toolbox.patches.for_os import walk
 
 
-def find_files_in_dir(input_dir, suffix_ls, b_relative_path=True, b_ignore_case=True):
+def find_files_in_dir(input_dir, suffix_ls=None, b_relative_path=True, b_ignore_case=True):
     """
         找出目录下带有给定后缀的所有文件的生成器
             主要利用了 for_os.walk 中的过滤规则进行实现
@@ -12,19 +12,24 @@ def find_files_in_dir(input_dir, suffix_ls, b_relative_path=True, b_ignore_case=
             b_relative_path:    <bool> 是否返回相对路径
             b_ignore_case:      <bool> 是否忽略大小写
     """
-    suffix_ls = tuple(set(suffix_ls))
-    suffix_ls = tuple(map(lambda x: x.lower(), suffix_ls)) if b_ignore_case else suffix_ls
-    for root, dirs, files in walk(top=input_dir, topdown=True,
-                                  ignore_s=[{
-                                      "func": lambda _, b_is_symlink, path: b_is_symlink or not (
-                                              path.lower() if b_ignore_case else path).endswith(suffix_ls),
-                                      "scope": ["files", ]
-                                  }]):
-        for file in files:
-            file_path = os.path.join(root, file)
-            if b_relative_path:
-                file_path = os.path.relpath(file_path, start=input_dir)
-            yield file_path
-
-
-
+    if suffix_ls is not None:
+        suffix_ls = tuple(set(suffix_ls))
+        suffix_ls = tuple(map(lambda x: x.lower(), suffix_ls)) if b_ignore_case else suffix_ls
+        for root, dirs, files in walk(top=input_dir, topdown=True,
+                                      ignore_s=[{
+                                          "func": lambda _, b_is_symlink, path: b_is_symlink or not (
+                                                  path.lower() if b_ignore_case else path).endswith(suffix_ls),
+                                          "scope": ["files", ]
+                                      }]):
+            for file in files:
+                file_path = os.path.join(root, file)
+                if b_relative_path:
+                    file_path = os.path.relpath(file_path, start=input_dir)
+                yield file_path
+    else:
+        for root, dirs, files in walk(top=input_dir, topdown=True):
+            for file in files:
+                file_path = os.path.join(root, file)
+                if b_relative_path:
+                    file_path = os.path.relpath(file_path, start=input_dir)
+                yield file_path

kevin_toolbox/patches/for_os/organize/__init__.py

@@ -0,0 +1 @@
+from .group_files_by_timestamp import group_files_by_timestamp

kevin_toolbox/patches/for_os/organize/group_files_by_timestamp.py

@@ -0,0 +1,90 @@
+import os
+from collections import defaultdict
+import warnings
+import time
+from kevin_toolbox.patches.for_os import find_files_in_dir, copy, remove
+
+# 获取文件的时间,返回time.struct_time格式
+get_timestamp_method_s = dict(
+    c=lambda file_path: time.localtime(max(os.path.getctime(file_path), 0)),
+    a=lambda file_path: time.localtime(max(os.path.getatime(file_path), 0)),
+    m=lambda file_path: time.localtime(max(os.path.getmtime(file_path), 0))
+)
+
+
+def group_files_by_timestamp(input_dir, output_dir=None, suffix_ls=None, b_ignore_case=True,
+                             grouping_rule=("%Y", "%m_%d"), timestamp_type="m", b_keep_source=True, b_verbose=False):
+    """
+        将 input_dir 中的文件按照时间戳信息进行分组，输出到 output_dir 中
+
+        参数：
+            input_dir:              输入目录
+            output_dir:             输出目录
+                                        当设置为 None 时，将进行空跑，不实际复制文件到 output_dir 中
+            suffix_ls:              <list of path/None> 指定要对带有哪些后缀的文件进行处理
+                                        默认为 None 表示对所有文件都进行处理
+            b_ignore_case:          <boolean> 忽略大小写
+                                        默认为 True
+            grouping_rule:          <str/list of str>分组规则
+                                        默认为 ("%Y", "%m_%d")，此时将按照年月日进行分组。
+                                        比如时间戳为 2016-03-20 11:45:39 的文件将被保存到 <output_dir>/2016/03_20 目录下
+                                        其他可选样式：
+                                            - "%Y_%m"   精确到月
+                                            - ("%Y", "%m_%d", "%H-%M-%S")   精确到秒
+                                            依次类推
+            timestamp_type:         使用哪个维度的时间戳
+                                        有以下可选值：
+                                            - "m"       文件的修改时间
+                                            - "a"       文件的最近访问时间
+                                            - "c"       文件的创建时间
+                                        默认为 "m"。
+            b_keep_source:          <boolean> 是否保留 input_dir 中的原始文件
+    """
+    if isinstance(grouping_rule, str):
+        grouping_rule = [grouping_rule]
+    assert timestamp_type in ['m', 'a', 'c']
+    global get_timestamp_method_s
+    os.makedirs(output_dir, exist_ok=True)
+
+    get_timestamp = get_timestamp_method_s[timestamp_type]
+    res_s = defaultdict(lambda: dict(src_ls=[], dst_ls=[], b_success_ls=[]))
+    for file in find_files_in_dir(input_dir=input_dir, suffix_ls=suffix_ls, b_ignore_case=b_ignore_case,
+                                  b_relative_path=True):
+        src = os.path.join(input_dir, file)
+        timestamp = get_timestamp(src)
+        group_name = tuple(f'{time.strftime(i, timestamp)}' for i in grouping_rule)
+        out_folder = os.path.join(output_dir, *group_name)
+        dst = os.path.join(out_folder, os.path.basename(file))
+        os.makedirs(out_folder, exist_ok=True)
+        b_success = False
+        try:
+            copy(src=src, dst=dst)
+            if b_verbose:
+                print(f'{file} -> {out_folder}')
+            b_success = True
+        except:
+            warnings.warn(f'failed to copy file {file} to {out_folder}')
+        if not b_keep_source:
+            remove(path=dst, ignore_errors=True)
+        res_s[group_name]['b_success_ls'].append(b_success)
+        res_s[group_name]['src_ls'].append(file)
+        res_s[group_name]['dst_ls'].append(os.path.relpath(path=dst, start=output_dir))
+    return res_s
+
+
+if __name__ == "__main__":
+    res = group_files_by_timestamp(suffix_ls=['.jpg', '.mp4', '.png', '.jpeg', '.mov', '.cr2', ".bmp"],
+                                   grouping_rule=("%Y-%m", "%Y_%m_%d"),
+                                   # "%Y-%m-%d %H:%M:%S" 2016-03-20 11:45:39 #"%a %b"  Sat Mar
+                                   input_dir="/home/SENSETIME/xukaiming/Desktop/my_repos/python_projects/kevin_toolbox/kevin_toolbox/developing/photo_organization/test/test_data",
+                                   output_dir="/home/SENSETIME/xukaiming/Desktop/my_repos/python_projects/kevin_toolbox/kevin_toolbox/developing/photo_organization/test/test_data1",
+                                   timestamp_type="m")
+    print(res)
+
+    # timestamp_type = input("分类标准：m for modifytime \n c for createtime\n a for accesstime\n")
+    # copy_path_root = os.path.join(output_dir, \
+    #                               'deal' + str(time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())))
+    # group_by_timestamp(suffix_ls=['.MP4', '.jpg', '.mp4', '.png', '.JPG', '.MOV', '.CR2'],
+    #                    grouping_rule="%Y_%m_%d",  # "%Y-%m-%d %H:%M:%S" 2016-03-20 11:45:39 #"%a %b"  Sat Mar
+    #                    input_dir=input("please input the root path\n"),
+    #                    output_dir=input("please input the target_path\n"), timestamp_type=timestamp_type)

kevin_toolbox/utils/variable.py

@@ -0,0 +1,6 @@
+import os
+
+root_dir = os.path.split(os.path.split(os.path.split(__file__)[0])[0])[0]
+
+if __name__ == '__main__':
+    print(root_dir)

kevin_toolbox_dev-1.4.6.dist-info/METADATA

@@ -0,0 +1,76 @@
+Metadata-Version: 2.1
+Name: kevin-toolbox-dev
+Version: 1.4.6
+Summary: 一个常用的工具代码包集合
+Home-page: https://github.com/cantbeblank96/kevin_toolbox
+Download-URL: https://github.com/username/your-package/archive/refs/tags/v1.0.0.tar.gz
+Author: kevin hsu
+Author-email: xukaiming1996@163.com
+License: MIT
+Keywords: mathematics,pytorch,numpy,machine-learning,algorithm
+Platform: UNKNOWN
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: torch (>=1.2.0)
+Requires-Dist: numpy (>=1.19.0)
+Provides-Extra: plot
+Requires-Dist: matplotlib (>=3.0) ; extra == 'plot'
+Provides-Extra: rest
+Requires-Dist: pytest (>=6.2.5) ; extra == 'rest'
+Requires-Dist: line-profiler (>=3.5) ; extra == 'rest'
+
+# kevin_toolbox
+
+一个通用的工具代码包集合
+
+
+
+环境要求
+
+```shell
+numpy>=1.19
+pytorch>=1.2
+```
+
+安装方法：
+
+```shell
+pip install kevin-toolbox  --no-dependencies
+```
+
+
+
+[项目地址 Repo](https://github.com/cantbeblank96/kevin_toolbox)
+
+[使用指南 User_Guide](./notes/User_Guide.md)
+
+[免责声明 Disclaimer](./notes/Disclaimer.md)
+
+[版本更新记录](./notes/Release_Record.md)：
+
+- v 1.4.6 （2025-01-24）【new feature】
+
+  - data_flow.file
+    - modify json_.read()，支持输入路径使用 ~ 表示家目录。
+    - 【new feature】add excel，该模块用于 excel 表格处理。
+      - write_with_matrix()：将矩阵写入到 excel 文件中
+
+  - patches.for_os
+    - modify find_files_in_dir()，支持 suffix_ls 设定 None 以表示不进行任何过滤。
+    - 【new feature】add organize，该模块用于文件整理。
+      - group_files_by_timestamp()：将 input_dir 中的文件按照时间戳信息进行分组，输出到 output_dir 中。
+    - 添加了对应的测试用例。
+  - env_info
+    - 【new feature】add check_validity_and_uninstall()，检查当前机器时间是否超过 expiration_timestamp 指定的有效期，若超过则卸载 package_name 对应的库。
+    - 【new feature】add check_version_and_update()，检查当前库的版本，并尝试更新。
+    - 以上函数均系从同名脚本中抽取出来。
+  - 以上修改，均已添加了对应的测试用例。
+  - developing
+    - 【new feature】add photo_album_organization，该模块包含一系列整理相册相关的脚本。
+      -  0_group_by_timestamp.py ：按照时间戳分组
+      -  1_merge_folders.py ：将文件数量较少的目录合并
+
+

{kevin_toolbox_dev-1.4.4.dist-info → kevin_toolbox_dev-1.4.6.dist-info}/RECORD

@@ -1,8 +1,8 @@
-kevin_toolbox/__init__.py,sha256=xs_PW9P4AhMhyKcprj8m2mOBXfhKslv-B4OCvUYBmmw,410
+kevin_toolbox/__init__.py,sha256=J3exMbMRFMTDP3RxpdeO_mXW0RvxbLdqQ_D0yxWUDmE,410
 kevin_toolbox/computer_science/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kevin_toolbox/computer_science/algorithm/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 kevin_toolbox/computer_science/algorithm/cache_manager/__init__.py,sha256=p2hddkZ1HfYF9-m2Hx-o9IotwQHd4QwDCePy2ADpTDA,41
-kevin_toolbox/computer_science/algorithm/cache_manager/cache_manager.py,sha256=e_mesTHjYlSV2VjXfmrA_osJRZhGe3nyS5SX1R5rfOQ,9151
+kevin_toolbox/computer_science/algorithm/cache_manager/cache_manager.py,sha256=yQ6bqOSPcTrq4FLyuxDERz7Yv9ZWsenjb48q6KRStj4,9771
 kevin_toolbox/computer_science/algorithm/cache_manager/variable.py,sha256=j6yLezcZxf7gNdxwUxP0PPy2zauVpTr6KFMCg66m9lo,835
 kevin_toolbox/computer_science/algorithm/cache_manager/cache/__init__.py,sha256=Tt8XgrZsHP3J3f6UucGNhGAN8L7HiVjLLO1JzICR2Mc,70
 kevin_toolbox/computer_science/algorithm/cache_manager/cache/cache_base.py,sha256=Jln4Ey0sUGrzUqpkHaMP0S_z6NUw43iLnwMbmm-8msg,2229
@@ -72,9 +72,11 @@ kevin_toolbox/data_flow/core/reader/file_iterative_reader.py,sha256=l6UMYnvWwqQm
 kevin_toolbox/data_flow/core/reader/unified_reader.py,sha256=l6JxPoDUOdx2ZIPX2WLXbGU3VZtTd1AeHn5q6L8GWAI,2453
 kevin_toolbox/data_flow/core/reader/unified_reader_base.py,sha256=4gIADdV8UKpt2yD8dZjQsXFcF75nJ83ooIae3D7bw2s,11783
 kevin_toolbox/data_flow/file/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+kevin_toolbox/data_flow/file/excel/__init__.py,sha256=5m_rmklI6n6yk4rSAEW39pxzYTiAPjC4q69v-oz7Zvs,82
+kevin_toolbox/data_flow/file/excel/write_excel_with_matrix.py,sha256=zrY_l0xCBpjqxm_9MoGpEXaZ4V_UwRMRgShessJ1sxA,5121
 kevin_toolbox/data_flow/file/json_/__init__.py,sha256=VAt8COS2tO3PJRuhSc43i35fEOlArFM_YahdTmEBaHE,85
-kevin_toolbox/data_flow/file/json_/read_json.py,sha256=BhAUCcagwPsSMaMeCJyyxDW3h9SGf1Dfvb0nXi6B_T8,2084
-kevin_toolbox/data_flow/file/json_/write_json.py,sha256=mWaxePr_QzfyeCb0hAy4xTKOGX7q0eFjep0jDqOqIgw,2379
+kevin_toolbox/data_flow/file/json_/read_json.py,sha256=RyCeNONMmvVOeX_F3kSSmED_nx4opipLe8OHJzXKZvQ,2151
+kevin_toolbox/data_flow/file/json_/write_json.py,sha256=uG6UnQ9KVhL_UWndGjvLLHF_UoGtOwVn4ADi1Gb1nRU,2417
 kevin_toolbox/data_flow/file/json_/converter/__init__.py,sha256=oQMgAgzELLq_f4LIIfz5E6l_E7g4lFsXqfmnJ3tPZTY,401
 kevin_toolbox/data_flow/file/json_/converter/convert_dict_key_to_number.py,sha256=SuSZj_HCqKZutHAJ5AttABnGBRZplPGQhMxJBt2Wlgc,559
 kevin_toolbox/data_flow/file/json_/converter/convert_ndarray_to_list.py,sha256=GALpC1MFJ4aMzs0FZIfJScYznfCP-gmhPeM8sWXGSWg,391
@@ -136,11 +138,15 @@ kevin_toolbox/developing/temperate/iterator_base.py,sha256=FYMJ49ltbBcV92tUThx8Z
 kevin_toolbox/developing/temperate/my_iterator.py,sha256=dpFq_3wa4W2Y4SZfVl93T8oAEoy3gDOkQUdXB4p-cBQ,1663
 kevin_toolbox/developing/temperate/my_iterator_base.py,sha256=gLv9zdM987BHRghTfADUTwwQcMOV4WZWx-8-QvlV2uY,1214
 kevin_toolbox/developing/temperate/sequence_map_base.py,sha256=ha1EIMhn9lBF05s9niHLTuxhRslOx5faOk8UIjhhxUk,217
-kevin_toolbox/env_info/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-kevin_toolbox/env_info/check_validity_and_uninstall.py,sha256=GskNfWwj2ak2AszV_0MMs2hBZb2tmRD0POO11UiouLM,879
-kevin_toolbox/env_info/check_version_and_update.py,sha256=og9ngoO6VhnlmUIkL0IZCyGXrI8rpZWRoN4t9FYGIkw,1953
+kevin_toolbox/env_info/__init__.py,sha256=8Io5RN5RcbEoMLHY4wfMa4pJxa1w0SMaXBN4v6k5CrM,134
+kevin_toolbox/env_info/check_validity_and_uninstall.py,sha256=FOLeVKRqqiFnAQpx_AmIc8D3UcJigzIx58XDTJgv_qw,1676
+kevin_toolbox/env_info/check_version_and_update.py,sha256=GsL5tqodh7bbcXmWRT1vWy9GFRhqHJmTgEgvJg4GT5E,2739
 kevin_toolbox/env_info/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kevin_toolbox/env_info/test/test_check_.py,sha256=wiIM_UVy_ksdq3la-pbF605Lhl5iW3a-0S_B74QzPT8,2054
+kevin_toolbox/env_info/test/test_variable_.py,sha256=n9To8UNfBSNey8Xy7relXcbrf0yX8ZoZzfJctd2fHBs,1657
 kevin_toolbox/env_info/test/test_version.py,sha256=xnF7RAcLSN3gpjIbVxFUV2-lmv0w7gOhdRa4XN0z0Q0,1728
+kevin_toolbox/env_info/variable_/__init__.py,sha256=qFs8ZZVBjAfj6IuUujYxaEnmXk6HEbtN6GXaIkuQhoM,81
+kevin_toolbox/env_info/variable_/env_vars_parser.py,sha256=8k6nBCD9oFsFu_Bq6usz3BF5NHr03c7SxFgbDj9EpP8,3781
 kevin_toolbox/env_info/version/__init__.py,sha256=PSrrvrYccfcE36IkOWG5kLQlKopfenQJ-4xilCdRULY,187
 kevin_toolbox/env_info/version/compare_version.py,sha256=rAksAR1OuOE5TrfJx3h5w7w5vftpcv_oJPHWGwuX7TI,2489
 kevin_toolbox/env_info/version/parse_version.py,sha256=QhYVO9hLZ8o4wdXWg5PBr0WIu5VayR-QFKQ_KyLDLgI,860
@@ -235,9 +241,9 @@ kevin_toolbox/nested_dict_list/copy_.py,sha256=MvzNRKm8htYpMe7Td1ao2-ZoaYVC_iNTG
 kevin_toolbox/nested_dict_list/count_leaf_node_nums.py,sha256=l67u47EvO1inoGinUqH6RZ7cHXwN0VcBQPUvSheqAvA,614
 kevin_toolbox/nested_dict_list/get_hash.py,sha256=Ygadnn5dnvIeE-9t39p2EwNKNRLzomL37ZsRD5daXxo,1286
 kevin_toolbox/nested_dict_list/get_nodes.py,sha256=doEcLPYOig4gGloGXEPlroXFcRWe5ovuH0RozsxYZ0U,3748
-kevin_toolbox/nested_dict_list/get_value.py,sha256=isvUhqSQyUNHBXgNuZX6_o2c84UV_SpjNjAYm2M3gd4,2083
+kevin_toolbox/nested_dict_list/get_value.py,sha256=IiAqQCphyv-pAZWuQRWm0anEwxYQOkC9CttY5ZlUbSs,2389
 kevin_toolbox/nested_dict_list/set_default.py,sha256=laSgGP1CbApNgFB9HZGCtxCG9fe7u1C-YOx9ZCoHJms,3460
-kevin_toolbox/nested_dict_list/set_value.py,sha256=pmSWzC0y0jBxk7yritsjKU2Q-PPMar0X3A9bF6uWvoQ,3470
+kevin_toolbox/nested_dict_list/set_value.py,sha256=AQ4foDtKo4JxyR---of-VSxjhRWfqkv6TrnQ4EoRo3M,3711
 kevin_toolbox/nested_dict_list/traverse.py,sha256=5_EirnYVy34JLfXxuTvb-mMjDeO1veyfLOcaVYcuGF8,6846
 kevin_toolbox/nested_dict_list/name_handler/__init__.py,sha256=P_pWq78oN6NdvWg2h6AduW_sUqbeaaVyoWWbW9kbgmU,107
 kevin_toolbox/nested_dict_list/name_handler/build_name.py,sha256=VPWyjE8i8l-4Zm4tkD06Ie4J2NCsmI32ecOxZQqqmok,989
@@ -245,10 +251,10 @@ kevin_toolbox/nested_dict_list/name_handler/escape_node.py,sha256=niT9MxmsyrSZYh
 kevin_toolbox/nested_dict_list/name_handler/parse_name.py,sha256=vUlAXPocpVSxtb3EnRi7U5K40Tz9plFG-_sbwLfYiy4,2280
 kevin_toolbox/nested_dict_list/serializer/__init__.py,sha256=79dd9l-mNz0bycFKjNm7YsfWPR-JsVx9NoG_Ofqy-HQ,153
 kevin_toolbox/nested_dict_list/serializer/enum_variable.py,sha256=RWPydtXI4adOJYGo_k5CWHSL0Odzj_bsahb24p1ranY,847
-kevin_toolbox/nested_dict_list/serializer/read.py,sha256=BjsEWYoyvEHgRKKVKw0suf1ukug2tAFLMCAmEnndqgg,2945
+kevin_toolbox/nested_dict_list/serializer/read.py,sha256=HaEJJw7hBVNmsIs348kaIyatHP77Kr-JHEwYqRwLrso,3202
 kevin_toolbox/nested_dict_list/serializer/saved_node_name_builder.py,sha256=qsD-rmDmVaKZP4owN3Wm3QY2Ksi71XlYETqw4VmIsSU,1011
 kevin_toolbox/nested_dict_list/serializer/variable.py,sha256=ZywG6obipRBCGY1cY42gdvsuWk8GLZXr6eCYcW7ZJ9c,392
-kevin_toolbox/nested_dict_list/serializer/write.py,sha256=MCGdAMxZJuQc5OD_wD50zs2AKurtA5-d3l0fOi_ikHw,22169
+kevin_toolbox/nested_dict_list/serializer/write.py,sha256=ZUYJlBXQbCkMW2UN3d29obskGGbTA-gm3dmuLLltxLI,24101
 kevin_toolbox/nested_dict_list/serializer/backends/__init__.py,sha256=8g7y-L3cmctxao616dVkGiot00FJzKNmNl_69V2bSmE,39
 kevin_toolbox/nested_dict_list/serializer/backends/_json_.py,sha256=oJXIc28yjxsD9ZJuw120pVHTVsTzCdaXEhVUSQeydq4,2145
 kevin_toolbox/nested_dict_list/serializer/backends/_ndl.py,sha256=3YkAq_Bqzehnw0kGxqxwtF6uUz0EV37tLI-1ROHjixY,1794
@@ -314,11 +320,13 @@ kevin_toolbox/patches/for_optuna/serialize/for_trial/dump.py,sha256=FT-Z1rzCNUYz
 kevin_toolbox/patches/for_optuna/serialize/for_trial/load.py,sha256=2fpeeHPKA9bT7CjQ6DVRXOarF6IAA6_f2pXbB1rXcvE,796
 kevin_toolbox/patches/for_os/__init__.py,sha256=OhGxHkzI-oBek6M07kkrRgTQfY42l1Y2nOIR95JYD-g,219
 kevin_toolbox/patches/for_os/copy.py,sha256=PWFLu15DpIA4JZxatvphHANNn2H3nC93qTbLLxDl5NU,1509
-kevin_toolbox/patches/for_os/find_files_in_dir.py,sha256=KtjUJTMTNivAoWXGYgVkuaXukU1ktEAA-u8peusznmU,1266
+kevin_toolbox/patches/for_os/find_files_in_dir.py,sha256=89bV4g-1-GBHwRr1K1KjJk7UvYM4LCvQmPVCVTZkX-g,1656
 kevin_toolbox/patches/for_os/pack.py,sha256=A6u4g3dfwXPtlU4gBcThNrktz6dO4DVi2wmQXytqfDI,656
 kevin_toolbox/patches/for_os/remove.py,sha256=PmwqzVJbyfdqwXn_T1F9d4Oar8CwQ2YFaqcZQkfnrnI,750
 kevin_toolbox/patches/for_os/unpack.py,sha256=d_fO7nPmExy1VIg7ADIMayCzjBBeFxvJLhIsulIRlzI,1047
 kevin_toolbox/patches/for_os/walk.py,sha256=LrtEeRUDwzZgu_zGZ-kPsFJd4D-8R8ECHW6WNdEsDSw,8376
+kevin_toolbox/patches/for_os/organize/__init__.py,sha256=GGSUvdHzYK9prCwz4XCm1lp4g4A_o9tuU_l3bNi5MMs,63
+kevin_toolbox/patches/for_os/organize/group_files_by_timestamp.py,sha256=CpGKMJ7GIywTgumo3TyM_SXgaeh0Gtl9xIkm6-YKJE4,5222
 kevin_toolbox/patches/for_os/path/__init__.py,sha256=M4XaYawTDj-SjwZ_bWS5D38lqzPujxvAtVEvzRLDhtU,108
 kevin_toolbox/patches/for_os/path/find_illegal_chars.py,sha256=QmqzeaeBY50of28qtvfEmnDW9xeVIfCXi6QVzLzngks,1416
 kevin_toolbox/patches/for_os/path/replace_illegal_chars.py,sha256=OhxndHEJ8xK-ip-sWYQehTNSho8eNFeKj2iwPHR02os,1672
@@ -344,7 +352,9 @@ kevin_toolbox/patches/for_torch/math/get_y_at_x.py,sha256=bfoVcasZ_tMdhR_1Me0Jli
 kevin_toolbox/patches/for_torch/math/my_around.py,sha256=ptpU3ids50gwf663EpHbw7raj9tNrDGBFZ5t_uMNH14,1378
 kevin_toolbox/patches/for_torch/nn/__init__.py,sha256=aJs3RMqRzQmd8KKDmQW9FxwCqS5yfPqEdg-m0PwlQro,39
 kevin_toolbox/patches/for_torch/nn/lambda_layer.py,sha256=KUuLiX_Dr4bvRmpAaCW5QTDWDcnMPRnw0jg4NNXTFhM,223
-kevin_toolbox_dev-1.4.4.dist-info/METADATA,sha256=klDj2qFLTgPXKcBByDV1Wb5OJbJuj5H995C_3LbrlSk,1504
-kevin_toolbox_dev-1.4.4.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-kevin_toolbox_dev-1.4.4.dist-info/top_level.txt,sha256=S5TeRGF-PwlhsaUEPTI-f2vWrpLmh3axpyI6v-Fi75o,14
-kevin_toolbox_dev-1.4.4.dist-info/RECORD,,
+kevin_toolbox/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kevin_toolbox/utils/variable.py,sha256=PxUmp9w4CKKcKHjgdVNF_Iaw5gwPPOd4aY_Oe5F9U1M,133
+kevin_toolbox_dev-1.4.6.dist-info/METADATA,sha256=G-yjDl5MP56GAhXAoPnGcqJPGqpuUiSt4X4B90dsP7c,2578
+kevin_toolbox_dev-1.4.6.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+kevin_toolbox_dev-1.4.6.dist-info/top_level.txt,sha256=S5TeRGF-PwlhsaUEPTI-f2vWrpLmh3axpyI6v-Fi75o,14
+kevin_toolbox_dev-1.4.6.dist-info/RECORD,,

kevin_toolbox_dev-1.4.4.dist-info/METADATA

@@ -1,58 +0,0 @@
-Metadata-Version: 2.1
-Name: kevin-toolbox-dev
-Version: 1.4.4
-Summary: 一个常用的工具代码包集合
-Home-page: https://github.com/cantbeblank96/kevin_toolbox
-Download-URL: https://github.com/username/your-package/archive/refs/tags/v1.0.0.tar.gz
-Author: kevin hsu
-Author-email: xukaiming1996@163.com
-License: MIT
-Keywords: mathematics,pytorch,numpy,machine-learning,algorithm
-Platform: UNKNOWN
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.6
-Description-Content-Type: text/markdown
-Requires-Dist: torch (>=1.2.0)
-Requires-Dist: numpy (>=1.19.0)
-Provides-Extra: plot
-Requires-Dist: matplotlib (>=3.0) ; extra == 'plot'
-Provides-Extra: rest
-Requires-Dist: pytest (>=6.2.5) ; extra == 'rest'
-Requires-Dist: line-profiler (>=3.5) ; extra == 'rest'
-
-# kevin_toolbox
-
-一个通用的工具代码包集合
-
-
-
-环境要求
-
-```shell
-numpy>=1.19
-pytorch>=1.2
-```
-
-安装方法：
-
-```shell
-pip install kevin-toolbox  --no-dependencies
-```
-
-
-
-[项目地址 Repo](https://github.com/cantbeblank96/kevin_toolbox)
-
-[使用指南 User_Guide](./notes/User_Guide.md)
-
-[免责声明 Disclaimer](./notes/Disclaimer.md)
-
-[版本更新记录](./notes/Release_Record.md)：
-
-- v 1.4.4 （2024-12-15）【bug fix】
-  - nested_dict_list.serializer
-    - fix bug in write() line 229，将判断目标是否存在时使用的 os.path.isfile 改为 os.path.exists 以支持目标是文件夹的情况。
-
-