kevin-toolbox-dev 1.3.3__py3-none-any.whl → 1.3.5__py3-none-any.whl

kevin_toolbox/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.3.3"
+__version__ = "1.3.5"
 
 
 import os
@@ -12,5 +12,5 @@ os.system(
 os.system(
     f'python {os.path.split(__file__)[0]}/env_info/check_validity_and_uninstall.py '
     f'--package_name kevin-toolbox-dev '
-    f'--expiration_timestamp 1727520805 --verbose 0'
+    f'--expiration_timestamp 1731518184 --verbose 0'
 )

kevin_toolbox/data_flow/file/json_/read_json.py

@@ -1,15 +1,17 @@
 import os
 import json
+from io import BytesIO, StringIO
 from kevin_toolbox.data_flow.file.json_.converter import integrate, unescape_tuple_and_set, unescape_non_str_dict_key
 from kevin_toolbox.nested_dict_list import traverse
 
 
-def read_json(file_path, converters=None, b_use_suggested_converter=False):
+def read_json(file_path=None, file_obj=None, converters=None, b_use_suggested_converter=False):
     """
         读取 json file
 
         参数：
             file_path
+            file_obj
             converters:                 <list of converters> 对读取内容中每个节点的处理方式
                                             转换器 converter 应该是一个形如 def(x): ... ; return x 的函数，具体可以参考
                                             json_.converter 中已实现的转换器
@@ -19,13 +21,17 @@ def read_json(file_path, converters=None, b_use_suggested_converter=False):
                                             默认为 False。
                     注意：当 converters 非 None，此参数失效，以 converters 中的具体设置为准
     """
-    assert os.path.isfile(file_path), f'file {file_path} not found'
+    assert file_path is not None or file_obj is not None
+    if file_path is not None:
+        assert os.path.isfile(file_path), f'file {file_path} not found'
+        file_obj = open(file_path, 'r')
+    elif isinstance(file_obj, (BytesIO,)):
+        file_obj = StringIO(file_obj.read().decode('utf-8'))
+    content = json.load(file_obj)
+
     if converters is None and b_use_suggested_converter:
         converters = [unescape_tuple_and_set, unescape_non_str_dict_key]
 
-    with open(file_path, 'r') as f:
-        content = json.load(f)
-
     if converters is not None:
         converter = integrate(converters)
         content = traverse(var=[content],

kevin_toolbox/data_flow/file/kevin_notation/kevin_notation_writer.py

@@ -5,7 +5,7 @@ import warnings
 from kevin_toolbox.data_flow.file.kevin_notation.converter import Converter, CONVERTER_FOR_WRITER
 from kevin_toolbox.data_flow.file import kevin_notation
 
-np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)
+warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)
 
 
 class Kevin_Notation_Writer:

kevin_toolbox/nested_dict_list/serializer/__init__.py

@@ -1,3 +1,4 @@
 from .write import write
 from .read import read
 from .enum_variable import Strictness_Level
+from .saved_node_name_builder import Saved_Node_Name_Builder

kevin_toolbox/nested_dict_list/serializer/backends/_ndl.py

@@ -27,7 +27,10 @@ class NDL(Backend_Base):
         """
             是否可以写
         """
-        return True
+        if "name" in kwargs:
+            return not os.path.exists(os.path.join(self.paras["folder"], f'{kwargs["name"]}'))
+        else:
+            return True
 
     def readable(self, name, **kwargs):
         """

kevin_toolbox/nested_dict_list/serializer/read.py

@@ -3,6 +3,7 @@ import time
 from kevin_toolbox.patches import for_os
 from kevin_toolbox.data_flow.file import json_
 import kevin_toolbox.nested_dict_list as ndl
+import tempfile
 
 
 def read(input_path, **kwargs):
@@ -16,26 +17,35 @@ def read(input_path, **kwargs):
 
     assert os.path.exists(input_path)
 
-    # 解压
-    temp_dir = None
-    if os.path.isfile(input_path) and input_path.endswith(".tar"):
-        while True:
-            temp_dir = os.path.join(os.path.dirname(input_path), f'temp{time.time()}')
-            if not os.path.isdir(temp_dir):
-                os.makedirs(temp_dir)
-                break
-        for_os.unpack(source=input_path, target=temp_dir)
-        input_path = os.path.join(temp_dir, os.listdir(temp_dir)[0])
+    with tempfile.TemporaryDirectory(dir=os.path.dirname(input_path)) as temp_dir:
+        if os.path.isfile(input_path) and input_path.endswith(".tar"): # 解压
+            for_os.unpack(source=input_path, target=temp_dir)
+            input_path = os.path.join(temp_dir, os.listdir(temp_dir)[0])
+        var = _read_unpacked_ndl(input_path, **kwargs)
+
+    return var
+
+
+def _read_unpacked_ndl(input_path, **kwargs):
+    """
+        读取 input_path 中保存的嵌套字典列表
+    """
+    from kevin_toolbox.nested_dict_list.serializer.variable import SERIALIZER_BACKEND
+
+    assert os.path.exists(input_path)
 
     # 读取 var
     var = json_.read(file_path=os.path.join(input_path, "var.json"), b_use_suggested_converter=True)
 
+    # 读取 record
+    record_s = dict()
+    if os.path.isfile(os.path.join(input_path, "record.json")):
+        record_s = json_.read(file_path=os.path.join(input_path, "record.json"), b_use_suggested_converter=True)
+
     # 读取被处理的节点
     processed_nodes = []
-    if os.path.isfile(os.path.join(input_path, "record.json")):
-        for name, value in ndl.get_nodes(
-                var=json_.read(file_path=os.path.join(input_path, "record.json"),
-                               b_use_suggested_converter=True)["processed"], level=-1, b_strict=True):
+    if record_s:
+        for name, value in ndl.get_nodes(var=record_s["processed"], level=-1, b_strict=True):
             if value:
                 processed_nodes.append(name)
     else:
@@ -57,8 +67,9 @@ def read(input_path, **kwargs):
             ndl.set_value(var=var, name=name, value=bk.read(**value))
 
     #
-    if temp_dir is not None:
-        for_os.remove(path=temp_dir, ignore_errors=True)
+    if record_s.get("b_keep_identical_relations", False):
+        from kevin_toolbox.nested_dict_list import value_parser
+        var = value_parser.replace_identical_with_reference(var=var, flag="same", b_reverse=True)
 
     return var
 

kevin_toolbox/nested_dict_list/serializer/saved_node_name_builder.py

@@ -0,0 +1,31 @@
+class Saved_Node_Name_Builder:
+    """
+        生成保存节点内容时的文件夹/文件名称
+    """
+
+    def __init__(self, format_):
+        try:
+            temp = format_.format(**{k: k + "_" * 3 for k in {"raw_name", "id", "hash_name", "count"}})
+            assert len(temp) > len(format_)
+        except:
+            raise ValueError(f'invalid saved_node_name_format {format_}')
+
+        self.format_ = format_
+        self.count = 0
+
+    def __call__(self, name, value):
+        from kevin_toolbox.nested_dict_list import get_hash
+
+        res = self.format_.format(
+            **{"raw_name": name, "id": id(value), "hash_name": get_hash(name, length=12), "count": self.count})
+        self.count += 1
+        return res
+
+
+if __name__ == '__main__':
+    bd = Saved_Node_Name_Builder(format_="{raw_name}_{count}_{hash_name}_{id}")
+    print(bd(":a@0", 1))
+    print(bd(":b:c", []))
+
+    # bd = Saved_Node_Name_Builder(format_="")
+    # bd = Saved_Node_Name_Builder(format_="{raw_name2}_{count}")

kevin_toolbox/nested_dict_list/serializer/write.py

@@ -1,22 +1,25 @@
 import os
 import time
 import warnings
+import tempfile
 import kevin_toolbox
 from kevin_toolbox.data_flow.file import json_
 from kevin_toolbox.patches import for_os
 import kevin_toolbox.nested_dict_list as ndl
 from kevin_toolbox.nested_dict_list.traverse import Traversal_Mode
 from .enum_variable import Strictness_Level
+from .saved_node_name_builder import Saved_Node_Name_Builder
 
 
 def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_pack_into_tar=True,
-          strictness_level=Strictness_Level.COMPATIBLE, **kwargs):
+          strictness_level=Strictness_Level.COMPATIBLE, saved_node_name_format='{count}_{hash_name}',
+          b_keep_identical_relations=False, **kwargs):
     """
         将输入的嵌套字典列表 var 的结构和节点值保存到文件中
             遍历 var，匹配并使用 settings 中设置的保存方式来对各部分结构/节点进行序列化
             将会生成一个文件夹或者 .tar 文件，其中包含：
                 - var.json：     用于保存结构、简单节点值、复杂节点值/结构的序列化方式
-                - nodes/目录：    其中包含一系列 <name>.<suffix> 文件或者 <name> 文件夹，其中包含复杂节点值/结构的序列化结果
+                - nodes/：       该目录中包含一系列 <name>.<suffix> 文件或者 <name> 文件夹，其中包含复杂节点值/结构的序列化结果
                 - record.json：   其中记录了：
                                         {
                                             "processed": ... # 对哪些节点/部分进行了处理
@@ -82,25 +85,56 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
                                             - "low" / Strictness_Level.IGNORE_FAILURE   匹配不完整，或者某些节点尝试过所有匹配到
                                                                                             的 backend 之后仍然无法写入
                                         默认是 "normal"
+            saved_node_name_format: <str> nodes/目录下节点文件/文件夹的命名方式。
+                                        基本结构为：  '{<part_0>}...{<part_1>}...'
+                                        其中 {} 内将根据 part 指定的类型进行自动填充。目前支持以下几种选项：
+                                            - "raw_name"        该节点对应位置的 name。
+                                            - "id"              该节点在当前内存中的 id。
+                                            - "hash_name"       该节点位置 name 的 hash 值。
+                                            - "count"           累加计算，表示是保存的第几个节点。
+                                        ！！注意：
+                                            "raw_name" 该选项在 v1.3.3 前被使用，但是由于其可能含有 : 和 / 等特殊符号，当以其作为文件夹名时，
+                                                可能会引发错误。因此对于 windows 用户，禁止使用该选项，对于 mac 和 linux 用户，同样也不建议使用该选项。
+                                            "id" 虽然具有唯一性，但是其值对于每次运行是随机的。
+                                            "hash_name" 有极低的可能会发生 hash 碰撞。
+                                        综合而言：
+                                            建议使用 "hash_name" 和 "count" 的组合。
+                                        默认值为：
+                                            '{count}_{hash_name}'
+            b_keep_identical_relations: <boolean> 是否保留不同节点之间的 id 相等关系。
+                                        具体而言，就是使用 value_parser.replace_identical_with_reference() 函数将具有相同 id 的多个节点，
+                                            替换为单个节点和其多个引用的形式。
+                                        对于 ndl 中存在大量具有相同 id 的重复节点的情况，使用该操作可以额外达到压缩的效果。
+                                        默认为 False
     """
     from kevin_toolbox.nested_dict_list.serializer.variable import SERIALIZER_BACKEND
 
-    #
+    # 检查参数
     traversal_mode = Traversal_Mode(traversal_mode)
     strictness_level = Strictness_Level(strictness_level)
-    os.makedirs(output_dir, exist_ok=True)
+    #
+    assert not os.path.exists(output_dir + ".tar" if b_pack_into_tar else output_dir), f'target already exists'
+    os.makedirs(os.path.dirname(output_dir), exist_ok=True)
+    temp_dir = tempfile.TemporaryDirectory(dir=os.path.dirname(output_dir))
+    temp_output_dir = os.path.join(temp_dir.name, os.path.basename(output_dir))
+    os.makedirs(temp_output_dir, exist_ok=True)
+    #
     var = ndl.copy_(var=var, b_deepcopy=False)
+    if b_keep_identical_relations:
+        from kevin_toolbox.nested_dict_list import value_parser
+        var = value_parser.replace_identical_with_reference(var=var, flag="same", b_reverse=False)
     if settings is None:
         settings = [{"match_cond": "<level>-1", "backend": (":skip:simple", ":numpy:npy", ":torch:tensor", ":pickle")}]
+    snn_builder = Saved_Node_Name_Builder(format_=saved_node_name_format)
 
     # 构建 processed_s
     #     为了避免重复处理节点/结构，首先构建与 var 具有相似结构的 processed_s 来记录处理处理进度。
     #     对于 processed_s，其节点值为 True 时表示该节点已经被处理，当节点值为 False 或者 list/dict 类型时表示该节点或者节点下面的结构中仍然
     #       存在未处理的部分。
     #     对于中间节点，只有其下所有叶节点都未处理时才会被匹配。
-    processed_s = dict()
+    processed_s = ndl.copy_(var=var, b_deepcopy=False, b_keep_internal_references=False)
     for n, _ in ndl.get_nodes(var=var, level=-1, b_strict=True):
-        ndl.set_value(var=processed_s, name=n, value=False, b_force=True)
+        ndl.set_value(var=processed_s, name=n, value=False, b_force=False)
     # processed_s_bak 用于记录 var 的原始结构
     processed_s_bak = ndl.copy_(var=processed_s, b_deepcopy=True)
     if "_hook_for_debug" in kwargs:
@@ -117,7 +151,7 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
         backend_name_ls = setting["backend"] if isinstance(setting["backend"], (list, tuple)) else [setting["backend"]]
         for i in backend_name_ls:
             if i not in backend_s:
-                backend_s[i] = SERIALIZER_BACKEND.get(name=i)(folder=os.path.join(output_dir, "nodes"))
+                backend_s[i] = SERIALIZER_BACKEND.get(name=i)(folder=os.path.join(temp_output_dir, "nodes"))
         #
         t_mode = Traversal_Mode(setting.get("traversal_mode", traversal_mode))
         # _process and  paras
@@ -126,27 +160,21 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
                 _process = _process_from_top_to_down
             else:
                 _process = _process_from_down_to_top
-            paras = dict(
-                var=var, processed_s=processed_s, match_cond=setting["match_cond"],
-                traversal_mode=t_mode, strictness_level=strictness_level
-            )
+            paras = dict(var=var, match_cond=setting["match_cond"], traversal_mode=t_mode)
         elif setting["match_cond"].startswith("<level>"):
             _process = _process_for_level
-            paras = dict(
-                var=var, processed_s=processed_s, processed_s_bak=processed_s_bak,
-                level=int(setting["match_cond"][7:]), strictness_level=strictness_level
-            )
+            paras = dict(var=var, processed_s_bak=processed_s_bak, level=int(setting["match_cond"][7:]))
         elif setting["match_cond"].startswith("<node>"):
             _process = _process_for_name
-            paras = dict(var=var, processed_s=processed_s, name=setting["match_cond"][6:],
-                         strictness_level=strictness_level)
+            paras = dict(var=var, name=setting["match_cond"][6:])
         else:
             raise ValueError(f'invalid match_cond: {setting["match_cond"]}')
         # 执行
         for i in backend_name_ls:
             # print(processed_s)
             # print(f'backend: {i}')
-            _process(backend=backend_s[i], **paras)
+            _process(backend=backend_s[i], strictness_level=strictness_level, processed_s=processed_s,
+                     snn_builder=snn_builder, **paras)
             if "_hook_for_debug" in kwargs:
                 kwargs["_hook_for_debug"]["processed"].append([i, ndl.copy_(var=processed_s, b_deepcopy=True)])
 
@@ -168,16 +196,19 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
             f'please check settings to make sure all nodes have been covered and can be deal with backend'
 
     # 保存 var 的结构
-    json_.write(content=var, file_path=os.path.join(output_dir, "var.json"), b_use_suggested_converter=True)
+    json_.write(content=var, file_path=os.path.join(temp_output_dir, "var.json"), b_use_suggested_converter=True)
     # 保存处理结果（非必要）
     json_.write(content=dict(processed=processed_s, raw_structure=processed_s_bak, timestamp=time.time(),
-                             kt_version=kevin_toolbox.__version__),
-                file_path=os.path.join(output_dir, "record.json"), b_use_suggested_converter=True)
+                             kt_version=kevin_toolbox.__version__,
+                             b_keep_identical_relations=b_keep_identical_relations),
+                file_path=os.path.join(temp_output_dir, "record.json"), b_use_suggested_converter=True)
 
     # 打包成 .tar 文件
     if b_pack_into_tar:
-        for_os.pack(source=output_dir)
-        for_os.remove(path=output_dir, ignore_errors=True)
+        for_os.pack(source=temp_output_dir)
+        os.rename(temp_output_dir + ".tar", output_dir + ".tar")
+    else:
+        os.rename(temp_output_dir, output_dir)
 
 
 def _judge_processed_or_not(processed_s, name):
@@ -196,13 +227,13 @@ def _judge_processed_or_not(processed_s, name):
     return b_processed
 
 
-def _process_for_level(var, processed_s, processed_s_bak, level, backend, strictness_level):
+def _process_for_level(var, processed_s, processed_s_bak, level, backend, strictness_level, snn_builder):
     for name, _ in ndl.get_nodes(var=processed_s_bak, level=level, b_strict=True):
         _process_for_name(var=var, processed_s=processed_s, name=name, backend=backend,
-                          strictness_level=strictness_level)
+                          strictness_level=strictness_level, snn_builder=snn_builder)
 
 
-def _process_for_name(var, processed_s, name, backend, strictness_level):
+def _process_for_name(var, processed_s, name, backend, strictness_level, snn_builder):
     if _judge_processed_or_not(processed_s=processed_s, name=name) is True:
         # has been processed
         return
@@ -212,8 +243,9 @@ def _process_for_name(var, processed_s, name, backend, strictness_level):
         return
 
     # write by backend
+    snn_name = snn_builder(name=name, value=value)
     try:
-        res = backend.write(name=name, var=value)
+        res = backend.write(name=snn_name, var=value)
     except:
         assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
             f'An error occurred when node {name} was saved using the first matched backend {backend}'
@@ -222,7 +254,7 @@ def _process_for_name(var, processed_s, name, backend, strictness_level):
     ndl.set_value(var=var, name=name, value=res, b_force=False)
 
 
-def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_mode, strictness_level):
+def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_mode, strictness_level, snn_builder):
     def match_cond_(parent_type, idx, value):
         nonlocal match_cond, processed_s
 
@@ -237,8 +269,9 @@ def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_m
         nonlocal processed_s, backend, strictness_level
 
         # write by backend
+        snn_name = snn_builder(name=idx, value=value)
         try:
-            res = backend.write(name=idx, var=value)
+            res = backend.write(name=snn_name, var=value)
         except:
             assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
                 f'An error occurred when node {name} was saved using the first matched backend {backend}'
@@ -250,7 +283,7 @@ def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_m
                  b_use_name_as_idx=True, traversal_mode=traversal_mode, b_traverse_matched_element=False)
 
 
-def _process_from_down_to_top(var, processed_s, match_cond, backend, traversal_mode, strictness_level):
+def _process_from_down_to_top(var, processed_s, match_cond, backend, traversal_mode, strictness_level, snn_builder):
     processed_s_raw, processed_s = processed_s, ndl.copy_(var=processed_s, b_deepcopy=True)
 
     def match_cond_(parent_type, idx, value):
@@ -268,8 +301,9 @@ def _process_from_down_to_top(var, processed_s, match_cond, backend, traversal_m
         nonlocal processed_s, backend, processed_s_raw, strictness_level
 
         # write by backend
+        snn_name = snn_builder(name=idx, value=value)
         try:
-            res = backend.write(name=idx, var=value)
+            res = backend.write(name=snn_name, var=value)
         except:
             assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
                 f'An error occurred when node {name} was saved using the first matched backend {backend}'
@@ -305,7 +339,7 @@ if __name__ == '__main__':
         {"match_cond": lambda _, __, value: not isinstance(value, (list, dict)),
          "backend": (":skip:simple",)},
     ]
-    write(var=var_, output_dir=os.path.join(os.path.dirname(__file__), "temp3"), traversal_mode="bfs",
+    write(var=var_, output_dir=os.path.join(os.path.dirname(__file__), "temp"), traversal_mode="bfs",
           b_pack_into_tar=True, settings=settings_, _hook_for_debug=_hook_for_debug)
 
     for bk_name, p in _hook_for_debug["processed"]:

kevin_toolbox/nested_dict_list/value_parser/__init__.py

@@ -2,3 +2,4 @@ from .cal_relation_between_references import cal_relation_between_references
 from .eval_references import eval_references
 from .parse_references import parse_references
 from .parse_and_eval_references import parse_and_eval_references
+from .replace_identical_with_reference import replace_identical_with_reference

kevin_toolbox/nested_dict_list/value_parser/replace_identical_with_reference.py

@@ -0,0 +1,127 @@
+from collections import defaultdict
+from kevin_toolbox.nested_dict_list import get_nodes, get_value, set_value
+from kevin_toolbox.nested_dict_list import value_parser
+
+
+def replace_identical_with_reference(var, flag="same", match_cond=None, b_reverse=False):
+    """
+        将具有相同 id 的多个节点，替换为单个节点和其多个引用的形式
+            一般用于去除冗余部分，压缩 ndl 的结构
+
+        参数：
+            var:
+            flag:                   <str> 引用标记头，表示该节点应该替换为指定节点的内容
+                                        默认为 "same"
+                                        注意，在正向过程中，如果遇到本身就是以该 flag 开头的字符串，会自动在前添加多一个 flag 以示区分，
+                                            然后在逆向过程中，遇到两个 flag 标记开头的字符串将删除一个，然后跳过不处理。
+            match_cond:             <func> 仅对匹配上（返回True视为匹配上）的节点进行处理
+                                        函数类型为 def(name, value)
+                                        其中：
+                                            name            该节点在结构体中的位置
+                                            value           节点的值
+                                        默认不对 int、float、bool、str、None 等类型进行处理
+            b_reverse:              <boolean> 是否进行逆向操作
+    """
+    if match_cond is None:
+        match_cond = lambda name, value: not isinstance(value, (int, float, bool, str, type(None)))
+    assert callable(match_cond)
+
+    if b_reverse:
+        return _reverse(var, flag)
+    else:
+        return _forward(var, flag, match_cond)
+
+
+def _forward(var, flag, match_cond):
+    id_to_height_s = defaultdict(set)
+    id_to_name_s = defaultdict(set)
+    height = 1
+    while True:
+        node_ls = get_nodes(var=var, level=-height, b_strict=True)
+        if not node_ls:
+            break
+        for name, value in node_ls:
+            if not match_cond(name, value):
+                continue
+            id_to_name_s[id(value)].add(name)
+            id_to_height_s[id(value)].add(height)
+        height += 1
+
+    #
+    for k, v in list(id_to_height_s.items()):
+        # 仅保留有多个节点对应的 id
+        if len(id_to_name_s[k]) <= 1:
+            id_to_height_s.pop(k)
+            id_to_name_s.pop(k)
+            continue
+        # 具有相同 id 的节点所处的高度应该相同
+        assert len(v) == 1, f'nodes {id_to_name_s[k]} have different heights: {v}'
+    # 按高度排序
+    id_vs_height = sorted([(k, v.pop()) for k, v in id_to_height_s.items()], key=lambda x: x[1], reverse=True)
+
+    # 从高到低，依次将具有相同 id 的节点替换为 单个节点和多个引用 的形式
+    temp = []
+    processed_name_set = set()
+    for k, _ in id_vs_height:
+        # 找出父节点仍然未被处理的节点（亦即仍然能够访问到的节点）
+        unprocessed_name_set = {n for n in id_to_name_s[k] if id(get_value(var=var, name=n, default=temp)) == k}
+        if len(unprocessed_name_set) <= 1:
+            continue
+        # 任选其一进行保留，其余改为引用
+        keep_name = unprocessed_name_set.pop()
+        for name in unprocessed_name_set:
+            try:
+                var = set_value(var=var, name=name, value=f'<{flag}>{{{keep_name}}}', b_force=False)
+            except:
+                breakpoint()
+        processed_name_set.update(unprocessed_name_set)
+
+    # 将叶节点中，未被处理过，且是 str，且以 flag 开头的字符串，添加多一个 flag，以示区分
+    for name, value in get_nodes(var=var, level=-1, b_strict=True):
+        if name not in processed_name_set and isinstance(value, str) and value.startswith(f'<{flag}>'):
+            var = set_value(var=var, name=name, value=f'<{flag}>' + value, b_force=False)
+
+    return var
+
+
+class _My_Str:
+    def __init__(self, s):
+        self.s = s
+
+
+def _reverse(var, flag):
+    # 找出叶节点中，带有2个以上 flag 标记的字符串，删除其中一个标记，并使用 _My_Str 包裹，以便与普通引用节点区分
+    for name, value in get_nodes(var=var, level=-1, b_strict=True):
+        if isinstance(value, str) and value.startswith(f'<{flag}><{flag}>'):
+            var = set_value(var=var, name=name, value=_My_Str(value[len(flag) + 2:]), b_force=False)
+    # 解释引用
+    var, _ = value_parser.parse_and_eval_references(var=var, flag=flag)
+    # 解除 _My_Str 包裹
+    for name, value in get_nodes(var=var, level=-1, b_strict=True):
+        if isinstance(value, _My_Str):
+            var = set_value(var=var, name=name, value=value.s, b_force=False)
+    return var
+
+
+if __name__ == '__main__':
+    import numpy as np
+    from kevin_toolbox.nested_dict_list import copy_
+
+    a = np.array([1, 2, 3])
+    b = np.ones((2, 3))
+    c = [a, b]
+    d = {"a": a, "b": b}
+    e = {"c1": c, "c2": c}
+    x = [e, a, d, c, "<same>{@1}", "<same><same>{@1}"]
+
+    print(x)
+
+    y = replace_identical_with_reference(var=copy_(x, b_deepcopy=True), flag="same")
+    print(y)
+
+    x1 = replace_identical_with_reference(var=y, flag="same", b_reverse=True)
+    print(x1)
+
+    from kevin_toolbox.patches.for_test import check_consistency
+
+    check_consistency(x, x1)

kevin_toolbox/patches/for_os/__init__.py

@@ -1,4 +1,5 @@
 from .remove import remove
+from .copy import copy
 from .pack import pack
 from .unpack import unpack
 from .walk import walk, Path_Ignorer, Ignore_Scope

kevin_toolbox/patches/for_os/copy.py

@@ -0,0 +1,33 @@
+import os
+import shutil
+from kevin_toolbox.patches.for_os import remove
+
+
+def copy(src, dst, follow_symlinks=True, remove_dst_if_exists=False):
+    """
+        复制文件/文件夹/软连接
+
+        参数：
+            follow_symlinks:        <boolean> 是否跟随符号链接
+                                    对于 src 是软连接或者 src 指向的目录下具有软连接的情况，
+                                        当设置为 True 时，会复制所有链接指向的实际文件或目录内容。
+                                        当设置为 False 时，则仅仅软连接本身，而不是指向的内容
+                                    默认为 True
+            remove_dst_if_exists:   <boolean> 当目标存在时，是否尝试进行移除
+    """
+    assert os.path.exists(src), f'failed to copy, src not exists: {src}'
+    if remove_dst_if_exists:
+        remove(path=dst, ignore_errors=True)
+    assert not os.path.exists(dst), f'failed to copy, dst exists: {dst}'
+
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+    if os.path.isdir(src):
+        if not follow_symlinks and os.path.islink(src):
+            # 如果是符号链接，并且我们跟随符号链接，则复制链接本身
+            os.symlink(os.readlink(src), dst)
+        else:
+            # 否则，递归复制目录
+            shutil.copytree(src, dst, symlinks=not follow_symlinks)
+    else:
+        # 复制文件
+        shutil.copy2(src, dst, follow_symlinks=follow_symlinks)

kevin_toolbox/patches/for_test/check_consistency.py

@@ -7,7 +7,8 @@ import kevin_toolbox.nested_dict_list as ndl
 def check_consistency(*args, tolerance=1e-7, require_same_shape=True):
     """
         检查 args 中多个变量之间是否一致
-            变量支持python的所有内置类型，以及复杂的 nested_dict_list 结构， array 等
+            变量支持 python 的所有内置类型，以及复杂的 nested_dict_list 结构， array 等
+            对于 array，不区分 numpy 的 array，torch 的 tensor，还是 tuple of number，只要其中的值相等，即视为相同。
 
         参数：
             tolerance:          <float> 判断 <np.number/np.bool_> 之间是否一致时，的容许误差。
@@ -28,13 +29,13 @@ def check_consistency(*args, tolerance=1e-7, require_same_shape=True):
         try:
             _check_item(*names_ls, tolerance=tolerance, require_same_shape=True)
         except AssertionError as e:
-            assert False, f'inputs <nested_dict_list> has different structure\nthe nodes that differ are:\n{e}'
+            raise AssertionError(f'inputs <nested_dict_list> has different structure\nthe nodes that differ are:\n{e}')
         for its in zip(names_ls[0], *values_ls):
             try:
                 _check_item(*its[1:], tolerance=tolerance, require_same_shape=require_same_shape)
             except AssertionError as e:
-                assert False, \
-                    f'value of nodes {its[0]} in inputs <nested_dict_list> are inconsistent\nthe difference is:\n{e}'
+                raise AssertionError(
+                    f'value of nodes {its[0]} in inputs <nested_dict_list> are inconsistent\nthe difference is:\n{e}')
     # 简单结构
     else:
         _check_item(*args, tolerance=tolerance, require_same_shape=require_same_shape)
@@ -45,44 +46,111 @@ def _check_item(*args, tolerance, require_same_shape):
         检查 args 中多个 array 之间是否一致
 
         工作流程：
-            本函数会首先将输入的 args 中的所有变量转换为 np.array;
-            然后使用 issubclass() 判断转换后得到的变量属于以下哪几种基本类型：
-                - 当所有变量都属于 np.number 数值（包含int、float等）或者 np.bool_ 布尔值时，
-                    将对变量两两求差，当差值小于给定的容许误差 tolerance 时，视为一致。
-                - 当所有变量都属于 np.flexible 可变长度类型（包含string等）或者 np.object 时，
-                    将使用==进行比较，当返回值都为 True 时，视为一致。
-                - 当变量的基本类型不一致（比如同时有np.number和np.flexible）时，
-                    直接判断为不一致。
-            numpy 中基本类型之间的继承关系参见： https://numpy.org.cn/reference/arrays/scalars.html
+            1. 对于 args 都是 tuple/list 且每个 tuple/list 的长度都一致的情况，将会拆分为对应各个元素递归进行比较。
+            2. 将 args 中的 tuple 和 tensor 分别转换为 list 和 numpy。
+            3. 检查 args 中是否有 np.array 或者 tensor，若有则根据 require_same_shape 判断其形状是否一致。
+            4. 先将输入的 args 中的所有变量转换为 np.array;
+                然后使用 issubclass() 判断转换后得到的变量属于以下哪几种基本类型：
+                    - 当所有变量都属于 np.number 数值（包含int、float等）或者 np.bool_ 布尔值时，
+                        将对变量两两求差，当差值小于给定的容许误差 tolerance 时，视为一致。
+                        注意：在比较过程中，若变量中存在 np.nan 值，将会首先比较有 np.nan 值的位置是否相等，然后再比较非 np.nan 值部分。
+                            亦即在相同位置上都具有 np.nan 视为相同。
+                            比如 [np.nan, np.nan, 3] 和 [np.nan, np.nan, 3] 会视为相等，
+                            [np.nan, np.nan] 和 np.nan 在 require_same_shape=False 时会被视为相等。
+                    - 当所有变量都属于 np.flexible 可变长度类型（包含string等）或者 np.object 时，
+                        将使用==进行比较，当返回值都为 True 时，视为一致。
+                    - 当变量的基本类型不一致（比如同时有np.number和np.flexible）时，
+                        直接判断为不一致。
+                numpy 中基本类型之间的继承关系参见： https://numpy.org.cn/reference/arrays/scalars.html
 
         参数：
             tolerance:          <float> 判断 <np.number/np.bool_> 之间是否一致时，的容许误差。
             require_same_shape: <boolean> 是否强制要求变量的形状一致。
-                                    当设置为 False 时，不同形状的变量可能因为 numpy 的 broadcast 机制而在比较前自动 reshape 为相同维度，进而可能通过比较。
+                                    注意：仅在原始 args 中含有 np.array 或者 tensor 的情况会采取 broadcast，亦即此时该参数才会起效。
+                                    当设置为 False 时，不同形状的变量可能因为 broadcast 机制而在比较前自动 reshape 为相同维度，进而通过比较。
     """
-    np.warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)
+    warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)
     warnings.filterwarnings("ignore", category=DeprecationWarning)
 
     assert len(args) >= 2
-    assert isinstance(tolerance, (int, float,))
+    assert isinstance(tolerance, (int, float,)) and tolerance >= 0
+    raw_args = args
+
+    # 当 args 都是 tuple/list 且每个 tuple 的长度都一致时，将会拆分为对应各个元素进行比较
+    if all([isinstance(i, (tuple, list)) for i in args]) and all([len(i) == len(args[0]) for i in args[1:]]):
+        for i, it_ls in enumerate(zip(*args)):
+            try:
+                _check_item(*it_ls, tolerance=tolerance, require_same_shape=require_same_shape)
+            except AssertionError as e:
+                raise AssertionError(f'elements {i} are inconsistent\nthe difference is:\n{e}')
+        return
+
+    #
+    args = ndl.traverse(var=list(args), match_cond=lambda _, __, v: isinstance(v, (tuple,)), action_mode="replace",
+                        converter=lambda _, v: list(v), b_traverse_matched_element=True)
+    args = ndl.traverse(var=list(args), match_cond=lambda _, __, v: torch.is_tensor(v), action_mode="replace",
+                        converter=lambda _, v: v.detach().cpu().numpy(), b_traverse_matched_element=False)
+    b_has_raw_array = any([isinstance(i, np.ndarray) for i in args])
 
-    args = [v.detach().cpu() if torch.is_tensor(v) else v for v in args]
-    args = [np.asarray(v) for v in args]
+    try:
+        args = [np.asarray(v) for v in args]  # if b_has_raw_array else [np.array(v, dtype=object) for v in args]
+    except Exception as e:
+        raise RuntimeError(f'{raw_args} cannot be converted to np.array, \n'
+                           f'because {e}')
 
-    for v in args[1:]:
+    # 比较形状
+    if b_has_raw_array:
         if require_same_shape:
-            assert args[0].shape == v.shape, \
-                f"{args[0]}, {v}, different shape: {args[0].shape}, {v.shape}"
-        if issubclass(args[0].dtype.type, (np.number, np.bool_,)):
-            # 数字类型
+            # 要求形状一致
+            for v in args[1:]:
+                assert args[0].shape == v.shape, \
+                    f"{args[0]}, {v}, different shape: {args[0].shape}, {v.shape}"
+        else:
+            # 否则要求至少能够进行 broadcast
+            for v in args[1:]:
+                try:
+                    np.broadcast_arrays(args[0], v)
+                except:
+                    raise AssertionError(f'{args[0]}, {v}, failed to broadcast')
+            # 如果都是空的 array，直接视为相等
+            if all([i.size == 0 for i in args]):
+                return
+    b_allow_broadcast = b_has_raw_array and not require_same_shape
+
+    # 比较值
+    if issubclass(args[0].dtype.type, (np.number, np.bool_,)):
+        # 数字类型
+        for v in args[1:]:
             assert issubclass(v.dtype.type, (np.number, np.bool_,))
-            if args[0].size > 0:
-                assert np.max(np.abs(args[0] - v.astype(dtype=float))) < tolerance, \
-                    f"{args[0]}, {v}, deviation: {np.max(np.abs(args[0] - v))}"
-        elif issubclass(args[0].dtype.type, (np.flexible, object,)):
-            # 可变长度类型
+            v_0, v_1 = args[0].astype(dtype=float), v.astype(dtype=float)
+            v_0, v_1 = np.broadcast_arrays(v_0, v_1) if b_allow_broadcast else (v_0, v_1)
+            assert v_0.shape == v_1.shape, \
+                f'{v_0}, {v_1}, different shape: {v_0.shape}, {v_1.shape}'
+            #
+            if v_0.size > 0:
+                try:
+                    if np.any(np.isnan(v)):
+                        assert np.all(np.isnan(v_0) == np.isnan(v_1))
+                        v_0 = np.nan_to_num(v_0, nan=1e10)
+                        v_1 = np.nan_to_num(v_1, nan=1e10)
+                    assert np.max(np.abs(v_0 - v_1)) < tolerance
+                except AssertionError:
+                    raise AssertionError(f"{args[0]}, {v}, deviation: {np.max(np.abs(args[0] - v))}")
+    elif issubclass(args[0].dtype.type, (np.flexible, object,)):
+        # 可变长度类型
+        for v in args[1:]:
             assert issubclass(v.dtype.type, (np.flexible, object,))
-            for i, j in zip(args[0].reshape(-1), v.reshape(-1)):
+            v_0, v_1 = np.broadcast_arrays(args[0], v) if b_allow_broadcast else (args[0], v)
+            assert v_0.shape == v_1.shape, \
+                f'{v_0}, {v_1}, different shape: {v_0.shape}, {v_1.shape}\n' + (
+                    '' if require_same_shape else
+                    f'\tMore details: \n'
+                    f'\t\tAlthough require_same_shape=False has been setted, broadcast failed because the variable at \n'
+                    f'\t\tthis position does not contain elements of type np.array and tensor.')
+            #
+            for i, j in zip(v_0.reshape(-1), v_1.reshape(-1)):
+                if i is j:
+                    continue
                 temp = i == j
                 if isinstance(temp, (bool,)):
                     assert temp, \
@@ -90,14 +158,17 @@ def _check_item(*args, tolerance, require_same_shape):
                 else:
                     assert temp.all(), \
                         f"{args[0]}, {v}, diff: {temp}"
-        else:
-            raise ValueError
+    else:
+        raise ValueError
 
 
 if __name__ == '__main__':
     a = np.array([[1, 2, 3]])
     b = np.array([[1, 2, 3]])
     c = {'d': 3, 'c': 4}
-    check_consistency([c, a], [c, b])
 
-    check_consistency(True, True)
+    # var = ((1, 2), (4))
+    #
+    # var = ndl.traverse(var=[var], match_cond=lambda _, __, v: isinstance(v, (tuple,)), action_mode="replace",
+    #                    converter=lambda _, v: list(v), b_traverse_matched_element=True)[0]
+    # print(var)

kevin_toolbox_dev-1.3.5.dist-info/METADATA

@@ -0,0 +1,74 @@
+Metadata-Version: 2.1
+Name: kevin-toolbox-dev
+Version: 1.3.5
+Summary: 一个常用的工具代码包集合
+Home-page: https://github.com/cantbeblank96/kevin_toolbox
+Download-URL: https://github.com/username/your-package/archive/refs/tags/v1.0.0.tar.gz
+Author: kevin hsu
+Author-email: xukaiming1996@163.com
+License: MIT
+Keywords: mathematics,pytorch,numpy,machine-learning,algorithm
+Platform: UNKNOWN
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: torch (>=1.2.0)
+Requires-Dist: numpy (>=1.19.0)
+Provides-Extra: plot
+Requires-Dist: matplotlib (>=3.0) ; extra == 'plot'
+Provides-Extra: rest
+Requires-Dist: pytest (>=6.2.5) ; extra == 'rest'
+Requires-Dist: line-profiler (>=3.5) ; extra == 'rest'
+
+# kevin_toolbox
+
+一个通用的工具代码包集合
+
+
+
+环境要求
+
+```shell
+numpy>=1.19
+pytorch>=1.2
+```
+
+安装方法：
+
+```shell
+pip install kevin-toolbox  --no-dependencies
+```
+
+
+
+[项目地址 Repo](https://github.com/cantbeblank96/kevin_toolbox)
+
+[使用指南 User_Guide](./notes/User_Guide.md)
+
+[免责声明 Disclaimer](./notes/Disclaimer.md)
+
+[版本更新记录](./notes/Release_Record.md)：
+
+- v 1.3.5 （2024-05-18）【bug fix】【new feature】
+  - patches
+    - for_os
+      - 【new feature】add copy()，无论是文件/目录/软连接都可以使用该函数进行复制。
+        - 同时支持follow_symlinks参数用于决定是否跟随符号链接复制其指向的内容，支持remove_dst_if_exists用于决定当目标存在时是否尝试进行移除。
+    - for_test
+      - 【bug fix】fix bug in check_consistency()，解决了以下问题：
+        - 对于含有 np.nan 值的array错误地一律都判断为不相等，修改后将相同位置的 np.nan 值视为相等。
+        - require_same_shape=False 时无法正常为不对齐的 可变长度类型 报错。
+        - 对于包含 requires_grad=True 的 tensor 的复杂 tuple 异常报错。
+    - 添加了对应的测试用例。
+  - nested_dict_list
+    - 【bug fix】modify temporary file management in write() and read()，加强对写入和读取时创建的临时文件的管理，保证异常退出时能够自动清理临时文件夹。
+  - data_flow.file
+    - json_
+      - 【new feature】modify read()，新增了 file_obj 参数，支持直接从 BytesIO 和 StringIO 中读取json文件。
+    - 添加了对应的测试用例。
+  - 在更高版本的numpy中已经没有 np.warnings 了，因此将所有 np.warnings 替换为 warnings。
+
+
+

{kevin_toolbox_dev-1.3.3.dist-info → kevin_toolbox_dev-1.3.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-kevin_toolbox/__init__.py,sha256=SLC9_d60jm2WkdRlGpjY4ED_icVGwHgWLxaZOqS0zbk,410
+kevin_toolbox/__init__.py,sha256=JzwJXkcT4vV0qixg5n7pd24pB0WTgH5FVJcfB3f6ZCQ,410
 kevin_toolbox/computer_science/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kevin_toolbox/computer_science/algorithm/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 kevin_toolbox/computer_science/algorithm/cache_manager/__init__.py,sha256=p2hddkZ1HfYF9-m2Hx-o9IotwQHd4QwDCePy2ADpTDA,41
@@ -71,7 +71,7 @@ kevin_toolbox/data_flow/core/reader/unified_reader.py,sha256=l6JxPoDUOdx2ZIPX2WL
 kevin_toolbox/data_flow/core/reader/unified_reader_base.py,sha256=4gIADdV8UKpt2yD8dZjQsXFcF75nJ83ooIae3D7bw2s,11783
 kevin_toolbox/data_flow/file/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 kevin_toolbox/data_flow/file/json_/__init__.py,sha256=VAt8COS2tO3PJRuhSc43i35fEOlArFM_YahdTmEBaHE,85
-kevin_toolbox/data_flow/file/json_/read_json.py,sha256=S-rlY-spIAJVyB-zPQOmKUDdtNAgedSmXHjkY_TNGUE,1808
+kevin_toolbox/data_flow/file/json_/read_json.py,sha256=BhAUCcagwPsSMaMeCJyyxDW3h9SGf1Dfvb0nXi6B_T8,2084
 kevin_toolbox/data_flow/file/json_/write_json.py,sha256=mWaxePr_QzfyeCb0hAy4xTKOGX7q0eFjep0jDqOqIgw,2379
 kevin_toolbox/data_flow/file/json_/converter/__init__.py,sha256=oQMgAgzELLq_f4LIIfz5E6l_E7g4lFsXqfmnJ3tPZTY,401
 kevin_toolbox/data_flow/file/json_/converter/convert_dict_key_to_number.py,sha256=SuSZj_HCqKZutHAJ5AttABnGBRZplPGQhMxJBt2Wlgc,559
@@ -84,7 +84,7 @@ kevin_toolbox/data_flow/file/json_/converter/unescape_tuple_and_set.py,sha256=3v
 kevin_toolbox/data_flow/file/kevin_notation/__init__.py,sha256=g9UUF9nJrOMc0ndCwVROQLsux4Or2yVMJMdhK5P9nRc,259
 kevin_toolbox/data_flow/file/kevin_notation/converter.py,sha256=5k_Yxw-fBKEkxFG0bnl1fRsz06MlUS-4f3gZ--bmDs8,3621
 kevin_toolbox/data_flow/file/kevin_notation/kevin_notation_reader.py,sha256=iN6nV8mMbifTbECNmjc-G2pzpxivhks5kCXGVdvS9fQ,8297
-kevin_toolbox/data_flow/file/kevin_notation/kevin_notation_writer.py,sha256=dyzkufi_SM-uU7cyI3yLFd8U4GnyRR5pyS88op6mhRY,16331
+kevin_toolbox/data_flow/file/kevin_notation/kevin_notation_writer.py,sha256=Py8kyyi9RgxCSpuGebNkaxc9g4S8TdtiNJNDrJyP2UA,16328
 kevin_toolbox/data_flow/file/kevin_notation/read.py,sha256=w0RE0WwTmWycEozJVshAiE0gMBxproBRwBEMLS6tB6c,544
 kevin_toolbox/data_flow/file/kevin_notation/write.py,sha256=gPabz_h2mtXqCTyBVzip_QSb6L4tsrNSqibFzuqsIv8,556
 kevin_toolbox/data_flow/file/kevin_notation/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -227,14 +227,15 @@ kevin_toolbox/nested_dict_list/name_handler/__init__.py,sha256=P_pWq78oN6NdvWg2h
 kevin_toolbox/nested_dict_list/name_handler/build_name.py,sha256=VPWyjE8i8l-4Zm4tkD06Ie4J2NCsmI32ecOxZQqqmok,989
 kevin_toolbox/nested_dict_list/name_handler/escape_node.py,sha256=niT9MxmsyrSZYhKXlWzdoKXVYhWRCR-kmQBkZopznpA,1163
 kevin_toolbox/nested_dict_list/name_handler/parse_name.py,sha256=vUlAXPocpVSxtb3EnRi7U5K40Tz9plFG-_sbwLfYiy4,2280
-kevin_toolbox/nested_dict_list/serializer/__init__.py,sha256=xLDfzSZIDNBssHArUEK84gF6WZFR64qJy0iOpM8LbP8,92
+kevin_toolbox/nested_dict_list/serializer/__init__.py,sha256=79dd9l-mNz0bycFKjNm7YsfWPR-JsVx9NoG_Ofqy-HQ,153
 kevin_toolbox/nested_dict_list/serializer/enum_variable.py,sha256=RWPydtXI4adOJYGo_k5CWHSL0Odzj_bsahb24p1ranY,847
-kevin_toolbox/nested_dict_list/serializer/read.py,sha256=8196AZW3y1eZNguw5fDi5gQmOtpJ0MdqhRNHZ5EWYHI,2577
+kevin_toolbox/nested_dict_list/serializer/read.py,sha256=Ms86R8jRvuvbIWiyL8WUYXvU9APRXe72w9q_YcOKUWM,2944
+kevin_toolbox/nested_dict_list/serializer/saved_node_name_builder.py,sha256=qsD-rmDmVaKZP4owN3Wm3QY2Ksi71XlYETqw4VmIsSU,1011
 kevin_toolbox/nested_dict_list/serializer/variable.py,sha256=ZywG6obipRBCGY1cY42gdvsuWk8GLZXr6eCYcW7ZJ9c,392
-kevin_toolbox/nested_dict_list/serializer/write.py,sha256=sQtryi7NbVWynOMWpKKkqxoTf9dxSGE4yC4ReIAxT8U,18145
+kevin_toolbox/nested_dict_list/serializer/write.py,sha256=VMwD579NBl0QZeHh9P4WuFzA3CdtuRaqLLu5S5NGt5E,21468
 kevin_toolbox/nested_dict_list/serializer/backends/__init__.py,sha256=8g7y-L3cmctxao616dVkGiot00FJzKNmNl_69V2bSmE,39
 kevin_toolbox/nested_dict_list/serializer/backends/_json_.py,sha256=oJXIc28yjxsD9ZJuw120pVHTVsTzCdaXEhVUSQeydq4,2145
-kevin_toolbox/nested_dict_list/serializer/backends/_ndl.py,sha256=QMF4DFAnt1sp35atds6t44nfCYuIOeGgW1-SPfJq6KM,1652
+kevin_toolbox/nested_dict_list/serializer/backends/_ndl.py,sha256=3YkAq_Bqzehnw0kGxqxwtF6uUz0EV37tLI-1ROHjixY,1794
 kevin_toolbox/nested_dict_list/serializer/backends/_numpy_bin.py,sha256=xiPFmPUTjy0X0R1E0N8mrByENhNb69QalHnbYQXFvTo,1470
 kevin_toolbox/nested_dict_list/serializer/backends/_numpy_npy.py,sha256=CF6R7ie68zA0TqAXBdxUgHKVDYtEPHfVXR9rMFBbsdw,1384
 kevin_toolbox/nested_dict_list/serializer/backends/_pickle_.py,sha256=yCe4BXwSU55fbBX-HbTWI5ReT9Hhd7knu5zqvgUw5t4,1282
@@ -243,11 +244,12 @@ kevin_toolbox/nested_dict_list/serializer/backends/_skip_simple.py,sha256=dS9kKh
 kevin_toolbox/nested_dict_list/serializer/backends/_torch_all.py,sha256=aw9M1Hep65lkvb8_QU3VkecE6q3l4UqXansh4lnMv7s,1282
 kevin_toolbox/nested_dict_list/serializer/backends/_torch_tensor.py,sha256=Wiimzc0XGxFRYJiipzVnUd27scaJZZTOYp5OxYo3cKg,1353
 kevin_toolbox/nested_dict_list/serializer/backends/backend_base.py,sha256=SZpLRhdSIykHZ_Ds3HX96lKLfXCyKzMQ_lx139XXBtc,1741
-kevin_toolbox/nested_dict_list/value_parser/__init__.py,sha256=KTXAUaounOQ8oWSX-XzJMIFoopgaodBAm9RmQGMBgP8,234
+kevin_toolbox/nested_dict_list/value_parser/__init__.py,sha256=MgbpkiXzVuA_i3VZ4VDNrQZfzpUZN3uaHrgOt_wq-4E,313
 kevin_toolbox/nested_dict_list/value_parser/cal_relation_between_references.py,sha256=m3P7q5-pbbWqtjOjJUT-499q4mCyjtFEtFpGgovShSg,3119
 kevin_toolbox/nested_dict_list/value_parser/eval_references.py,sha256=YQyOm3awKVRusXxSNObjJ2yPf0oE4gleobOn_RN_nzU,2301
 kevin_toolbox/nested_dict_list/value_parser/parse_and_eval_references.py,sha256=RQEDFFNAhQQcX9H8curwja-pI2gKZlVx4M2qeneBOhA,2370
 kevin_toolbox/nested_dict_list/value_parser/parse_references.py,sha256=G470xNzrRpYlS5To8R5yV0M6nX4iE5LLMp_eV49bh3Y,2116
+kevin_toolbox/nested_dict_list/value_parser/replace_identical_with_reference.py,sha256=4nd5q3qTi4sTfOqHF0-HyLMDS9a5x8wgwETfpDI8jh8,5419
 kevin_toolbox/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kevin_toolbox/patches/for_logging/__init__.py,sha256=xymF6mjwY4Cin7CoEwanFY5ZVk8oY0pDLqjZAGa7_Rg,39
 kevin_toolbox/patches/for_logging/build_logger.py,sha256=0UoRMKaERd8wlHGTiNR3dbLQRAtXcb0QZuGkX2oFgGo,3071
@@ -280,13 +282,14 @@ kevin_toolbox/patches/for_optuna/serialize/for_study/load.py,sha256=DwnDheJx5IJp
 kevin_toolbox/patches/for_optuna/serialize/for_trial/__init__.py,sha256=YSROCv89O84gzzRP2NfVl0Xhd4_RsdoqU3Nnr-vhWhk,46
 kevin_toolbox/patches/for_optuna/serialize/for_trial/dump.py,sha256=FT-Z1rzCNUYz5St1Yc0bfDWAV10WaV41rlbWIS72Jp0,982
 kevin_toolbox/patches/for_optuna/serialize/for_trial/load.py,sha256=2fpeeHPKA9bT7CjQ6DVRXOarF6IAA6_f2pXbB1rXcvE,796
-kevin_toolbox/patches/for_os/__init__.py,sha256=Q6GzEPWggV4EovJdrertlL2l__W7k0zUEpuQIQDcA7A,128
+kevin_toolbox/patches/for_os/__init__.py,sha256=jx4UCTxEW2QhLfgfPhxcgUrftvs_2420IAQJso82DeQ,151
+kevin_toolbox/patches/for_os/copy.py,sha256=PWFLu15DpIA4JZxatvphHANNn2H3nC93qTbLLxDl5NU,1509
 kevin_toolbox/patches/for_os/pack.py,sha256=A6u4g3dfwXPtlU4gBcThNrktz6dO4DVi2wmQXytqfDI,656
 kevin_toolbox/patches/for_os/remove.py,sha256=PmwqzVJbyfdqwXn_T1F9d4Oar8CwQ2YFaqcZQkfnrnI,750
 kevin_toolbox/patches/for_os/unpack.py,sha256=d_fO7nPmExy1VIg7ADIMayCzjBBeFxvJLhIsulIRlzI,1047
 kevin_toolbox/patches/for_os/walk.py,sha256=LrtEeRUDwzZgu_zGZ-kPsFJd4D-8R8ECHW6WNdEsDSw,8376
 kevin_toolbox/patches/for_test/__init__.py,sha256=sFr2VZD1zk8Vtjq2_F8uE4xNovJF6yDY8j1YND5XAw0,49
-kevin_toolbox/patches/for_test/check_consistency.py,sha256=5zwjHc0M3A-RTYcgZpIugm6-L-m2pDSG496dCkg8Q4Q,5038
+kevin_toolbox/patches/for_test/check_consistency.py,sha256=cerf4NywkvWYMvuJUjimfRRVU7D9vL30jTAX0NxxRoM,9422
 kevin_toolbox/patches/for_torch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kevin_toolbox/patches/for_torch/compatible/__init__.py,sha256=7l4FPTILBR8EmStJNHIsbwfFkZpe4xkJZtrlyC8riL8,75
 kevin_toolbox/patches/for_torch/compatible/concat.py,sha256=vFJn9B437gxDOa5Ze_gXxZewiClqoiIkWeAktDEcJK4,393
@@ -302,7 +305,7 @@ kevin_toolbox/patches/for_torch/math/get_y_at_x.py,sha256=bfoVcasZ_tMdhR_1Me0Jli
 kevin_toolbox/patches/for_torch/math/my_around.py,sha256=ptpU3ids50gwf663EpHbw7raj9tNrDGBFZ5t_uMNH14,1378
 kevin_toolbox/patches/for_torch/nn/__init__.py,sha256=aJs3RMqRzQmd8KKDmQW9FxwCqS5yfPqEdg-m0PwlQro,39
 kevin_toolbox/patches/for_torch/nn/lambda_layer.py,sha256=KUuLiX_Dr4bvRmpAaCW5QTDWDcnMPRnw0jg4NNXTFhM,223
-kevin_toolbox_dev-1.3.3.dist-info/METADATA,sha256=LkQQ3jJCLlHGvHeAnCCGC_uiNz6qoWLFNSt_K00N0qs,2228
-kevin_toolbox_dev-1.3.3.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-kevin_toolbox_dev-1.3.3.dist-info/top_level.txt,sha256=S5TeRGF-PwlhsaUEPTI-f2vWrpLmh3axpyI6v-Fi75o,14
-kevin_toolbox_dev-1.3.3.dist-info/RECORD,,
+kevin_toolbox_dev-1.3.5.dist-info/METADATA,sha256=ftmjWCBMUjdd756VCDmVAXgSXT1hIBWpZpvzngyAWCQ,2636
+kevin_toolbox_dev-1.3.5.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+kevin_toolbox_dev-1.3.5.dist-info/top_level.txt,sha256=S5TeRGF-PwlhsaUEPTI-f2vWrpLmh3axpyI6v-Fi75o,14
+kevin_toolbox_dev-1.3.5.dist-info/RECORD,,

kevin_toolbox_dev-1.3.3.dist-info/METADATA

@@ -1,75 +0,0 @@
-Metadata-Version: 2.1
-Name: kevin-toolbox-dev
-Version: 1.3.3
-Summary: 一个常用的工具代码包集合
-Home-page: https://github.com/cantbeblank96/kevin_toolbox
-Download-URL: https://github.com/username/your-package/archive/refs/tags/v1.0.0.tar.gz
-Author: kevin hsu
-Author-email: xukaiming1996@163.com
-License: MIT
-Keywords: mathematics,pytorch,numpy,machine-learning,algorithm
-Platform: UNKNOWN
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.6
-Description-Content-Type: text/markdown
-Requires-Dist: torch (>=1.2.0)
-Requires-Dist: numpy (>=1.19.0)
-Provides-Extra: plot
-Requires-Dist: matplotlib (>=3.0) ; extra == 'plot'
-Provides-Extra: rest
-Requires-Dist: pytest (>=6.2.5) ; extra == 'rest'
-Requires-Dist: line-profiler (>=3.5) ; extra == 'rest'
-
-# kevin_toolbox
-
-一个通用的工具代码包集合
-
-
-
-环境要求
-
-```shell
-numpy>=1.19
-pytorch>=1.2
-```
-
-安装方法：
-
-```shell
-pip install kevin-toolbox  --no-dependencies
-```
-
-
-
-[项目地址 Repo](https://github.com/cantbeblank96/kevin_toolbox)
-
-[使用指南 User_Guide](./notes/User_Guide.md)
-
-[免责声明 Disclaimer](./notes/Disclaimer.md)
-
-[版本更新记录](./notes/Release_Record.md)：
-
-- v 1.3.3 （2024-04-01）【bug fix】【new feature】
-  - math.utils
-    - 【bug fix】更正拼写错误，将原有的 spilt_integer_most_evenly() 中的 "spilt" 改正为 "split"，新的函数名为 split_integer_most_evenly
-
-  - patches.for_numpy.random
-    - 【bug fix】modify get_rng()，改进以避免遇到 rng 中不存在的函数时报错。
-
-  - data_flow.file
-    - json_
-      - 【new feature】将 escape_tuple() 改成 escape_tuple_and_set()，新增对 set 进行处理。
-      - 【new feature】将 unescape_tuple() 改成 unescape_tuple_and_set()。
-
-    - core.reader
-      - 【new feature】为 File_Iterative_Reader 新增了 file_obj 参数，支持直接输入文件对象。
-
-    - kevin_notation
-      - 【new feature】根据 File_Iterative_Reader 的修改，为 Kevin_Notation_Reader 和 read() 对应增加 file_obj 参数，以支持直接输入文件对象。
-      - 补充了对应的测试用例。
-
-
-
-