kevin-toolbox-dev 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

kevin_toolbox/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.3.2"
+__version__ = "1.3.4"
 
 
 import os
@@ -12,5 +12,5 @@ os.system(
 os.system(
     f'python {os.path.split(__file__)[0]}/env_info/check_validity_and_uninstall.py '
     f'--package_name kevin-toolbox-dev '
-    f'--expiration_timestamp 1725197084 --verbose 0'
+    f'--expiration_timestamp 1727961379 --verbose 0'
 )

kevin_toolbox/data_flow/core/reader/file_iterative_reader.py

@@ -1,4 +1,5 @@
 import os
+import copy
 
 
 class File_Iterative_Reader:
@@ -12,11 +13,14 @@ class File_Iterative_Reader:
             设定关键参数
             必要参数：
                 file_path:  文件路径
+                file_obj:   文件对象
+                    注意！！以上两个参数指定其一即可，同时指定时候，以后者为准。
             读取模式相关参数：
                 paras_for_open:     open() 函数的补充参数
                 mode:       读取模式，默认为 "lines"
                                 "lines"：  按行数计算批次大小
                                 "bytes"：  按字节数计算
+                    注意！！以上两个参数在指定了 file_obj 参数后将失效。
                 chunk_size: 批次大小
                                 默认为 1k
                                 当为-1时，读取整个文件
@@ -51,6 +55,7 @@ class File_Iterative_Reader:
         paras = {
             # 必要参数
             "file_path": None,
+            "file_obj": None,
             # 读取模式相关参数
             "paras_for_open": dict(mode="r", encoding='utf-8'),
             "mode": "lines",
@@ -74,16 +79,21 @@ class File_Iterative_Reader:
         assert mode in ["lines", "bytes"]
         paras["chunk_size"] = int(paras["chunk_size"])
         paras["loop_num"] = int(paras["loop_num"]) - 1
-        #
-        file_path = paras["file_path"]
-        assert isinstance(file_path, (str,)) and os.path.exists(file_path), \
-            Exception(f"Error: file {file_path} not exists!")
-        #
-        paras_for_open = paras["paras_for_open"]
-        assert isinstance(paras_for_open, (dict,))
 
         # 获取文件对象
-        self.file = open(file_path, **paras_for_open)
+        if paras["file_obj"] is None:
+            assert isinstance(paras["file_path"], (str,)) and os.path.isfile(paras["file_path"]), \
+                Exception(f'Error: file {paras["file_path"]} not exists!')
+            #
+            assert isinstance(paras["paras_for_open"], (dict,))
+            self.file = open(paras["file_path"], **paras["paras_for_open"])
+        else:
+            # 拷贝对象，防止修改外部对象
+            try:
+                self.file = copy.deepcopy(paras["file_obj"])
+            except:
+                self.file = open(paras["file_obj"].name, mode=paras["file_obj"].mode)
+
         # 选择相应模式
         self.__read_func = {"lines": self.__read_lines, "bytes": self.__read_bytes}[mode]
         self.__jump_func = {"lines": self.__jump_lines, "bytes": self.__jump_bytes}[mode]
@@ -225,9 +235,34 @@ class File_Iterative_Reader:
 if __name__ == "__main__":
     import numpy as np
 
-    reader = File_Iterative_Reader(file_path="developing/test_data.txt", chunk_size=2, drop=True, loop_num=2,
+    print("使用 file_path")
+    reader = File_Iterative_Reader(file_path="test/test_data/test_data.txt", chunk_size=2, drop=True, loop_num=2,
                                    pre_jump_size=3, convert_func=lambda x: np.array(x))
     for i in reader:
         print(i)
 
     del reader
+
+    print("使用 file_obj")
+    reader = File_Iterative_Reader(
+        file_obj=open("test/test_data/test_data.txt", "r"), chunk_size=2, drop=True, loop_num=2,
+        pre_jump_size=3, convert_func=lambda x: np.array(x))
+    for i in reader:
+        print(i)
+
+    del reader
+
+    print("从字符串构建文件对象作为 file_obj")
+    from io import StringIO
+
+    file_obj = StringIO(initial_value=open("test/test_data/test_data.txt", "r").read())
+    reader = File_Iterative_Reader(
+        file_obj=file_obj, chunk_size=2, drop=True, loop_num=2,
+        pre_jump_size=3, convert_func=lambda x: np.array(x))
+    for i in reader:
+        print(i)
+
+    print("证明不会修改外部对象")
+    print(file_obj.read())
+
+    del reader

kevin_toolbox/data_flow/core/reader/unified_reader.py

@@ -49,7 +49,7 @@ if __name__ == '__main__':
     print(reader.read([3, 3]).shape)
     print(reader.find(1))
 
-    reader = UReader(file_path="test_data.txt", chunk_size=2, folder_path="./temp/233")
+    reader = UReader(file_path="test/test_data/test_data.txt", chunk_size=2, folder_path="./temp/233")
 
     print(reader.read(2, 7))
     # del reader
@@ -67,7 +67,7 @@ if __name__ == '__main__':
 
     print(reader.find('data/6/horse_race_pan/2132020102319002000161_43_4.bmp'))
 
-    reader = Reader_for_files(file_path="test_data.txt", chunk_size=2, pre_jump_size=2, jump_size=2)
+    reader = Reader_for_files(file_path="test/test_data/test_data.txt", chunk_size=2, pre_jump_size=2, jump_size=2)
 
     for i in reader:
         print(2333, i)

kevin_toolbox/data_flow/file/json_/converter/__init__.py

@@ -2,6 +2,6 @@ from .convert_dict_key_to_number import convert_dict_key_to_number
 from .convert_ndarray_to_list import convert_ndarray_to_list
 from .escape_non_str_dict_key import escape_non_str_dict_key
 from .unescape_non_str_dict_key import unescape_non_str_dict_key
-from .escape_tuple import escape_tuple
-from .unescape_tuple import unescape_tuple
+from .escape_tuple_and_set import escape_tuple_and_set
+from .unescape_tuple_and_set import unescape_tuple_and_set
 from .integrate import integrate

kevin_toolbox/data_flow/file/json_/converter/escape_tuple_and_set.py

@@ -0,0 +1,23 @@
+def escape_tuple_and_set(x):
+    """
+        将 tuple 和 set 进行转义
+            转义：     x ==> f"<eval>{x}"
+            反转义：   f"<eval>{x}" ==> x
+
+        为什么要进行转义？
+            由于 json 中会将 tuple 作为 list 进行保存，同时也无法保存 set，因此在保存过程中会丢失相应信息。
+    """
+    if isinstance(x, (tuple, set,)) or (isinstance(x, (str,)) and x.startswith("<eval>")):
+        return f'<eval>{x}'
+    else:
+        return x
+
+
+if __name__ == '__main__':
+    print(escape_tuple_and_set((1, 2, "\'1\'")))
+    # <eval>(1, 2, "'1'")
+    print(escape_tuple_and_set({"1", (1, 2, 3), 233}))
+    # <eval>{'1', 233, (1, 2, 3)}
+
+    print(escape_tuple_and_set("<eval>233"))
+    # <eval><eval>233

kevin_toolbox/data_flow/file/json_/converter/{unescape_tuple.py → unescape_tuple_and_set.py}

@@ -1,11 +1,11 @@
-def unescape_tuple(x):
+def unescape_tuple_and_set(x):
     """
-        将 tuple 进行反转义
+        将 tuple 和 set 进行反转义
             转义：     x ==> f"<eval>{x}"
             反转义：   f"<eval>{x}" ==> x
 
         为什么要进行转义？
-            由于 json 中会将 tuple 作为 list 进行保存，因此在保存过程中会丢失相应信息。
+            由于 json 中会将 tuple 作为 list 进行保存，同时也无法保存 set，因此在保存过程中会丢失相应信息。
     """
     if isinstance(x, str) and x.startswith("<eval>"):
         x = x[6:]
@@ -17,7 +17,9 @@ def unescape_tuple(x):
 
 
 if __name__ == '__main__':
-    print(unescape_tuple("<eval>(1, 2, \"'1'\")"))
+    print(unescape_tuple_and_set("<eval>(1, 2, \"'1'\")"))
     # (1, 2, "\'1\'")
-    print(unescape_tuple("<eval><eval>233"))
+    print(unescape_tuple_and_set("<eval>{'1', 233, (1, 2, 3)}"))
+    # {'1', 233, (1, 2, 3)}
+    print(unescape_tuple_and_set("<eval><eval>233"))
     # "<eval>233"

kevin_toolbox/data_flow/file/json_/read_json.py

@@ -1,6 +1,6 @@
 import os
 import json
-from kevin_toolbox.data_flow.file.json_.converter import integrate, unescape_tuple, unescape_non_str_dict_key
+from kevin_toolbox.data_flow.file.json_.converter import integrate, unescape_tuple_and_set, unescape_non_str_dict_key
 from kevin_toolbox.nested_dict_list import traverse
 
 
@@ -14,14 +14,14 @@ def read_json(file_path, converters=None, b_use_suggested_converter=False):
                                             转换器 converter 应该是一个形如 def(x): ... ; return x 的函数，具体可以参考
                                             json_.converter 中已实现的转换器
             b_use_suggested_converter:  <boolean> 是否使用建议的转换器
-                                            建议使用 unescape/escape_non_str_dict_key 和 unescape/escape_tuple 这两对转换器，
+                                            建议使用 unescape/escape_non_str_dict_key 和 unescape/escape_tuple_and_set 这两对转换器，
                                             可以避免因 json 的读取/写入而丢失部分信息。
                                             默认为 False。
                     注意：当 converters 非 None，此参数失效，以 converters 中的具体设置为准
     """
     assert os.path.isfile(file_path), f'file {file_path} not found'
     if converters is None and b_use_suggested_converter:
-        converters = [unescape_tuple, unescape_non_str_dict_key]
+        converters = [unescape_tuple_and_set, unescape_non_str_dict_key]
 
     with open(file_path, 'r') as f:
         content = json.load(f)

kevin_toolbox/data_flow/file/json_/write_json.py

@@ -1,7 +1,7 @@
 import os
 import json
 import copy
-from kevin_toolbox.data_flow.file.json_.converter import integrate, escape_tuple, escape_non_str_dict_key
+from kevin_toolbox.data_flow.file.json_.converter import integrate, escape_tuple_and_set, escape_non_str_dict_key
 from kevin_toolbox.nested_dict_list import traverse
 
 
@@ -18,7 +18,7 @@ def write_json(content, file_path, sort_keys=False, converters=None, b_use_sugge
                                             转换器 converter 应该是一个形如 def(x): ... ; return x 的函数，具体可以参考
                                             json_.converter 中已实现的转换器
             b_use_suggested_converter:  <boolean> 是否使用建议的转换器
-                                            建议使用 unescape/escape_non_str_dict_key 和 unescape/escape_tuple 这两对转换器，
+                                            建议使用 unescape/escape_non_str_dict_key 和 unescape/escape_tuple_and_set 这两对转换器，
                                             可以避免因 json 的读取/写入而丢失部分信息。
                                             默认为 False。
                     注意：当 converters 非 None，此参数失效，以 converters 中的具体设置为准
@@ -26,7 +26,7 @@ def write_json(content, file_path, sort_keys=False, converters=None, b_use_sugge
     assert isinstance(file_path, (str, type(None)))
 
     if converters is None and b_use_suggested_converter:
-        converters = [escape_tuple, escape_non_str_dict_key]
+        converters = [escape_tuple_and_set, escape_non_str_dict_key]
 
     if converters is not None:
         converter = integrate(converters)

kevin_toolbox/data_flow/file/kevin_notation/kevin_notation_reader.py

@@ -15,6 +15,8 @@ class Kevin_Notation_Reader:
 
             必要参数：
                 file_path:          <string> 文件路径
+                file_obj:           <file object> 文件对象
+                    以上参数2选一，具体参见 File_Iterative_Reader
             读取相关参数：
                 chunk_size:         <integer> 每次读取多少行数据
                 beg：                <integer> 开始读取的位置
@@ -28,6 +30,7 @@ class Kevin_Notation_Reader:
         paras = {
             # 必要参数
             "file_path": None,
+            "file_obj": None,
             # 读取相关参数
             "chunk_size": 100,
             "beg": 0,
@@ -38,9 +41,7 @@ class Kevin_Notation_Reader:
         paras.update(kwargs)
 
         # 校验参数
-        assert isinstance(paras["file_path"], (str,)) and os.path.isfile(paras["file_path"]), \
-            f'file not exists :{paras["file_path"]}'
-        #
+        #   file_path 和 file_obj 交给 File_Iterative_Reader 校验
         assert isinstance(paras["chunk_size"], (int,)) and (paras["chunk_size"] > 0 or paras["chunk_size"] == -1)
         assert isinstance(paras["beg"], (int,)) and paras["beg"] >= 0
         assert isinstance(paras["converter"], (Converter, dict,))
@@ -48,7 +49,7 @@ class Kevin_Notation_Reader:
         self.paras = paras
 
         # 读取开头
-        self.reader = File_Iterative_Reader(file_path=self.paras["file_path"],
+        self.reader = File_Iterative_Reader(file_path=self.paras["file_path"], file_obj=self.paras["file_obj"],
                                             pre_jump_size=self.paras["beg"],
                                             filter_=lambda x: x != "\n" and not x.startswith("//"),  # 去除注释
                                             map_func=lambda x: x.rsplit("\n", 1)[0].split("//", 1)[0],
@@ -64,7 +65,7 @@ class Kevin_Notation_Reader:
         del self.reader
 
         # 读取内容
-        self.reader = File_Iterative_Reader(file_path=self.paras["file_path"],
+        self.reader = File_Iterative_Reader(file_path=self.paras["file_path"], file_obj=self.paras["file_obj"],
                                             pre_jump_size=self.paras["beg"] + offset,
                                             filter_=lambda x: x != "\n" and not x.startswith("//"),  # 去除注释
                                             map_func=lambda x: x.rsplit("\n", 1)[0].split("//", 1)[0],

kevin_toolbox/data_flow/file/kevin_notation/read.py

@@ -1,11 +1,13 @@
 from kevin_toolbox.data_flow.file import kevin_notation
 
 
-def read(file_path):
+def read(file_path=None, file_obj=None):
     """
         读取整个文件的快捷接口
     """
-    with kevin_notation.Reader(file_path=file_path, chunk_size=-1) as reader:
+    assert file_path is not None or file_obj is not None
+
+    with kevin_notation.Reader(file_path=file_path, chunk_size=-1, file_obj=file_obj) as reader:
         # metadata
         metadata = reader.metadata
         # content

kevin_toolbox/data_flow/file/kevin_notation/test/test_kevin_notation.py

@@ -134,11 +134,23 @@ def test_writer_1(expected_metadata, expected_content, file_path):
                          zip(metadata_ls, content_ls, file_path_ls))
 def test_read(expected_metadata, expected_content, file_path):
     print("test read()")
-    # 读取
+    # 使用 file_path 读取
     metadata, content = kevin_notation.read(file_path=file_path)
+
+    # 使用 file_obj 读取
+    file_obj = open(file_path, "r")
+    metadata_1, content_1 = kevin_notation.read(file_obj=file_obj)
+    assert len(file_obj.read()) > 0  # 不影响输入的 file_obj
+
+    # 使用字符串构造 file_obj 读取
+    from io import StringIO
+    file_obj = StringIO(initial_value=open(file_path, "r").read())
+    metadata_2, content_2 = kevin_notation.read(file_obj=file_obj)
+    assert len(file_obj.read()) > 0
+
     # 检验
-    check_consistency(expected_metadata, metadata)
-    check_consistency(expected_content, content)
+    check_consistency(expected_metadata, metadata, metadata_1, metadata_2)
+    check_consistency(expected_content, content, content_1, content_2)
 
 
 @pytest.mark.parametrize("expected_metadata, expected_content, file_path",

kevin_toolbox/data_flow/file/markdown/generate_table.py

@@ -1,4 +1,4 @@
-from kevin_toolbox.math.utils import spilt_integer_most_evenly
+from kevin_toolbox.math.utils import split_integer_most_evenly
 
 
 def generate_table(content_s, orientation="vertical", chunk_nums=None, chunk_size=None, b_allow_misaligned_values=False,
@@ -60,7 +60,7 @@ def generate_table(content_s, orientation="vertical", chunk_nums=None, chunk_siz
     # 按照 chunk_nums 或者 chunk_size 对表格进行分割
     if chunk_nums is not None or chunk_size is not None:
         if chunk_nums is not None:
-            split_len_ls = spilt_integer_most_evenly(x=max_length, group_nums=chunk_nums)
+            split_len_ls = split_integer_most_evenly(x=max_length, group_nums=chunk_nums)
         else:
             split_len_ls = [chunk_size] * (max_length // chunk_size)
             if max_length % chunk_size != 0:

kevin_toolbox/math/utils/__init__.py

@@ -2,4 +2,4 @@ from .get_function_table_for_array_and_tensor import get_function_table_for_arra
 from .convert_dtype import convert_dtype
 from .get_crop_by_box import get_crop_by_box
 from .set_crop_by_box import set_crop_by_box
-from .spilt_integer_most_evenly import spilt_integer_most_evenly
+from .split_integer_most_evenly import split_integer_most_evenly

kevin_toolbox/math/utils/{spilt_integer_most_evenly.py → split_integer_most_evenly.py}

@@ -1,7 +1,7 @@
 import numpy as np
 
 
-def spilt_integer_most_evenly(x, group_nums):
+def split_integer_most_evenly(x, group_nums):
     assert isinstance(x, (int, np.integer,)) and x >= 0 and group_nums > 0
 
     res = np.ones(group_nums, dtype=int) * (x // group_nums)
@@ -10,4 +10,4 @@ def spilt_integer_most_evenly(x, group_nums):
 
 
 if __name__ == '__main__':
-    print(spilt_integer_most_evenly(x=100, group_nums=7))
+    print(split_integer_most_evenly(x=100, group_nums=7))

kevin_toolbox/nested_dict_list/serializer/__init__.py

@@ -1,3 +1,4 @@
 from .write import write
 from .read import read
 from .enum_variable import Strictness_Level
+from .saved_node_name_builder import Saved_Node_Name_Builder

kevin_toolbox/nested_dict_list/serializer/read.py

@@ -29,13 +29,15 @@ def read(input_path, **kwargs):
 
     # 读取 var
     var = json_.read(file_path=os.path.join(input_path, "var.json"), b_use_suggested_converter=True)
+    # 读取 record
+    record_s = dict()
+    if os.path.isfile(os.path.join(input_path, "record.json")):
+        record_s = json_.read(file_path=os.path.join(input_path, "record.json"), b_use_suggested_converter=True)
 
     # 读取被处理的节点
     processed_nodes = []
-    if os.path.isfile(os.path.join(input_path, "record.json")):
-        for name, value in ndl.get_nodes(
-                var=json_.read(file_path=os.path.join(input_path, "record.json"),
-                               b_use_suggested_converter=True)["processed"], level=-1, b_strict=True):
+    if record_s:
+        for name, value in ndl.get_nodes(var=record_s["processed"], level=-1, b_strict=True):
             if value:
                 processed_nodes.append(name)
     else:
@@ -56,6 +58,11 @@ def read(input_path, **kwargs):
             bk = SERIALIZER_BACKEND.get(name=value.pop("backend"))(folder=os.path.join(input_path, "nodes"))
             ndl.set_value(var=var, name=name, value=bk.read(**value))
 
+    #
+    if record_s.get("b_keep_identical_relations", False):
+        from kevin_toolbox.nested_dict_list import value_parser
+        var = value_parser.replace_identical_with_reference(var=var, flag="same", b_reverse=True)
+
     #
     if temp_dir is not None:
         for_os.remove(path=temp_dir, ignore_errors=True)

kevin_toolbox/nested_dict_list/serializer/saved_node_name_builder.py

@@ -0,0 +1,31 @@
+class Saved_Node_Name_Builder:
+    """
+        生成保存节点内容时的文件夹/文件名称
+    """
+
+    def __init__(self, format_):
+        try:
+            temp = format_.format(**{k: k + "_" * 3 for k in {"raw_name", "id", "hash_name", "count"}})
+            assert len(temp) > len(format_)
+        except:
+            raise ValueError(f'invalid saved_node_name_format {format_}')
+
+        self.format_ = format_
+        self.count = 0
+
+    def __call__(self, name, value):
+        from kevin_toolbox.nested_dict_list import get_hash
+
+        res = self.format_.format(
+            **{"raw_name": name, "id": id(value), "hash_name": get_hash(name, length=12), "count": self.count})
+        self.count += 1
+        return res
+
+
+if __name__ == '__main__':
+    bd = Saved_Node_Name_Builder(format_="{raw_name}_{count}_{hash_name}_{id}")
+    print(bd(":a@0", 1))
+    print(bd(":b:c", []))
+
+    # bd = Saved_Node_Name_Builder(format_="")
+    # bd = Saved_Node_Name_Builder(format_="{raw_name2}_{count}")

kevin_toolbox/nested_dict_list/serializer/write.py

@@ -7,16 +7,18 @@ from kevin_toolbox.patches import for_os
 import kevin_toolbox.nested_dict_list as ndl
 from kevin_toolbox.nested_dict_list.traverse import Traversal_Mode
 from .enum_variable import Strictness_Level
+from .saved_node_name_builder import Saved_Node_Name_Builder
 
 
 def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_pack_into_tar=True,
-          strictness_level=Strictness_Level.COMPATIBLE, **kwargs):
+          strictness_level=Strictness_Level.COMPATIBLE, saved_node_name_format='{count}_{hash_name}',
+          b_keep_identical_relations=False, **kwargs):
     """
         将输入的嵌套字典列表 var 的结构和节点值保存到文件中
             遍历 var，匹配并使用 settings 中设置的保存方式来对各部分结构/节点进行序列化
             将会生成一个文件夹或者 .tar 文件，其中包含：
                 - var.json：     用于保存结构、简单节点值、复杂节点值/结构的序列化方式
-                - nodes/目录：    其中包含一系列 <name>.<suffix> 文件或者 <name> 文件夹，其中包含复杂节点值/结构的序列化结果
+                - nodes/：       该目录中包含一系列 <name>.<suffix> 文件或者 <name> 文件夹，其中包含复杂节点值/结构的序列化结果
                 - record.json：   其中记录了：
                                         {
                                             "processed": ... # 对哪些节点/部分进行了处理
@@ -82,25 +84,50 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
                                             - "low" / Strictness_Level.IGNORE_FAILURE   匹配不完整，或者某些节点尝试过所有匹配到
                                                                                             的 backend 之后仍然无法写入
                                         默认是 "normal"
+            saved_node_name_format: <str> nodes/目录下节点文件/文件夹的命名方式。
+                                        基本结构为：  '{<part_0>}...{<part_1>}...'
+                                        其中 {} 内将根据 part 指定的类型进行自动填充。目前支持以下几种选项：
+                                            - "raw_name"        该节点对应位置的 name。
+                                            - "id"              该节点在当前内存中的 id。
+                                            - "hash_name"       该节点位置 name 的 hash 值。
+                                            - "count"           累加计算，表示是保存的第几个节点。
+                                        ！！注意：
+                                            "raw_name" 该选项在 v1.3.3 前被使用，但是由于其可能含有 : 和 / 等特殊符号，当以其作为文件夹名时，
+                                                可能会引发错误。因此对于 windows 用户，禁止使用该选项，对于 mac 和 linux 用户，同样也不建议使用该选项。
+                                            "id" 虽然具有唯一性，但是其值对于每次运行是随机的。
+                                            "hash_name" 有极低的可能会发生 hash 碰撞。
+                                        综合而言：
+                                            建议使用 "hash_name" 和 "count" 的组合。
+                                        默认值为：
+                                            '{count}_{hash_name}'
+            b_keep_identical_relations: <boolean> 是否保留不同节点之间的 id 相等关系。
+                                        具体而言，就是使用 value_parser.replace_identical_with_reference() 函数将具有相同 id 的多个节点，
+                                            替换为单个节点和其多个引用的形式。
+                                        对于 ndl 中存在大量具有相同 id 的重复节点的情况，使用该操作可以额外达到压缩的效果。
+                                        默认为 False
     """
     from kevin_toolbox.nested_dict_list.serializer.variable import SERIALIZER_BACKEND
 
-    #
+    # 检查参数
     traversal_mode = Traversal_Mode(traversal_mode)
     strictness_level = Strictness_Level(strictness_level)
     os.makedirs(output_dir, exist_ok=True)
     var = ndl.copy_(var=var, b_deepcopy=False)
+    if b_keep_identical_relations:
+        from kevin_toolbox.nested_dict_list import value_parser
+        var = value_parser.replace_identical_with_reference(var=var, flag="same", b_reverse=False)
     if settings is None:
         settings = [{"match_cond": "<level>-1", "backend": (":skip:simple", ":numpy:npy", ":torch:tensor", ":pickle")}]
+    snn_builder = Saved_Node_Name_Builder(format_=saved_node_name_format)
 
     # 构建 processed_s
     #     为了避免重复处理节点/结构，首先构建与 var 具有相似结构的 processed_s 来记录处理处理进度。
     #     对于 processed_s，其节点值为 True 时表示该节点已经被处理，当节点值为 False 或者 list/dict 类型时表示该节点或者节点下面的结构中仍然
     #       存在未处理的部分。
     #     对于中间节点，只有其下所有叶节点都未处理时才会被匹配。
-    processed_s = dict()
+    processed_s = ndl.copy_(var=var, b_deepcopy=False, b_keep_internal_references=False)
     for n, _ in ndl.get_nodes(var=var, level=-1, b_strict=True):
-        ndl.set_value(var=processed_s, name=n, value=False, b_force=True)
+        ndl.set_value(var=processed_s, name=n, value=False, b_force=False)
     # processed_s_bak 用于记录 var 的原始结构
     processed_s_bak = ndl.copy_(var=processed_s, b_deepcopy=True)
     if "_hook_for_debug" in kwargs:
@@ -126,27 +153,21 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
                 _process = _process_from_top_to_down
             else:
                 _process = _process_from_down_to_top
-            paras = dict(
-                var=var, processed_s=processed_s, match_cond=setting["match_cond"],
-                traversal_mode=t_mode, strictness_level=strictness_level
-            )
+            paras = dict(var=var, match_cond=setting["match_cond"], traversal_mode=t_mode)
         elif setting["match_cond"].startswith("<level>"):
             _process = _process_for_level
-            paras = dict(
-                var=var, processed_s=processed_s, processed_s_bak=processed_s_bak,
-                level=int(setting["match_cond"][7:]), strictness_level=strictness_level
-            )
+            paras = dict(var=var, processed_s_bak=processed_s_bak, level=int(setting["match_cond"][7:]))
         elif setting["match_cond"].startswith("<node>"):
             _process = _process_for_name
-            paras = dict(var=var, processed_s=processed_s, name=setting["match_cond"][6:],
-                         strictness_level=strictness_level)
+            paras = dict(var=var, name=setting["match_cond"][6:])
         else:
             raise ValueError(f'invalid match_cond: {setting["match_cond"]}')
         # 执行
         for i in backend_name_ls:
             # print(processed_s)
             # print(f'backend: {i}')
-            _process(backend=backend_s[i], **paras)
+            _process(backend=backend_s[i], strictness_level=strictness_level, processed_s=processed_s,
+                     snn_builder=snn_builder, **paras)
             if "_hook_for_debug" in kwargs:
                 kwargs["_hook_for_debug"]["processed"].append([i, ndl.copy_(var=processed_s, b_deepcopy=True)])
 
@@ -171,7 +192,8 @@ def write(var, output_dir, settings=None, traversal_mode=Traversal_Mode.BFS, b_p
     json_.write(content=var, file_path=os.path.join(output_dir, "var.json"), b_use_suggested_converter=True)
     # 保存处理结果（非必要）
     json_.write(content=dict(processed=processed_s, raw_structure=processed_s_bak, timestamp=time.time(),
-                             kt_version=kevin_toolbox.__version__),
+                             kt_version=kevin_toolbox.__version__,
+                             b_keep_identical_relations=b_keep_identical_relations),
                 file_path=os.path.join(output_dir, "record.json"), b_use_suggested_converter=True)
 
     # 打包成 .tar 文件
@@ -196,13 +218,13 @@ def _judge_processed_or_not(processed_s, name):
     return b_processed
 
 
-def _process_for_level(var, processed_s, processed_s_bak, level, backend, strictness_level):
+def _process_for_level(var, processed_s, processed_s_bak, level, backend, strictness_level, snn_builder):
     for name, _ in ndl.get_nodes(var=processed_s_bak, level=level, b_strict=True):
         _process_for_name(var=var, processed_s=processed_s, name=name, backend=backend,
-                          strictness_level=strictness_level)
+                          strictness_level=strictness_level, snn_builder=snn_builder)
 
 
-def _process_for_name(var, processed_s, name, backend, strictness_level):
+def _process_for_name(var, processed_s, name, backend, strictness_level, snn_builder):
     if _judge_processed_or_not(processed_s=processed_s, name=name) is True:
         # has been processed
         return
@@ -212,8 +234,9 @@ def _process_for_name(var, processed_s, name, backend, strictness_level):
         return
 
     # write by backend
+    snn_name = snn_builder(name=name, value=value)
     try:
-        res = backend.write(name=name, var=value)
+        res = backend.write(name=snn_name, var=value)
     except:
         assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
             f'An error occurred when node {name} was saved using the first matched backend {backend}'
@@ -222,7 +245,7 @@ def _process_for_name(var, processed_s, name, backend, strictness_level):
     ndl.set_value(var=var, name=name, value=res, b_force=False)
 
 
-def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_mode, strictness_level):
+def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_mode, strictness_level, snn_builder):
     def match_cond_(parent_type, idx, value):
         nonlocal match_cond, processed_s
 
@@ -237,8 +260,9 @@ def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_m
         nonlocal processed_s, backend, strictness_level
 
         # write by backend
+        snn_name = snn_builder(name=idx, value=value)
         try:
-            res = backend.write(name=idx, var=value)
+            res = backend.write(name=snn_name, var=value)
         except:
             assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
                 f'An error occurred when node {name} was saved using the first matched backend {backend}'
@@ -250,7 +274,7 @@ def _process_from_top_to_down(var, processed_s, match_cond, backend, traversal_m
                  b_use_name_as_idx=True, traversal_mode=traversal_mode, b_traverse_matched_element=False)
 
 
-def _process_from_down_to_top(var, processed_s, match_cond, backend, traversal_mode, strictness_level):
+def _process_from_down_to_top(var, processed_s, match_cond, backend, traversal_mode, strictness_level, snn_builder):
     processed_s_raw, processed_s = processed_s, ndl.copy_(var=processed_s, b_deepcopy=True)
 
     def match_cond_(parent_type, idx, value):
@@ -268,8 +292,9 @@ def _process_from_down_to_top(var, processed_s, match_cond, backend, traversal_m
         nonlocal processed_s, backend, processed_s_raw, strictness_level
 
         # write by backend
+        snn_name = snn_builder(name=idx, value=value)
         try:
-            res = backend.write(name=idx, var=value)
+            res = backend.write(name=snn_name, var=value)
         except:
             assert strictness_level in (Strictness_Level.IGNORE_FAILURE, Strictness_Level.COMPATIBLE), \
                 f'An error occurred when node {name} was saved using the first matched backend {backend}'
@@ -305,7 +330,7 @@ if __name__ == '__main__':
         {"match_cond": lambda _, __, value: not isinstance(value, (list, dict)),
          "backend": (":skip:simple",)},
     ]
-    write(var=var_, output_dir=os.path.join(os.path.dirname(__file__), "temp3"), traversal_mode="bfs",
+    write(var=var_, output_dir=os.path.join(os.path.dirname(__file__), "temp"), traversal_mode="bfs",
           b_pack_into_tar=True, settings=settings_, _hook_for_debug=_hook_for_debug)
 
     for bk_name, p in _hook_for_debug["processed"]:

kevin_toolbox/nested_dict_list/value_parser/__init__.py

@@ -2,3 +2,4 @@ from .cal_relation_between_references import cal_relation_between_references
 from .eval_references import eval_references
 from .parse_references import parse_references
 from .parse_and_eval_references import parse_and_eval_references
+from .replace_identical_with_reference import replace_identical_with_reference

kevin_toolbox/nested_dict_list/value_parser/replace_identical_with_reference.py

@@ -0,0 +1,127 @@
+from collections import defaultdict
+from kevin_toolbox.nested_dict_list import get_nodes, get_value, set_value
+from kevin_toolbox.nested_dict_list import value_parser
+
+
+def replace_identical_with_reference(var, flag="same", match_cond=None, b_reverse=False):
+    """
+        将具有相同 id 的多个节点，替换为单个节点和其多个引用的形式
+            一般用于去除冗余部分，压缩 ndl 的结构
+
+        参数：
+            var:
+            flag:                   <str> 引用标记头，表示该节点应该替换为指定节点的内容
+                                        默认为 "same"
+                                        注意，在正向过程中，如果遇到本身就是以该 flag 开头的字符串，会自动在前添加多一个 flag 以示区分，
+                                            然后在逆向过程中，遇到两个 flag 标记开头的字符串将删除一个，然后跳过不处理。
+            match_cond:             <func> 仅对匹配上（返回True视为匹配上）的节点进行处理
+                                        函数类型为 def(name, value)
+                                        其中：
+                                            name            该节点在结构体中的位置
+                                            value           节点的值
+                                        默认不对 int、float、bool、str、None 等类型进行处理
+            b_reverse:              <boolean> 是否进行逆向操作
+    """
+    if match_cond is None:
+        match_cond = lambda name, value: not isinstance(value, (int, float, bool, str, type(None)))
+    assert callable(match_cond)
+
+    if b_reverse:
+        return _reverse(var, flag)
+    else:
+        return _forward(var, flag, match_cond)
+
+
+def _forward(var, flag, match_cond):
+    id_to_height_s = defaultdict(set)
+    id_to_name_s = defaultdict(set)
+    height = 1
+    while True:
+        node_ls = get_nodes(var=var, level=-height, b_strict=True)
+        if not node_ls:
+            break
+        for name, value in node_ls:
+            if not match_cond(name, value):
+                continue
+            id_to_name_s[id(value)].add(name)
+            id_to_height_s[id(value)].add(height)
+        height += 1
+
+    #
+    for k, v in list(id_to_height_s.items()):
+        # 仅保留有多个节点对应的 id
+        if len(id_to_name_s[k]) <= 1:
+            id_to_height_s.pop(k)
+            id_to_name_s.pop(k)
+            continue
+        # 具有相同 id 的节点所处的高度应该相同
+        assert len(v) == 1, f'nodes {id_to_name_s[k]} have different heights: {v}'
+    # 按高度排序
+    id_vs_height = sorted([(k, v.pop()) for k, v in id_to_height_s.items()], key=lambda x: x[1], reverse=True)
+
+    # 从高到低，依次将具有相同 id 的节点替换为 单个节点和多个引用 的形式
+    temp = []
+    processed_name_set = set()
+    for k, _ in id_vs_height:
+        # 找出父节点仍然未被处理的节点（亦即仍然能够访问到的节点）
+        unprocessed_name_set = {n for n in id_to_name_s[k] if id(get_value(var=var, name=n, default=temp)) == k}
+        if len(unprocessed_name_set) <= 1:
+            continue
+        # 任选其一进行保留，其余改为引用
+        keep_name = unprocessed_name_set.pop()
+        for name in unprocessed_name_set:
+            try:
+                var = set_value(var=var, name=name, value=f'<{flag}>{{{keep_name}}}', b_force=False)
+            except:
+                breakpoint()
+        processed_name_set.update(unprocessed_name_set)
+
+    # 将叶节点中，未被处理过，且是 str，且以 flag 开头的字符串，添加多一个 flag，以示区分
+    for name, value in get_nodes(var=var, level=-1, b_strict=True):
+        if name not in processed_name_set and isinstance(value, str) and value.startswith(f'<{flag}>'):
+            var = set_value(var=var, name=name, value=f'<{flag}>' + value, b_force=False)
+
+    return var
+
+
+class _My_Str:
+    def __init__(self, s):
+        self.s = s
+
+
+def _reverse(var, flag):
+    # 找出叶节点中，带有2个以上 flag 标记的字符串，删除其中一个标记，并使用 _My_Str 包裹，以便与普通引用节点区分
+    for name, value in get_nodes(var=var, level=-1, b_strict=True):
+        if isinstance(value, str) and value.startswith(f'<{flag}><{flag}>'):
+            var = set_value(var=var, name=name, value=_My_Str(value[len(flag) + 2:]), b_force=False)
+    # 解释引用
+    var, _ = value_parser.parse_and_eval_references(var=var, flag=flag)
+    # 解除 _My_Str 包裹
+    for name, value in get_nodes(var=var, level=-1, b_strict=True):
+        if isinstance(value, _My_Str):
+            var = set_value(var=var, name=name, value=value.s, b_force=False)
+    return var
+
+
+if __name__ == '__main__':
+    import numpy as np
+    from kevin_toolbox.nested_dict_list import copy_
+
+    a = np.array([1, 2, 3])
+    b = np.ones((2, 3))
+    c = [a, b]
+    d = {"a": a, "b": b}
+    e = {"c1": c, "c2": c}
+    x = [e, a, d, c, "<same>{@1}", "<same><same>{@1}"]
+
+    print(x)
+
+    y = replace_identical_with_reference(var=copy_(x, b_deepcopy=True), flag="same")
+    print(y)
+
+    x1 = replace_identical_with_reference(var=y, flag="same", b_reverse=True)
+    print(x1)
+
+    from kevin_toolbox.patches.for_test import check_consistency
+
+    check_consistency(x, x1)

kevin_toolbox/patches/for_numpy/random/get_rng.py

@@ -8,11 +8,57 @@ class DEFAULT_RNG:
 for name in np.random.__all__:
     setattr(DEFAULT_RNG, name, getattr(np.random, name))
 
+func_map_s = {
+    'rand': lambda rng, *arg, **kwargs: rng.random(size=arg, **kwargs),
+    'randint': 'integers',
+    'randn': lambda rng, *arg, **kwargs: rng.normal(size=arg, **kwargs),
+    'random_integers': 'integers',
+    'random_sample': 'random',
+    'ranf': 'random'
+}
+
+
+class My_RNG:
+    def __init__(self, rng):
+        self._rng = rng
+
+        for name in np.random.__all__:
+            setattr(DEFAULT_RNG, name, getattr(np.random, name))
+
+    # self.key
+    def __getattr__(self, key):
+        if "_rng" not in self.__dict__:
+            # _rng 未被设置，未完成初始化。
+            return super().__getattr__(key)
+        else:
+            res = getattr(self._rng, key, None)
+            if res is None and key in func_map_s:
+                if callable(func_map_s[key]):
+                    res = lambda *arg, **kwargs: func_map_s[key](self._rng, *arg, **kwargs)
+                else:
+                    res = getattr(self._rng, func_map_s[key], None)
+            #
+            if res is None:
+                raise AttributeError(f"attribute '{key}' not found in {type(self)}")
+            else:
+                return res
+
 
 def get_rng(seed=None, rng=None, **kwargs):
     if seed is not None:
-        rng = np.random.default_rng(seed=seed)
+        # 注意，随机生成器相较于 numpy.random 有部分属性缺失：
+        #   ['get_state', 'rand', 'randint', 'randn', 'random_integers', 'random_sample', 'ranf', 'sample', 'seed',
+        #   'set_state', 'Generator', 'RandomState', 'SeedSequence', 'MT19937', 'Philox', 'PCG64', 'PCG64DXSM',
+        #   'SFC64', 'default_rng', 'BitGenerator']
+        rng = My_RNG(rng=np.random.default_rng(seed=seed))
     if rng is not None:
         return rng
     else:
         return DEFAULT_RNG
+
+
+if __name__ == '__main__':
+    a = get_rng(seed=2)
+
+    # 尝试访问随机生成器中缺失的部分方法
+    print(a.randn(2, 3))

kevin_toolbox_dev-1.3.4.dist-info/METADATA

@@ -0,0 +1,67 @@
+Metadata-Version: 2.1
+Name: kevin-toolbox-dev
+Version: 1.3.4
+Summary: 一个常用的工具代码包集合
+Home-page: https://github.com/cantbeblank96/kevin_toolbox
+Download-URL: https://github.com/username/your-package/archive/refs/tags/v1.0.0.tar.gz
+Author: kevin hsu
+Author-email: xukaiming1996@163.com
+License: MIT
+Keywords: mathematics,pytorch,numpy,machine-learning,algorithm
+Platform: UNKNOWN
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: torch (>=1.2.0)
+Requires-Dist: numpy (>=1.19.0)
+Provides-Extra: plot
+Requires-Dist: matplotlib (>=3.0) ; extra == 'plot'
+Provides-Extra: rest
+Requires-Dist: pytest (>=6.2.5) ; extra == 'rest'
+Requires-Dist: line-profiler (>=3.5) ; extra == 'rest'
+
+# kevin_toolbox
+
+一个通用的工具代码包集合
+
+
+
+环境要求
+
+```shell
+numpy>=1.19
+pytorch>=1.2
+```
+
+安装方法：
+
+```shell
+pip install kevin-toolbox  --no-dependencies
+```
+
+
+
+[项目地址 Repo](https://github.com/cantbeblank96/kevin_toolbox)
+
+[使用指南 User_Guide](./notes/User_Guide.md)
+
+[免责声明 Disclaimer](./notes/Disclaimer.md)
+
+[版本更新记录](./notes/Release_Record.md)：
+
+- v 1.3.4 （2024-04-06）【bug fix】【new feature】
+  - nested_dict_list
+    - 【new feature】add replace_identical_with_reference() to value_parser，新增该函数用于将具有相同 id 的多个节点，替换为单个节点和其多个引用的形式。一般用于去除冗余部分，压缩 ndl 的结构。
+    - 【bug fix】【new feature】fix bug in write()，添加了 saved_node_name_format 参数控制 nodes 下文件名的生成。
+      - bug：在 v1.3.3 前直接使用原始的 node_name 来作为 nodes/ 目录下的文件名，这导致当 node_name 中带有特殊字符时，比如 "/"（在linux下） 和 ":"（在windows下），将会导致保存失败。
+      - fix：使用 saved_node_name_format 指定生成文件名的方式，默认方式 '{count}_{hash_name}' 可以避免出现特殊字符。
+    - 【bug fix】fix bug in write()
+      - bug：在 v1.3.3 前 processed_s 通过 ndl.set_value() 来逐个节点构建，但是由于根据节点名创建的结果可能和原结构存在差异（详见 ndl.set_value() 中b_force参数的介绍），因此导致 processed_s 和 var 结构不一致，导致出错。
+      - fix：使用 ndl.copy_() 来创建结构与 var 一致的 processed_s。
+    - 【new feature】add b_keep_identical_relations to write()，增加该参数用于决定是否保留不同节点之间的 id 相等关系。
+    - 添加了对应的测试用例。
+
+
+

{kevin_toolbox_dev-1.3.2.dist-info → kevin_toolbox_dev-1.3.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-kevin_toolbox/__init__.py,sha256=a_ziVpZsIzbKH9fSWA7nVVgLggqfgGyTsMeZPce6VXI,410
+kevin_toolbox/__init__.py,sha256=ryG5DAc-XozLnxNQjuTKTRynJFnwdQHKBZ2vaN1upx4,410
 kevin_toolbox/computer_science/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kevin_toolbox/computer_science/algorithm/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 kevin_toolbox/computer_science/algorithm/cache_manager/__init__.py,sha256=p2hddkZ1HfYF9-m2Hx-o9IotwQHd4QwDCePy2ADpTDA,41
@@ -66,29 +66,29 @@ kevin_toolbox/data_flow/core/cache/cache_manager_for_iterator.py,sha256=DqTNiZbn
 kevin_toolbox/data_flow/core/cache/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kevin_toolbox/data_flow/core/cache/test/test_cache_manager_for_iterator.py,sha256=OjJEEmvWtsGCPFXp2NQP2lpUGFXe9Zr2EpQSfRTDfRI,1147
 kevin_toolbox/data_flow/core/reader/__init__.py,sha256=i2118MlsNSJHG5u6ZuPcN3NW2LZmYUtDO7dNEdnOncI,146
-kevin_toolbox/data_flow/core/reader/file_iterative_reader.py,sha256=Xl-OMOR8YDgKK3Cfz2rR8exOG4Ce6QpwsZjx3BB2oC0,8827
-kevin_toolbox/data_flow/core/reader/unified_reader.py,sha256=k1-I4rj5bjPix-_jni7ICnL0JWjmtHsD_qpqbCH0p6k,2423
+kevin_toolbox/data_flow/core/reader/file_iterative_reader.py,sha256=l6UMYnvWwqQm3zEA2g5fDA1oJeKLlbRYnzrRJvuCxBg,10158
+kevin_toolbox/data_flow/core/reader/unified_reader.py,sha256=l6JxPoDUOdx2ZIPX2WLXbGU3VZtTd1AeHn5q6L8GWAI,2453
 kevin_toolbox/data_flow/core/reader/unified_reader_base.py,sha256=4gIADdV8UKpt2yD8dZjQsXFcF75nJ83ooIae3D7bw2s,11783
 kevin_toolbox/data_flow/file/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 kevin_toolbox/data_flow/file/json_/__init__.py,sha256=VAt8COS2tO3PJRuhSc43i35fEOlArFM_YahdTmEBaHE,85
-kevin_toolbox/data_flow/file/json_/read_json.py,sha256=XzygeWCxDeO95Q0Wl2Z8lXgfVUCdEyLMph4NpN-nh44,1784
-kevin_toolbox/data_flow/file/json_/write_json.py,sha256=F7wkolZkggAh6H8YExcVxmy-109BliYTtNYmr0JI_-E,2355
-kevin_toolbox/data_flow/file/json_/converter/__init__.py,sha256=jQoEaS57dl33O0XguHBEilkVmn0q-LiStKWSiqXZNLc,369
+kevin_toolbox/data_flow/file/json_/read_json.py,sha256=S-rlY-spIAJVyB-zPQOmKUDdtNAgedSmXHjkY_TNGUE,1808
+kevin_toolbox/data_flow/file/json_/write_json.py,sha256=mWaxePr_QzfyeCb0hAy4xTKOGX7q0eFjep0jDqOqIgw,2379
+kevin_toolbox/data_flow/file/json_/converter/__init__.py,sha256=oQMgAgzELLq_f4LIIfz5E6l_E7g4lFsXqfmnJ3tPZTY,401
 kevin_toolbox/data_flow/file/json_/converter/convert_dict_key_to_number.py,sha256=SuSZj_HCqKZutHAJ5AttABnGBRZplPGQhMxJBt2Wlgc,559
 kevin_toolbox/data_flow/file/json_/converter/convert_ndarray_to_list.py,sha256=GALpC1MFJ4aMzs0FZIfJScYznfCP-gmhPeM8sWXGSWg,391
 kevin_toolbox/data_flow/file/json_/converter/escape_non_str_dict_key.py,sha256=83qwH_-v4A5UvSxdctE1TBdxw8PFewoctKEX5nECNG8,809
-kevin_toolbox/data_flow/file/json_/converter/escape_tuple.py,sha256=2WnfkV3bJv2-4L1JoOh9VposRpWVqJTyapKNTRmQxyE,605
+kevin_toolbox/data_flow/file/json_/converter/escape_tuple_and_set.py,sha256=u4i8bG0Hgszh-3lEsmBmP4VWVOfJBfgXRyGxm1L3PvA,763
 kevin_toolbox/data_flow/file/json_/converter/integrate.py,sha256=uEnRMqt4hTCVHL8p3tH7jir7mYSo3vGhZ4WS_Qi4Rms,377
 kevin_toolbox/data_flow/file/json_/converter/unescape_non_str_dict_key.py,sha256=KDBjxJHn8q9dqM-kbtbUunjpc-6shZypYSyihVtkzgI,808
-kevin_toolbox/data_flow/file/json_/converter/unescape_tuple.py,sha256=ByoCmB28RciFR9JxKw-octRYl_jdYkBJ4o_nNvPizUY,661
+kevin_toolbox/data_flow/file/json_/converter/unescape_tuple_and_set.py,sha256=3vCFr6CtNZxiVR1nRVDcuodIzHRfrmDsvBPCWtnWtBs,814
 kevin_toolbox/data_flow/file/kevin_notation/__init__.py,sha256=g9UUF9nJrOMc0ndCwVROQLsux4Or2yVMJMdhK5P9nRc,259
 kevin_toolbox/data_flow/file/kevin_notation/converter.py,sha256=5k_Yxw-fBKEkxFG0bnl1fRsz06MlUS-4f3gZ--bmDs8,3621
-kevin_toolbox/data_flow/file/kevin_notation/kevin_notation_reader.py,sha256=Am1lJDkI-9gOLLqmwiSgWvggLZB4g4dd9y_KyIPipek,8149
+kevin_toolbox/data_flow/file/kevin_notation/kevin_notation_reader.py,sha256=iN6nV8mMbifTbECNmjc-G2pzpxivhks5kCXGVdvS9fQ,8297
 kevin_toolbox/data_flow/file/kevin_notation/kevin_notation_writer.py,sha256=dyzkufi_SM-uU7cyI3yLFd8U4GnyRR5pyS88op6mhRY,16331
-kevin_toolbox/data_flow/file/kevin_notation/read.py,sha256=cljU8Rv-yyiH2OYwycPVBwUwc1X8njgRjLCfbcQvNWw,447
+kevin_toolbox/data_flow/file/kevin_notation/read.py,sha256=w0RE0WwTmWycEozJVshAiE0gMBxproBRwBEMLS6tB6c,544
 kevin_toolbox/data_flow/file/kevin_notation/write.py,sha256=gPabz_h2mtXqCTyBVzip_QSb6L4tsrNSqibFzuqsIv8,556
 kevin_toolbox/data_flow/file/kevin_notation/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kevin_toolbox/data_flow/file/kevin_notation/test/test_kevin_notation.py,sha256=MxDeJuyjmSeAyK7tuL1Axc2IQHApNiFoC_MkSFmsIP4,6838
+kevin_toolbox/data_flow/file/kevin_notation/test/test_kevin_notation.py,sha256=ab402r5LGyNz4XW2SnjvicNtQqBAAHZTaGfYMNdMExI,7345
 kevin_toolbox/data_flow/file/kevin_notation/test/test_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kevin_toolbox/data_flow/file/kevin_notation/test/test_data/data_0.py,sha256=CKRb86O3JV9lkGrMtyJzEH041o0xABfT32Zo4GQ5Qis,324
 kevin_toolbox/data_flow/file/kevin_notation/test/test_data/data_1.py,sha256=Xs8oFJqwi0uPOJewulij7DY0iMEp6dWBMiiDIwPlm4s,176
@@ -96,7 +96,7 @@ kevin_toolbox/data_flow/file/kevin_notation/test/test_data/data_all.py,sha256=cv
 kevin_toolbox/data_flow/file/markdown/__init__.py,sha256=iTZTBvcEUehBdcWxzFQEW4iEcXbAQkdkEmENiGtBjqs,125
 kevin_toolbox/data_flow/file/markdown/generate_link.py,sha256=9okSyCFIDQW5T35a6-epVyoCkCL1vFH5215P5MRXfYk,304
 kevin_toolbox/data_flow/file/markdown/generate_list.py,sha256=Gv5BcqWE4M4w8ADN8NX5LyD9DxILXTQtJvcazi_NuyE,1006
-kevin_toolbox/data_flow/file/markdown/generate_table.py,sha256=Xh9IRqLaeVoXDoFF1u6temMgf4MRng6pawJiUDknEdg,7209
+kevin_toolbox/data_flow/file/markdown/generate_table.py,sha256=Ct2gfnciBv0GGZHOKlIHyTlE7KqXsL0L5vBRCrQnOpI,7209
 kevin_toolbox/developing/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 kevin_toolbox/developing/general_matrix_multiplication.py,sha256=Ie9c8mYBYR-Bg7CjU4L1dsOxXsxnx1jz-rA7_ez7vjg,2089
 kevin_toolbox/developing/test.py,sha256=6Y23SY3FJVrvZmiiXKNPKv84lhVRW-XyjNeecj9lLYA,241
@@ -208,12 +208,12 @@ kevin_toolbox/math/transform/dct/dct_calculator.py,sha256=4goxXUMvYF3gkDH-lLFwX8
 kevin_toolbox/math/transform/dct/generate_dct_trans_matrix.py,sha256=za6eaq0QoxZSiCj-hZ2dYjvpxMXcNABAHbVig811ER4,4305
 kevin_toolbox/math/transform/scaling_and_shift/__init__.py,sha256=UXJcL7hyz1ag5rzz5eTvua36i76P4L2M94hhfV436pI,29
 kevin_toolbox/math/transform/scaling_and_shift/scaling.py,sha256=HypsgC1KYsUmti6I0nEYzQpL1_343hXtfppDJ3ucje0,1280
-kevin_toolbox/math/utils/__init__.py,sha256=grmWqvpoC47t3OeB-giB4KHAZbuRkN26u4ambMSBgvQ,289
+kevin_toolbox/math/utils/__init__.py,sha256=ceXYQaTpFeQD2QPutLz__bJlq3ECNbHUCnoLGUPHKok,289
 kevin_toolbox/math/utils/convert_dtype.py,sha256=WW8KE5NlkjZ76BM48_cRClm41lZRybe6xnBBc4qEY70,1943
 kevin_toolbox/math/utils/get_crop_by_box.py,sha256=oiM14a0jsyxOnVn5n0pU0chdNRc5VOw3_KbS7CArdfw,1620
 kevin_toolbox/math/utils/get_function_table_for_array_and_tensor.py,sha256=hZrXb3427SMyMsUBcz5DWF2FhV-bSWzBqx5cvrvk_8w,1553
 kevin_toolbox/math/utils/set_crop_by_box.py,sha256=NzW7M26Av097RchLUAhaO84ETLV121uxsNot_U6otLw,1775
-kevin_toolbox/math/utils/spilt_integer_most_evenly.py,sha256=H0H7FixHsYtPVElIfVpuH7s7kaQ0JQI0r0Eu_HCvfwY,345
+kevin_toolbox/math/utils/split_integer_most_evenly.py,sha256=6hTWKXYx3YlotNMaw8cmecWO0A4C_Ny2kxgN9asiN9A,345
 kevin_toolbox/nested_dict_list/__init__.py,sha256=ALcn1tYdBdDWUHyIQj588UfHgrAwbUcZu_bN2v-cEAc,333
 kevin_toolbox/nested_dict_list/copy_.py,sha256=MvzNRKm8htYpMe7Td1ao2-ZoaYVC_iNTG7O2SBVrJKE,6144
 kevin_toolbox/nested_dict_list/count_leaf_node_nums.py,sha256=l67u47EvO1inoGinUqH6RZ7cHXwN0VcBQPUvSheqAvA,614
@@ -227,11 +227,12 @@ kevin_toolbox/nested_dict_list/name_handler/__init__.py,sha256=P_pWq78oN6NdvWg2h
 kevin_toolbox/nested_dict_list/name_handler/build_name.py,sha256=VPWyjE8i8l-4Zm4tkD06Ie4J2NCsmI32ecOxZQqqmok,989
 kevin_toolbox/nested_dict_list/name_handler/escape_node.py,sha256=niT9MxmsyrSZYhKXlWzdoKXVYhWRCR-kmQBkZopznpA,1163
 kevin_toolbox/nested_dict_list/name_handler/parse_name.py,sha256=vUlAXPocpVSxtb3EnRi7U5K40Tz9plFG-_sbwLfYiy4,2280
-kevin_toolbox/nested_dict_list/serializer/__init__.py,sha256=xLDfzSZIDNBssHArUEK84gF6WZFR64qJy0iOpM8LbP8,92
+kevin_toolbox/nested_dict_list/serializer/__init__.py,sha256=79dd9l-mNz0bycFKjNm7YsfWPR-JsVx9NoG_Ofqy-HQ,153
 kevin_toolbox/nested_dict_list/serializer/enum_variable.py,sha256=RWPydtXI4adOJYGo_k5CWHSL0Odzj_bsahb24p1ranY,847
-kevin_toolbox/nested_dict_list/serializer/read.py,sha256=8196AZW3y1eZNguw5fDi5gQmOtpJ0MdqhRNHZ5EWYHI,2577
+kevin_toolbox/nested_dict_list/serializer/read.py,sha256=yessvu7msmP2kV3ZhOTVmI2ENI-R1-TdhVgZdS8eWDk,2843
+kevin_toolbox/nested_dict_list/serializer/saved_node_name_builder.py,sha256=qsD-rmDmVaKZP4owN3Wm3QY2Ksi71XlYETqw4VmIsSU,1011
 kevin_toolbox/nested_dict_list/serializer/variable.py,sha256=ZywG6obipRBCGY1cY42gdvsuWk8GLZXr6eCYcW7ZJ9c,392
-kevin_toolbox/nested_dict_list/serializer/write.py,sha256=sQtryi7NbVWynOMWpKKkqxoTf9dxSGE4yC4ReIAxT8U,18145
+kevin_toolbox/nested_dict_list/serializer/write.py,sha256=kP_sM-NtI3vJT7KwBZsz_ReZprfVefzUFplW7Kh0zVQ,21024
 kevin_toolbox/nested_dict_list/serializer/backends/__init__.py,sha256=8g7y-L3cmctxao616dVkGiot00FJzKNmNl_69V2bSmE,39
 kevin_toolbox/nested_dict_list/serializer/backends/_json_.py,sha256=oJXIc28yjxsD9ZJuw120pVHTVsTzCdaXEhVUSQeydq4,2145
 kevin_toolbox/nested_dict_list/serializer/backends/_ndl.py,sha256=QMF4DFAnt1sp35atds6t44nfCYuIOeGgW1-SPfJq6KM,1652
@@ -243,11 +244,12 @@ kevin_toolbox/nested_dict_list/serializer/backends/_skip_simple.py,sha256=dS9kKh
 kevin_toolbox/nested_dict_list/serializer/backends/_torch_all.py,sha256=aw9M1Hep65lkvb8_QU3VkecE6q3l4UqXansh4lnMv7s,1282
 kevin_toolbox/nested_dict_list/serializer/backends/_torch_tensor.py,sha256=Wiimzc0XGxFRYJiipzVnUd27scaJZZTOYp5OxYo3cKg,1353
 kevin_toolbox/nested_dict_list/serializer/backends/backend_base.py,sha256=SZpLRhdSIykHZ_Ds3HX96lKLfXCyKzMQ_lx139XXBtc,1741
-kevin_toolbox/nested_dict_list/value_parser/__init__.py,sha256=KTXAUaounOQ8oWSX-XzJMIFoopgaodBAm9RmQGMBgP8,234
+kevin_toolbox/nested_dict_list/value_parser/__init__.py,sha256=MgbpkiXzVuA_i3VZ4VDNrQZfzpUZN3uaHrgOt_wq-4E,313
 kevin_toolbox/nested_dict_list/value_parser/cal_relation_between_references.py,sha256=m3P7q5-pbbWqtjOjJUT-499q4mCyjtFEtFpGgovShSg,3119
 kevin_toolbox/nested_dict_list/value_parser/eval_references.py,sha256=YQyOm3awKVRusXxSNObjJ2yPf0oE4gleobOn_RN_nzU,2301
 kevin_toolbox/nested_dict_list/value_parser/parse_and_eval_references.py,sha256=RQEDFFNAhQQcX9H8curwja-pI2gKZlVx4M2qeneBOhA,2370
 kevin_toolbox/nested_dict_list/value_parser/parse_references.py,sha256=G470xNzrRpYlS5To8R5yV0M6nX4iE5LLMp_eV49bh3Y,2116
+kevin_toolbox/nested_dict_list/value_parser/replace_identical_with_reference.py,sha256=4nd5q3qTi4sTfOqHF0-HyLMDS9a5x8wgwETfpDI8jh8,5419
 kevin_toolbox/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kevin_toolbox/patches/for_logging/__init__.py,sha256=xymF6mjwY4Cin7CoEwanFY5ZVk8oY0pDLqjZAGa7_Rg,39
 kevin_toolbox/patches/for_logging/build_logger.py,sha256=0UoRMKaERd8wlHGTiNR3dbLQRAtXcb0QZuGkX2oFgGo,3071
@@ -263,7 +265,7 @@ kevin_toolbox/patches/for_numpy/linalg/entropy.py,sha256=PSdwkzySvWF4h4Xi27w2kvL
 kevin_toolbox/patches/for_numpy/linalg/normalize.py,sha256=7qstt__rwUkk3jRJOQoneBp9YdfhYQtWfh6PZoWbvaA,625
 kevin_toolbox/patches/for_numpy/linalg/softmax.py,sha256=M4a3jyKZBMFdiC_sPqO7AVts6AnEju8WbLc_GNSEtQ4,2095
 kevin_toolbox/patches/for_numpy/random/__init__.py,sha256=f1nOm2jr-4x5ZW80S5VzvIAtag0aQTGiYVzxgGG1Oq8,149
-kevin_toolbox/patches/for_numpy/random/get_rng.py,sha256=BZ8b8ILUGGs6sZLcECN4faiYF6xyys5aYaL-VaJUAGk,337
+kevin_toolbox/patches/for_numpy/random/get_rng.py,sha256=QblrMKg4OFVy-C4A6rQ-zq26uDKzhMifKTFUlyW3Ksw,1999
 kevin_toolbox/patches/for_numpy/random/truncated_multivariate_normal.py,sha256=ZOaFZrzFI-ty7oRwUXxLx7pJMLTPPAAN28_AM6LeUDk,5859
 kevin_toolbox/patches/for_numpy/random/truncated_normal.py,sha256=uej3SQnLu0HsBTD47Yrgft1NpnsoEIOm925H9ipS8UQ,3726
 kevin_toolbox/patches/for_numpy/random/variable.py,sha256=Sam-QZgkx9_IvHzZhpUrXld0izP5MZfMBiM8qRWcB6M,231
@@ -302,7 +304,7 @@ kevin_toolbox/patches/for_torch/math/get_y_at_x.py,sha256=bfoVcasZ_tMdhR_1Me0Jli
 kevin_toolbox/patches/for_torch/math/my_around.py,sha256=ptpU3ids50gwf663EpHbw7raj9tNrDGBFZ5t_uMNH14,1378
 kevin_toolbox/patches/for_torch/nn/__init__.py,sha256=aJs3RMqRzQmd8KKDmQW9FxwCqS5yfPqEdg-m0PwlQro,39
 kevin_toolbox/patches/for_torch/nn/lambda_layer.py,sha256=KUuLiX_Dr4bvRmpAaCW5QTDWDcnMPRnw0jg4NNXTFhM,223
-kevin_toolbox_dev-1.3.2.dist-info/METADATA,sha256=VYS2lZKUfiljCvzTlyCEm38wPd_Ai-46l8555JR1jM4,5059
-kevin_toolbox_dev-1.3.2.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-kevin_toolbox_dev-1.3.2.dist-info/top_level.txt,sha256=S5TeRGF-PwlhsaUEPTI-f2vWrpLmh3axpyI6v-Fi75o,14
-kevin_toolbox_dev-1.3.2.dist-info/RECORD,,
+kevin_toolbox_dev-1.3.4.dist-info/METADATA,sha256=NANoXo3yczTlDCHbDo38F_QLlPf0LBYWIwMS4fRHipY,2710
+kevin_toolbox_dev-1.3.4.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+kevin_toolbox_dev-1.3.4.dist-info/top_level.txt,sha256=S5TeRGF-PwlhsaUEPTI-f2vWrpLmh3axpyI6v-Fi75o,14
+kevin_toolbox_dev-1.3.4.dist-info/RECORD,,

kevin_toolbox/data_flow/file/json_/converter/escape_tuple.py

@@ -1,20 +0,0 @@
-def escape_tuple(x):
-    """
-        将 tuple 进行转义
-            转义：     x ==> f"<eval>{x}"
-            反转义：   f"<eval>{x}" ==> x
-
-        为什么要进行转义？
-            由于 json 中会将 tuple 作为 list 进行保存，因此在保存过程中会丢失相应信息。
-    """
-    if isinstance(x, tuple) or (isinstance(x, (str,)) and x.startswith("<eval>")):
-        return f'<eval>{x}'
-    else:
-        return x
-
-
-if __name__ == '__main__':
-    print(escape_tuple((1, 2, "\'1\'")))
-    # <eval>(1, 2, "'1'")
-    print(escape_tuple("<eval>233"))
-    # <eval><eval>233

kevin_toolbox_dev-1.3.2.dist-info/METADATA

@@ -1,96 +0,0 @@
-Metadata-Version: 2.1
-Name: kevin-toolbox-dev
-Version: 1.3.2
-Summary: 一个常用的工具代码包集合
-Home-page: https://github.com/cantbeblank96/kevin_toolbox
-Download-URL: https://github.com/username/your-package/archive/refs/tags/v1.0.0.tar.gz
-Author: kevin hsu
-Author-email: xukaiming1996@163.com
-License: MIT
-Keywords: mathematics,pytorch,numpy,machine-learning,algorithm
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.6
-Description-Content-Type: text/markdown
-Requires-Dist: torch (>=1.2.0)
-Requires-Dist: numpy (>=1.19.0)
-Provides-Extra: plot
-Requires-Dist: matplotlib (>=3.0) ; extra == 'plot'
-Provides-Extra: rest
-Requires-Dist: pytest (>=6.2.5) ; extra == 'rest'
-Requires-Dist: line-profiler (>=3.5) ; extra == 'rest'
-
-# kevin_toolbox
-
-一个通用的工具代码包集合
-
-
-
-环境要求
-
-```shell
-numpy>=1.19
-pytorch>=1.2
-```
-
-安装方法：
-
-```shell
-pip install kevin-toolbox  --no-dependencies
-```
-
-
-
-[项目地址 Repo](https://github.com/cantbeblank96/kevin_toolbox)
-
-[使用指南 User_Guide](./notes/User_Guide.md)
-
-[免责声明 Disclaimer](./notes/Disclaimer.md)
-
-[版本更新记录](./notes/Release_Record.md)：
-
-- v 1.3.2 （2024-03-05）【bug fix】【new feature】
-  - patches
-    - for_optuna.serialize
-      - 【bug fix】fix bug in for_study.dump()，使用 try except 来捕抓并跳过使用 getattr(study, k) 读取 study 中属性时产生的错误。（比如单变量优化时的best_trials参数） 
-      - 【bug fix】fix bug in for_study.dump()，避免意外修改 study 中的属性。
-        - 添加了对应的测试用例。
-    - for_matplotlib
-      - 【new feature】add generate_color_list()，用于生成指定数量的颜色列表，支持对指定颜色的排除。
-    - for_numpy
-      - 【new feature】add linalg.entropy()，用于计算分布的熵。
-      - 【new feature】add random，添加了用于随机生成的模块，其中包含：
-        - get_rng()，获取默认随机生成器or根据指定的seed构建随机生成器。
-        - truncated_normal()，从截断的高斯分布中进行随机采样。
-        - truncated_multivariate_normal()，从截断的多维高斯分布中进行随机采样。
-        - 添加了测试用例。
-  - nested_dict_list
-    - 【bug fix】fix bug in get_nodes()，修复了遍历非叶节点时，当节点下的叶节点均不存在时会异常跳过该节点的问题。
-      - 添加了对应的测试用例。
-    - 【new feature】modify set_default()，修改后将默认返回对应name的值，而不是返回整体的var，从而与python dict的setdefault函数的行为对齐。特别地，也支持通过设置b_return_var参数来获取 var。
-      - 修改了对应的测试用例。
-  - computer_science.algorithm
-    - registration.Registry
-      - 【new feature】改进 self.collect_from_paths() 函数。
-        - 增加一个检查用于避免待搜索路径包含调用该函数的文件。如果包含，则报错，同时提示这样会导致  collect_from_paths() 函数被无限递归调用，从而引起死循环。并建议将该函数的调用位置放置在待搜索路径外，或者使用 ignore_s 将其进行屏蔽。 
-        - 添加了测试用例。
-      - 【bug fix】fix bug in self.get()，之前 get() 函数中只从 `self._item_to_add`和 `self._path_to_collect` 中加载一次注册成员，但是加载的过程中，可能后面因为对 `self._path_to_collect` 的加载，又往 `self._item_to_add` 中添加了待处理内容，导致不能完全加载。该问题已修复。
-        - 添加了测试用例。
-    - 【new feature】add cache_manager，新增了 cache_manager 模块用于进行缓存管理。
-      - 其中主要包含三个部分：
-        - 缓存数据结构：cache_manager.cache 和 `cache_manager.variable.CACHE_BUILDER_REGISTRY`
-          - 基类：`Cache_Base`
-          - 基于内存的缓存结构：`Memo_Cache`，注册名 `":in_memory:Memo_Cache"` 等等。
-        - 缓存更新策略：cache_manager.strategy 和 `cache_manager.variable.CACHE_STRATEGY_REGISTRY`
-          - 基类：`Strategy_Base`
-          - 删除最后一次访问时间最久远的部分：`FIFO_Strategy`，注册名 `":by_initial_time:FIFO_Strategy"` 等等。
-          - 删除访问频率最低的部分：`LFU_Strategy`，注册名 `":by_counts:LFU_Strategy"` 等等。
-          - 删除最后一次访问时间最久远的部分：`LRU_Strategy`，注册名 `":by_last_time:LRU_Strategy"` 等等。
-          - 删除访问频率最低的部分：`LST_Strategy`，注册名 `":by_survival_time:LST_Strategy"` 等等。
-        - 缓存管理器：Cache_Manager（主要用这个）
-      - 添加了测试用例。
-  - data_flow.core.cache
-    - modify Cache_Manager_for_Iterator，使用新增的 cache_manager 模块替换 Cache_Manager_for_Iterator 中基于内存的缓存。相关参数有修改。
-      - 添加了测试用例。
-