kevin-toolbox-dev 1.3.1__py3-none-any.whl → 1.3.3__py3-none-any.whl

kevin_toolbox/computer_science/algorithm/cache_manager/test/test_cache_manager.py

@@ -0,0 +1,197 @@
+import pytest
+import time
+import random
+from kevin_toolbox.patches.for_test import check_consistency
+from kevin_toolbox.computer_science.algorithm.cache_manager import Cache_Manager
+
+
+def test_cache_manager_with_lru_strategy():
+    print("test Cache_Manager with LRU_Strategy")
+
+    strategy = ":by_last_time:LRU"
+    for cache in [":in_memory:Memo", ]:
+        cache_manager = Cache_Manager(upper_bound=3, refactor_size=2, strategy=strategy, cache=cache)
+
+        # 添加数据 a、b、c
+        #   优先级依次为 c、b、a
+        cache_manager.add(key="a", value=1)
+        cache_manager.add(key="b", value=2)
+        cache_manager.add(key="c", value=3)
+        #
+        for k, v in {"a": 1, "b": 2, "c": 3}.items():
+            assert cache_manager.has(key=k)
+            check_consistency(cache_manager.cache.read(key=k), v)  # 不影响 metadata
+
+        # 重新添加 a，并访问 a
+        #   优先级变为 a、c、b
+        cache_manager.add(key="a", value=3)
+        check_consistency(cache_manager.get(key="a"), 3)
+        #
+        for k, v in {"a": 3, "b": 2, "c": 3}.items():
+            assert cache_manager.has(key=k)
+            check_consistency(cache_manager.cache.read(key=k), v)  # 不影响 metadata
+        for k in ["d", "e"]:
+            assert not cache_manager.has(key=k)
+            with pytest.raises(KeyError):
+                cache_manager.get(key=k)
+
+        # 添加数据 d
+        #   优先级变为 d、a、c、b，超过upper_bound，触发重整到refactor_size大小，变为 d、a
+        cache_manager.add(key="d", value=4)
+        #
+        for k, v in {"a": 3, "d": 4}.items():
+            assert cache_manager.has(key=k)
+            check_consistency(cache_manager.cache.read(key=k), v)  # 不影响 metadata
+        for k in ["b", "c"]:
+            assert not cache_manager.has(key=k)
+            with pytest.raises(KeyError):
+                cache_manager.get(key=k)
+
+        # 继续添加 e、f、g
+        #    重整后优先级依次为 g、f、e
+        cache_manager.add(key="e", value=5)
+        cache_manager.add(key="f", value=6)
+        cache_manager.add(key="g", value=7)
+        #
+        for k, v in {"e": 5, "f": 6, "g": 7}.items():
+            assert cache_manager.has(key=k)
+        for k in ["a", "d"]:
+            assert not cache_manager.has(key=k)
+            with pytest.raises(KeyError):
+                cache_manager.get(key=k)
+
+
+def test_cache_manager_with_fifo_strategy():
+    print("test Cache_Manager with FIFO_Strategy")
+
+    strategy = ":by_initial_time:FIFO"
+    for cache in [":in_memory:Memo", ]:
+        cache_manager = Cache_Manager(upper_bound=3, refactor_size=2, strategy=strategy, cache=cache)
+
+        # 添加数据 a、b、c
+        #   优先级依次为 c、b、a
+        cache_manager.add(key="a", value=1)
+        cache_manager.add(key="b", value=2)
+        cache_manager.add(key="c", value=3)
+        #
+        for k, v in {"a": 1, "b": 2, "c": 3}.items():
+            check_consistency(cache_manager.get(key=k), v)
+
+        # 无论后面怎么读取，优先级都不变
+        for k, v in {"b": 2, "c": 3, "a": 1}.items():
+            for _ in range(random.randint(3, 6)):
+                cache_manager.get(key=k)
+            cache_manager.add(key=k, value=v)
+
+        # 添加数据 d
+        #   优先级变为 d、c、b、a，超过upper_bound，触发重整到refactor_size大小，变为 d、c
+        cache_manager.add(key="d", value=4)
+        #
+        for k, v in {"c": 3, "d": 4}.items():
+            assert cache_manager.has(key=k)
+            check_consistency(cache_manager.cache.read(key=k), v)  # 不影响 metadata
+        for k in ["b", "a"]:
+            assert not cache_manager.has(key=k)
+            with pytest.raises(KeyError):
+                cache_manager.get(key=k)
+
+        # 继续添加 e、f、g
+        #    重整后优先级依次为 g、f、e
+        cache_manager.add(key="e", value=5)
+        cache_manager.add(key="f", value=6)
+        cache_manager.add(key="g", value=7)
+        #
+        for k, v in {"e": 5, "f": 6, "g": 7}.items():
+            assert cache_manager.has(key=k)
+        for k in ["a", "d"]:
+            assert not cache_manager.has(key=k)
+            with pytest.raises(KeyError):
+                cache_manager.get(key=k)
+
+
+def test_cache_manager_with_lfu_strategy():
+    print("test Cache_Manager with LFU_Strategy")
+
+    strategy = ":by_counts:LFU"
+    for cache in [":in_memory:Memo", ]:
+        cache_manager = Cache_Manager(upper_bound=3, refactor_size=2, strategy=strategy, cache=cache)
+
+        # 添加数据 a、b、c，然后各访问 2、3、4
+        #   优先级依次为 c、b、a
+        cache_manager.add(key="a", value=1)
+        cache_manager.add(key="b", value=2)
+        cache_manager.add(key="c", value=3)
+        #
+        for k, v in {"a": 1, "b": 2, "c": 3}.items():
+            for _ in range(v + 1):
+                check_consistency(cache_manager.get(key=k), v)
+        check_consistency(cache_manager.get(key="d", default_factory=lambda: "fuck"), "fuck")
+
+        # 重新添加 c
+        #   优先级变为 b、a、c
+        cache_manager.add(key="c", value=4, b_allow_overwrite=True)
+        check_consistency(cache_manager.get(key="c"), 4)
+        #
+        for k, v in {"a": 1, "b": 2, "c": 4}.items():
+            assert cache_manager.has(key=k)
+            check_consistency(cache_manager.cache.read(key=k), v)  # 不影响 metadata
+        for k in ["d", "e"]:
+            assert not cache_manager.has(key=k)
+            with pytest.raises(KeyError):
+                cache_manager.get(key=k)
+
+        # 添加数据 d
+        #   优先级变为 b、a、c、d，超过upper_bound，触发重整到refactor_size大小，变为 b、a
+        cache_manager.get(key="d", default=4, b_add_if_not_found=True)
+        #
+        for k, v in {"a": 1, "b": 2}.items():
+            assert cache_manager.has(key=k)
+            check_consistency(cache_manager.cache.read(key=k), v)  # 不影响 metadata
+        for k in ["c", "d"]:
+            assert not cache_manager.has(key=k)
+            with pytest.raises(KeyError):
+                cache_manager.get(key=k)
+
+        # 继续添加 e、f、g，访问一次 e
+        #    重整后优先级依次为 b、a、g
+        cache_manager.get(key="e", default=5, b_add_if_not_found=True)
+        cache_manager.add(key="f", value=6)
+        cache_manager.add(key="g", value=7)
+        #
+        for k, v in {"g": 7, "a": 1, "b": 2}.items():
+            assert cache_manager.has(key=k)
+        for k in ["f", "e"]:
+            assert not cache_manager.has(key=k)
+            with pytest.raises(KeyError):
+                cache_manager.get(key=k)
+
+
+def test_cache_manager_with_lst_strategy():
+    print("test Cache_Manager with LST_Strategy")
+
+    strategy = ":by_survival_time:LST"
+    for cache in [":in_memory:Memo", ]:
+        cache_manager = Cache_Manager(upper_bound=3, refactor_size=2, strategy=strategy, cache=cache)
+
+        # 添加数据 a、b、c，然后读取2次 b，读取1次 c，读取3次 a
+        #   优先级依次为 a、c、b
+        cache_manager.add(key="a", value=1)
+        cache_manager.add(key="b", value=2)
+        cache_manager.add(key="c", value=3)
+        #
+        for k, v in {"b": 2, "c": 1, "a": 3}.items():
+            for _ in range(v):
+                time.sleep(0.1)
+                cache_manager.get(key=k)
+
+        # 添加数据 d
+        #   优先级变为 a、c、b、d，超过upper_bound，触发重整到refactor_size大小，变为 a、c
+        cache_manager.get(key="d", default=4, b_add_if_not_found=True)
+        #
+        for k, v in {"a": 1, "c": 3}.items():
+            assert cache_manager.has(key=k)
+            check_consistency(cache_manager.cache.read(key=k), v)  # 不影响 metadata
+        for k in ["b", "d"]:
+            assert not cache_manager.has(key=k)
+            with pytest.raises(KeyError):
+                cache_manager.get(key=k)

kevin_toolbox/computer_science/algorithm/cache_manager/test/test_cache_strategy.py

@@ -0,0 +1,129 @@
+import pytest
+import random
+import time
+from kevin_toolbox.patches.for_test import check_consistency
+from kevin_toolbox.computer_science.algorithm.cache_manager.strategy import Strategy_Base
+from kevin_toolbox.computer_science.algorithm.cache_manager.variable import CACHE_STRATEGY_REGISTRY
+
+
+def test_lru_strategy():
+    print("test LRU_Strategy")
+
+    strategy = CACHE_STRATEGY_REGISTRY.get(name=":by_last_time:LRU")()  # type: Strategy_Base
+
+    # 写入 1、2、3、4
+    #   按照 last_time 排序，优先级为 4、3、2、1
+    for i in range(1, 5):
+        strategy.notified_by_write_of_cache(key=i, value=i, metadata=None)
+    #
+    expected_orders = [4, 3, 2, 1]
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))
+    # 移除并重写 3
+    #   优先级变为 3、4、2、1
+    strategy.notified_by_remove_of_cache(key=3, metadata=None)
+    strategy.notified_by_write_of_cache(key=3, value=3, metadata=None)
+    #
+    expected_orders = [3, 4, 2, 1]
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))
+    # 读取 1
+    #   优先级变为 1、3、4、2
+    strategy.notified_by_read_of_cache(key=1, value=1, metadata=None)
+    #
+    expected_orders = [1, 3, 4, 2]
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))
+
+
+def test_fifo_strategy():
+    print("test FIFO_Strategy")
+
+    strategy = CACHE_STRATEGY_REGISTRY.get(name=":by_initial_time:FIFO")()  # type: Strategy_Base
+
+    # 写入 1、2、3、4
+    #   按照 initial_time 排序，优先级为 4、3、2、1
+    for i in range(1, 5):
+        strategy.notified_by_write_of_cache(key=i, value=i, metadata=None)
+    #
+    expected_orders = [4, 3, 2, 1]
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))
+
+    # 无论后面怎么读取，优先级都不变
+    for i in range(1, 5):
+        for _ in range(random.randint(3, 6)):
+            strategy.notified_by_read_of_cache(key=i, value=i, metadata=None)
+    #
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))
+
+    # 重写 1，删除 4
+    #   优先级变为 1、3、2
+    strategy.notified_by_remove_of_cache(key=1, metadata=None)
+    strategy.notified_by_write_of_cache(key=1, value=3, metadata=None)
+    strategy.notified_by_remove_of_cache(key=4, metadata=None)
+    #
+    expected_orders = [1, 3, 2]
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))
+
+
+def test_lfu_strategy():
+    print("test LFU_Strategy")
+
+    strategy = CACHE_STRATEGY_REGISTRY.get(name=":by_counts:LFU")()  # type: Strategy_Base
+
+    # 写入 1、2、3、4
+    #   按照 counts 排序，优先级为 4==3==2==1
+    for i in range(1, 5):
+        strategy.notified_by_write_of_cache(key=i, value=i, metadata=None)
+    # 读取2次 1，读取1次 3，读取3次 2
+    #   counts变为 4:0 3:1 2:3 1:2
+    #   优先级变为 2、1、3、4
+    for key, counts in {1: 2, 3: 1, 2: 3}.items():
+        for i in range(counts):
+            strategy.notified_by_read_of_cache(key=key, value=key, metadata=None)
+    #
+    expected_orders = [2, 1, 3, 4]
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))
+    # 移除并重写 3，读取一次 4
+    #   优先级变为 2、1、4、3
+    strategy.notified_by_remove_of_cache(key=3, metadata=None)
+    strategy.notified_by_write_of_cache(key=3, value=3, metadata=None)
+    strategy.notified_by_read_of_cache(key=4, value=4, metadata=None)
+    #
+    expected_orders = [2, 1, 4, 3]
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))
+
+
+def test_lst_strategy():
+    print("test LST_Strategy")
+
+    strategy = CACHE_STRATEGY_REGISTRY.get(name=":by_survival_time:LST")()  # type: Strategy_Base
+
+    # 写入 1、2、3、4
+    #   按照 survival_time 排序，优先级为 4==3==2==1
+    for i in range(1, 5):
+        strategy.notified_by_write_of_cache(key=i, value=i, metadata={"survival_time": 0})
+    init_time = time.time()
+    # 依次 读取2次 1，读取1次 3，读取3次 2
+    #   优先级变为 2、3、1、4
+    for key, counts in {1: 2, 3: 1, 2: 3}.items():
+        for i in range(counts):
+            strategy.notified_by_read_of_cache(key=key, value=key, metadata={"survival_time": time.time() - init_time})
+    #
+    expected_orders = [2, 3, 1, 4]
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))
+    # 移除并重写 3，读取一次 4
+    #   优先级变为 4、2、1、3
+    strategy.notified_by_remove_of_cache(key=3, metadata=None)
+    strategy.notified_by_write_of_cache(key=3, value=3, metadata={"survival_time": 0})
+    strategy.notified_by_read_of_cache(key=4, value=4, metadata={"survival_time": time.time() - init_time})
+    #
+    expected_orders = [4, 2, 1, 3]
+    for i in range(0, 5):
+        check_consistency(strategy.suggest(refactor_size=i), list(reversed(expected_orders[i:])))

kevin_toolbox/computer_science/algorithm/cache_manager/variable.py

@@ -0,0 +1,28 @@
+import os
+from kevin_toolbox.computer_science.algorithm.registration import Registry
+
+ignore_s = [
+    {
+        "func": lambda _, __, path: os.path.basename(path) in ["temp", "test", "__pycache__",
+                                                               "_old_version"],
+        "scope": ["root", "dirs"]
+    },
+]
+
+# 包含缓存管理or更新策略
+CACHE_STRATEGY_REGISTRY = Registry(uid="CACHE_STRATEGY_REGISTRY")
+
+CACHE_STRATEGY_REGISTRY.collect_from_paths(
+    path_ls=[os.path.join(os.path.dirname(__file__), "strategy"), ],
+    ignore_s=ignore_s,
+    b_execute_now=False
+)
+
+# 包含缓存构建器
+CACHE_BUILDER_REGISTRY = Registry(uid="CACHE_BUILDER_REGISTRY")
+
+CACHE_BUILDER_REGISTRY.collect_from_paths(
+    path_ls=[os.path.join(os.path.dirname(__file__), "cache"), ],
+    ignore_s=ignore_s,
+    b_execute_now=False
+)

kevin_toolbox/computer_science/algorithm/registration/registry.py

@@ -180,14 +180,16 @@ class Registry:
                                     找不到时，若无默认值则报错，否则将返回默认值
         """
         # 加载待注册成员
-        if len(self._item_to_add) > 0:
-            for i in self._item_to_add:
-                self.add(**i)
-            self._item_to_add.clear()
-        if len(self._path_to_collect) > 0:
-            for i in self._path_to_collect:
-                self.collect_from_paths(**i)
-            self._path_to_collect.clear()
+        while self._item_to_add or self._path_to_collect:
+            if len(self._item_to_add) > 0:
+                for i in self._item_to_add:
+                    self.add(**i)
+                self._item_to_add.clear()
+            if len(self._path_to_collect) > 0:
+                for i in self._path_to_collect:
+                    i.setdefault("caller_file", inspect.stack()[1].filename)
+                    self.collect_from_paths(**i)
+                self._path_to_collect.clear()
 
         return ndl.get_value(var=self.database, name=name, b_pop=b_pop, **kwargs)
 
@@ -220,7 +222,7 @@ class Registry:
 
     # -------------------- 通过路径添加 --------------------- #
 
-    def collect_from_paths(self, path_ls=None, ignore_s=None, b_execute_now=False):
+    def collect_from_paths(self, path_ls=None, ignore_s=None, b_execute_now=False, **kwargs):
         """
             遍历 path_ls 下的所有模块，并自动导入其中主要被注册的部分
                 比如被 register() 装饰器包裹或者通过 add() 添加的部分
@@ -243,14 +245,15 @@ class Registry:
                             1. 在当前脚本中显式导入该实例前，调用了其他脚本执行了该实例的 collect_from_paths() 函数，且设置 b_execute_now=True，
                                 此时若导入的成员中有类，且该类继承自某个父类，且在初始化时使用了 super(xx,self).__init__ 继承初始化函数，将出现
                                 TypeError: super(type, obj): obj must be an instance or subtype of type 的错误
-                            2. 在模块的 __init__.py 文件中使用 collect_from_paths()
+                            2. 在模块的 __init__.py 文件中使用 collect_from_paths() 或间接通过 get() 调用 collect_from_paths()
+                            3. collect_from_paths() 函数中的搜索路径中包含了调用该函数的文件位置。
                         为了避免情况 1，应该尽量避免设置 b_execute_now=True。
                             或者省略 super(xx,self).__init__ 中的参数改为 super().__init__
         """
         # 检查调用位置
-        caller_frame = inspect.stack()[1]
-        assert os.path.basename(caller_frame.filename) != "__init__.py", \
-            f'calling Registry.collect_from_paths() in __init__.py is forbidden, file: {caller_frame.filename}.\n' \
+        caller_file = kwargs.get("caller_file", inspect.stack()[1].filename)
+        assert os.path.basename(caller_file) != "__init__.py", \
+            f'calling Registry.collect_from_paths() in __init__.py is forbidden, file: {caller_file}.\n' \
             f'you can call it in other files, and then import the result of the call in __init__.py'
 
         # 根据 ignore_s 构建 Path_Ignorer
@@ -259,7 +262,7 @@ class Registry:
         #
         if not b_execute_now:
             self._path_to_collect.append(
-                dict(path_ls=path_ls, ignore_s=path_ignorer, b_execute_now=True))
+                dict(path_ls=path_ls, ignore_s=path_ignorer, b_execute_now=True, caller_file=caller_file))
             return
 
         #
@@ -280,16 +283,35 @@ class Registry:
             #   （快速判断）判断该模块所在目录是否在 path_set 中
             if loader.path not in path_set:
                 continue
+            # is_pkg:
+            #   - 为 True 时表示当前遍历到的模块是一个包（即一个包含其他模块或子包的目录）
+            #   - 为 False 时表示当前模块是一个普通的 Python 模块（文件），不包含其他模块或子包。
             if is_pkg:
-                #   若不是 package，判断是否满足 Path_Ignorer 中的 dirs 对应的规则
+                #   若是目录形式的 package，判断是否满足 Path_Ignorer 中的 dirs 对应的规则
                 path = os.path.dirname(loader.find_module(module_name).path)
                 if path_ignorer(Ignore_Scope.DIRS, True, os.path.islink(path), path):
                     continue
             else:
-                #   若该模块是 package，判断该模块的文件路径是否满足 Path_Ignorer 中的 files 对应的规则
+                #   若该模块是 module，判断该模块的文件路径是否满足 Path_Ignorer 中的 files 对应的规则
                 path = loader.find_module(module_name).path
                 if path_ignorer(Ignore_Scope.FILES, False, os.path.islink(path), path):
                     continue
+                #   若该模块与调用的文件相同，则报错。
+                if path == caller_file:
+                    # collect_from_paths() 函数中的搜索路径不应该包含调用该函数文件。
+                    #   因为这样将会导致该函数被自己无限递归调用。
+                    # 要避免这样的错误，你可以选择：
+                    #   1. 将该函数的调用位置放置在待搜索路径外；
+                    #   2. 使用 ignore_s 参数来避免加载该函数的调用位置。
+                    raise RuntimeError(
+                        f'Registry.collect_from_paths(): \n'
+                        f'\tThe search path in a function should not include the file location from which it is called. \n'
+                        f'\tBecause this will cause the function to be called infinitely recursively by itself. \n'
+                        f'To avoid such errors, you can choose: \n'
+                        f'\t1. Place the calling location of this function ({path}) outside the path to be searched; \n'
+                        f'\t2. Use the ignore_s parameter to avoid searching the calling location of the function, '
+                        f'such as {{"func": lambda _, __, path: path == "{path}", "scope": ["files",]}}'
+                    )
             # 加载模块
             module = loader.find_module(module_name).load_module(module_name)
             # 选择遍历过程中第一次找到的 Registry 实例

kevin_toolbox/data_flow/core/cache/__init__.py

@@ -1 +1 @@
-from .cache_manager_for_iterator import Cache_Manager_for_Iterator, Strategies
+from .cache_manager_for_iterator import Cache_Manager_for_Iterator