kevin-toolbox-dev 1.3.1__py3-none-any.whl → 1.3.3__py3-none-any.whl

kevin_toolbox/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.3.1"
+__version__ = "1.3.3"
 
 
 import os
@@ -12,5 +12,5 @@ os.system(
 os.system(
     f'python {os.path.split(__file__)[0]}/env_info/check_validity_and_uninstall.py '
     f'--package_name kevin-toolbox-dev '
-    f'--expiration_timestamp 1719996516 --verbose 0'
+    f'--expiration_timestamp 1727520805 --verbose 0'
 )

kevin_toolbox/computer_science/algorithm/cache_manager/__init__.py

@@ -0,0 +1 @@
+from .cache_manager import Cache_Manager

kevin_toolbox/computer_science/algorithm/cache_manager/cache/__init__.py

@@ -0,0 +1,2 @@
+from .cache_base import Cache_Base
+from .memo_cache import Memo_Cache

kevin_toolbox/computer_science/algorithm/cache_manager/cache/cache_base.py

@@ -0,0 +1,89 @@
+from abc import ABC, abstractmethod
+
+
+class Cache_Base(ABC):
+    """
+        缓存结构的基类
+
+        要求实现：
+            - 读取 _read_freely()
+            - 写入 _write_freely()
+            - 清除单个条目 _remove_freely()
+            - 判断是否命中 has()
+            - 获取缓存已占空间 len()
+            - 清空所有内容 clear()
+        等方法。
+
+        对外提供：
+            - 只读取已有条目 read()
+            - 只写入不存在条目 write()
+            - 只清除已存在的单个条目 remove()
+            - 判断是否命中 has()
+            - 获取缓存已占空间 len()
+            - 清空所有内容 clear()
+        等接口。
+    """
+
+    def read(self, key):
+        """只允许读取已存在的条目"""
+        if self.has(key=key):
+            return self._read_freely(key=key)
+        else:
+            raise KeyError(f"key {key} not found")
+
+    def write(self, key, value):
+        """只允许写入不已存在的条目"""
+        if not self.has(key=key):
+            self._write_freely(key=key, value=value)
+        else:
+            raise KeyError(f"key {key} already exists")
+
+    def remove(self, key):
+        """只允许删除已存在的条目"""
+        if self.has(key=key):
+            self._remove_freely(key=key)
+        else:
+            raise KeyError(f"key {key} not found")
+
+    @abstractmethod
+    def _read_freely(self, key):
+        return self.read(key=key)
+
+    @abstractmethod
+    def _write_freely(self, key, value):
+        pass
+
+    @abstractmethod
+    def _remove_freely(self, key):
+        pass
+
+    @abstractmethod
+    def has(self, key) -> bool:
+        pass
+
+    @abstractmethod
+    def len(self) -> int:
+        pass
+
+    @abstractmethod
+    def clear(self):
+        pass
+
+    def __getitem__(self, key):
+        # self[key]
+        return self.read(key=key)
+
+    def __setitem__(self, key, value):
+        # self[key] = value
+        self.write(key=key, value=value)
+
+    def __contains__(self, key):
+        # if key in self:
+        #     return True
+        # else:
+        #     return False
+        return self.has(key=key)
+
+    def __len__(self):
+        # len(self)
+        return self.len()

kevin_toolbox/computer_science/algorithm/cache_manager/cache/memo_cache.py

@@ -0,0 +1,36 @@
+from kevin_toolbox.computer_science.algorithm.cache_manager.cache import Cache_Base
+from kevin_toolbox.computer_science.algorithm.cache_manager.variable import CACHE_BUILDER_REGISTRY
+
+
+@CACHE_BUILDER_REGISTRY.register()
+class Memo_Cache(Cache_Base):
+    """
+        基于内存的缓存结构
+    """
+    name = ":in_memory:Memo_Cache"
+
+    def __init__(self, **kwargs):
+        self.cache_s = dict()
+
+    def _read_freely(self, key):
+        return self.cache_s[key]
+
+    def _write_freely(self, key, value):
+        self.cache_s[key] = value
+
+    def _remove_freely(self, key):
+        self.cache_s.pop(key)
+
+    def has(self, key):
+        return key in self.cache_s
+
+    def len(self):
+        return len(self.cache_s)
+
+    def clear(self):
+        self.cache_s.clear()
+
+
+# 添加其他别名
+for name in [":in_memory:Memo", ":in_memory:MC", ":in_memory:memory"]:
+    CACHE_BUILDER_REGISTRY.add(obj=Memo_Cache, name=name, b_force=False, b_execute_now=False)

kevin_toolbox/computer_science/algorithm/cache_manager/cache_manager.py

@@ -0,0 +1,218 @@
+import time
+from .strategy import Strategy_Base
+from .cache import Cache_Base
+from .variable import CACHE_STRATEGY_REGISTRY, CACHE_BUILDER_REGISTRY
+
+
+class Cache_Manager:
+    """
+        缓存管理器
+
+        提供以下接口：
+            - 添加条目 add(key, value, b_allow_overwrite)
+            - 获取条目 get(key, b_add_if_not_found, default_factory, default)
+            - 删除并返回条目 pop(key)
+            - 判断是否有该条目 has()
+            - 清空所有内容 clear()
+
+        并支持以下用法：
+            通过 len(.) 获取缓存大小，通过 in 操作符判断是否有某个条目
+
+
+        相关变量：
+            self.cache：     缓存
+            self.metadata_s：缓存的属性数据
+                                包含以下字段，各字段将自动更新
+                                例如：{
+                                        <key>:  {
+                                            "last_time": xxx,      # 最近读取时间
+                                            "initial_time": xxx,   # 最初读取时间
+                                            "survival_time": xxx,  # survival_time:=last_time-initial_time
+                                            "counts": xxx,         # 读取次数
+                                        },
+                                       ...
+                                    }
+            self.strategy：  缓存更新策略
+    """
+
+    def __init__(self, **kwargs):
+        """
+            参数：
+                upper_bound:            <int> 当缓存容量超过该值时，触发重整（抛弃部分条目以将容量降低到 refactor_size 指定的大小）
+                refactor_size:          <int/float> 内存重整后的容量大小
+                                            当设置为 int 时，表示具体容量的大小；
+                                            当设置为 float 时表示占 upper_bound 的比例
+                                            默认为 0.5
+                strategy:               <str/dict/Strategy_Base> 管理策略
+                cache:                  <str/dict/Cache_Base> 缓存种类
+        """
+        # 默认参数
+        paras = {
+            "upper_bound": None,
+            "refactor_size": 0.5,
+            "strategy": ":by_last_time:LRU",
+            "cache": ":in_memory:Memo",
+        }
+
+        # 获取参数
+        paras.update(kwargs)
+
+        # 校验参数
+        assert isinstance(paras["upper_bound"], int) and paras["upper_bound"] > 0, \
+            "upper_bound must be a positive integer"
+        if isinstance(paras["refactor_size"], float):
+            paras["refactor_size"] = int(paras["refactor_size"] * paras["upper_bound"])
+        assert 0 <= paras["refactor_size"] <= paras["upper_bound"], \
+            "refactor_size must be less than upper_bound and bigger than zero"
+        # strategy
+        assert isinstance(paras["strategy"], (str, dict, Strategy_Base)), \
+            "strategy must be a string, dict of paras or a Strategy_Base object"
+        if isinstance(paras["strategy"], str):
+            strategy = CACHE_STRATEGY_REGISTRY.get(name=paras["strategy"])()
+        elif isinstance(paras["strategy"], dict):
+            strategy = CACHE_STRATEGY_REGISTRY.get(name=paras["strategy"]["name"])(
+                **paras["strategy"].get("paras", dict()))
+        else:
+            strategy = paras["strategy"]
+        # cache
+        assert isinstance(paras["cache"], (str, dict, Cache_Base)), \
+            "cache must be a string, dict of paras or a Cache_Base object"
+        if isinstance(paras["cache"], str):
+            cache = CACHE_BUILDER_REGISTRY.get(name=paras["cache"])()
+        elif isinstance(paras["cache"], dict):
+            cache = CACHE_BUILDER_REGISTRY.get(name=paras["cache"]["name"])(**paras["cache"].get("paras", dict()))
+        else:
+            cache = paras["cache"]
+
+        self.paras = paras
+        self.strategy = strategy  # type:Strategy_Base
+        self.cache = cache  # type:Cache_Base
+        self.metadata_s = dict()  # 保存条目的相关信息
+
+    def add(self, key, value, b_allow_overwrite=True):
+        """
+            以 key 为查询键，将 value 添加到缓存中
+
+            参数:
+                key:
+                value:
+                b_allow_overwrite:      <boolean> 是否允许覆写缓存中已有的条目
+                                            默认为 True
+        """
+        # 判断是否需要进行覆写
+        if self.cache.has(key=key):
+            if id(value) != id(self.cache.read(key=key)):
+                if b_allow_overwrite:
+                    self.__remove_of_cache(key=key)
+                else:
+                    raise KeyError(f'key: {key} already exists, modification of existing entries is prohibited')
+            else:
+                return
+        # 记录到缓存
+        self.__write_of_cache(key=key, value=value)
+
+    def __write_of_cache(self, key, value):
+        """
+            向缓存中新增不存在的条目
+        """
+        temp = time.time()
+        metadata = {
+            "initial_time": temp,  # 最初读取时间（加入缓存的时间）
+            "last_time": temp,  # 最近读取时间
+            "survival_time": 0.0,  # 最后一次读取与加入缓存时间之差，last_time - initial_time
+            "counts": 0,  # 读取次数
+        }
+        #
+        self.cache.write(key=key, value=value)
+        self.metadata_s[key] = metadata
+        # 通知策略管理器
+        self.strategy.notified_by_write_of_cache(key=key, value=value, metadata=metadata)
+
+        # 判断是否重构
+        if len(self.cache) <= self.paras["upper_bound"]:
+            return
+        # 从策略管理器获取建议
+        #   根据建议删除条目
+        suggest_keys = self.strategy.suggest(refactor_size=self.paras["refactor_size"])
+        assert len(suggest_keys) == len(self.cache) - self.paras["refactor_size"], \
+            f'expect {len(self.cache) - self.paras["refactor_size"]} deletion suggestions, but got {len(suggest_keys)}'
+        for key in suggest_keys:
+            self.__remove_of_cache(key=key)
+
+    def __remove_of_cache(self, key):
+        """
+            从缓存中删除存在的条目
+        """
+        self.cache.remove(key=key)
+        metadata = self.metadata_s.pop(key)
+        # 通知策略管理器
+        self.strategy.notified_by_remove_of_cache(key=key, metadata=metadata)
+
+    def get(self, key, b_add_if_not_found=False, default_factory=None, **kwargs):
+        """
+            获取 key 对应的值
+
+            参数:
+                key:                    <hashable>
+                default:                默认值
+                                            当 key 在缓存中不存在时返回
+                default_factory:        <callable> 用于产生默认值的函数
+                                            当 key 在缓存中不存在时返回该函数的结果
+                                            使用该参数相对于 default 能延迟默认值的产生（如果不需要用到默认值就不生成），提高效率
+                        注意！！上面两个参数同时指定时，将以前者为准。
+                b_add_if_not_found:     <boolean> 当 key 在缓存中不存在时，是否添加到缓存中
+                                            默认为 False
+        """
+        if self.cache.has(key=key):
+            value = self.__read_of_cache(key=key)
+        elif "default" in kwargs or callable(default_factory):
+            value = kwargs["default"] if "default" in kwargs else default_factory()
+            if b_add_if_not_found:
+                self.__write_of_cache(key=key, value=value)
+        else:
+            raise KeyError(key)
+        return value
+
+    def pop(self, key):
+        """
+            从缓存中删除 key 对应的条目，并返回该条目的值
+        """
+        if self.cache.has(key=key):
+            value = self.__read_of_cache(key=key)
+            self.__remove_of_cache(key=key)
+        else:
+            raise KeyError(key)
+        return value
+
+    def __read_of_cache(self, key):
+        """
+            读取缓存中 已经存在的 条目
+        """
+        value = self.cache.read(key=key)
+        metadata = self.metadata_s[key]
+        # 更新 metadata
+        metadata["last_time"] = time.time()
+        metadata["survival_time"] = metadata["last_time"] - metadata["initial_time"]
+        metadata["counts"] += 1
+        # 通知策略管理器
+        self.strategy.notified_by_read_of_cache(key=key, value=value, metadata=metadata)
+
+        return value
+
+    def has(self, key):
+        """
+            判断 key 是否在缓存中
+                注意！！不会更新 metadata
+        """
+        return self.cache.has(key=key)
+
+    def clear(self):
+        self.cache.clear()
+        self.metadata_s.clear()
+        self.strategy.notified_by_clear_of_cache()
+
+    def __len__(self):
+        return len(self.cache)
+
+    def __contains__(self, key):
+        return self.has(key)

kevin_toolbox/computer_science/algorithm/cache_manager/strategy/__init__.py

@@ -0,0 +1,5 @@
+from .strategy_base import Strategy_Base
+from .lru_strategy import LRU_Strategy
+from .lfu_strategy import LFU_Strategy
+from .fifo_strategy import FIFO_Strategy
+from .lst_strategy import LST_Strategy

kevin_toolbox/computer_science/algorithm/cache_manager/strategy/fifo_strategy.py

@@ -0,0 +1,21 @@
+from collections import OrderedDict
+from kevin_toolbox.computer_science.algorithm.cache_manager.strategy import LRU_Strategy
+from kevin_toolbox.computer_science.algorithm.cache_manager.variable import CACHE_STRATEGY_REGISTRY
+
+
+@CACHE_STRATEGY_REGISTRY.register()
+class FIFO_Strategy(LRU_Strategy):
+    """
+        删除最后一次访问时间最久远的部分
+        drop items with smaller initial_time
+    """
+
+    name = ":by_initial_time:FIFO_Strategy"
+
+    def notified_by_read_of_cache(self, key, value, metadata):
+        pass
+
+
+# 添加其他别名
+for name in [":by_initial_time:FIFO", ":by_initial_time:First_In_First_Out", ":by_initial_time:drop_smaller"]:
+    CACHE_STRATEGY_REGISTRY.add(obj=FIFO_Strategy, name=name, b_force=False, b_execute_now=False)

kevin_toolbox/computer_science/algorithm/cache_manager/strategy/lfu_strategy.py

@@ -0,0 +1,80 @@
+import heapq
+from kevin_toolbox.computer_science.algorithm.cache_manager.strategy import Strategy_Base
+from kevin_toolbox.computer_science.algorithm.cache_manager.variable import CACHE_STRATEGY_REGISTRY
+
+
+class Score_to_Key:
+    def __init__(self, score, key):
+        self.score = score
+        self.key = key
+
+    def __lt__(self, other):
+        return self.score < other.score
+
+    def __eq__(self, other):
+        return self.score == other.score
+
+    def __gt__(self, other):
+        return self.score > other.score
+
+    def __repr__(self):
+        return f"Counts_to_Key(key:{self.key}, score:{self.score})"
+
+
+@CACHE_STRATEGY_REGISTRY.register()
+class LFU_Strategy(Strategy_Base):
+    """
+        删除访问频率最低的部分
+        drop items with smaller counts
+    """
+
+    name = ":by_counts:LFU_Strategy"
+
+    def __init__(self):
+        self.order_ls = list()  # 最小堆
+        self.record_s = dict()
+
+    def notified_by_write_of_cache(self, key, value, metadata):
+        assert (metadata is None or metadata["counts"] == 0) and key not in self.record_s
+        self.record_s[key] = Score_to_Key(score=0, key=key)
+        heapq.heappush(self.order_ls, self.record_s[key])
+
+    def notified_by_read_of_cache(self, key, value, metadata):
+        assert metadata is None or metadata["counts"] == self.record_s[key].score + 1
+        self.record_s[key].score += 1
+
+    def notified_by_remove_of_cache(self, key, metadata):
+        temp = self.record_s.pop(key)
+        temp.score = -1
+        heapq.heapify(self.order_ls)
+        temp = heapq.heappop(self.order_ls)
+        assert temp.key == key
+
+    def notified_by_clear_of_cache(self):
+        self.record_s.clear()
+        self.order_ls.clear()
+
+    def suggest(self, refactor_size):
+        heapq.heapify(self.order_ls)
+        res = []
+        temp_ls = []
+        for _ in range(len(self.order_ls) - refactor_size):
+            temp = heapq.heappop(self.order_ls)
+            res.append(temp.key)
+            temp_ls.append(temp)
+        self.order_ls = temp_ls + self.order_ls
+        return res
+
+    def clear(self):
+        self.notified_by_clear_of_cache()
+
+
+# 添加其他别名
+for name in [":by_counts:LFU", ":by_counts:Least_Frequently_Used", ":by_counts:drop_smaller"]:
+    CACHE_STRATEGY_REGISTRY.add(obj=LFU_Strategy, name=name, b_force=False, b_execute_now=False)
+
+if __name__ == "__main__":
+    max_heap = []
+    for i, j in [(-5, "a"), (-2, "adafd"), (-7, "ad"), (-1, "a"), (-10, "fasfde")]:
+        heapq.heappush(max_heap, Score_to_Key(i, j))
+    print(max_heap)

kevin_toolbox/computer_science/algorithm/cache_manager/strategy/lru_strategy.py

@@ -0,0 +1,43 @@
+from collections import OrderedDict
+from kevin_toolbox.computer_science.algorithm.cache_manager.strategy import Strategy_Base
+from kevin_toolbox.computer_science.algorithm.cache_manager.variable import CACHE_STRATEGY_REGISTRY
+
+
+@CACHE_STRATEGY_REGISTRY.register()
+class LRU_Strategy(Strategy_Base):
+    """
+        删除最后一次访问时间最久远的部分
+        drop items with smaller last_time
+    """
+
+    name = ":by_last_time:LRU_Strategy"
+
+    def __init__(self):
+        self.record_s = OrderedDict()
+
+    def notified_by_write_of_cache(self, key, value, metadata):
+        self.record_s[key] = None
+
+    def notified_by_read_of_cache(self, key, value, metadata):
+        self.record_s.pop(key)
+        self.record_s[key] = None
+
+    def notified_by_remove_of_cache(self, key, metadata):
+        self.record_s.pop(key)
+
+    def notified_by_clear_of_cache(self):
+        self.record_s.clear()
+
+    def suggest(self, refactor_size):
+        if refactor_size == 0:
+            return list(self.record_s.keys())
+        else:
+            return list(self.record_s.keys())[:-refactor_size]
+
+    def clear(self):
+        self.record_s.clear()
+
+
+# 添加其他别名
+for name in [":by_last_time:LRU", ":by_last_time:Least_Recently_Used", ":by_last_time:drop_smaller"]:
+    CACHE_STRATEGY_REGISTRY.add(obj=LRU_Strategy, name=name, b_force=False, b_execute_now=False)

kevin_toolbox/computer_science/algorithm/cache_manager/strategy/lst_strategy.py

@@ -0,0 +1,26 @@
+import heapq
+from kevin_toolbox.computer_science.algorithm.cache_manager.strategy import Strategy_Base, LFU_Strategy
+from kevin_toolbox.computer_science.algorithm.cache_manager.strategy.lfu_strategy import Score_to_Key
+from kevin_toolbox.computer_science.algorithm.cache_manager.variable import CACHE_STRATEGY_REGISTRY
+
+
+@CACHE_STRATEGY_REGISTRY.register()
+class LST_Strategy(LFU_Strategy):
+    """
+        删除访问频率最低的部分
+        drop items with smaller survival_time
+    """
+
+    name = ":by_survival_time:LST_Strategy"
+
+    def notified_by_write_of_cache(self, key, value, metadata):
+        self.record_s[key] = Score_to_Key(score=metadata["survival_time"], key=key)
+        heapq.heappush(self.order_ls, self.record_s[key])
+
+    def notified_by_read_of_cache(self, key, value, metadata):
+        self.record_s[key].score = metadata["survival_time"]
+
+
+# 添加其他别名
+for name in [":by_survival_time:LST", ":by_survival_time:Least_Survival_Time", ":by_survival_time:drop_smaller"]:
+    CACHE_STRATEGY_REGISTRY.add(obj=LST_Strategy, name=name, b_force=False, b_execute_now=False)

kevin_toolbox/computer_science/algorithm/cache_manager/strategy/strategy_base.py

@@ -0,0 +1,45 @@
+from abc import ABC, abstractmethod
+
+
+class Strategy_Base(ABC):
+    """
+        缓存管理策略的基类
+
+        要求实现：
+            - notified_by_write_of_cache()
+            - notified_by_read_of_cache()
+            - notified_by_remove_of_cache()
+            - notified_by_clear_of_cache()
+            - 返回需要删除的条目的键的列表 suggest()
+            - 清空所有内容 clear()
+        等方法。
+
+        其中 notified_by_xxx_of_cache 系列方法用于接受在 Cache_Manager 中的通讯，比如 notified_by_write_of_cache 方法
+            将在 Cache_Manager 中每次调用 cache 的 write 方法时被调用。利用这些通讯接口，使得策略管理器可以提前追踪、感知条目的变化，
+            从而进行一些预处理操作，以免在后续需要其提出重构建议时，才当场进行过大的计算量。
+    """
+
+    @abstractmethod
+    def notified_by_write_of_cache(self, key, value, metadata):
+        pass
+
+    @abstractmethod
+    def notified_by_read_of_cache(self, key, value, metadata):
+        pass
+
+    @abstractmethod
+    def notified_by_remove_of_cache(self, key, metadata):
+        pass
+
+    @abstractmethod
+    def notified_by_clear_of_cache(self):
+        self.clear()
+
+    @abstractmethod
+    def suggest(self, refactor_size) -> list:
+        """返回需要删除的条目的键的列表"""
+        pass
+
+    @abstractmethod
+    def clear(self):
+        pass

kevin_toolbox/computer_science/algorithm/cache_manager/test/test_cache_builder.py

@@ -0,0 +1,37 @@
+import pytest
+from kevin_toolbox.patches.for_test import check_consistency
+from kevin_toolbox.computer_science.algorithm.cache_manager.cache import Cache_Base
+from kevin_toolbox.computer_science.algorithm.cache_manager.variable import CACHE_BUILDER_REGISTRY
+
+
+def test_memo_cache():
+    print("test Memo_Cache")
+
+    cache = CACHE_BUILDER_REGISTRY.get(name=":in_memory:Memo")()  # type: Cache_Base
+
+    # 读写
+    cache.write(key="1", value=1)
+    cache[2] = "2"
+    check_consistency(cache["1"], 1)
+    check_consistency(cache.read(2), "2")
+    # 禁止重复写入
+    with pytest.raises(KeyError):
+        cache.write(key="1", value=2)
+    # 禁止读取/删除不存在条目
+    with pytest.raises(KeyError):
+        cache.read(key=1)
+    with pytest.raises(KeyError):
+        cache.remove(key=1)
+
+    # 容量判断
+    check_consistency(len(cache), 2)
+
+    # 删除与命中判断
+    check_consistency(cache.has("1"), True)
+    check_consistency(cache.has(2), True)
+    cache.remove(2)
+    check_consistency(cache.has(2), False)
+
+    # 清空
+    cache.clear()
+    check_consistency(len(cache), 0)