isage-middleware 0.1.1__py3-none-any.whl → 0.1.3.1__py3-none-any.whl

sage/middleware/components/neuromem/storage_engine/vector_storage.py

@@ -0,0 +1,85 @@
+# file sage/core/sage.middleware.services.neuromem./storage_engine/vector_storage.py
+# python -m sage.core.sage.middleware.services.neuromem..storage_engine.vector_storage
+
+import json
+import numpy as np
+from typing import Any, Optional, List
+
+from sage.middleware.components.neuromem.storage_engine.kv_backend.base_kv_backend import BaseKVBackend
+from sage.middleware.components.neuromem.storage_engine.kv_backend.dict_kv_backend import DictKVBackend
+
+
+class VectorStorage:
+    """
+    Simple vector storage based on hash IDs.
+    基于哈希ID的简单向量存储器。
+    """
+
+    def __init__(self, backend: Optional[BaseKVBackend] = None):
+        self.backend = backend or DictKVBackend()
+        
+    def get_all_ids(self) -> List[str]:
+        return self.backend.get_all_keys()
+    
+    def has(self, item_id: str) -> bool:
+        return self.backend.has(item_id)
+    
+    def delete(self, item_id: str):
+        self.backend.delete(item_id)
+        
+    def store(self, hash_id: str, vector: Any):
+        self.backend.set(hash_id, vector)
+
+    def get(self, hash_id: str) -> Any:
+        return self.backend.get(hash_id)
+
+    def clear(self):
+        self.backend.clear()
+
+    def store_to_disk(self, path: str):
+        # type: ignore for IDE/static checker
+        save_dict = {k: v.tolist() if isinstance(v, np.ndarray) else v for k, v in self.backend._store.items()}  # type: ignore
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(save_dict, f, ensure_ascii=False, indent=2)
+
+    def load_from_disk(self, path: str):
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        # type: ignore for IDE/static checker
+        for k, v in data.items():
+            self.backend._store[k] = np.array(v, dtype="float32")  # type: ignore
+
+    def clear_disk_data(self, path: str):
+        if not hasattr(self.backend, "clear_disk_data"):
+            raise NotImplementedError("Backend does not support clear_disk_data")
+        self.backend.clear_disk_data(path)
+
+if __name__ == "__main__":
+    import hashlib
+
+    vs = VectorStorage()
+    vector = [1, 2, 3]
+    vector_id = hashlib.sha256(str(vector).encode()).hexdigest()
+    disk_path = "test_vector_storage.json"
+
+    # 存储并保存到磁盘
+    vs.store(vector_id, vector)
+    print("Step 1 | Retrieved:", vs.get(vector_id))
+    vs.store_to_disk(disk_path)
+    print(f"Step 2 | Data has been saved to {disk_path}")
+
+    # 清空内存
+    vs.clear()
+    print("Step 3 | After clear (should be None):", vs.get(vector_id))
+
+    # 等待用户输入 yes 再读取
+    user_input = input("Step 4 | Enter 'yes' to load data from disk: ")
+    if user_input.strip().lower() == 'yes':
+        vs.load_from_disk(disk_path)
+        print("Step 5 | After reload, Retrieved:", vs.get(vector_id))
+    else:
+        print("Step 5 | Skipped loading from disk.")
+
+    # 删除磁盘文件
+    vs.clear_disk_data(disk_path)
+    print(f"Step 6 | Disk file {disk_path} has been deleted.")

sage/middleware/components/neuromem/tests/__init__.py

@@ -0,0 +1,56 @@
+"""
+SAGE - Streaming-Augmented Generative Execution
+"""
+
+# 动态版本加载
+def _load_version():
+    """从 sage-common 包加载版本信息"""
+    try:
+        # 优先从 sage-common 包加载版本
+        from sage.common._version import __version__
+        return {
+            'version': __version__,
+            'author': 'SAGE Team',
+            'email': 'shuhao_zhang@hust.edu.cn'
+        }
+    except ImportError:
+        # 如果 sage-common 不可用，从项目根目录加载（开发环境）
+        try:
+            from pathlib import Path
+            current_file = Path(__file__).resolve()
+            # 根据当前文件位置计算到项目根目录的层数
+            parts = current_file.parts
+            sage_index = -1
+            for i, part in enumerate(parts):
+                if part == 'SAGE':
+                    sage_index = i
+                    break
+            
+            if sage_index >= 0:
+                root_dir = Path(*parts[:sage_index+1])
+                version_file = root_dir / "_version.py"
+                
+                if version_file.exists():
+                    version_globals = {}
+                    with open(version_file, 'r', encoding='utf-8') as f:
+                        exec(f.read(), version_globals)
+                    return {
+                        'version': version_globals.get('__version__', '0.1.3'),
+                        'author': version_globals.get('__author__', 'SAGE Team'),
+                        'email': version_globals.get('__email__', 'shuhao_zhang@hust.edu.cn')
+                    }
+        except Exception:
+            pass
+    
+    # 最后的默认值
+    return {
+        'version': '0.1.3',
+        'author': 'SAGE Team', 
+        'email': 'shuhao_zhang@hust.edu.cn'
+    }
+
+# 加载信息
+_info = _load_version()
+__version__ = _info['version']
+__author__ = _info['author']
+__email__ = _info['email']

sage/middleware/components/neuromem/tests/core_test/__init__.py

@@ -0,0 +1,56 @@
+"""
+SAGE - Streaming-Augmented Generative Execution
+"""
+
+# 动态版本加载
+def _load_version():
+    """从 sage-common 包加载版本信息"""
+    try:
+        # 优先从 sage-common 包加载版本
+        from sage.common._version import __version__
+        return {
+            'version': __version__,
+            'author': 'SAGE Team',
+            'email': 'shuhao_zhang@hust.edu.cn'
+        }
+    except ImportError:
+        # 如果 sage-common 不可用，从项目根目录加载（开发环境）
+        try:
+            from pathlib import Path
+            current_file = Path(__file__).resolve()
+            # 根据当前文件位置计算到项目根目录的层数
+            parts = current_file.parts
+            sage_index = -1
+            for i, part in enumerate(parts):
+                if part == 'SAGE':
+                    sage_index = i
+                    break
+            
+            if sage_index >= 0:
+                root_dir = Path(*parts[:sage_index+1])
+                version_file = root_dir / "_version.py"
+                
+                if version_file.exists():
+                    version_globals = {}
+                    with open(version_file, 'r', encoding='utf-8') as f:
+                        exec(f.read(), version_globals)
+                    return {
+                        'version': version_globals.get('__version__', '0.1.3'),
+                        'author': version_globals.get('__author__', 'SAGE Team'),
+                        'email': version_globals.get('__email__', 'shuhao_zhang@hust.edu.cn')
+                    }
+        except Exception:
+            pass
+    
+    # 最后的默认值
+    return {
+        'version': '0.1.3',
+        'author': 'SAGE Team', 
+        'email': 'shuhao_zhang@hust.edu.cn'
+    }
+
+# 加载信息
+_info = _load_version()
+__version__ = _info['version']
+__author__ = _info['author']
+__email__ = _info['email']

sage/middleware/components/neuromem/tests/core_test/collection_test/__init__.py

@@ -0,0 +1,56 @@
+"""
+SAGE - Streaming-Augmented Generative Execution
+"""
+
+# 动态版本加载
+def _load_version():
+    """从 sage-common 包加载版本信息"""
+    try:
+        # 优先从 sage-common 包加载版本
+        from sage.common._version import __version__
+        return {
+            'version': __version__,
+            'author': 'SAGE Team',
+            'email': 'shuhao_zhang@hust.edu.cn'
+        }
+    except ImportError:
+        # 如果 sage-common 不可用，从项目根目录加载（开发环境）
+        try:
+            from pathlib import Path
+            current_file = Path(__file__).resolve()
+            # 根据当前文件位置计算到项目根目录的层数
+            parts = current_file.parts
+            sage_index = -1
+            for i, part in enumerate(parts):
+                if part == 'SAGE':
+                    sage_index = i
+                    break
+            
+            if sage_index >= 0:
+                root_dir = Path(*parts[:sage_index+1])
+                version_file = root_dir / "_version.py"
+                
+                if version_file.exists():
+                    version_globals = {}
+                    with open(version_file, 'r', encoding='utf-8') as f:
+                        exec(f.read(), version_globals)
+                    return {
+                        'version': version_globals.get('__version__', '0.1.3'),
+                        'author': version_globals.get('__author__', 'SAGE Team'),
+                        'email': version_globals.get('__email__', 'shuhao_zhang@hust.edu.cn')
+                    }
+        except Exception:
+            pass
+    
+    # 最后的默认值
+    return {
+        'version': '0.1.3',
+        'author': 'SAGE Team', 
+        'email': 'shuhao_zhang@hust.edu.cn'
+    }
+
+# 加载信息
+_info = _load_version()
+__version__ = _info['version']
+__author__ = _info['author']
+__email__ = _info['email']

sage/middleware/components/neuromem/tests/core_test/collection_test/kv_collection_test.py

@@ -0,0 +1,60 @@
+import pytest
+from sage.middleware.components.neuromem.memory_collection.kv_collection import KVMemoryCollection
+from sage.middleware.components.neuromem.memory_collection.base_collection import get_default_data_dir
+
+@pytest.fixture
+def setup_kv_collection():
+    col = KVMemoryCollection(name="demo")
+
+    col.insert("Hello Jack.", {})
+    col.insert("Hello Tom.", {})
+    col.insert("Hello Alice.", {})
+    col.insert("Jack and Tom say hi.", {})
+    col.insert("Alice in Wonderland.", {})
+    col.insert("Jacky is not Jack.", {})
+
+    col.create_index("global_index")
+    return col
+
+
+def test_kv_retrieve(setup_kv_collection):
+    col = setup_kv_collection
+
+    res1 = col.retrieve("Jack", topk=3, index_name="global_index")
+    assert set(res1) == {"Hello Jack.", "Jack and Tom say hi.", "Jacky is not Jack."}
+
+    res2 = col.retrieve("Alice", topk=2, index_name="global_index")
+    assert set(res2) == {"Hello Alice.", "Alice in Wonderland."}
+
+    res3 = col.retrieve("Tom", topk=2, index_name="global_index")
+    assert set(res3) == {"Hello Tom.", "Jack and Tom say hi."}
+
+
+def test_kv_update_and_delete(setup_kv_collection):
+    col = setup_kv_collection
+
+    col.delete("Hello Tom.")
+    res4 = col.retrieve("Tom", index_name="global_index")
+    assert "Hello Tom." not in res4
+
+    col.update("Hello Jack.", "Hello Jacky.", {}, "global_index")
+    res5 = col.retrieve("Jacky", index_name="global_index")
+
+    assert {"Jacky is not Jack.", "Hello Jacky."}.issubset(set(res5))
+
+
+def test_kv_persistence(setup_kv_collection):
+    col = setup_kv_collection
+    store_path = get_default_data_dir()
+    col_name = "demo"
+    col.store(store_path)
+
+    del col
+
+    col2 = KVMemoryCollection.load(col_name)
+    res = col2.retrieve("Jacky", index_name="global_index")
+
+
+    assert "Jacky is not Jack." in res
+
+    KVMemoryCollection.clear(col_name)

sage/middleware/components/neuromem/tests/core_test/collection_test/vdb_collection_test.py

@@ -0,0 +1,88 @@
+# file sage/sage_tests/neuromem_test/core_test/collection_test/vdb_collection_test.py
+
+import os
+import time
+import pytest
+from sage.middleware.components.neuromem.memory_collection.base_collection import get_default_data_dir
+from sage.utils.embedding_methods.mockembedder import MockTextEmbedder
+from sage.middleware.components.neuromem.memory_collection.vdb_collection import VDBMemoryCollection
+
+
+def almost_equal_dict(d1, d2, float_tol=1e-3):
+    # 只对所有值都是float的dict做容忍，否则严格等价
+    if d1.keys() != d2.keys():
+        return False
+    for k in d1:
+        v1, v2 = d1[k], d2[k]
+        if isinstance(v1, float) and isinstance(v2, float):
+            if abs(v1 - v2) > float_tol:
+                return False
+        else:
+            if v1 != v2:
+                return False
+    return True
+
+@pytest.fixture
+def setup_vdb_collection():
+    default_model = MockTextEmbedder(fixed_dim=16)
+    col = VDBMemoryCollection("vdb_demo", default_model, 16)
+    col.add_metadata_field("source")
+    col.add_metadata_field("lang")
+    col.add_metadata_field("timestamp")
+
+    current_time = time.time()
+    texts = [
+        ("hello world", {"source": "user", "lang": "en", "timestamp": current_time - 3600}),
+        ("你好，世界", {"source": "user", "lang": "zh", "timestamp": current_time - 1800}),
+        ("bonjour le monde", {"source": "web", "lang": "fr", "timestamp": current_time}),
+    ]
+    for t, meta in texts:
+        col.insert(t, meta)
+
+    col.create_index("global_index")
+    col.create_index("en_index", metadata_filter_func=lambda m: m.get("lang") == "en", description="English only")
+    col.create_index("user_index", metadata_filter_func=lambda m: m.get("source") == "user")
+    return col, current_time
+
+def test_vdb_persist(setup_vdb_collection):
+    col, current_time = setup_vdb_collection
+
+    # 检索校验
+    res = col.retrieve("hello", topk=3, index_name="global_index")
+    assert set(res) & {"hello world"} == {"hello world"}
+
+    res = col.retrieve("hello", index_name="en_index")
+    assert set(res) & {"hello world"} == {"hello world"}
+
+    res = col.retrieve("你好", index_name="user_index")
+    assert set(res) & {"你好，世界"} == {"你好，世界"}
+
+    # 持久化保存
+    store_path = get_default_data_dir()
+    col_name = "vdb_demo"
+    col.store(store_path)
+
+    # 清除内存对象
+    del col
+
+    # 恢复对象并回归测试
+    default_model2 = MockTextEmbedder(fixed_dim=16)
+    col2 = VDBMemoryCollection.load(col_name, embedding_model=default_model2)
+
+    # 再检索
+    res = col2.retrieve("hello", index_name="global_index")
+    assert set(res) & {"hello world"} == {"hello world"}
+
+    res = col2.retrieve("你好", index_name="user_index")
+    assert set(res) & {"你好，世界"} == {"你好，世界"}
+
+    # 校验metadata一致性
+    meta = col2.metadata_storage.get(col2._get_stable_id("hello world"))
+    assert almost_equal_dict(meta, {"source": "user", "lang": "en", "timestamp": current_time - 3600})
+
+    # 校验索引条件
+    idx_meta = col2.indexes["en_index"]
+    assert idx_meta.get("description", "") == "English only"
+
+    # 删除磁盘数据
+    VDBMemoryCollection.clear(col_name)

sage/middleware/components/neuromem/tests/core_test/manager_test.py

@@ -0,0 +1,154 @@
+# file sage/sage_tests/neuromem_test/core_test/manager_test.py
+# python -m sage.sage_tests.neuromem_test.core_test.manager_test
+
+from time import sleep
+
+
+def test_vdb(do_reload=True, do_delete=True):
+    from sage.middleware.components.neuromem.memory_manager import MemoryManager
+    from sage.utils.embedding_methods.mockembedder import MockTextEmbedder
+    import os
+
+    def colored(text, color):
+        colors = {"green": "\033[92m", "red": "\033[91m", "yellow": "\033[93m", "reset": "\033[0m"}
+        return colors.get(color, "") + text + colors["reset"]
+        # return text  # 自动化测试不需要彩色输出
+
+    def print_test_case(desc, expected, actual):
+        status = "通过" if expected == actual or (isinstance(expected, set) and set(expected) == set(actual)) else "不通过"
+        print(f"【{desc}】")
+        print(f"预期结果：{expected}")
+        print(f"实际结果：{actual}")
+        print(f"测试情况：{status}\n")
+
+    # 1. 创建 MemoryManager 和 VDBCollection
+    mgr = MemoryManager()
+    embedder = MockTextEmbedder(fixed_dim=16)
+    col = mgr.create_collection(name="test_vdb", backend_type="VDB", description="测试VDB集合", embedding_model=embedder, dim=16)
+    col.add_metadata_field("tag")  # 注册元数据字段tag
+
+    # 2. 插入数据
+    col.insert("Alpha", {"tag": "A"})
+    col.insert("Beta", {"tag": "B"})
+    col.insert("Gamma", {"tag": "A"})
+
+    # 3. 创建索引
+    col.create_index("global_index")
+    col.create_index("tag_A_index", metadata_filter_func=lambda m: m.get("tag") == "A")
+
+    # 4. 检索测试
+    res1 = col.retrieve("Alpha", topk=1, index_name="global_index")
+    print_test_case("检索Alpha", ["Alpha"], res1)
+
+    res2 = set(col.retrieve("Alpha", topk=5, index_name="tag_A_index"))  # Alpha 的向量检索，tag为A
+    print_test_case("tag_A_index 检索", {'Gamma', 'Alpha', 'Beta'}, res2)
+
+    # 5. 持久化
+    mgr.store_collection()
+    print("数据已保存到磁盘！")
+    data_dir = mgr.data_dir
+    print("目录为：", os.path.join(data_dir, "vdb_collection", "test_vdb"))
+
+    # 6. 清空对象
+    del mgr, col
+    print("内存对象已清除。")
+
+    # 7. 读取持久化数据，自动测试时直接执行
+    if do_reload:
+    from sage.middleware.components.neuromem.memory_manager import MemoryManager
+        embedder2 = MockTextEmbedder(fixed_dim=16)
+        mgr2 = MemoryManager()
+        col2 = mgr2.connect_collection("test_vdb", embedding_model=embedder2)
+        res3 = set(col2.retrieve("Alpha", topk=5, index_name="tag_A_index"))
+        print_test_case("恢复后tag_A_index 检索", {'Gamma', 'Alpha', 'Beta'}, res3)
+    else:
+        print("跳过加载测试。")
+
+    sleep(10)  # 等待一会儿，确保所有操作完成
+    # 8. 删除所有数据
+    if do_delete:
+    from sage.middleware.components.neuromem.memory_collection.vdb_collection import VDBMemoryCollection
+        VDBMemoryCollection.clear("test_vdb", data_dir)
+        manager_json = os.path.join(data_dir, "manager.json")
+        if os.path.exists(manager_json):
+            os.remove(manager_json)
+        print("所有数据已删除！")
+    else:
+        print("未执行删除。")
+
+# 作为测试脚本被自动调用
+if __name__ == "__main__":
+    test_vdb()
+
+
+# def kvtest():
+#     from sage.core.sage.middleware.services.neuromem..memory_manager import MemoryManager
+#     import os
+
+#     def colored(text, color):
+#         colors = {"green": "\033[92m", "red": "\033[91m", "yellow": "\033[93m", "reset": "\033[0m"}
+#         return colors.get(color, "") + text + colors["reset"]
+
+#     def print_test_case(desc, expected, actual):
+#         status = "通过" if expected == actual or (isinstance(expected, set) and set(expected) == set(actual)) else "不通过"
+#         color = "green" if status == "通过" else "red"
+#         print(f"【{desc}】")
+#         print(f"预期结果：{expected}")
+#         print(f"实际结果：{actual}")
+#         print(f"测试情况：{colored(status, color)}\n")
+
+#     # 1. 创建 MemoryManager 和 KVCollection
+#     mgr = MemoryManager()
+#     col = mgr.create_collection(name="test_kv", backend_type="KV", description="测试KV集合")
+#     col.add_metadata_field("tag")  # 注册元数据字段tag
+
+#     # 2. 插入数据
+#     col.insert("Alpha", {"tag": "A"})
+#     col.insert("Beta", {"tag": "B"})
+#     col.insert("Gamma", {"tag": "A"})
+
+#     # 3. 创建索引
+#     col.create_index("global_index")
+#     col.create_index("tag_A_index", metadata_filter_func=lambda m: m.get("tag") == "A")
+
+#     # 4. 检索测试
+#     res1 = col.retrieve("Alpha", topk=1, index_name="global_index")
+#     print_test_case("检索Alpha", ["Alpha"], res1)
+
+#     res2 = set(col.retrieve("a", topk=5, index_name="tag_A_index"))  # 检索包含'a'的文本，tag为A
+#     print_test_case("tag_A_index 检索", {"Alpha", "Gamma"}, res2)
+
+#     # 5. 持久化
+#     mgr.store_collection()
+#     print(colored("数据已保存到磁盘！", "yellow"))
+#     data_dir = mgr.data_dir
+#     print("目录为：", os.path.join(data_dir, "kv_collection", "test_kv"))
+
+#     # 6. 清空对象
+#     del mgr, col
+#     print(colored("内存对象已清除。", "yellow"))
+
+#     # 7. 读取持久化数据
+#     user_input = input(colored("输入 yes 加载刚才保存的数据: ", "yellow"))
+#     if user_input.strip().lower() == "yes":
+#         mgr2 = MemoryManager()  # 会自动加载 manager.json 和 collection
+#         col2 = mgr2.get_collection("test_kv")
+#         # 检查是否正常恢复
+#         res3 = set(col2.retrieve("a", topk=5, index_name="tag_A_index"))
+#         print_test_case("恢复后tag_A_index 检索", {"Alpha", "Gamma"}, res3)
+#     else:
+#         print(colored("跳过加载测试。", "yellow"))
+
+#     # 8. 删除所有数据
+#     user_input = input(colored("输入 yes 删除磁盘所有数据: ", "yellow"))
+#     if user_input.strip().lower() == "yes":
+#         from sage.core.sage.middleware.services.neuromem.memory_collection.kv_collection import KVMemoryCollection
+#         KVMemoryCollection.clear("test_kv", data_dir)
+#         manager_json = os.path.join(data_dir, "manager.json")
+#         if os.path.exists(manager_json):
+#             os.remove(manager_json)
+#         print(colored("所有数据已删除！", "green"))
+#     else:
+#         print(colored("未执行删除。", "yellow"))
+
+# kvtest()