isage-middleware 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

sage/middleware/components/neuromem/storage_engine/__init__.py

@@ -0,0 +1,56 @@
+"""
+SAGE - Streaming-Augmented Generative Execution
+"""
+
+# 动态版本加载
+def _load_version():
+    """从 sage-common 包加载版本信息"""
+    try:
+        # 优先从 sage-common 包加载版本
+        from sage.common._version import __version__
+        return {
+            'version': __version__,
+            'author': 'SAGE Team',
+            'email': 'shuhao_zhang@hust.edu.cn'
+        }
+    except ImportError:
+        # 如果 sage-common 不可用，从项目根目录加载（开发环境）
+        try:
+            from pathlib import Path
+            current_file = Path(__file__).resolve()
+            # 根据当前文件位置计算到项目根目录的层数
+            parts = current_file.parts
+            sage_index = -1
+            for i, part in enumerate(parts):
+                if part == 'SAGE':
+                    sage_index = i
+                    break
+            
+            if sage_index >= 0:
+                root_dir = Path(*parts[:sage_index+1])
+                version_file = root_dir / "_version.py"
+                
+                if version_file.exists():
+                    version_globals = {}
+                    with open(version_file, 'r', encoding='utf-8') as f:
+                        exec(f.read(), version_globals)
+                    return {
+                        'version': version_globals.get('__version__', '0.1.4'),
+                        'author': version_globals.get('__author__', 'SAGE Team'),
+                        'email': version_globals.get('__email__', 'shuhao_zhang@hust.edu.cn')
+                    }
+        except Exception:
+            pass
+    
+    # 最后的默认值
+    return {
+        'version': '0.1.4',
+        'author': 'SAGE Team', 
+        'email': 'shuhao_zhang@hust.edu.cn'
+    }
+
+# 加载信息
+_info = _load_version()
+__version__ = _info['version']
+__author__ = _info['author']
+__email__ = _info['email']

sage/middleware/components/neuromem/storage_engine/kv_backend/__init__.py

@@ -0,0 +1,56 @@
+"""
+SAGE - Streaming-Augmented Generative Execution
+"""
+
+# 动态版本加载
+def _load_version():
+    """从 sage-common 包加载版本信息"""
+    try:
+        # 优先从 sage-common 包加载版本
+        from sage.common._version import __version__
+        return {
+            'version': __version__,
+            'author': 'SAGE Team',
+            'email': 'shuhao_zhang@hust.edu.cn'
+        }
+    except ImportError:
+        # 如果 sage-common 不可用，从项目根目录加载（开发环境）
+        try:
+            from pathlib import Path
+            current_file = Path(__file__).resolve()
+            # 根据当前文件位置计算到项目根目录的层数
+            parts = current_file.parts
+            sage_index = -1
+            for i, part in enumerate(parts):
+                if part == 'SAGE':
+                    sage_index = i
+                    break
+            
+            if sage_index >= 0:
+                root_dir = Path(*parts[:sage_index+1])
+                version_file = root_dir / "_version.py"
+                
+                if version_file.exists():
+                    version_globals = {}
+                    with open(version_file, 'r', encoding='utf-8') as f:
+                        exec(f.read(), version_globals)
+                    return {
+                        'version': version_globals.get('__version__', '0.1.4'),
+                        'author': version_globals.get('__author__', 'SAGE Team'),
+                        'email': version_globals.get('__email__', 'shuhao_zhang@hust.edu.cn')
+                    }
+        except Exception:
+            pass
+    
+    # 最后的默认值
+    return {
+        'version': '0.1.4',
+        'author': 'SAGE Team', 
+        'email': 'shuhao_zhang@hust.edu.cn'
+    }
+
+# 加载信息
+_info = _load_version()
+__version__ = _info['version']
+__author__ = _info['author']
+__email__ = _info['email']

sage/middleware/components/neuromem/storage_engine/kv_backend/base_kv_backend.py

@@ -0,0 +1,65 @@
+# file: sage/core/sage.middleware.services.neuromem./storage_engine/kv_backend/base_kv_backend.py
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+class BaseKVBackend(ABC):
+    """
+    Abstract base class for key-value backends.
+    抽象基类，用于定义 KV 存储后端接口规范。
+    """
+    @abstractmethod
+    def get_all_keys(self) -> list[str]:
+        pass
+
+    @abstractmethod
+    def has(self, key: str) -> bool:
+        """
+        Check whether the key exists.
+        检查指定键是否存在。
+        """
+        pass
+
+    @abstractmethod
+    def get(self, key: str) -> Any:
+        """
+        Retrieve value by key.
+        根据键获取对应的值。
+        """
+        pass
+
+    @abstractmethod
+    def set(self, key: str, value: Any):
+        """
+        Set a key-value pair.
+        存储键值对。
+        """
+        pass
+
+    @abstractmethod
+    def delete(self, key: str):
+        """
+        Delete a key-value pair.
+        删除指定键及其对应的值。
+        """
+        pass
+
+    @abstractmethod
+    def clear(self):
+        """
+        Clear the entire store.
+        清空所有键值对。
+        """
+        pass
+    
+    @abstractmethod
+    def load_data_to_memory(self, path: str):
+        pass
+    
+    @abstractmethod
+    def store_data_to_disk(self, path: str):
+        pass
+
+    @abstractmethod
+    def clear_disk_data(self, path: str):
+        pass

sage/middleware/components/neuromem/storage_engine/kv_backend/dict_kv_backend.py

@@ -0,0 +1,54 @@
+# file: sage/core/sage.middleware.services.neuromem./storage_engine/kv_backend/dict_kv_backend.py
+
+import os
+import json
+from typing import Any, Dict
+from sage.middleware.components.neuromem.storage_engine.kv_backend.base_kv_backend import BaseKVBackend
+
+class DictKVBackend(BaseKVBackend):
+    """
+    In-memory KV backend using a Python dictionary.
+    """
+
+    def __init__(self):
+        self._store: Dict[str, Any] = {}
+
+    def has(self, key: str) -> bool:
+        return key in self._store
+
+    def get(self, key: str) -> Any:
+        return self._store.get(key)
+
+    def set(self, key: str, value: Any):
+        self._store[key] = value
+
+    def delete(self, key: str):
+        self._store.pop(key, None)
+
+    def clear(self):
+        self._store.clear()
+
+    def get_all_keys(self):
+        """
+        Get all keys in the store.
+        获取所有存储的key。
+        """
+        return list(self._store.keys())
+
+    def store_data_to_disk(self, path: str):
+        """将当前内存数据存储为 JSON 文件"""
+        with open(path, 'w', encoding='utf-8') as f:
+            # 用 ensure_ascii=False 保证 utf-8 兼容中文，indent=2 可读性高
+            json.dump(self._store, f, ensure_ascii=False, indent=2)
+
+    def load_data_to_memory(self, path: str):
+        """从指定 JSON 文件加载数据到内存（覆盖当前 _store）"""
+        if not os.path.exists(path):
+            raise FileNotFoundError(f"File '{path}' does not exist.")
+        with open(path, 'r', encoding='utf-8') as f:
+            self._store = json.load(f)
+
+    def clear_disk_data(self, path: str):
+        """删除指定 JSON 文件"""
+        if os.path.exists(path):
+            os.remove(path)

sage/middleware/components/neuromem/storage_engine/metadata_storage.py

@@ -0,0 +1,260 @@
+# file sage/core/sage.middleware.services.neuromem./storage_engine/metadata_storage.py
+# python -m sage.core.sage.middleware.services.neuromem..storage_engine.metadata_storage
+
+from typing import Dict, Any, Optional, List
+
+from sage.middleware.components.neuromem.storage_engine.kv_backend.base_kv_backend import BaseKVBackend
+from sage.middleware.components.neuromem.storage_engine.kv_backend.dict_kv_backend import DictKVBackend
+
+
+class MetadataStorage:
+    """
+    A lightweight metadata manager that handles field registration,
+    validation, and per-item metadata storage.
+    简单的元数据管理器，用于处理字段注册、字段校验和每条数据的元数据存储。
+
+    Attributes:
+        fields (set): Set of registered metadata field names
+        backend (BaseKVBackend): Backend storage implementation
+    """
+
+    def __init__(self, backend: Optional[BaseKVBackend] = None):
+        # Registered metadata fields 已注册的元数据字段名集合
+        self.fields = set()
+        # 底层存储后端，默认是内存字典
+        self.backend = backend or DictKVBackend()
+
+    def add_field(self, field_name: str) -> None:
+        """
+        Register a new metadata field. If the field is already registered, 
+        this operation is ignored.
+        注册新的元数据字段。如果字段已存在，则忽略此操作。
+
+        Args:
+            field_name: Name of the field to register
+            
+        Raises:
+            ValueError: If field_name is None or empty
+        """
+        if not field_name or not isinstance(field_name, str):
+            raise ValueError("Field name must be a non-empty string")
+        self.fields.add(field_name)
+
+    def has_field(self, field_name: str) -> bool:
+        """
+        Check if a field is registered.
+        检查字段是否已注册。
+
+        Args:
+            field_name: Name of the field to check
+
+        Returns:
+            bool: True if the field is registered, False otherwise
+        """
+        return field_name in self.fields
+
+    def validate_fields(self, metadata: Dict[str, Any]) -> None:
+        """
+        Validate that all fields in the metadata are registered.
+        验证所有元数据字段是否已注册。
+
+        Args:
+            metadata: Metadata dictionary to validate
+
+        Raises:
+            ValueError: If any field is not registered or if metadata is invalid
+        """
+        if not isinstance(metadata, dict):
+            raise ValueError("Metadata must be a dictionary")
+            
+        unregistered = set(metadata.keys()) - self.fields
+        if unregistered:
+            raise ValueError(f"Unregistered metadata fields: {unregistered}")
+
+    def store(self, item_id: str, metadata: Dict[str, Any]) -> None:
+        """
+        Store metadata for an item.
+        存储条目的元数据。
+
+        Args:
+            item_id: ID of the item
+            metadata: Metadata dictionary to store
+
+        Raises:
+            ValueError: If item_id is invalid or metadata validation fails
+        """
+        if not item_id or not isinstance(item_id, str):
+            raise ValueError("Item ID must be a non-empty string")
+        if metadata is None:
+            metadata = {}
+        elif not isinstance(metadata, dict):
+            raise ValueError("Metadata must be a dictionary")
+
+        self.validate_fields(metadata)
+        self.backend.set(item_id, metadata.copy())
+        
+    def get_all_ids(self) -> List[str]:
+        """
+        Get all stored item IDs.
+        获取所有已存储的条目ID。
+
+        Returns:
+            List of item IDs
+        """
+        return self.backend.get_all_keys()
+
+    def get(self, item_id: str) -> Dict[str, Any]:
+        """
+        Get metadata for an item.
+        获取条目的元数据。
+
+        Args:
+            item_id: ID of the item
+
+        Returns:
+            Metadata dictionary, empty dict if item doesn't exist
+            
+        Raises:
+            ValueError: If item_id is invalid
+        """
+        if not item_id or not isinstance(item_id, str):
+            raise ValueError("Item ID must be a non-empty string")
+        return self.backend.get(item_id) or {}
+
+    def has(self, item_id: str) -> bool:
+        """
+        Check if an item has metadata stored.
+        检查条目是否有存储的元数据。
+
+        Args:
+            item_id: ID of the item to check
+            
+        Returns:
+            bool: True if item has metadata, False otherwise
+            
+        Raises:
+            ValueError: If item_id is invalid
+        """
+        if not item_id or not isinstance(item_id, str):
+            raise ValueError("Item ID must be a non-empty string")
+        return self.backend.has(item_id)
+
+    def delete(self, item_id: str) -> None:
+        """
+        Delete metadata for an item.
+        删除条目的元数据。
+
+        Args:
+            item_id: ID of the item to delete
+            
+        Raises:
+            ValueError: If item_id is invalid
+        """
+        if not item_id or not isinstance(item_id, str):
+            raise ValueError("Item ID must be a non-empty string")
+        self.backend.delete(item_id)
+
+    def clear(self) -> None:
+        """Clear all metadata and registered fields."""
+        self.fields.clear()
+        self.backend.clear()
+        
+    def store_to_disk(self, path: str) -> None:
+        """
+        Store all data to disk as JSON file.
+        将所有数据存储到磁盘JSON文件。
+
+        Args:
+            path: Path to the JSON file
+            
+        Raises:
+            NotImplementedError: If backend doesn't support disk operations
+            ValueError: If path is invalid
+        """
+        if not path or not isinstance(path, str):
+            raise ValueError("Path must be a non-empty string")
+        if not hasattr(self.backend, "store_data_to_disk"):
+            raise NotImplementedError("Backend does not support store_data_to_disk")
+        self.backend.store_data_to_disk(path)
+
+    def load_from_disk(self, path: str) -> None:
+        """
+        Load all data from disk JSON file (overwrites memory).
+        从磁盘JSON文件加载所有数据（覆盖内存）。
+
+        Args:
+            path: Path to the JSON file
+            
+        Raises:
+            NotImplementedError: If backend doesn't support disk operations
+            ValueError: If path is invalid
+            FileNotFoundError: If file doesn't exist
+        """
+        if not path or not isinstance(path, str):
+            raise ValueError("Path must be a non-empty string")
+        if not hasattr(self.backend, "load_data_to_memory"):
+            raise NotImplementedError("Backend does not support load_data_to_memory")
+        self.backend.load_data_to_memory(path)
+
+    def clear_disk_data(self, path: str) -> None:
+        """
+        Delete the disk JSON file.
+        删除磁盘JSON文件。
+
+        Args:
+            path: Path to the JSON file
+            
+        Raises:
+            NotImplementedError: If backend doesn't support disk operations
+            ValueError: If path is invalid
+        """
+        if not path or not isinstance(path, str):
+            raise ValueError("Path must be a non-empty string")
+        if not hasattr(self.backend, "clear_disk_data"):
+            raise NotImplementedError("Backend does not support clear_disk_data")
+        self.backend.clear_disk_data(path)
+
+if __name__ == "__main__":
+    import os
+
+    metadata_store = MetadataStorage()
+    disk_path = "test_metadata_storage.json"
+
+    # 注册元数据字段
+    metadata_store.add_field("author")
+    metadata_store.add_field("topic")
+
+    # 构造示例数据
+    item_id = "abc123"
+    metadata = {
+        "author": "Alice",
+        "topic": "AI"
+    }
+
+    # 存储元数据
+    metadata_store.store(item_id, metadata)
+    print("Step 1 | Retrieved metadata:", metadata_store.get(item_id))
+
+    # 保存到磁盘
+    metadata_store.store_to_disk(disk_path)
+    print(f"Step 2 | Metadata saved to {disk_path}")
+
+    # 清空内存
+    metadata_store.clear()
+    print("Step 3 | After clear, retrieved:", metadata_store.get(item_id))
+
+    # 等待用户输入 yes 再加载
+    user_input = input("Step 4 | Enter 'yes' to load metadata from disk: ")
+    if user_input.strip().lower() == 'yes':
+        metadata_store.load_from_disk(disk_path)
+        print("Step 5 | After reload, retrieved:", metadata_store.get(item_id))
+    else:
+        print("Step 5 | Skipped loading from disk.")
+
+    # 删除磁盘文件
+    metadata_store.clear_disk_data(disk_path)
+    print(f"Step 6 | Disk file {disk_path} has been deleted.")
+
+    # 可选：检查磁盘上确实没这个文件
+    print("Step 7 | File exists after deletion?:", os.path.exists(disk_path))
+

sage/middleware/components/neuromem/storage_engine/text_storage.py

@@ -0,0 +1,106 @@
+# file sage/core/sage.middleware.services.neuromem./storage_engine/text_storage.py
+# python -m sage.core.sage.middleware.services.neuromem..storage_engine.text_storage
+
+from typing import Optional, List
+
+from sage.middleware.components.neuromem.storage_engine.kv_backend.base_kv_backend import BaseKVBackend
+from sage.middleware.components.neuromem.storage_engine.kv_backend.dict_kv_backend import DictKVBackend
+
+
+class TextStorage:
+    """
+    Simple raw text storage based on externally provided IDs.
+
+    简单的原始文本存储器，由外部提供稳定 ID（不生成哈希）。
+    """
+
+    def __init__(self, backend: Optional[BaseKVBackend] = None):
+        # 支持自定义后端，默认用内存字典
+        self.backend = backend or DictKVBackend()
+        
+    def get_all_ids(self) -> List[str]:
+        return self.backend.get_all_keys()
+
+    def has(self, item_id: str) -> bool:
+        return self.backend.has(item_id)
+
+    def delete(self, item_id: str):
+        self.backend.delete(item_id)
+        
+    def store(self, item_id: str, text: str):
+        """
+        Store text under a given item_id.
+
+        使用外部提供的 ID 存储原始文本。
+        """
+        self.backend.set(item_id, text)
+
+    def get(self, item_id: str) -> str:
+        """
+        Retrieve text using item_id.
+
+        使用 ID 获取文本内容。
+        """
+        return self.backend.get(item_id) or ""
+
+    def clear(self):
+        """
+        Clear all stored text.
+
+        清空所有存储内容。
+        """
+        self.backend.clear()
+
+    def store_to_disk(self, path: str):
+        """存储所有数据到磁盘 json 文件"""
+        # 要求 backend 有该方法
+        if not hasattr(self.backend, "store_data_to_disk"):
+            raise NotImplementedError("Backend does not support store_data_to_disk")
+        self.backend.store_data_to_disk(path)
+
+    def load_from_disk(self, path: str):
+        """从磁盘 json 文件加载所有数据（覆盖内存）"""
+        if not hasattr(self.backend, "load_data_to_memory"):
+            raise NotImplementedError("Backend does not support load_data_to_memory")
+        self.backend.load_data_to_memory(path)
+
+    def clear_disk_data(self, path: str):
+        """删除磁盘上的 json 文件"""
+        if not hasattr(self.backend, "clear_disk_data"):
+            raise NotImplementedError("Backend does not support clear_disk_data")
+        self.backend.clear_disk_data(path)
+
+"""测试预期输出
+Retrieved: hello world
+After clear: 
+"""
+
+if __name__ == "__main__":
+    import hashlib
+
+    ts = TextStorage()
+    text = "hello world"
+    text_id = hashlib.sha256(text.encode()).hexdigest()
+    disk_path = "test_text_storage.json"
+
+    # 1. 存储并保存到磁盘
+    ts.store(text_id, text)
+    print("Step 1 | Retrieved:", ts.get(text_id))
+    ts.store_to_disk(disk_path)
+    print(f"Data has been saved to {disk_path}")
+
+    # 2. 清空内存
+    ts.clear()
+    print("Step 2 | After clear (should be empty):", ts.get(text_id))
+
+    # 3. 等待用户输入 yes 再读取
+    user_input = input("Step 3 | Enter 'yes' to load data from disk: ")
+    if user_input.strip().lower() == 'yes':
+        ts.load_from_disk(disk_path)
+        print("Step 3 | After reload, Retrieved:", ts.get(text_id))
+    else:
+        print("Step 3 | Skipped loading from disk.")
+
+    # 4. 删除磁盘数据
+    ts.clear_disk_data(disk_path)
+    print(f"Step 4 | Disk file {disk_path} has been deleted.")