isage-middleware 0.1.0__py3-none-any.whl

sage/middleware/services/kv/examples/kv_demo.py

@@ -0,0 +1,213 @@
+"""
+KV Service API 使用示例
+展示如何正确使用KV微服务的API接口
+"""
+import time
+from sage.core.api.local_environment import LocalEnvironment
+from sage.middleware.services.kv.kv_service import create_kv_service_factory
+from sage.middleware.api.kv_api import KVServiceAPI
+
+
+def test_kv_service_api():
+    """测试KV服务API的正确使用方式"""
+    print("🚀 KV Service API Demo")
+    print("=" * 50)
+    
+    # 创建环境
+    env = LocalEnvironment("kv_service_demo")
+    
+    # 注册KV服务 - 内存后端
+    kv_factory = create_kv_service_factory(
+        service_name="demo_kv_service",
+        backend_type="memory",
+        max_size=1000,
+        ttl_seconds=300  # 5分钟过期
+    )
+    env.register_service_factory("demo_kv_service", kv_factory)
+    
+    print("✅ KV Service registered with memory backend")
+    
+    # 在实际应用中，你需要启动环境并获取服务代理
+    # env.submit()  # 启动环境
+    # kv_service = env.get_service_proxy("demo_kv_service")
+    
+    # 这里我们演示API接口的预期使用方式
+    demonstrate_kv_api_usage()
+
+
+def demonstrate_kv_api_usage():
+    """演示KV服务API的标准使用模式"""
+    print("\n📝 KV Service API Usage Patterns:")
+    print("-" * 40)
+    
+    # 展示API接口
+    print("💡 KV Service API Interface:")
+    print("   class KVServiceAPI:")
+    print("     - put(key: str, value: Any) -> bool")
+    print("     - get(key: str) -> Optional[Any]")
+    print("     - delete(key: str) -> bool")
+    print("     - exists(key: str) -> bool")
+    print("     - list_keys(prefix: Optional[str] = None) -> List[str]")
+    print("     - size() -> int")
+    print("     - clear() -> bool")
+    
+    print("\n📋 Standard Usage Example:")
+    usage_code = '''
+# 1. 获取服务代理
+kv_service = env.get_service_proxy("demo_kv_service")
+
+# 2. 基本CRUD操作
+# 存储数据
+success = kv_service.put("user:123", {
+    "name": "Alice", 
+    "age": 30, 
+    "email": "alice@example.com"
+})
+
+# 读取数据
+user_data = kv_service.get("user:123")
+exists = kv_service.exists("user:123")
+
+# 3. 批量操作
+# 存储会话数据
+session_data = {
+    "user_id": "123",
+    "timestamp": time.time(),
+    "activity": "browsing_products"
+}
+kv_service.put("session:abc", session_data)
+
+# 列出所有用户相关的键
+user_keys = kv_service.list_keys("user:")
+session_keys = kv_service.list_keys("session:")
+
+# 4. 管理操作
+total_items = kv_service.size()
+cleanup_success = kv_service.delete("session:abc")
+
+# 5. 错误处理
+try:
+    result = kv_service.get("non_existent_key")
+    if result is None:
+        print("Key not found")
+except Exception as e:
+    print(f"Error accessing KV service: {e}")
+'''
+    print(usage_code)
+    
+    # 模拟执行结果
+    print("🎯 Expected Results:")
+    operations = [
+        ("put('user:123', user_data)", "True"),
+        ("get('user:123')", "{'name': 'Alice', 'age': 30, 'email': '...'}"),
+        ("exists('user:123')", "True"),
+        ("list_keys('user:')", "['user:123']"),
+        ("size()", "2"),
+        ("delete('session:abc')", "True"),
+        ("get('non_existent_key')", "None"),
+    ]
+    
+    for operation, result in operations:
+        print(f"   {operation:<30} -> {result}")
+
+
+def test_kv_advanced_patterns():
+    """演示KV服务的高级使用模式"""
+    print("\n🔧 Advanced KV Usage Patterns:")
+    print("-" * 40)
+    
+    advanced_patterns = '''
+# 1. 缓存模式
+class UserCache:
+    def __init__(self, kv_service: KVServiceAPI):
+        self.kv = kv_service
+        self.cache_ttl = 300  # 5分钟
+    
+    def get_user(self, user_id: str):
+        cache_key = f"user_cache:{user_id}"
+        cached_user = self.kv.get(cache_key)
+        
+        if cached_user is None:
+            # 从数据库加载
+            user = load_user_from_db(user_id)
+            # 存入缓存
+            self.kv.put(cache_key, user)
+            return user
+        return cached_user
+
+# 2. 分布式锁模式
+class DistributedLock:
+    def __init__(self, kv_service: KVServiceAPI):
+        self.kv = kv_service
+    
+    def acquire_lock(self, resource_id: str, timeout: int = 30):
+        lock_key = f"lock:{resource_id}"
+        lock_value = {"acquired_at": time.time(), "timeout": timeout}
+        
+        if not self.kv.exists(lock_key):
+            return self.kv.put(lock_key, lock_value)
+        return False
+    
+    def release_lock(self, resource_id: str):
+        lock_key = f"lock:{resource_id}"
+        return self.kv.delete(lock_key)
+
+# 3. 配置管理模式
+class ConfigManager:
+    def __init__(self, kv_service: KVServiceAPI):
+        self.kv = kv_service
+        self.config_prefix = "config:"
+    
+    def get_config(self, key: str, default=None):
+        config_key = f"{self.config_prefix}{key}"
+        value = self.kv.get(config_key)
+        return value if value is not None else default
+    
+    def set_config(self, key: str, value):
+        config_key = f"{self.config_prefix}{key}"
+        return self.kv.put(config_key, value)
+    
+    def list_all_configs(self):
+        return self.kv.list_keys(self.config_prefix)
+'''
+    print(advanced_patterns)
+
+
+def test_kv_with_redis():
+    """演示KV服务的Redis后端配置"""
+    print("\n🔧 Redis Backend Configuration:")
+    
+    redis_config_example = '''
+# Redis后端配置示例
+redis_kv_factory = create_kv_service_factory(
+    service_name="redis_kv_service",
+    backend_type="redis",
+    redis_url="redis://localhost:6379",
+    redis_db=0,
+    connection_pool_size=10,
+    ttl_seconds=3600,  # 1小时默认TTL
+    max_size=1000000   # 最大键数量
+)
+
+# 使用方式完全相同
+env.register_service_factory("redis_kv", redis_kv_factory)
+
+# API调用方式不变
+redis_kv = env.get_service_proxy("redis_kv")
+redis_kv.put("persistent_key", {"data": "stored_in_redis"})
+'''
+    
+    print(redis_config_example)
+    print("✅ Redis KV factory configuration shown")
+    print("   - 连接: redis://localhost:6379")
+    print("   - TTL: 1小时")
+    print("   - 持久化存储")
+    print("   - 相同的API接口")
+
+
+if __name__ == "__main__":
+    test_kv_service_api()
+    test_kv_advanced_patterns()
+    test_kv_with_redis()
+    print("\n🎯 KV Service API demo completed!")
+    print("\n📚 Next: Check VDB and Memory service API examples")

sage/middleware/services/kv/kv_service.py

@@ -0,0 +1,306 @@
+"""
+KV Service - 键值存储微服务
+提供键值存储功能的服务任务，集成到SAGE DAG中
+"""
+from typing import Dict, Any, Optional, List, TYPE_CHECKING
+from dataclasses import dataclass
+import json
+import time
+import logging
+
+from sage.core.api.service.base_service import BaseService
+
+
+if TYPE_CHECKING:
+    from sage.core.factory.service_factory import ServiceFactory
+    from sage.kernel import ServiceContext
+
+
+@dataclass
+class KVConfig:
+    """KV服务配置"""
+    backend_type: str = "memory"  # "memory" 或 "redis"
+    redis_url: Optional[str] = None
+    max_size: int = 10000
+    ttl_seconds: Optional[int] = None
+
+
+class MemoryKVBackend:
+    """内存KV后端"""
+    
+    def __init__(self, max_size: int = 10000, ttl_seconds: Optional[int] = None):
+        self.store: Dict[str, Any] = {}
+        self.timestamps: Dict[str, float] = {}
+        self.max_size = max_size
+        self.ttl_seconds = ttl_seconds
+        self.logger = logging.getLogger(__name__)
+    
+    def get(self, key: str) -> Optional[Any]:
+        """获取值"""
+        if self._is_expired(key):
+            self.delete(key)
+            return None
+        return self.store.get(key)
+    
+    def put(self, key: str, value: Any) -> bool:
+        """存储值"""
+        try:
+            if len(self.store) >= self.max_size and key not in self.store:
+                # 清理过期项或最老的项
+                self._evict_if_needed()
+            
+            self.store[key] = value
+            self.timestamps[key] = time.time()
+            return True
+        except Exception as e:
+            self.logger.error(f"Error storing key {key}: {e}")
+            return False
+    
+    def delete(self, key: str) -> bool:
+        """删除值"""
+        removed = key in self.store
+        self.store.pop(key, None)
+        self.timestamps.pop(key, None)
+        return removed
+    
+    def list_keys(self, prefix: str = "") -> List[str]:
+        """列出所有键"""
+        # 先清理过期键
+        expired_keys = [k for k in self.store.keys() if self._is_expired(k)]
+        for k in expired_keys:
+            self.delete(k)
+        
+        if prefix:
+            return [k for k in self.store.keys() if k.startswith(prefix)]
+        return list(self.store.keys())
+    
+    def size(self) -> int:
+        """获取存储大小"""
+        return len(self.store)
+    
+    def clear(self) -> None:
+        """清空存储"""
+        self.store.clear()
+        self.timestamps.clear()
+    
+    def _is_expired(self, key: str) -> bool:
+        """检查键是否过期"""
+        if self.ttl_seconds is None:
+            return False
+        
+        timestamp = self.timestamps.get(key)
+        if timestamp is None:
+            return True
+        
+        return time.time() - timestamp > self.ttl_seconds
+    
+    def _evict_if_needed(self):
+        """在需要时清理存储"""
+        # 先尝试清理过期项
+        expired_keys = [k for k in self.store.keys() if self._is_expired(k)]
+        for k in expired_keys:
+            self.delete(k)
+        
+        # 如果还是太满，删除最老的项
+        if len(self.store) >= self.max_size:
+            # 按时间排序，删除最老的
+            sorted_keys = sorted(self.timestamps.items(), key=lambda x: x[1])
+            keys_to_remove = len(sorted_keys) - self.max_size + 1
+            for key, _ in sorted_keys[:keys_to_remove]:
+                self.delete(key)
+
+
+class RedisKVBackend:
+    """Redis KV后端"""
+    
+    def __init__(self, redis_url: str, ttl_seconds: Optional[int] = None):
+        try:
+            import redis
+            self.redis_client = redis.from_url(redis_url)
+            self.ttl_seconds = ttl_seconds
+            self.logger = logging.getLogger(__name__)
+            
+            # 测试连接
+            self.redis_client.ping()
+            self.logger.info(f"Connected to Redis: {redis_url}")
+        except ImportError:
+            raise ImportError("Redis package not installed. Run: pip install redis")
+        except Exception as e:
+            raise ConnectionError(f"Failed to connect to Redis: {e}")
+    
+    def get(self, key: str) -> Optional[Any]:
+        """获取值"""
+        try:
+            value = self.redis_client.get(key)
+            if value is None:
+                return None
+            return json.loads(value.decode('utf-8'))
+        except Exception as e:
+            self.logger.error(f"Error getting key {key}: {e}")
+            return None
+    
+    def put(self, key: str, value: Any) -> bool:
+        """存储值"""
+        try:
+            serialized = json.dumps(value, ensure_ascii=False)
+            if self.ttl_seconds:
+                return self.redis_client.setex(key, self.ttl_seconds, serialized)
+            else:
+                return self.redis_client.set(key, serialized)
+        except Exception as e:
+            self.logger.error(f"Error storing key {key}: {e}")
+            return False
+    
+    def delete(self, key: str) -> bool:
+        """删除值"""
+        try:
+            return bool(self.redis_client.delete(key))
+        except Exception as e:
+            self.logger.error(f"Error deleting key {key}: {e}")
+            return False
+    
+    def list_keys(self, prefix: str = "") -> List[str]:
+        """列出所有键"""
+        try:
+            pattern = f"{prefix}*" if prefix else "*"
+            keys = self.redis_client.keys(pattern)
+            return [key.decode('utf-8') for key in keys]
+        except Exception as e:
+            self.logger.error(f"Error listing keys: {e}")
+            return []
+    
+    def size(self) -> int:
+        """获取数据库大小"""
+        try:
+            return self.redis_client.dbsize()
+        except Exception as e:
+            self.logger.error(f"Error getting size: {e}")
+            return 0
+    
+    def clear(self) -> None:
+        """清空数据库"""
+        try:
+            self.redis_client.flushdb()
+        except Exception as e:
+            self.logger.error(f"Error clearing database: {e}")
+
+
+class KVService(BaseService):
+    """
+    KV服务任务
+    
+    提供键值存储功能，可以在SAGE DAG中作为服务节点使用
+    支持内存和Redis后端
+    """
+    
+    def __init__(self, service_factory: 'ServiceFactory', ctx: 'ServiceContext' = None):
+        super().__init__(service_factory, ctx)
+        
+        # 从service_factory获取配置
+        self.config: KVConfig = getattr(service_factory, 'config', KVConfig())
+        
+        # 初始化后端
+        if self.config.backend_type == "redis":
+            if not self.config.redis_url:
+                raise ValueError("Redis URL required for Redis backend")
+            self.backend = RedisKVBackend(self.config.redis_url, self.config.ttl_seconds)
+            self.logger.info(f"KV Service '{self.service_name}' initialized with Redis backend")
+        else:
+            self.backend = MemoryKVBackend(self.config.max_size, self.config.ttl_seconds)
+            self.logger.info(f"KV Service '{self.service_name}' initialized with memory backend")
+    
+    def _start_service_instance(self):
+        """启动KV服务实例"""
+        self.logger.info(f"KV Service '{self.service_name}' started")
+    
+    def _stop_service_instance(self):
+        """停止KV服务实例"""
+        self.logger.info(f"KV Service '{self.service_name}' stopped")
+    
+    # KV操作方法 - 这些方法可以通过服务调用机制被调用
+    
+    def get(self, key: str) -> Optional[Any]:
+        """获取键值"""
+        self.logger.debug(f"GET key: {key}")
+        result = self.backend.get(key)
+        self.logger.debug(f"GET result: {result is not None}")
+        return result
+    
+    def put(self, key: str, value: Any) -> bool:
+        """存储键值"""
+        self.logger.debug(f"PUT key: {key}")
+        result = self.backend.put(key, value)
+        self.logger.debug(f"PUT result: {result}")
+        return result
+    
+    def delete(self, key: str) -> bool:
+        """删除键值"""
+        self.logger.debug(f"DELETE key: {key}")
+        result = self.backend.delete(key)
+        self.logger.debug(f"DELETE result: {result}")
+        return result
+    
+    def list_keys(self, prefix: str = "") -> List[str]:
+        """列出键"""
+        self.logger.debug(f"LIST keys with prefix: {prefix}")
+        result = self.backend.list_keys(prefix)
+        self.logger.debug(f"LIST result: {len(result)} keys")
+        return result
+    
+    def size(self) -> int:
+        """获取存储大小"""
+        result = self.backend.size()
+        self.logger.debug(f"SIZE result: {result}")
+        return result
+    
+    def clear(self) -> None:
+        """清空存储"""
+        self.logger.debug("CLEAR storage")
+        self.backend.clear()
+    
+    def stats(self) -> Dict[str, Any]:
+        """获取服务统计信息"""
+        base_stats = self.get_statistics()
+        kv_stats = {
+            "backend_type": self.config.backend_type,
+            "storage_size": self.backend.size(),
+            "max_size": getattr(self.backend, 'max_size', None),
+            "ttl_seconds": self.config.ttl_seconds
+        }
+        base_stats.update(kv_stats)
+        return base_stats
+
+
+# 工厂函数，用于在DAG中创建KV服务
+def create_kv_service_factory(
+    service_name: str = "kv_service",
+    backend_type: str = "memory",
+    redis_url: Optional[str] = None,
+    max_size: int = 10000,
+    ttl_seconds: Optional[int] = None
+):
+    """
+    创建KV服务工厂
+    
+    Args:
+        service_name: 服务名称
+        backend_type: 后端类型 ("memory" 或 "redis")
+        redis_url: Redis连接URL (当backend_type="redis"时需要)
+        max_size: 最大存储条目数 (仅内存后端)
+        ttl_seconds: 数据过期时间(秒)
+    
+    Returns:
+        ServiceFactory: 可以用于注册到环境的服务工厂
+    """
+    from sage.core.factory.service_factory import ServiceFactory
+    
+    config = KVConfig(
+        backend_type=backend_type,
+        redis_url=redis_url,
+        max_size=max_size,
+        ttl_seconds=ttl_seconds
+    )
+    
+    factory = ServiceFactory(service_name, KVService)
+    factory.config = config
+    return factory

sage/middleware/services/kv/search_engine/base_kv_index.py

@@ -0,0 +1,75 @@
+# file sage/core/sage.service.memory./search_engine/kv_index/base_kv_index.py
+
+from abc import ABC, abstractmethod
+from typing import List, Any, Dict
+
+class BaseKVIndex(ABC):
+    def __init__(
+        self,
+        name: str,
+        texts: List[str],
+        ids: List[str]
+    ):
+        """
+        初始化索引基类。
+        Initialize the base class for KV Index.
+        """
+        self.name = name
+
+    @abstractmethod
+    def insert(self, text: str, id: str) -> None:
+        """
+        插入一条新数据。
+        Insert a new entry.
+        """
+        pass
+
+    @abstractmethod
+    def delete(self, id: str) -> None:
+        """
+        删除指定id的数据。
+        Delete an entry by id.
+        """
+        pass
+
+    @abstractmethod
+    def search(self, query: str, topk: int = 10) -> List[str]:
+        """
+        检索相关数据，返回最相关的id列表。
+        Search for relevant entries and return the most relevant ids.
+        """
+        pass
+
+    @abstractmethod
+    def update(self, id: str, new_text: str) -> None:
+        """
+        更新指定id的数据内容。
+        Update the entry corresponding to the given id.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def load(cls, name: str, root_path: str) -> "BaseKVIndex":
+        """
+        加载索引实例。
+        Load the index instance.
+        """
+        pass
+
+    @abstractmethod
+    def store(self, root_path: str) -> Dict[str, Any]:
+        """
+        存储索引数据到指定目录。
+        Store the index data to the specified directory.
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def clear(root_path: str, name: str) -> None:
+        """
+        删除指定名称下的所有索引数据。
+        Remove all index data under the specified name.
+        """
+        pass