isage-middleware 0.1.3.1__py3-none-any.whl → 0.2.0.0__py3-none-any.whl

sage/middleware/components/sage_db/sage_db.py

@@ -1,187 +0,0 @@
-"""
-SAGE DB - High-performance vector database with FAISS backend
-
-This module provides a Python interface to the SAGE DB vector database,
-which supports efficient similarity search with metadata filtering.
-"""
-
-import numpy as np
-from typing import List, Dict, Any, Optional, Callable, Tuple, Union
-try:
-    from . import _sage_db
-except ImportError:
-    import _sage_db
-
-# Re-export C++ classes and enums
-IndexType = _sage_db.IndexType
-DistanceMetric = _sage_db.DistanceMetric
-QueryResult = _sage_db.QueryResult
-SearchParams = _sage_db.SearchParams
-DatabaseConfig = _sage_db.DatabaseConfig
-SageDBException = _sage_db.SageDBException
-
-class SageDB:
-    """
-    High-performance vector database with FAISS backend.
-    
-    Supports efficient similarity search, metadata filtering, and hybrid search.
-    """
-    
-    def __init__(self, dimension: int, 
-                 index_type: IndexType = IndexType.AUTO,
-                 metric: DistanceMetric = DistanceMetric.L2):
-        """
-        Initialize a new SageDB instance.
-        
-        Args:
-            dimension: Vector dimension
-            index_type: Index type for similarity search
-            metric: Distance metric
-        """
-        self._db = _sage_db.create_database(dimension, index_type, metric)
-    
-    @classmethod
-    def from_config(cls, config: DatabaseConfig):
-        """Create SageDB from configuration object."""
-        instance = cls.__new__(cls)
-        instance._db = _sage_db.create_database(config)
-        return instance
-    
-    def add(self, vector: Union[List[float], np.ndarray], 
-            metadata: Optional[Dict[str, str]] = None) -> int:
-        """Add a single vector with optional metadata."""
-        if isinstance(vector, np.ndarray):
-            vector = vector.tolist()
-        return self._db.add(vector, metadata or {})
-    
-    def add_batch(self, vectors: Union[List[List[float]], np.ndarray],
-                  metadata: Optional[List[Dict[str, str]]] = None) -> List[int]:
-        """Add multiple vectors with optional metadata."""
-        if isinstance(vectors, np.ndarray):
-            if len(vectors.shape) != 2:
-                raise ValueError("Vectors array must be 2-dimensional")
-            return _sage_db.add_numpy(self._db, vectors, metadata or [])
-        else:
-            return self._db.add_batch(vectors, metadata or [])
-    
-    def search(self, query: Union[List[float], np.ndarray], 
-               k: int = 10, include_metadata: bool = True) -> List[QueryResult]:
-        """Search for similar vectors."""
-        if isinstance(query, np.ndarray):
-            return _sage_db.search_numpy(self._db, query, SearchParams(k))
-        return self._db.search(query, k, include_metadata)
-    
-    def search_with_params(self, query: Union[List[float], np.ndarray],
-                          params: SearchParams) -> List[QueryResult]:
-        """Search with custom parameters."""
-        if isinstance(query, np.ndarray):
-            return _sage_db.search_numpy(self._db, query, params)
-        return self._db.search(query, params)
-    
-    def filtered_search(self, query: Union[List[float], np.ndarray],
-                       params: SearchParams,
-                       filter_fn: Callable[[Dict[str, str]], bool]) -> List[QueryResult]:
-        """Search with metadata filtering."""
-        if isinstance(query, np.ndarray):
-            query = query.tolist()
-        return self._db.filtered_search(query, params, filter_fn)
-    
-    def search_by_metadata(self, query: Union[List[float], np.ndarray],
-                          params: SearchParams,
-                          metadata_key: str,
-                          metadata_value: str) -> List[QueryResult]:
-        """Search with specific metadata constraint."""
-        if isinstance(query, np.ndarray):
-            query = query.tolist()
-        return self._db.query_engine().search_with_metadata(query, params, metadata_key, metadata_value)
-    
-    def hybrid_search(self, query: Union[List[float], np.ndarray],
-                     params: SearchParams,
-                     text_query: str = "",
-                     vector_weight: float = 0.7,
-                     text_weight: float = 0.3) -> List[QueryResult]:
-        """Hybrid vector and text search."""
-        if isinstance(query, np.ndarray):
-            query = query.tolist()
-        return self._db.query_engine().hybrid_search(query, params, text_query, vector_weight, text_weight)
-    
-    def build_index(self):
-        """Build/train the search index."""
-        self._db.build_index()
-    
-    def train_index(self, training_vectors: Optional[Union[List[List[float]], np.ndarray]] = None):
-        """Train the index with training data."""
-        if training_vectors is None:
-            self._db.train_index()
-        elif isinstance(training_vectors, np.ndarray):
-            training_list = [training_vectors[i].tolist() for i in range(training_vectors.shape[0])]
-            self._db.train_index(training_list)
-        else:
-            self._db.train_index(training_vectors)
-    
-    def is_trained(self) -> bool:
-        """Check if the index is trained."""
-        return self._db.is_trained()
-    
-    def set_metadata(self, vector_id: int, metadata: Dict[str, str]) -> bool:
-        """Set metadata for a vector."""
-        return self._db.set_metadata(vector_id, metadata)
-    
-    def get_metadata(self, vector_id: int) -> Optional[Dict[str, str]]:
-        """Get metadata for a vector."""
-        return self._db.get_metadata(vector_id)
-    
-    def find_by_metadata(self, key: str, value: str) -> List[int]:
-        """Find vectors by metadata key-value."""
-        return self._db.find_by_metadata(key, value)
-    
-    def save(self, filepath: str):
-        """Save database to disk."""
-        self._db.save(filepath)
-    
-    def load(self, filepath: str):
-        """Load database from disk."""
-        self._db.load(filepath)
-    
-    @property
-    def size(self) -> int:
-        """Number of vectors in the database."""
-        return self._db.size()
-    
-    @property
-    def dimension(self) -> int:
-        """Vector dimension."""
-        return self._db.dimension()
-    
-    @property
-    def index_type(self) -> IndexType:
-        """Index type."""
-        return self._db.index_type()
-    
-    def get_search_stats(self) -> Dict[str, Any]:
-        """Get last search statistics."""
-        stats = self._db.query_engine().get_last_search_stats()
-        return {
-            'total_candidates': stats.total_candidates,
-            'filtered_candidates': stats.filtered_candidates,
-            'final_results': stats.final_results,
-            'search_time_ms': stats.search_time_ms,
-            'filter_time_ms': stats.filter_time_ms,
-            'total_time_ms': stats.total_time_ms
-        }
-
-# Convenience functions
-def create_database(dimension: int, 
-                   index_type: IndexType = IndexType.AUTO,
-                   metric: DistanceMetric = DistanceMetric.L2) -> SageDB:
-    """Create a new SageDB instance."""
-    return SageDB(dimension, index_type, metric)
-
-def create_database_from_config(config: DatabaseConfig) -> SageDB:
-    """Create SageDB from configuration."""
-    return SageDB.from_config(config)
-
-__all__ = [
-    'SageDB', 'IndexType', 'DistanceMetric', 'QueryResult', 'SearchParams',
-    'DatabaseConfig', 'SageDBException', 'create_database', 'create_database_from_config'
-]

sage/middleware/components/sage_db/tests/test_python.py

@@ -1,144 +0,0 @@
-import sys
-import os
-import numpy as np
-
-# Add the build directory to the path
-sys.path.append('/home/shuhao/SAGE/sage_ext/sage_db/build')
-
-try:
-    import sage_db_py
-    from sage_db import SageDatabase
-    print("✅ Import successful!")
-except ImportError as e:
-    print(f"❌ Import failed: {e}")
-    print("Make sure to build the module first with ./build.sh")
-    sys.exit(1)
-
-def test_python_api():
-    """Test the Python API wrapper"""
-    print("\n🐍 Testing Python API...")
-    
-    # Create database
-    db = SageDatabase(dimension=64)
-    
-    # Add single vector
-    vector = np.random.random(64).astype(np.float32)
-    metadata = {"text": "test vector", "category": "example"}
-    
-    vector_id = db.add_vector(vector, metadata)
-    print(f"Added vector with ID: {vector_id}")
-    
-    # Add batch
-    batch_vectors = np.random.random((10, 64)).astype(np.float32)
-    batch_metadata = [{"text": f"vector_{i}", "batch": "test"} for i in range(10)]
-    
-    batch_ids = db.add_batch(batch_vectors, batch_metadata)
-    print(f"Added batch of {len(batch_ids)} vectors")
-    
-    # Search
-    query = np.random.random(64).astype(np.float32)
-    results = db.search(query, k=3)
-    
-    print(f"Search returned {len(results)} results:")
-    for i, result in enumerate(results):
-        print(f"  {i+1}. ID: {result['id']}, Distance: {result['distance']:.4f}")
-        print(f"      Metadata: {result['metadata']}")
-    
-    # Metadata search
-    category_results = db.find_by_metadata("category", "example")
-    print(f"Found {len(category_results)} vectors with category='example'")
-    
-    print("✅ Python API test passed!")
-
-def test_pybind11_interface():
-    """Test the direct pybind11 interface"""
-    print("\n🔧 Testing pybind11 interface...")
-    
-    # Test vector store
-    vector_store = sage_db_py.VectorStore(32)
-    
-    # Add vectors
-    vec1 = np.array([1.0] * 32, dtype=np.float32)
-    vec2 = np.array([2.0] * 32, dtype=np.float32)
-    
-    id1 = vector_store.add(vec1)
-    id2 = vector_store.add(vec2)
-    
-    print(f"Added vectors with IDs: {id1}, {id2}")
-    print(f"Vector store size: {vector_store.size()}")
-    
-    # Search
-    query = np.array([1.1] * 32, dtype=np.float32)
-    results = vector_store.search(query, 2)
-    
-    print(f"Search results: {len(results)} items")
-    for result in results:
-        print(f"  ID: {result['id']}, Distance: {result['distance']:.4f}")
-    
-    # Test metadata store
-    metadata_store = sage_db_py.MetadataStore()
-    
-    metadata_store.set(id1, {"name": "first", "type": "test"})
-    metadata_store.set(id2, {"name": "second", "type": "test"})
-    
-    meta1 = metadata_store.get(id1)
-    print(f"Metadata for ID {id1}: {meta1}")
-    
-    type_matches = metadata_store.find_by_key_value("type", "test")
-    print(f"Found {len(type_matches)} vectors with type='test'")
-    
-    print("✅ pybind11 interface test passed!")
-
-def benchmark_python():
-    """Simple performance test"""
-    print("\n📊 Python performance benchmark...")
-    
-    import time
-    
-    dimension = 128
-    num_vectors = 1000
-    
-    db = SageDatabase(dimension=dimension)
-    
-    # Generate random data
-    vectors = np.random.random((num_vectors, dimension)).astype(np.float32)
-    metadata = [{"id": str(i), "batch": "benchmark"} for i in range(num_vectors)]
-    
-    # Time batch addition
-    start_time = time.time()
-    ids = db.add_batch(vectors, metadata)
-    add_time = time.time() - start_time
-    
-    print(f"Added {num_vectors} vectors in {add_time:.3f} seconds")
-    print(f"Rate: {num_vectors / add_time:.1f} vectors/second")
-    
-    # Time searches
-    num_queries = 100
-    query_vectors = np.random.random((num_queries, dimension)).astype(np.float32)
-    
-    start_time = time.time()
-    for query in query_vectors:
-        results = db.search(query, k=10)
-    search_time = time.time() - start_time
-    
-    print(f"Performed {num_queries} searches in {search_time:.3f} seconds")
-    print(f"Rate: {num_queries / search_time:.1f} searches/second")
-    
-    print("✅ Performance benchmark completed!")
-
-if __name__ == "__main__":
-    print("🧪 SAGE DB Python Test Suite")
-    print("============================")
-    
-    try:
-        test_python_api()
-        test_pybind11_interface()
-        benchmark_python()
-        
-        print("\n🎉 All Python tests passed!")
-        
-    except Exception as e:
-        print(f"\n❌ Test failed: {e}")
-        import traceback
-        traceback.print_exc()
-        sys.exit(1)

sage/middleware/examples/api_usage_tutorial.py

@@ -1,339 +0,0 @@
-"""
-SAGE Services API Usage Tutorial
-展示如何正确使用SAGE微服务的API接口
-"""
-import asyncio
-import numpy as np
-from typing import List, Dict, Any
-
-# 导入API接口
-from sage.middleware.api import KVServiceAPI, VDBServiceAPI, MemoryServiceAPI, GraphServiceAPI
-
-# 导入具体服务实现和工厂函数
-from sage.middleware.services import (
-    KVService, VDBService, MemoryService, GraphService,
-    create_kv_service_factory, create_vdb_service_factory,
-    create_memory_service_factory, create_graph_service_factory,
-)
-from sage.core.api.local_environment import LocalEnvironment
-
-
-class ServiceAPITutorial:
-    """SAGE服务API使用教程"""
-    
-    def __init__(self):
-        self.env = LocalEnvironment("api_tutorial_demo")
-        self.setup_services()
-    
-    def setup_services(self):
-        """设置所有服务"""
-        print("🚀 Setting up SAGE Services for API Tutorial")
-        print("=" * 60)
-        
-        # 注册KV服务
-        kv_factory = create_kv_service_factory(
-            service_name="tutorial_kv",
-            backend_type="memory",
-            max_size=10000
-        )
-        self.env.register_service_factory("tutorial_kv", kv_factory)
-        
-        # 注册VDB服务
-        vdb_factory = create_vdb_service_factory(
-            service_name="tutorial_vdb",
-            embedding_dimension=384,
-            max_vectors=10000
-        )
-        self.env.register_service_factory("tutorial_vdb", vdb_factory)
-        
-        # 注册Memory服务
-        memory_factory = create_memory_service_factory(
-            service_name="tutorial_memory",
-            kv_service_name="tutorial_kv",
-            vdb_service_name="tutorial_vdb"
-        )
-        self.env.register_service_factory("tutorial_memory", memory_factory)
-        
-        print("✅ All services registered successfully")
-    
-    def demonstrate_kv_api(self):
-        """演示KV API的正确使用方式"""
-        print("\n📦 KV Service API Tutorial")
-        print("-" * 40)
-        
-        # 在实际应用中，你会从环境中获取服务代理
-        # 这里我们模拟API调用的预期行为
-        
-        print("💡 KV Service API Interface:")
-        print("   - put(key, value) -> bool")
-        print("   - get(key) -> Any")
-        print("   - delete(key) -> bool")
-        print("   - exists(key) -> bool")
-        print("   - list_keys(prefix) -> List[str]")
-        print("   - size() -> int")
-        print("   - clear() -> bool")
-        
-        print("\n📝 Expected Usage Pattern:")
-        usage_example = '''
-# 获取KV服务代理
-kv_service = env.get_service_proxy("tutorial_kv")
-
-# 基本操作
-success = kv_service.put("user:123", {"name": "Alice", "age": 30})
-user_data = kv_service.get("user:123")
-exists = kv_service.exists("user:123")
-
-# 批量操作
-keys = kv_service.list_keys("user:")
-total_size = kv_service.size()
-
-# 删除操作
-deleted = kv_service.delete("user:123")
-'''
-        print(usage_example)
-        
-        # 模拟执行结果
-        print("🎯 Expected Results:")
-        print("   put() -> True")
-        print("   get() -> {'name': 'Alice', 'age': 30}")
-        print("   exists() -> True")
-        print("   list_keys('user:') -> ['user:123']")
-        print("   size() -> 1")
-        print("   delete() -> True")
-    
-    def demonstrate_vdb_api(self):
-        """演示VDB API的正确使用方式"""
-        print("\n🗂️ VDB Service API Tutorial")
-        print("-" * 40)
-        
-        print("💡 VDB Service API Interface:")
-        print("   - add_vectors(documents) -> List[str]")
-        print("   - search(query_vector, top_k, threshold) -> List[Dict]")
-        print("   - get_vector(doc_id) -> Dict")
-        print("   - delete_vectors(doc_ids) -> bool")
-        print("   - update_vector(doc_id, document) -> bool")
-        print("   - count() -> int")
-        print("   - save_index(path) -> bool")
-        print("   - load_index(path) -> bool")
-        
-        print("\n📝 Expected Usage Pattern:")
-        usage_example = '''
-# 获取VDB服务代理
-vdb_service = env.get_service_proxy("tutorial_vdb")
-
-# 添加向量文档
-documents = [{
-    "id": "doc_001",
-    "vector": [0.1, 0.2, 0.3, ...],  # 384维向量
-    "text": "这是一个示例文档",
-    "metadata": {"category": "example"}
-}]
-doc_ids = vdb_service.add_vectors(documents)
-
-# 向量搜索
-query_vector = [0.1, 0.2, 0.3, ...]  # 查询向量
-results = vdb_service.search(
-    query_vector=query_vector,
-    top_k=5,
-    similarity_threshold=0.8
-)
-
-# 获取特定文档
-document = vdb_service.get_vector("doc_001")
-
-# 更新和删除
-updated = vdb_service.update_vector("doc_001", new_document)
-deleted = vdb_service.delete_vectors(["doc_001"])
-'''
-        print(usage_example)
-        
-        print("🎯 Expected Results:")
-        print("   add_vectors() -> ['doc_001']")
-        print("   search() -> [{'id': 'doc_001', 'score': 0.95, ...}]")
-        print("   get_vector() -> {'id': 'doc_001', 'vector': [...], ...}")
-        print("   count() -> 1")
-    
-    def demonstrate_memory_api(self):
-        """演示Memory API的正确使用方式"""
-        print("\n🧠 Memory Service API Tutorial")
-        print("-" * 40)
-        
-        print("💡 Memory Service API Interface (High-level):")
-        print("   - store_memory(content, vector, session_id, ...) -> str")
-        print("   - retrieve_memories(query_vector, session_id, ...) -> List[Dict]")
-        print("   - get_memory(memory_id) -> Dict")
-        print("   - delete_memory(memory_id) -> bool")
-        print("   - search_memories(query, session_id, ...) -> List[Dict]")
-        print("   - get_session_memories(session_id) -> List[Dict]")
-        print("   - clear_session_memories(session_id) -> bool")
-        
-        print("\n📝 Expected Usage Pattern:")
-        usage_example = '''
-# 获取Memory服务代理（高级编排服务）
-memory_service = env.get_service_proxy("tutorial_memory")
-
-# 存储记忆（自动同时存储到KV和VDB）
-memory_id = memory_service.store_memory(
-    content="用户询问了关于Python的问题",
-    vector=[0.1, 0.2, 0.3, ...],  # 内容的向量表示
-    session_id="session_123",
-    memory_type="conversation",
-    metadata={"topic": "programming", "language": "python"}
-)
-
-# 检索相关记忆（基于向量相似性）
-related_memories = memory_service.retrieve_memories(
-    query_vector=[0.1, 0.2, 0.3, ...],
-    session_id="session_123",
-    top_k=5
-)
-
-# 文本搜索记忆
-search_results = memory_service.search_memories(
-    query="Python",
-    session_id="session_123",
-    top_k=10
-)
-
-# 获取会话的所有记忆
-session_memories = memory_service.get_session_memories("session_123")
-'''
-        print(usage_example)
-        
-        print("🎯 Expected Results:")
-        print("   store_memory() -> 'memory_uuid_123'")
-        print("   retrieve_memories() -> [{'id': 'memory_uuid_123', 'score': 0.92, ...}]")
-        print("   search_memories() -> [{'id': 'memory_uuid_123', 'content': '...', ...}]")
-    
-    def demonstrate_api_composition(self):
-        """演示如何组合使用多个API"""
-        print("\n🔗 API Composition Tutorial")
-        print("-" * 40)
-        
-        print("💡 实际应用场景: 智能问答系统")
-        
-        composition_example = '''
-class IntelligentQASystem:
-    """智能问答系统 - 展示API组合使用"""
-    
-    def __init__(self, env):
-        # 获取各种服务代理
-        self.kv_service = env.get_service_proxy("tutorial_kv")
-        self.vdb_service = env.get_service_proxy("tutorial_vdb")
-        self.memory_service = env.get_service_proxy("tutorial_memory")
-    
-    async def process_question(self, user_id: str, question: str):
-        """处理用户问题"""
-        
-        # 1. 从KV服务获取用户上下文
-        user_context = self.kv_service.get(f"user_context:{user_id}")
-        
-        # 2. 将问题转换为向量（使用embedding工具）
-        question_vector = embed_text(question)  # 假设的embedding函数
-        
-        # 3. 使用Memory服务检索相关记忆
-        related_memories = self.memory_service.retrieve_memories(
-            query_vector=question_vector,
-            session_id=user_context.get("session_id"),
-            top_k=5
-        )
-        
-        # 4. 直接在VDB中搜索知识库
-        knowledge_results = self.vdb_service.search(
-            query_vector=question_vector,
-            top_k=10,
-            similarity_threshold=0.7
-        )
-        
-        # 5. 生成回答（使用LLM）
-        answer = generate_answer(question, related_memories, knowledge_results)
-        
-        # 6. 存储新的对话记忆
-        self.memory_service.store_memory(
-            content=f"Q: {question}\\nA: {answer}",
-            vector=question_vector,
-            session_id=user_context.get("session_id"),
-            memory_type="qa_pair"
-        )
-        
-        # 7. 更新用户上下文
-        user_context["last_question"] = question
-        self.kv_service.put(f"user_context:{user_id}", user_context)
-        
-        return answer
-'''
-        print(composition_example)
-        
-        print("\n🎯 这个例子展示了:")
-        print("   ✅ KV服务用于快速的上下文存取")
-        print("   ✅ VDB服务用于语义相似性搜索")
-        print("   ✅ Memory服务用于高级记忆管理")
-        print("   ✅ 各服务协同工作完成复杂任务")
-    
-    def demonstrate_error_handling(self):
-        """演示API错误处理最佳实践"""
-        print("\n⚠️ Error Handling Best Practices")
-        print("-" * 40)
-        
-        error_handling_example = '''
-# 正确的错误处理方式
-try:
-    # 尝试获取服务代理
-    kv_service = env.get_service_proxy("tutorial_kv")
-    
-    # API调用
-    result = kv_service.get("some_key")
-    if result is None:
-        print("Key not found")
-    else:
-        print(f"Retrieved: {result}")
-        
-except ServiceNotAvailableError:
-    print("KV service is not available")
-except ServiceTimeoutError:
-    print("KV service call timed out")
-except Exception as e:
-    print(f"Unexpected error: {e}")
-
-# 批量操作的错误处理
-documents = [...]  # 大量文档
-batch_size = 100
-
-for i in range(0, len(documents), batch_size):
-    batch = documents[i:i+batch_size]
-    try:
-        doc_ids = vdb_service.add_vectors(batch)
-        print(f"Successfully added batch {i//batch_size + 1}")
-    except Exception as e:
-        print(f"Failed to add batch {i//batch_size + 1}: {e}")
-        # 可以选择重试或跳过
-'''
-        print(error_handling_example)
-    
-    def run_tutorial(self):
-        """运行完整的API教程"""
-        print("🎓 SAGE Services API Complete Tutorial")
-        print("=" * 60)
-        
-        self.demonstrate_kv_api()
-        self.demonstrate_vdb_api()
-        self.demonstrate_memory_api()
-        self.demonstrate_api_composition()
-        self.demonstrate_error_handling()
-        
-        print("\n🎉 Tutorial Complete!")
-        print("\n📚 Next Steps:")
-        print("   1. 查看具体服务的实现代码")
-        print("   2. 运行真实的集成测试")
-        print("   3. 在你的项目中集成SAGE服务")
-        print("   4. 参考API文档了解更多细节")
-
-
-def main():
-    """主函数"""
-    tutorial = ServiceAPITutorial()
-    tutorial.run_tutorial()
-
-
-if __name__ == "__main__":
-    main()