isage-middleware 0.2.4.3__cp311-none-any.whl

sage/middleware/components/sage_db/__init__.py

@@ -0,0 +1,116 @@
+"""SageVDB compatibility layer for SAGE.
+
+SageVDB has been migrated to an independent PyPI package.
+
+Installation:
+    pip install isage-vdb
+
+This module re-exports SageVDB classes from the sagevdb package
+for backward-compatible import paths within SAGE.
+
+Important:
+    - PyPI package name: isage-vdb (with hyphen and 'i' prefix)
+    - Python import name: sagevdb (no 'i', no hyphen)
+
+For detailed migration information, see:
+    docs-public/docs_src/dev-notes/cross-layer/sagedb-independence-migration.md
+"""
+
+import warnings
+
+# Re-export everything from sagevdb (Python import name, PyPI: isage-vdb)
+_SAGE_DB_AVAILABLE = False
+try:
+    from sagevdb import (
+        DatabaseConfig,
+        DistanceMetric,
+        IndexType,
+        MetadataStore,
+        QueryEngine,
+        QueryResult,
+        SageVDB,
+        SageVDBException,
+        SearchParams,
+        SearchStats,
+        VectorStore,
+        add_numpy,
+        create_database,
+        distance_metric_to_string,
+        index_type_to_string,
+        search_numpy,
+        string_to_distance_metric,
+        string_to_index_type,
+    )
+
+    _SAGE_DB_AVAILABLE = True
+except ImportError as e:
+    # Don't warn on import - only when actually trying to use SageVDB
+    # Store error message for later use
+    _SAGE_DB_IMPORT_ERROR = str(e)
+    pass
+    # Provide stub exports to prevent ImportError
+    SageVDB = None
+    IndexType = None
+    DistanceMetric = None
+    QueryResult = None
+    SearchParams = None
+    SearchStats = None
+    DatabaseConfig = None
+    MetadataStore = None
+    QueryEngine = None
+    VectorStore = None
+    SageVDBException = None
+    create_database = None
+    add_numpy = None
+    search_numpy = None
+    distance_metric_to_string = None
+    index_type_to_string = None
+    string_to_distance_metric = None
+    string_to_index_type = None
+
+# Import backend adapters
+try:
+    from .backend import SageVDBBackend  # noqa: F401
+except ImportError:
+    SageVDBBackend = None
+
+__all__ = [
+    # Core classes (may be None if not installed)
+    "SageVDB",
+    "IndexType",
+    "DistanceMetric",
+    "QueryResult",
+    "SearchParams",
+    "SearchStats",
+    "DatabaseConfig",
+    "MetadataStore",
+    "QueryEngine",
+    "VectorStore",
+    "SageVDBException",
+    # Factory functions
+    "create_database",
+    # Numpy utilities
+    "add_numpy",
+    "search_numpy",
+    # Conversion utilities
+    "distance_metric_to_string",
+    "index_type_to_string",
+    "string_to_distance_metric",
+    "string_to_index_type",
+    # Backend adapters
+    "SageVDBBackend",
+    # Availability flag
+    "_SAGE_DB_AVAILABLE",
+]
+
+
+def __getattr__(name):
+    """Provide friendly error message when SageVDB is not installed"""
+    if name in __all__ and not _SAGE_DB_AVAILABLE:
+        raise ImportError(
+            f"Cannot import '{name}' from sage.middleware.components.sage_db. "
+            "SageVDB is not installed. Please install it using:\n"
+            "  pip install isage-vdb\n"
+            "Note: PyPI package name is 'isage-vdb', Python import name is 'sagevdb'"
+        )
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")

sage/middleware/components/sage_db/backend.py

@@ -0,0 +1,136 @@
+"""SageVDB Backend Adapter - VectorStore implementation using SageVDB
+
+This module provides a VectorStore adapter for SageVDB, enabling it to be used
+with the unified IndexBuilder interface.
+
+Layer: L4 (sage-middleware)
+Dependencies: isage-vdb (PyPI package, Python import: sagevdb)
+
+SageVDB is now an independent PyPI package. Install with: pip install isage-vdb
+"""
+
+from pathlib import Path
+from typing import Any
+
+from sagevdb import SageVDB
+
+
+class SageVDBBackend:
+    """VectorStore adapter for SageVDB (C++ vector database).
+
+    This class wraps SageVDB to conform to the VectorStore Protocol,
+    enabling it to be used with IndexBuilder via dependency injection.
+
+    Architecture:
+        - Implements VectorStore Protocol from L3 (sage-libs)
+        - Uses SageVDB from isage-vdb (PyPI package, Python: sagevdb)
+        - Injected into IndexBuilder by L5 (sage-cli)
+
+    Example:
+        >>> from sage.libs.rag.index_builder import IndexBuilder
+        >>> from sage.middleware.components.sage_db import SageVDBBackend
+        >>>
+        >>> def factory(path: Path, dim: int):
+        ...     return SageVDBBackend(path, dim)
+        >>>
+        >>> builder = IndexBuilder(backend_factory=factory)
+    """
+
+    def __init__(self, persist_path: Path, dim: int):
+        """Initialize SageVDB backend.
+
+        Args:
+            persist_path: Path where index will be saved
+            dim: Vector dimension
+        """
+        self.db = SageVDB(dim)
+        self.persist_path = persist_path
+        self.dim = dim
+        self._count = 0
+
+    def add(self, vector: list[float], metadata: dict[str, Any]) -> None:
+        """Add vector with metadata to SageVDB.
+
+        Args:
+            vector: Dense vector embedding
+            metadata: Associated metadata
+        """
+        self.db.add(vector, metadata)
+        self._count += 1
+
+    def build_index(self) -> None:
+        """Build SageVDB index for efficient search."""
+        self.db.build_index()
+
+    def save(self, path: str) -> None:
+        """Persist SageVDB index to disk.
+
+        Args:
+            path: Absolute path to save location
+        """
+        self.db.save(path)
+
+    def load(self, path: str) -> None:
+        """Load SageVDB index from disk.
+
+        Args:
+            path: Absolute path to load from
+        """
+        self.db.load(path)
+
+    def search(
+        self,
+        query_vector: list[float],
+        top_k: int = 5,
+        filter_metadata: dict[str, Any] | None = None,
+    ) -> list[dict[str, Any]]:
+        """Search for nearest neighbors in SageVDB.
+
+        Args:
+            query_vector: Query embedding
+            top_k: Number of results to return
+            filter_metadata: Optional metadata filters
+
+        Returns:
+            List of search results with metadata and scores
+        """
+        # SageVDB search returns QueryResult objects
+        results = self.db.search(query_vector, top_k=top_k)
+
+        # Convert to standard format
+        formatted_results = []
+        for result in results:
+            formatted_results.append(
+                {
+                    "vector": result.vector,
+                    "metadata": result.metadata,
+                    "score": result.distance,  # or result.score
+                    "id": result.id,
+                }
+            )
+
+        # Apply metadata filter if provided
+        if filter_metadata:
+            formatted_results = [
+                r
+                for r in formatted_results
+                if all(r["metadata"].get(k) == v for k, v in filter_metadata.items())
+            ]
+
+        return formatted_results
+
+    def get_dim(self) -> int:
+        """Get vector dimension.
+
+        Returns:
+            Vector dimension
+        """
+        return self.dim
+
+    def count(self) -> int:
+        """Get total number of vectors.
+
+        Returns:
+            Total vector count
+        """
+        return self._count

sage/middleware/components/sage_db/service.py

@@ -0,0 +1,15 @@
+"""
+SageDB Middleware Service
+
+This module provides the middleware service interface for SageDB,
+wrapping the Python bindings from the sageDB C++ core.
+"""
+
+# Micro-service wrapper
+from .python.micro_service.sage_db_service import SageDBService, SageDBServiceConfig
+from .python.multimodal_sage_db import MultimodalSageDB
+
+# Core Python bindings
+from .python.sage_db import SageDB
+
+__all__ = ["SageDB", "MultimodalSageDB", "SageDBService", "SageDBServiceConfig"]

sage/middleware/components/sage_flow/__init__.py

@@ -0,0 +1,76 @@
+"""SageFlow compatibility layer for SAGE.
+
+SageFlow has been migrated to an independent PyPI package.
+
+Installation:
+    pip install isage-flow
+
+This module re-exports SageFlow classes from the isage-flow package
+for backward-compatible import paths within SAGE, and provides
+SAGE-specific services and wrappers.
+
+For detailed migration information, see:
+    docs-public/docs_src/dev-notes/cross-layer/sageflow-independence-migration.md
+"""
+
+import warnings
+
+# Import from PyPI package (isage-flow)
+_SAGE_FLOW_AVAILABLE = False
+try:
+    from sage_flow import (
+        DataType,
+        SimpleStreamSource,
+        Stream,
+        StreamEnvironment,
+        VectorData,
+        VectorRecord,
+        __author__,
+        __email__,
+        __version__,
+    )
+
+    _SAGE_FLOW_AVAILABLE = True
+except ImportError as e:
+    # Don't fail immediately - allow graceful degradation
+    warnings.warn(
+        f"SAGE Flow not available: {e}\n"
+        "Install with: pip install isage-flow\n"
+        "Some advanced streaming features will be unavailable.",
+        UserWarning,
+        stacklevel=2,
+    )
+    # Provide stub exports to prevent ImportError
+    DataType = None
+    SimpleStreamSource = None
+    Stream = None
+    StreamEnvironment = None
+    VectorData = None
+    VectorRecord = None
+    __version__ = "unavailable"
+    __author__ = "IntelliStream Team"
+    __email__ = "shuhao_zhang@hust.edu.cn"
+
+# SAGE-specific services (kept in SAGE repo)
+# Only import if sage_flow is available
+if _SAGE_FLOW_AVAILABLE:
+    from .python.micro_service.sage_flow_service import SageFlowService
+else:
+    SageFlowService = None
+
+__all__ = [
+    # Core API from isage-flow (may be None if not installed)
+    "StreamEnvironment",
+    "Stream",
+    "SimpleStreamSource",
+    "VectorData",
+    "VectorRecord",
+    "DataType",
+    "__version__",
+    "__author__",
+    "__email__",
+    # SAGE-specific services (may be None if isage-flow not installed)
+    "SageFlowService",
+    # Availability flag
+    "_SAGE_FLOW_AVAILABLE",
+]

sage/middleware/components/sage_flow/python/__init__.py

@@ -0,0 +1,14 @@
+"""Python bindings and wrappers for SAGE-Flow live here.
+
+This package is intended to house all Python-side modules for the component.
+"""
+
+# Try to import the C++ extension module
+# If this fails, sage_flow.py will handle the fallback logic
+try:
+    from . import _sage_flow  # noqa: F401  # type: ignore[import-not-found]
+
+    __all__ = ["_sage_flow"]
+except ImportError:
+    # sage_flow.py will handle finding and importing the .so file
+    __all__ = []

sage/middleware/components/sage_flow/python/micro_service/__init__.py

@@ -0,0 +1,4 @@
+# Expose SageFlowService from submodule
+from .sage_flow_service import SageFlowService
+
+__all__ = ["SageFlowService"]

sage/middleware/components/sage_flow/python/micro_service/sage_flow_service.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import queue
+import threading
+import time
+from dataclasses import dataclass
+
+import numpy as np
+
+from sage.middleware.components.sage_flow.python.sage_flow import (
+    SimpleStreamSource,
+    StreamEnvironment,
+)
+
+
+@dataclass
+class _Record:
+    uid: int
+    vec: np.ndarray
+
+
+class SageFlowService:
+    """
+    A minimal micro-service wrapper for SAGE-Flow used by examples.
+
+    - push(uid, vec): enqueue vector for processing
+    - run(): drain queue, feed to flow, and execute once
+    """
+
+    def __init__(self, dim: int = 4, dtype: str = "Float32") -> None:
+        self.dim = dim
+        self.dtype = dtype
+        self._q: queue.Queue[_Record] = queue.Queue()
+        self._env = StreamEnvironment()
+        self._source = SimpleStreamSource("sage_flow_service_source")
+        self._lock = threading.Lock()
+        self._added_to_env = False
+        # Note: don't add to env yet; defer until a sink is attached
+
+    # API expected by examples
+    def push(self, uid: int, vec: np.ndarray) -> None:
+        if not isinstance(vec, np.ndarray):
+            vec = np.asarray(vec, dtype=np.float32)
+        vec = vec.astype(np.float32, copy=False)
+        if vec.ndim != 1 or vec.shape[0] != self.dim:
+            raise ValueError(f"vector shape must be ({self.dim},)")
+        self._q.put(_Record(uid=int(uid), vec=vec))
+
+    def run(self) -> None:
+        # Drain queue into source, then execute once
+        drained = 0
+        with self._lock:
+            while True:
+                try:
+                    rec = self._q.get_nowait()
+                except queue.Empty:
+                    break
+                ts = int(time.time() * 1000)
+                self._source.addRecord(rec.uid, ts, rec.vec)
+                drained += 1
+        if drained:
+            # If user hasn't attached sinks, add source to env once so execution proceeds
+            if not self._added_to_env:
+                # Attach a default printing sink for visibility
+                self._source.write_sink_py(
+                    "default_print_sink",
+                    lambda uid, ts: print(f"[svc sink] uid={uid}, ts={ts}", flush=True),
+                )
+                self._env.addStream(self._source)
+                self._added_to_env = True
+            self._env.execute()
+
+    def set_sink(self, callback, name: str = "py_sink") -> None:
+        """Attach a Python sink callback for visible outputs.
+
+        Args:
+            callback: Callable taking (uid: int, ts: int)
+            name: Sink name, defaults to 'py_sink'.
+        """
+        self._source.write_sink_py(name, callback)
+        if not self._added_to_env:
+            self._env.addStream(self._source)
+            self._added_to_env = True
+
+    # Optional: expose environment for advanced integrations
+    @property
+    def env(self) -> StreamEnvironment:
+        return self._env

sage/middleware/components/sage_flow/python/sage_flow.py

@@ -0,0 +1,30 @@
+"""
+SAGE Flow - High-performance vector stream processing engine (Python wrapper)
+
+This module re-exports SageFlow classes from the isage-flow PyPI package.
+"""
+
+# Re-export all classes from isage-flow
+from sage_flow import (
+    DataType,
+    SimpleStreamSource,
+    Stream,
+    StreamEnvironment,
+    VectorData,
+    VectorRecord,
+    __author__,
+    __email__,
+    __version__,
+)
+
+__all__ = [
+    "__version__",
+    "__author__",
+    "__email__",
+    "StreamEnvironment",
+    "Stream",
+    "SimpleStreamSource",
+    "VectorData",
+    "VectorRecord",
+    "DataType",
+]

sage/middleware/components/sage_flow/service.py

@@ -0,0 +1,14 @@
+"""
+SageFlow Middleware Service
+
+This module provides the middleware service interface for SageFlow,
+wrapping the Python bindings from the sageFlow C++ core.
+"""
+
+# Micro-service wrapper
+from .python.micro_service.sage_flow_service import SageFlowService
+
+# Core Python bindings
+from .python.sage_flow import SageFlow
+
+__all__ = ["SageFlow", "SageFlowService"]

sage/middleware/components/sage_mem/__init__.py

@@ -0,0 +1,83 @@
+"""
+SAGE-Mem: Memory Management Component for SAGE
+
+Provides memory management capabilities for RAG applications.
+This is a namespace package that can contain multiple memory implementations:
+- neuromem: Brain-inspired memory system (from isage-neuromem package)
+- future implementations can be added here
+
+Usage:
+    # Method 1: Import from neuromem subpackage (recommended)
+    from sage.middleware.components.sage_mem.neuromem import MemoryManager
+
+    # Method 2: Convenience imports from sage_mem root (if neuromem is installed)
+    from sage.middleware.components.sage_mem import MemoryManager
+"""
+
+# This is a namespace package - allow subpackages from different distributions
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)
+
+# Convenience re-exports from neuromem (optional, requires isage-neuromem installed)
+# These are lazy-loaded to avoid import errors if neuromem is not installed
+_NEUROMEM_AVAILABLE = False
+
+try:
+    # Try to import from the neuromem subpackage first (supports namespace merging)
+    from sage.middleware.components.sage_mem.neuromem import (
+        BaseMemoryCollection,
+        GraphMemoryCollection,
+        KVMemoryCollection,
+        MemoryManager,
+        VDBMemoryCollection,
+    )
+
+    try:
+        from sage.middleware.components.sage_mem.neuromem.services import (
+            BaseMemoryService,
+            MemoryServiceRegistry,
+            NeuromemServiceFactory,
+        )
+    except ImportError:
+        # Services might not be available in all neuromem versions
+        pass
+
+    # SimpleGraphIndex is in search_engine, not memory_collection
+    try:
+        from sage.middleware.components.sage_mem.neuromem.search_engine.graph_index import (
+            SimpleGraphIndex,
+        )
+    except ImportError:
+        SimpleGraphIndex = None
+
+    __all__ = [
+        # Core neuromem components
+        "MemoryManager",
+        "BaseMemoryCollection",
+        "VDBMemoryCollection",
+        "KVMemoryCollection",
+        "GraphMemoryCollection",
+        # Services (if available)
+        "BaseMemoryService",
+        "MemoryServiceRegistry",
+        "NeuromemServiceFactory",
+    ]
+
+    if SimpleGraphIndex is not None:
+        __all__.append("SimpleGraphIndex")
+
+    _NEUROMEM_AVAILABLE = True
+
+except (ImportError, FileNotFoundError, ModuleNotFoundError):
+    # Neuromem not installed - provide helpful error message via __getattr__
+    def __getattr__(name):
+        """Provide friendly error message when neuromem is not installed"""
+        raise ImportError(
+            f"Cannot import '{name}' from sage.middleware.components.sage_mem. "
+            "NeuroMem is not installed. Please install it using:\n"
+            "  pip install isage-neuromem\n"
+            "or install sage-middleware with neuromem support:\n"
+            "  pip install isage-middleware[neuromem]"
+        )
+
+    __all__ = []
+    _NEUROMEM_AVAILABLE = False

sage/middleware/components/sage_sias/__init__.py

@@ -0,0 +1,59 @@
+"""SIAS - Streaming Importance-Aware Agent System.
+
+This middleware component provides sample importance and continual learning
+capabilities for agent systems. It integrates with NeuroMem for memory-based
+importance scoring and experience replay.
+
+Core Components:
+- CoresetSelector: Importance-aware sample selection (loss_topk, diversity, hybrid)
+- OnlineContinualLearner: Experience replay with importance weighting
+- SelectionSummary: Statistics for selection operations
+
+Usage:
+    from sage.middleware.components.sage_sias import (
+        CoresetSelector,
+        OnlineContinualLearner,
+        SelectionSummary,
+    )
+
+    # Coreset selection
+    selector = CoresetSelector(strategy="hybrid")
+    selected = selector.select(samples, target_size=1000)
+
+    # Continual learning with replay
+    learner = OnlineContinualLearner(buffer_size=2048, replay_ratio=0.25)
+    batch = learner.update_buffer(new_samples)
+
+Future Components (planned):
+- StreamingImportanceScorer: I(x) = α·L_grad + β·D_ctx + γ·T_exec
+- ReflectiveMemoryStore: Experience storage with pattern extraction (uses NeuroMem)
+- AdaptiveExecutor: Pre/post verification and localized replanning
+- MultiAgentRouter: Task decomposition and agent collaboration
+
+Note:
+    SIAS is placed in sage-middleware (L4) rather than sage-libs (L3) because
+    it depends on NeuroMem memory system and potentially SageVDB for
+    importance-based retrieval.
+"""
+
+from sage.middleware.components.sage_sias.continual_learner import (
+    OnlineContinualLearner,
+)
+from sage.middleware.components.sage_sias.coreset_selector import (
+    CoresetSelector,
+    SelectionSummary,
+)
+from sage.middleware.components.sage_sias.types import (
+    SampleProtocol,
+    SIASSample,
+)
+
+__all__ = [
+    # Core components
+    "CoresetSelector",
+    "OnlineContinualLearner",
+    "SelectionSummary",
+    # Types
+    "SIASSample",
+    "SampleProtocol",
+]