isage-middleware 0.1.3.1__py3-none-any.whl → 0.1.5.1__py3-none-any.whl

sage/middleware/components/sage_db/cmake/FindBLASLAPACK.cmake

@@ -0,0 +1,125 @@
+# FindBLASLAPACK.cmake
+# 增强的 BLAS/LAPACK 查找模块，支持多种环境
+# 适用于 Ubuntu, CentOS, macOS, Conda 等环境
+
+# 设置查找路径
+list(APPEND CMAKE_PREFIX_PATH
+    ${CMAKE_INSTALL_PREFIX}
+    $ENV{CONDA_PREFIX}
+    /usr/local
+    /usr
+    /opt/local
+    /opt/homebrew
+)
+
+# 设置库文件搜索路径
+set(BLAS_LAPACK_SEARCH_PATHS
+    # Conda 环境
+    $ENV{CONDA_PREFIX}/lib
+    $ENV{CONDA_PREFIX}/lib64
+    
+    # 系统标准路径
+    /usr/lib
+    /usr/lib64
+    /usr/lib/x86_64-linux-gnu
+    /usr/lib/i386-linux-gnu
+    /usr/lib/aarch64-linux-gnu
+    /usr/lib/arm-linux-gnueabihf
+    
+    # macOS 路径
+    /usr/local/lib
+    /opt/local/lib
+    /opt/homebrew/lib
+    /System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Versions/Current
+    
+    # 其他可能路径
+    /opt/intel/mkl/lib/intel64
+    /opt/atlas/lib
+)
+
+# 尝试不同的 BLAS 实现
+set(BLAS_VENDOR_OPTIONS "OpenBLAS" "ATLAS" "Intel10_64lp" "Generic")
+
+foreach(VENDOR ${BLAS_VENDOR_OPTIONS})
+    set(BLA_VENDOR ${VENDOR})
+    find_package(BLAS QUIET)
+    if(BLAS_FOUND)
+        message(STATUS "Found BLAS implementation: ${VENDOR}")
+        break()
+    endif()
+endforeach()
+
+# 如果标准查找失败，手动搜索
+if(NOT BLAS_FOUND)
+    message(STATUS "Standard BLAS search failed, trying manual search...")
+    
+    # 手动查找 OpenBLAS
+    find_library(OPENBLAS_LIB 
+        NAMES openblas libopenblas
+        PATHS ${BLAS_LAPACK_SEARCH_PATHS}
+        NO_DEFAULT_PATH
+    )
+    
+    if(OPENBLAS_LIB)
+        set(BLAS_LIBRARIES ${OPENBLAS_LIB})
+        set(BLAS_FOUND TRUE)
+        message(STATUS "Found OpenBLAS manually: ${OPENBLAS_LIB}")
+    else()
+        # 尝试查找系统 BLAS
+        find_library(BLAS_LIB
+            NAMES blas libblas cblas libcblas
+            PATHS ${BLAS_LAPACK_SEARCH_PATHS}
+            NO_DEFAULT_PATH
+        )
+        
+        if(BLAS_LIB)
+            set(BLAS_LIBRARIES ${BLAS_LIB})
+            set(BLAS_FOUND TRUE)
+            message(STATUS "Found system BLAS manually: ${BLAS_LIB}")
+        endif()
+    endif()
+endif()
+
+# 查找 LAPACK
+find_package(LAPACK QUIET)
+
+if(NOT LAPACK_FOUND)
+    message(STATUS "Standard LAPACK search failed, trying manual search...")
+    
+    # 手动查找 LAPACK
+    find_library(LAPACK_LIB
+        NAMES lapack liblapack
+        PATHS ${BLAS_LAPACK_SEARCH_PATHS}
+        NO_DEFAULT_PATH
+    )
+    
+    if(LAPACK_LIB)
+        set(LAPACK_LIBRARIES ${LAPACK_LIB})
+        set(LAPACK_FOUND TRUE)
+        message(STATUS "Found LAPACK manually: ${LAPACK_LIB}")
+    endif()
+endif()
+
+# 结果报告
+if(BLAS_FOUND AND LAPACK_FOUND)
+    message(STATUS "✅ BLAS and LAPACK found successfully")
+    message(STATUS "   BLAS libraries: ${BLAS_LIBRARIES}")
+    message(STATUS "   LAPACK libraries: ${LAPACK_LIBRARIES}")
+    set(HAVE_BLAS_LAPACK TRUE CACHE BOOL "BLAS and LAPACK available")
+else()
+    message(WARNING "❌ BLAS or LAPACK not found")
+    if(NOT BLAS_FOUND)
+        message(WARNING "   BLAS not found")
+    endif()
+    if(NOT LAPACK_FOUND)
+        message(WARNING "   LAPACK not found")
+    endif()
+    message(STATUS "")
+    message(STATUS "🔧 To install BLAS/LAPACK:")
+    message(STATUS "   Ubuntu/Debian: sudo apt-get install libopenblas-dev liblapack-dev")
+    message(STATUS "   CentOS/RHEL: sudo yum install openblas-devel lapack-devel")
+    message(STATUS "   macOS: brew install openblas lapack")
+    message(STATUS "   Conda: conda install -c conda-forge openblas liblapack")
+    message(STATUS "")
+    set(HAVE_BLAS_LAPACK FALSE CACHE BOOL "BLAS and LAPACK not available")
+endif()

sage/middleware/components/sage_db/cmake/gperftools.cmake

@@ -0,0 +1,24 @@
+# Resolve gperftools libraries when building sage_db stand-alone.
+# This script assumes the boolean option ENABLE_GPERFTOOLS controls whether we attempt lookup.
+
+if(NOT ENABLE_GPERFTOOLS)
+    message(STATUS "gperftools support disabled for sage_db")
+    set(SAGE_GPERFTOOLS_LIBS "")
+    return()
+endif()
+
+find_library(SAGE_GPERFTOOLS_PROFILER_LIB profiler)
+find_library(SAGE_GPERFTOOLS_TCMALLOC_LIB tcmalloc)
+
+if(SAGE_GPERFTOOLS_PROFILER_LIB AND SAGE_GPERFTOOLS_TCMALLOC_LIB)
+    set(SAGE_GPERFTOOLS_LIBS ${SAGE_GPERFTOOLS_PROFILER_LIB} ${SAGE_GPERFTOOLS_TCMALLOC_LIB})
+    message(STATUS "gperftools found for sage_db: ${SAGE_GPERFTOOLS_LIBS}")
+else()
+    if(NOT SAGE_GPERFTOOLS_PROFILER_LIB)
+        message(WARNING "Requested gperftools but libprofiler not found; disabling ENABLE_GPERFTOOLS")
+    elseif(NOT SAGE_GPERFTOOLS_TCMALLOC_LIB)
+        message(WARNING "Requested gperftools but libtcmalloc not found; disabling ENABLE_GPERFTOOLS")
+    endif()
+    set(SAGE_GPERFTOOLS_LIBS "")
+    set(ENABLE_GPERFTOOLS OFF CACHE BOOL "Enable gperftools profiling support" FORCE)
+endif()

sage/middleware/components/sage_db/cmake/pybind11_dependency.cmake

@@ -0,0 +1,27 @@
+# Standalone pybind11 dependency resolver for sage_db.
+# When the build is orchestrated by the Python superbuild, a shared dependency
+# file can be provided via SAGE_COMMON_DEPS_FILE. Otherwise this fallback keeps
+# the project self-contained.
+
+if(NOT TARGET pybind11::module)
+    find_package(pybind11 CONFIG QUIET)
+    if(pybind11_FOUND)
+        message(STATUS "Using system pybind11 ${pybind11_VERSION} from ${pybind11_CONFIG}")
+    else()
+        include(FetchContent)
+        if(NOT DEFINED pybind11_POPULATED)
+            set(_sage_db_pybind11_tag "v2.13.0" CACHE STRING "Pinned pybind11 version for sage_db")
+            FetchContent_Declare(
+                pybind11
+                GIT_REPOSITORY https://github.com/pybind/pybind11.git
+                GIT_TAG ${_sage_db_pybind11_tag}
+            )
+        endif()
+        FetchContent_MakeAvailable(pybind11)
+        message(STATUS "Fetched pybind11 ${_sage_db_pybind11_tag} for sage_db")
+    endif()
+endif()
+
+if(NOT COMMAND pybind11_add_module)
+    message(FATAL_ERROR "pybind11_add_module is unavailable even after resolving pybind11 dependency")
+endif()

sage/middleware/components/sage_db/examples/faiss_plugin_demo.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Showcase for the FAISS ANNS plugin with graceful fallback when unavailable."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Dict, List, Tuple
+
+import numpy as np
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+PACKAGES_ROOT = REPO_ROOT / "packages"
+if str(PACKAGES_ROOT) not in sys.path:
+    sys.path.insert(0, str(PACKAGES_ROOT))
+
+try:
+    import _sage_db
+except ImportError as exc:  # pragma: no cover - documented failure mode
+    raise SystemExit(
+        "❌ Unable to import _sage_db. Build the native extension first:\n"
+        "   cd packages/sage-middleware/src/sage/middleware/components/sage_db && ./build.sh"
+    ) from exc
+
+
+def build_faiss_config(dimension: int = 8) -> _sage_db.DatabaseConfig:
+    cfg = _sage_db.DatabaseConfig(dimension)
+    cfg.metric = _sage_db.DistanceMetric.COSINE
+    cfg.index_type = _sage_db.IndexType.AUTO
+    cfg.anns_algorithm = "FAISS"
+    cfg.anns_build_params = {
+        "index_type": "ivf_flat",
+        "metric": "cosine",
+        "nlist": "16",
+        "auto_threshold_ivf": "200",
+    }
+    cfg.anns_query_params = {
+        "nprobe": "4",
+        "efSearch": "64",
+    }
+    return cfg
+
+
+def build_fallback_config(dimension: int = 8) -> _sage_db.DatabaseConfig:
+    cfg = _sage_db.DatabaseConfig(dimension)
+    cfg.metric = _sage_db.DistanceMetric.COSINE
+    cfg.index_type = _sage_db.IndexType.AUTO
+    cfg.anns_algorithm = "brute_force"
+    cfg.anns_build_params = {"metric": "cosine"}
+    cfg.anns_query_params = {"nprobe": "1"}
+    return cfg
+
+
+def generate_vectors(
+    dim: int, count: int = 64
+) -> Tuple[np.ndarray, List[Dict[str, str]]]:
+    rng = np.random.default_rng(seed=13)
+    raw = rng.normal(size=(count, dim)).astype("float32")
+    # Normalise to unit vectors for cosine distance
+    norms = np.linalg.norm(raw, axis=1, keepdims=True) + 1e-12
+    vectors = raw / norms
+    metadata: List[Dict[str, str]] = []
+    for idx in range(count):
+        metadata.append(
+            {
+                "id": f"vec-{idx}",
+                "bucket": str(idx // 8),
+                "parity": "even" if idx % 2 == 0 else "odd",
+            }
+        )
+    return vectors, metadata
+
+
+def ingest_numpy_batch(
+    db: _sage_db.SageDB, vectors: np.ndarray, metadata: List[Dict[str, str]]
+) -> None:
+    print(f"➕ ingesting {len(vectors)} vectors via add_batch (numpy)")
+    _sage_db.add_numpy(db, vectors, metadata)
+    print("   stored vectors:", db.size())
+
+
+def run_demo(db: _sage_db.SageDB, query: np.ndarray) -> None:
+    params = _sage_db.SearchParams(5)
+    params.include_metadata = True
+    params.nprobe = 4
+
+    print("\n🔍 Top-5 cosine neighbours for query vector")
+    results = _sage_db.search_numpy(db, query, params)
+    for rank, res in enumerate(results, start=1):
+        print(f"  {rank}. id={res.id:>3} score={res.score:.4f} metadata={res.metadata}")
+
+    engine = db.query_engine()
+    stats = engine.get_last_search_stats()
+    print(
+        f"   stats: candidates={stats.total_candidates} filter={stats.filtered_candidates} "
+        f"time={stats.total_time_ms:.3f}ms"
+    )
+
+
+def main() -> None:
+    vectors, metadata = generate_vectors(dim=8)
+    query = vectors[0] * 0.95 + vectors[1] * 0.05
+
+    print("=== SageDB FAISS plugin demo ===")
+
+    cfg = build_faiss_config(dimension=vectors.shape[1])
+    try:
+        db = _sage_db.SageDB(cfg)
+        ingest_numpy_batch(db, vectors, metadata)
+        db.build_index()
+        print("✅ FAISS index built successfully")
+    except (_sage_db.SageDBException, RuntimeError) as exc:
+        print("⚠️ FAISS backend unavailable, falling back to brute_force:\n   ", exc)
+        cfg = build_fallback_config(dimension=vectors.shape[1])
+        db = _sage_db.SageDB(cfg)
+        ingest_numpy_batch(db, vectors, metadata)
+        db.build_index()
+        print("✅ fallback brute_force index built")
+
+    run_demo(db, query)
+
+    print("\nℹ️ resolved config:")
+    print(
+        {
+            "algorithm": db.config().anns_algorithm,
+            "metric": db.config().metric.name,
+            "anns_build_params": dict(db.config().anns_build_params),
+            "anns_query_params": dict(db.config().anns_query_params),
+        }
+    )
+
+
+if __name__ == "__main__":
+    main()

sage/middleware/components/sage_db/examples/quickstart.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Minimal SageDB quickstart demo for the pluggable ANNS stack."""
+
+from __future__ import annotations
+
+import json
+import math
+import random
+import sys
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Dict, List
+
+import numpy as np
+
+# Ensure we can import the locally built extension/module without installation.
+REPO_ROOT = Path(__file__).resolve().parents[1]
+PACKAGES_ROOT = REPO_ROOT / "packages"
+if str(PACKAGES_ROOT) not in sys.path:
+    sys.path.insert(0, str(PACKAGES_ROOT))
+
+try:
+    import _sage_db
+except ImportError as exc:  # pragma: no cover - the example should exit cleanly
+    raise SystemExit(
+        "❌ Unable to import _sage_db. Please build the native extension first:\n"
+        "   cd packages/sage-middleware/src/sage/middleware/components/sage_db && ./build.sh"
+    ) from exc
+
+
+def build_config(dimension: int = 4) -> _sage_db.DatabaseConfig:
+    """Create a DatabaseConfig wired to the default brute-force plugin."""
+    cfg = _sage_db.DatabaseConfig(dimension)
+    cfg.metric = _sage_db.DistanceMetric.L2
+    cfg.index_type = _sage_db.IndexType.AUTO  # delegate selection to the plugin
+    cfg.anns_algorithm = "brute_force"
+    cfg.anns_build_params = {"metric": "l2"}
+    cfg.anns_query_params = {"nprobe": "1"}
+    return cfg
+
+
+def make_demo_vectors() -> List[List[float]]:
+    """Create a tiny set of 4D vectors arranged on a circle for easy inspection."""
+    base_angles = [0.0, math.pi / 4, math.pi / 2, 3 * math.pi / 4, math.pi]
+    vectors: List[List[float]] = []
+    for idx, angle in enumerate(base_angles):
+        vectors.append(
+            [
+                math.cos(angle),
+                math.sin(angle),
+                0.25 * (-1) ** idx,
+                0.1 * idx,
+            ]
+        )
+    return vectors
+
+
+def add_vectors(db: _sage_db.SageDB, vectors: List[List[float]]) -> List[int]:
+    """Insert vectors with metadata and return their ids."""
+    ids: List[int] = []
+    rng = random.Random(42)
+    for i, vec in enumerate(vectors, start=1):
+        metadata: Dict[str, str] = {
+            "label": f"demo-{i}",
+            "quadrant": str((i - 1) % 4 + 1),
+            "color": rng.choice(["red", "green", "blue", "yellow"]),
+        }
+        vec_id = db.add(vec, metadata)
+        ids.append(vec_id)
+        print(f"➕ inserted vector #{i} -> id={vec_id}, metadata={metadata}")
+    return ids
+
+
+def run_search(db: _sage_db.SageDB, query: List[float]) -> None:
+    params = _sage_db.SearchParams(k=3)
+    params.include_metadata = True
+    params.nprobe = 1
+
+    print("\n🔎 running 3-NN search for:", query)
+    results = db.search(query, params)
+    for rank, result in enumerate(results, start=1):
+        print(
+            f"  {rank}. id={result.id:>3}  score={result.score:.4f}  metadata={result.metadata}"
+        )
+
+
+def demonstrate_updates(db: _sage_db.SageDB, target_id: int) -> None:
+    print(
+        f"\n🛠️ updating vector id={target_id} to move it closer to the query direction"
+    )
+    new_vector = [0.8, 0.6, 0.0, 0.0]
+    db.update(target_id, new_vector, {"label": "demo-updated", "quadrant": "1"})
+    print("   metadata now:", db.get_metadata(target_id))
+
+
+def demonstrate_persistence(db: _sage_db.SageDB, query: List[float]) -> None:
+    print("\n💾 saving database to a temporary location and loading it back")
+    with TemporaryDirectory() as tmp:
+        db_path = Path(tmp) / "demo_db"
+        db.save(str(db_path))
+        print(f"   saved artefacts to {db_path}")
+
+        # Restore using a fresh instance with the same configuration
+        cfg = _sage_db.DatabaseConfig(db.dimension())
+        cfg.metric = db.config().metric
+        cfg.index_type = db.config().index_type
+        cfg.anns_algorithm = db.config().anns_algorithm
+        cfg.anns_build_params = dict(db.config().anns_build_params)
+        cfg.anns_query_params = dict(db.config().anns_query_params)
+
+        reloaded = _sage_db.SageDB(cfg)
+        reloaded.load(str(db_path))
+        print("   reloaded DB size:", reloaded.size())
+        run_search(reloaded, query)
+
+
+def main() -> None:
+    print("=== SageDB Quickstart ===")
+    cfg = build_config()
+    db = _sage_db.SageDB(cfg)
+
+    vectors = make_demo_vectors()
+    ids = add_vectors(db, vectors)
+
+    db.build_index()
+    print("📦 total vectors:", db.size())
+
+    query = [0.9, 0.3, 0.0, 0.0]
+    run_search(db, query)
+
+    demonstrate_updates(db, ids[0])
+    run_search(db, query)
+
+    print("\n🗑️ removing the furthest vector and rebuilding the index")
+    db.remove(ids[-1])
+    db.build_index()
+    print("   size after removal:", db.size())
+    run_search(db, query)
+
+    demonstrate_persistence(db, query)
+
+    print("\n📄 current configuration:")
+    print(
+        json.dumps(
+            {
+                "dimension": db.dimension(),
+                "metric": db.config().metric.name,
+                "anns_algorithm": db.config().anns_algorithm,
+                "anns_build_params": dict(db.config().anns_build_params),
+                "anns_query_params": dict(db.config().anns_query_params),
+            },
+            indent=2,
+            ensure_ascii=False,
+        )
+    )
+
+
+if __name__ == "__main__":
+    np.random.seed(0)
+    main()

sage/middleware/components/sage_db/python/__init__.py

@@ -2,55 +2,11 @@
 SAGE - Streaming-Augmented Generative Execution
 """
 
-# 动态版本加载
-def _load_version():
-    """从 sage-common 包加载版本信息"""
-    try:
-        # 优先从 sage-common 包加载版本
-        from sage.common._version import __version__
-        return {
-            'version': __version__,
-            'author': 'SAGE Team',
-            'email': 'shuhao_zhang@hust.edu.cn'
-        }
-    except ImportError:
-        # 如果 sage-common 不可用，从项目根目录加载（开发环境）
-        try:
-            from pathlib import Path
-            current_file = Path(__file__).resolve()
-            # 根据当前文件位置计算到项目根目录的层数
-            parts = current_file.parts
-            sage_index = -1
-            for i, part in enumerate(parts):
-                if part == 'SAGE':
-                    sage_index = i
-                    break
-            
-            if sage_index >= 0:
-                root_dir = Path(*parts[:sage_index+1])
-                version_file = root_dir / "_version.py"
-                
-                if version_file.exists():
-                    version_globals = {}
-                    with open(version_file, 'r', encoding='utf-8') as f:
-                        exec(f.read(), version_globals)
-                    return {
-                        'version': version_globals.get('__version__', '0.1.3'),
-                        'author': version_globals.get('__author__', 'SAGE Team'),
-                        'email': version_globals.get('__email__', 'shuhao_zhang@hust.edu.cn')
-                    }
-        except Exception:
-            pass
-    
-    # 最后的默认值
-    return {
-        'version': '0.1.3',
-        'author': 'SAGE Team', 
-        'email': 'shuhao_zhang@hust.edu.cn'
-    }
-
-# 加载信息
-_info = _load_version()
-__version__ = _info['version']
-__author__ = _info['author']
-__email__ = _info['email']
+# 直接从本包的_version模块加载版本信息
+try:
+    from sage.middleware._version import __author__, __email__, __version__
+except ImportError:
+    # 备用硬编码版本
+    __version__ = "0.1.4"
+    __author__ = "IntelliStream Team"
+    __email__ = "shuhao_zhang@hust.edu.cn"

sage/middleware/components/sage_db/python/micro_service/__init__.py

@@ -0,0 +1,8 @@
+"""SAGE-DB micro_service package.
+
+Provides a minimal SageDBService used by examples and quick demos.
+"""
+
+from .sage_db_service import SageDBService  # noqa: F401
+
+__all__ = ["SageDBService"]

sage/middleware/components/sage_db/python/micro_service/sage_db_service.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+from ..sage_db import DatabaseConfig, IndexType, SageDB
+
+
+@dataclass
+class SageDBServiceConfig:
+    dimension: int = 4
+    index_type: str = "AUTO"  # one of IndexType names
+
+
+class SageDBService:
+    """
+    A minimal micro-service style wrapper for SageDB used by examples.
+
+    Methods:
+      - add(vector: np.ndarray | list, metadata: dict) -> int
+      - add_batch(vectors: np.ndarray | list[list], metadata_list: list[dict]) -> list[int]
+      - search(query: np.ndarray | list, k: int = 5) -> list[dict]
+    """
+
+    def __init__(self, dimension: int = 4, index_type: str = "AUTO") -> None:
+        try:
+            idx_enum = getattr(IndexType, index_type)
+        except AttributeError:
+            idx_enum = IndexType.AUTO
+        cfg = DatabaseConfig(dimension)
+        cfg.index_type = idx_enum
+        self._db = SageDB.from_config(cfg)
+        self._dim = dimension
+
+    def add(
+        self,
+        vector: np.ndarray | List[float],
+        metadata: Optional[Dict[str, str]] = None,
+    ) -> int:
+        if not isinstance(vector, np.ndarray):
+            vector = np.asarray(vector, dtype=np.float32)
+        if vector.ndim != 1 or vector.shape[0] != self._dim:
+            raise ValueError(f"vector shape must be ({self._dim},)")
+        return self._db.add(vector, metadata or {})
+
+    def add_batch(
+        self,
+        vectors: np.ndarray | List[List[float]],
+        metadata_list: Optional[List[Dict[str, str]]] = None,
+    ) -> List[int]:
+        if isinstance(vectors, list):
+            arr = np.asarray(vectors, dtype=np.float32)
+        else:
+            arr = vectors.astype(np.float32, copy=False)
+        if arr.ndim != 2 or arr.shape[1] != self._dim:
+            raise ValueError(f"vectors shape must be (N, {self._dim})")
+        return self._db.add_batch(arr, metadata_list or [])
+
+    def search(
+        self, query: np.ndarray | List[float], k: int = 5, include_metadata: bool = True
+    ) -> List[Dict[str, Any]]:
+        if not isinstance(query, np.ndarray):
+            query = np.asarray(query, dtype=np.float32)
+        results = self._db.search(query, k=k, include_metadata=include_metadata)
+        formatted = []
+        for r in results:
+            formatted.append(
+                {
+                    "id": int(r.id),
+                    "score": float(r.score),
+                    "metadata": dict(r.metadata) if include_metadata else {},
+                }
+            )
+        return formatted
+
+    def stats(self) -> Dict[str, Any]:
+        s = self._db.get_search_stats()
+        return {
+            "size": self._db.size,
+            "dimension": self._db.dimension,
+            **s,
+        }