deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/rag/pipelines/llamaindex/vector_store.py

@@ -0,0 +1,288 @@
+"""Vector-store backend selection for the LlamaIndex RAG pipeline.
+
+This module is the single seam between DeepTutor's LlamaIndex pipeline and the
+concrete vector store implementation. It exists so the rest of the pipeline
+(ingestion, storage, retrieval) never has to know *how* vectors are stored.
+
+Why it exists
+-------------
+LlamaIndex's built-in ``SimpleVectorStore`` keeps every embedding in a JSON
+document and performs a pure-Python, O(N) brute-force scan on each query. On a
+large knowledge base that is re-parsed and re-scanned per query, which pins a
+CPU core and makes retrieval take minutes (issue #552).
+
+This seam swaps in a FAISS index instead:
+
+* New (and re-indexed) knowledge bases are persisted as a binary FAISS index
+  (exact ``IndexFlatIP`` by default, or opt-in HNSW for large corpora), loaded
+  once and searched without a Python-side scan.
+* Legacy ``SimpleVectorStore`` knowledge bases stay fully readable, so upgrading
+  never breaks an existing index. Re-indexing one rebuilds it as FAISS for the
+  full speed-up.
+
+FAISS is optional. When ``faiss`` / ``llama-index-vector-stores-faiss`` are not
+importable, every path falls back to ``SimpleVectorStore`` so retrieval keeps
+working (just without the speed-up).
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+from fsspec.implementations.local import LocalFileSystem
+from llama_index.core import StorageContext, load_index_from_storage
+from llama_index.core.vector_stores.simple import DEFAULT_VECTOR_STORE, NAMESPACE_SEP
+import numpy as np
+
+from .config import HNSW_VECTOR_INDEX, VectorIndexConfig
+
+logger = logging.getLogger(__name__)
+
+BACKEND_FAISS = "faiss"
+BACKEND_SIMPLE = "simple"
+
+# The default vector store is always persisted under this filename, whether it
+# holds a SimpleVectorStore JSON document or a binary FAISS index. The first
+# byte disambiguates the two: "{" => JSON/simple, anything else => FAISS.
+DEFAULT_VECTOR_STORE_FILENAME = f"{DEFAULT_VECTOR_STORE}{NAMESPACE_SEP}vector_store.json"
+
+_COSINE_FAISS_CLS: Optional[type] = None
+
+
+def _faiss_modules() -> tuple[Any, Any]:
+    """Return ``(faiss, FaissVectorStore)`` or ``(None, None)`` when unavailable."""
+    try:
+        import faiss
+        from llama_index.vector_stores.faiss import FaissVectorStore
+    except Exception:  # pragma: no cover - exercised only without faiss installed
+        return None, None
+    return faiss, FaissVectorStore
+
+
+def faiss_available() -> bool:
+    """True when both the FAISS library and its LlamaIndex integration import."""
+    faiss, faiss_store_cls = _faiss_modules()
+    return faiss is not None and faiss_store_cls is not None
+
+
+def _normalize(embedding: Any) -> list[float]:
+    """L2-normalize an embedding so inner product equals cosine similarity."""
+    vector = np.asarray(embedding, dtype="float32")
+    norm = float(np.linalg.norm(vector))
+    if norm > 0:
+        vector = vector / norm
+    return vector.tolist()
+
+
+def faiss_write_index(index: Any, persist_path: str) -> None:
+    """Persist a FAISS index through a Python byte stream (Unicode-path safe).
+
+    Stock ``faiss.write_index`` is a SWIG passthrough that hands the path
+    straight to C++ ``fopen``. On Windows that is the *narrow* ANSI API, while
+    SWIG encodes the Python string as UTF-8 — so any non-ASCII path (a Chinese
+    knowledge-base name, or a ``C:\\Users\\张三`` home directory) fails to open
+    and index rebuilds crash. Serializing to bytes in memory and letting Python
+    write the file sidesteps this: CPython opens files via the wide ``_wfopen``
+    API on Windows, so Unicode paths work. The byte payload is identical to
+    ``write_index`` output, so indexes stay cross-readable with stock FAISS.
+    """
+    import faiss
+
+    payload = faiss.serialize_index(index)
+    with open(persist_path, "wb") as handle:
+        handle.write(payload.tobytes())
+
+
+def faiss_read_index(persist_path: str) -> Any:
+    """Load a FAISS index written by :func:`faiss_write_index` (or stock FAISS).
+
+    The on-disk format is identical either way, so this also reads indexes
+    persisted by an older ``faiss.write_index`` call. Reading the bytes with
+    Python ``open`` keeps the load path Unicode-safe on Windows, mirroring
+    :func:`faiss_write_index`.
+    """
+    import faiss
+
+    with open(persist_path, "rb") as handle:
+        buffer = np.frombuffer(handle.read(), dtype="uint8")
+    return faiss.deserialize_index(buffer)
+
+
+def _cosine_faiss_cls() -> Optional[type]:
+    """Return a memoized FaissVectorStore subclass that ranks by cosine.
+
+    Stock ``FaissVectorStore`` normalizes neither on add nor on query, so an
+    ``IndexFlatIP`` would rank by raw dot product. Normalizing both sides makes
+    inner-product ranking identical to the cosine similarity SimpleVectorStore
+    used, preserving retrieval behaviour after the backend swap.
+    """
+    global _COSINE_FAISS_CLS
+    if _COSINE_FAISS_CLS is not None:
+        return _COSINE_FAISS_CLS
+
+    _, faiss_store_cls = _faiss_modules()
+    if faiss_store_cls is None:
+        return None
+
+    class _CosineFaissVectorStore(faiss_store_cls):  # type: ignore[valid-type, misc]
+        """FAISS store that L2-normalizes vectors for cosine ranking.
+
+        It also overrides persistence to route through :func:`faiss_write_index`
+        / :func:`faiss_read_index` instead of the stock path-based
+        ``faiss.write_index`` / ``read_index``, which cannot handle non-ASCII
+        paths on Windows. The two concerns are independent: cosine ranking
+        shapes *what* is stored, the IO override shapes *how* it reaches disk.
+        """
+
+        def add(self, nodes: list[Any], **kwargs: Any) -> list[str]:
+            for node in nodes:
+                node.embedding = _normalize(node.get_embedding())
+            return super().add(nodes, **kwargs)
+
+        def query(self, query: Any, **kwargs: Any) -> Any:
+            if getattr(query, "query_embedding", None) is not None:
+                query.query_embedding = _normalize(query.query_embedding)
+            return super().query(query, **kwargs)
+
+        def persist(self, persist_path: str, fs: Any = None) -> None:
+            if fs is not None and not isinstance(fs, LocalFileSystem):
+                raise NotImplementedError("FAISS only supports local storage for now.")
+            dirpath = os.path.dirname(persist_path)
+            if dirpath:
+                os.makedirs(dirpath, exist_ok=True)
+            faiss_write_index(self._faiss_index, persist_path)
+
+        @classmethod
+        def from_persist_path(cls, persist_path: str, fs: Any = None) -> Any:
+            if fs is not None and not isinstance(fs, LocalFileSystem):
+                raise NotImplementedError("FAISS only supports local storage for now.")
+            if not os.path.exists(persist_path):
+                raise ValueError(f"No existing FAISS index found at {persist_path}.")
+            return cls(faiss_index=faiss_read_index(persist_path))
+
+    _COSINE_FAISS_CLS = _CosineFaissVectorStore
+    return _COSINE_FAISS_CLS
+
+
+def _uniform_dimension(embeddings: Iterable[Any]) -> Optional[int]:
+    """Return the shared embedding dimension, or None if missing/ragged.
+
+    Mixed dimensions (e.g. text + multimodal image vectors) cannot live in a
+    single fixed-width FAISS index, so callers fall back to SimpleVectorStore.
+    """
+    dimension: Optional[int] = None
+    for embedding in embeddings:
+        if embedding is None:
+            return None
+        length = len(embedding)
+        if dimension is None:
+            dimension = length
+        elif length != dimension:
+            return None
+    return dimension if dimension and dimension > 0 else None
+
+
+def _new_faiss_index(faiss: Any, dimension: int, config: VectorIndexConfig) -> Any:
+    """Construct a cosine-compatible FAISS index for ``config``."""
+    if config.type == HNSW_VECTOR_INDEX:
+        index = faiss.IndexHNSWFlat(
+            dimension,
+            max(1, int(config.hnsw_m)),
+            faiss.METRIC_INNER_PRODUCT,
+        )
+        index.hnsw.efConstruction = max(1, int(config.hnsw_ef_construction))
+        index.hnsw.efSearch = max(1, int(config.hnsw_ef_search))
+        return index
+    return faiss.IndexFlatIP(dimension)
+
+
+def new_faiss_storage_context(
+    dimension: int, index_config: VectorIndexConfig | None = None
+) -> Optional[StorageContext]:
+    """Return a StorageContext whose default store is a fresh cosine FAISS index.
+
+    Returns None when FAISS is unavailable or the dimension is invalid, so
+    callers fall back to the default SimpleVectorStore StorageContext.
+    """
+    faiss, _ = _faiss_modules()
+    cosine_cls = _cosine_faiss_cls()
+    if faiss is None or cosine_cls is None or dimension <= 0:
+        return None
+    store = cosine_cls(
+        faiss_index=_new_faiss_index(faiss, dimension, index_config or VectorIndexConfig())
+    )
+    return StorageContext.from_defaults(vector_store=store)
+
+
+def storage_context_for_nodes(
+    nodes: list[Any], index_config: VectorIndexConfig | None = None
+) -> Optional[StorageContext]:
+    """Choose the write-time StorageContext for a set of embedded nodes.
+
+    Returns a FAISS-backed context when every node shares one embedding
+    dimension and FAISS is available; otherwise None (use the SimpleVectorStore
+    default). ``None`` keeps multimodal / mixed-dimension and faiss-less installs
+    on the original code path.
+    """
+    if not faiss_available():
+        return None
+    dimension = _uniform_dimension(getattr(node, "embedding", None) for node in nodes)
+    if dimension is None:
+        return None
+    return new_faiss_storage_context(dimension, index_config)
+
+
+def detect_backend(storage_dir: Path) -> str:
+    """Detect a persisted index's vector backend from its default store file."""
+    path = Path(storage_dir) / DEFAULT_VECTOR_STORE_FILENAME
+    try:
+        with open(path, "rb") as handle:
+            head = handle.read(1)
+    except OSError:
+        return BACKEND_SIMPLE
+    return BACKEND_SIMPLE if head[:1] == b"{" else BACKEND_FAISS
+
+
+def load_index(storage_dir: Path) -> Any:
+    """Load a persisted index for retrieval.
+
+    FAISS-persisted versions load their binary index directly. Legacy
+    SimpleVectorStore versions load unchanged and stay queryable; re-indexing
+    such a knowledge base rebuilds it as FAISS for the full speed-up.
+    """
+    storage_dir = Path(storage_dir)
+    from .embedding_adapter import current_embedding
+
+    if detect_backend(storage_dir) == BACKEND_FAISS:
+        cosine_cls = _cosine_faiss_cls()
+        if cosine_cls is None:
+            raise RuntimeError(
+                "This knowledge base was indexed with FAISS but the 'faiss-cpu' "
+                "package is not installed. Install it (pip install faiss-cpu) or "
+                "re-index the knowledge base to query it again."
+            )
+        vector_store = cosine_cls.from_persist_dir(str(storage_dir))
+        context = StorageContext.from_defaults(
+            persist_dir=str(storage_dir), vector_store=vector_store
+        )
+        return load_index_from_storage(context, embed_model=current_embedding())
+
+    context = StorageContext.from_defaults(persist_dir=str(storage_dir))
+    return load_index_from_storage(context, embed_model=current_embedding())
+
+
+__all__ = [
+    "BACKEND_FAISS",
+    "BACKEND_SIMPLE",
+    "DEFAULT_VECTOR_STORE_FILENAME",
+    "detect_backend",
+    "faiss_available",
+    "faiss_read_index",
+    "faiss_write_index",
+    "load_index",
+    "new_faiss_storage_context",
+    "storage_context_for_nodes",
+]

deeptutor/services/rag/pipelines/modes.py

@@ -0,0 +1,48 @@
+"""Shared retrieval-mode resolution for mode-aware pipelines (LightRAG, GraphRAG).
+
+Resolution order, first valid wins:
+
+1. an explicit ``mode`` kwarg (a one-off override on the search call),
+2. the KB's own ``search_mode`` (``kb_config.json`` → ``knowledge_bases[kb]``),
+3. the engine's global default mode set from the engine card
+   (``kb_config.json`` → ``defaults.provider_modes[provider]``),
+4. the engine's built-in default.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Optional, Sequence
+
+
+def resolve_kb_mode(
+    kb_base_dir: str | Path,
+    kb_name: Optional[str],
+    provider: str,
+    *,
+    explicit: Any = None,
+    supported: Sequence[str],
+    default: str,
+) -> str:
+    candidates: list[Any] = [explicit]
+    try:
+        cfg_path = Path(kb_base_dir) / "kb_config.json"
+        if cfg_path.exists():
+            data = json.loads(cfg_path.read_text(encoding="utf-8"))
+            if kb_name:
+                entry = data.get("knowledge_bases", {}).get(kb_name, {})
+                candidates.append(entry.get("search_mode"))
+            candidates.append(data.get("defaults", {}).get("provider_modes", {}).get(provider))
+    except Exception:  # pragma: no cover - defensive
+        pass
+
+    supported_set = {m.lower() for m in supported}
+    for candidate in candidates:
+        norm = (str(candidate) if candidate else "").strip().lower()
+        if norm in supported_set:
+            return norm
+    return default
+
+
+__all__ = ["resolve_kb_mode"]

deeptutor/services/rag/pipelines/pageindex/__init__.py

@@ -0,0 +1,47 @@
+"""PageIndex Cloud and OSS knowledge-base lifecycle.
+
+Both providers use turn-scoped PageIndex SDK tools: Cloud delegates its transport
+to the SDK, while OSS binds the same interface to the KB's Local Library. Neither
+provider answers through ``RAGPipeline.search``.
+"""
+
+from __future__ import annotations
+
+from .pipeline import PageIndexPipeline
+
+
+def _bound_pageindex_provider(kb_name: str | None) -> str | None:
+    if not kb_name:
+        return None
+    try:
+        from deeptutor.multi_user.knowledge_access import resolve_kb
+        from deeptutor.services.rag.provider_binding import resolve_bound_provider
+
+        resource = resolve_kb(kb_name, require_write=False)
+        return resolve_bound_provider(resource.base_dir, resource.name)
+    except Exception:
+        return None
+
+
+def is_pageindex_kb(kb_name: str | None) -> bool:
+    from deeptutor.services.rag.factory import PAGEINDEX_OSS_PROVIDER, PAGEINDEX_PROVIDER
+
+    return _bound_pageindex_provider(kb_name) in {PAGEINDEX_PROVIDER, PAGEINDEX_OSS_PROVIDER}
+
+
+def validate_pageindex_oss_selection(knowledge_bases: list[str]) -> None:
+    from deeptutor.services.rag.factory import PAGEINDEX_OSS_PROVIDER
+
+    selected = [
+        name
+        for name in dict.fromkeys(str(item or "").strip() for item in knowledge_bases)
+        if name and _bound_pageindex_provider(name) == PAGEINDEX_OSS_PROVIDER
+    ]
+    if len(selected) > 1:
+        raise ValueError(
+            "Select at most one PageIndex OSS knowledge base per request. "
+            f"Selected: {', '.join(selected)}"
+        )
+
+
+__all__ = ["PageIndexPipeline", "is_pageindex_kb", "validate_pageindex_oss_selection"]

deeptutor/services/rag/pipelines/pageindex/client.py

@@ -0,0 +1,146 @@
+"""Async adapter around the synchronous PageIndex SDK clients."""
+
+from __future__ import annotations
+
+import asyncio
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from deeptutor.services.provider_registry import find_by_name, strip_provider_prefix
+
+from .config import PageIndexConfig
+
+
+def _sdk_types():
+    from pageindex import PageIndexCloudClient, PageIndexLocalClient
+
+    return PageIndexCloudClient, PageIndexLocalClient
+
+
+@lru_cache(maxsize=1)
+def _cloud_sdk_client(api_key: str):
+    """Reuse the SDK's MCP bridge until the global Cloud key changes."""
+    cloud_type, _ = _sdk_types()
+    return cloud_type(api_key)
+
+
+def _prefixed_model(prefix: str, model: str) -> str:
+    return model if model.startswith(f"{prefix}/") else f"{prefix}/{model}"
+
+
+def resolve_oss_sdk_config() -> tuple[str, dict[str, Any]]:
+    """Translate DeepTutor's active LLM into PageIndex's indexing lane."""
+    from deeptutor.services.config import resolve_llm_runtime_config
+
+    cfg = resolve_llm_runtime_config()
+    model = str(getattr(cfg, "model", "") or "").strip()
+    binding = str(
+        getattr(cfg, "binding", None) or getattr(cfg, "provider_name", None) or "openai"
+    ).strip()
+    spec = find_by_name(binding)
+    if not model:
+        raise RuntimeError(
+            "PageIndex OSS needs an active LLM. Configure one under Settings → Catalog."
+        )
+    if (
+        spec is None
+        or spec.is_oauth
+        or spec.backend
+        in {
+            "openai_codex",
+            "github_copilot",
+            "codebuddy",
+        }
+    ):
+        raise RuntimeError(
+            "PageIndex OSS indexing needs an API-key or local LLM profile; "
+            "the active OAuth-only provider cannot be used."
+        )
+
+    resolved_model = strip_provider_prefix(model, spec)
+    prefix = {
+        "anthropic": "anthropic",
+        "azure_openai": "azure",
+        "openai_compat": "openai",
+    }.get(spec.backend)
+    if prefix is None:
+        raise RuntimeError("The active LLM transport is not supported by PageIndex OSS indexing.")
+
+    sdk_model = _prefixed_model(prefix, resolved_model)
+    backend: dict[str, Any] = {}
+    api_key = str(getattr(cfg, "api_key", "") or "").strip()
+    base_url = str(getattr(cfg, "base_url", "") or "").strip()
+    api_version = str(getattr(cfg, "api_version", "") or "").strip()
+    headers = getattr(cfg, "extra_headers", None)
+
+    if spec.backend == "openai_compat":
+        # PageIndex treats ``openai/...`` as the OpenAI-compatible fast path;
+        # this also preserves gateway model ids such as anthropic/claude-*.
+        backend["api_key"] = api_key or "sk-no-key-required"
+    elif api_key:
+        backend["api_key"] = api_key
+
+    if base_url:
+        backend["api_base"] = base_url
+    if api_version and spec.backend != "openai_compat":
+        backend["api_version"] = api_version
+    if isinstance(headers, dict) and headers:
+        if spec.backend == "openai_compat":
+            backend["default_headers"] = dict(headers)
+        else:
+            backend["extra_headers"] = dict(headers)
+
+    return sdk_model, backend
+
+
+class PageIndexClient:
+    """Small async facade used by the DeepTutor RAG lifecycle."""
+
+    def __init__(self, sdk_client: Any) -> None:
+        self.sdk_client = sdk_client
+
+    @classmethod
+    def cloud(cls, config: PageIndexConfig) -> "PageIndexClient":
+        return cls(_cloud_sdk_client(config.api_key))
+
+    @classmethod
+    def local(cls, storage_path: str | Path) -> "PageIndexClient":
+        _, local_type = _sdk_types()
+        model, backend = resolve_oss_sdk_config()
+        return cls(
+            local_type(
+                storage_path=str(storage_path),
+                index_model=model,
+                summary_model=model,
+                index_backend=backend,
+            )
+        )
+
+    @classmethod
+    def local_read(cls, storage_path: str | Path) -> "PageIndexClient":
+        """Open an existing Local Library without resolving indexing credentials."""
+        _, local_type = _sdk_types()
+        return cls(local_type(storage_path=str(storage_path)))
+
+    async def submit_document(self, file_path: str | Path, *, mode: str | None = None) -> str:
+        result = await asyncio.to_thread(
+            self.sdk_client.submit_document,
+            str(file_path),
+            mode=mode,
+            wait=True,
+        )
+        doc_id = result.get("doc_id") if isinstance(result, dict) else None
+        if not doc_id:
+            raise RuntimeError(f"PageIndex submit_document returned no doc_id: {result!r}")
+        return str(doc_id)
+
+    async def delete_document(self, doc_id: str) -> bool:
+        await asyncio.to_thread(self.sdk_client.delete_document, doc_id)
+        return True
+
+
+__all__ = [
+    "PageIndexClient",
+    "resolve_oss_sdk_config",
+]

deeptutor/services/rag/pipelines/pageindex/config.py

@@ -0,0 +1,49 @@
+"""Resolve PageIndex Cloud credentials and OSS readiness.
+
+The Cloud key lives in ``data/.../settings/pageindex.json`` (managed by
+``RuntimeSettingsService``), surfaced to users under Knowledge → RAG pipeline
+settings. A single deployment credential is shared by every ``pageindex`` KB;
+the SDK owns the official Cloud endpoint.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class PageIndexConfig:
+    api_key: str
+
+
+def get_pageindex_config(*, require_key: bool = True) -> PageIndexConfig:
+    """Load the active PageIndex credential.
+
+    Raises when ``require_key`` and the key is empty, so callers fail with a
+    clear, actionable message instead of an opaque 401 from the API.
+    """
+    from deeptutor.services.config import get_runtime_settings_service
+
+    settings = get_runtime_settings_service().load_pageindex()
+    api_key = str(settings.get("api_key") or "").strip()
+    if require_key and not api_key:
+        raise RuntimeError(
+            "PageIndex API key is not configured. Add it under "
+            "Knowledge → RAG pipeline settings before using a PageIndex knowledge base."
+        )
+    return PageIndexConfig(api_key=api_key)
+
+
+def is_pageindex_configured() -> bool:
+    """Best-effort check used to flag the provider as ready in the UI."""
+    try:
+        return bool(get_pageindex_config(require_key=False).api_key)
+    except Exception:
+        return False
+
+
+__all__ = [
+    "PageIndexConfig",
+    "get_pageindex_config",
+    "is_pageindex_configured",
+]