deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/rag/pipelines/lightrag_server/client.py

@@ -0,0 +1,149 @@
+"""Thin async HTTP client for an external LightRAG server's REST API.
+
+We talk to the documented endpoints directly (``httpx`` only) — the calls map
+1:1 onto our retrieval-only contract:
+
+* ``POST /query`` with ``only_need_context=True`` — return the grounded context
+  the server retrieved, WITHOUT its own generation. DeepTutor's chat loop keeps
+  ownership of the answer; the server is used purely as a retriever.
+* ``GET /auth-status`` — reachability + whether the server requires an API key
+  (whitelisted on the server, so it answers without credentials).
+* ``GET /documents/pipeline_status`` — an auth-gated, side-effect-free call used
+  only to validate that a configured API key is accepted.
+
+Mirrors :class:`PageIndexClient`: a fresh :class:`httpx.AsyncClient` per call so
+the object is safe to construct once and reuse, and an injectable ``transport``
+so tests can stub the wire without a live server.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+
+import httpx
+
+from .config import LightRagServerConfig
+
+logger = logging.getLogger(__name__)
+
+
+class LightRagServerAPIError(RuntimeError):
+    """Raised when the LightRAG server returns an error or unexpected payload."""
+
+
+class LightRagServerClient:
+    """Stateless wrapper over an external LightRAG server's REST API."""
+
+    def __init__(
+        self,
+        config: LightRagServerConfig,
+        *,
+        timeout: float = 60.0,
+        transport: Optional[httpx.AsyncBaseTransport] = None,
+    ) -> None:
+        self._config = config
+        self._timeout = timeout
+        self._transport = transport
+
+    def _open(self) -> httpx.AsyncClient:
+        headers = {"Accept": "application/json"}
+        if self._config.api_key:
+            # LightRAG server authenticates with an ``X-API-Key`` header
+            # (its ``LIGHTRAG_API_KEY``); absent when the server runs open.
+            headers["X-API-Key"] = self._config.api_key
+        return httpx.AsyncClient(
+            base_url=self._config.base_url,
+            headers=headers,
+            timeout=self._timeout,
+            transport=self._transport,
+        )
+
+    @staticmethod
+    def _json(resp: httpx.Response) -> dict[str, Any]:
+        if resp.status_code >= 400:
+            raise LightRagServerAPIError(
+                f"LightRAG server returned {resp.status_code}: {resp.text[:300]}"
+            )
+        try:
+            data = resp.json()
+        except Exception as exc:  # pragma: no cover - defensive
+            raise LightRagServerAPIError(
+                f"LightRAG server returned a non-JSON response: {exc}"
+            ) from exc
+        if not isinstance(data, dict):
+            raise LightRagServerAPIError(f"LightRAG server returned unexpected payload: {data!r}")
+        return data
+
+    # ----- retrieval ------------------------------------------------------
+
+    async def query_context(self, query: str, mode: str) -> dict[str, Any]:
+        """Retrieve grounded context for ``query`` without server-side generation.
+
+        Returns ``{"content": <context string>, "sources": [...]}``. ``sources``
+        are derived from the server's ``references`` list when present (one entry
+        per cited source file); an older server that omits references yields an
+        empty list rather than an error.
+        """
+        async with self._open() as client:
+            resp = await client.post(
+                "/query",
+                json={"query": query, "mode": mode, "only_need_context": True},
+            )
+        data = self._json(resp)
+        content = str(data.get("response") or "")
+        sources = _sources_from_references(data.get("references"))
+        return {"content": content, "sources": sources}
+
+    # ----- probing --------------------------------------------------------
+
+    async def auth_status(self) -> dict[str, Any]:
+        """Fetch ``/auth-status`` (no credentials) to probe reachability.
+
+        The presence of LightRAG-specific keys (``auth_configured`` /
+        ``core_version``) doubles as a "this really is a LightRAG server" signal.
+        """
+        async with self._open() as client:
+            resp = await client.get("/auth-status")
+        return self._json(resp)
+
+    async def verify_key(self) -> bool:
+        """Return whether the configured API key is accepted by the server.
+
+        Hits the auth-gated, read-only ``/documents/pipeline_status``: a 2xx
+        means the key (or open access) is valid; 401/403 means it was rejected.
+        Any other transport error propagates to the caller.
+        """
+        async with self._open() as client:
+            resp = await client.get("/documents/pipeline_status")
+        if resp.status_code in (401, 403):
+            return False
+        if resp.status_code >= 400:
+            raise LightRagServerAPIError(
+                f"LightRAG server returned {resp.status_code}: {resp.text[:300]}"
+            )
+        return True
+
+
+def _sources_from_references(references: Any) -> list[dict[str, Any]]:
+    """Map a LightRAG ``references`` list into DeepTutor's ``sources`` shape."""
+    if not isinstance(references, list):
+        return []
+    sources: list[dict[str, Any]] = []
+    for ref in references:
+        if not isinstance(ref, dict):
+            continue
+        file_path = str(ref.get("file_path") or "").strip()
+        ref_id = str(ref.get("reference_id") or "").strip()
+        if not file_path and not ref_id:
+            continue
+        source: dict[str, Any] = {}
+        if ref_id:
+            source["id"] = ref_id
+        if file_path:
+            source["file_path"] = file_path
+        sources.append(source)
+    return sources
+
+
+__all__ = ["LightRagServerClient", "LightRagServerAPIError"]

deeptutor/services/rag/pipelines/lightrag_server/config.py

@@ -0,0 +1,68 @@
+"""Per-KB connection config and retrieval modes for the LightRAG Server engine.
+
+Unlike the hosted PageIndex engine (one global account key shared by every KB),
+each LightRAG Server KB points at its OWN server instance — a standalone LightRAG
+server's ``--workspace`` is fixed at startup, so one server instance = one
+workspace = one knowledge base. The endpoint (base URL + optional API key) is
+therefore stored per-KB in ``kb_config.json`` (the same place the KB's
+``search_mode`` lives), exactly like a ``linked`` KB stores its ``external_path``.
+
+This module is the single seam that reads that binding into a typed config; it
+holds no global state and imports no HTTP client (the client lives in
+``client.py``).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+# LightRAG's ``/query`` retrieval modes. ``bypass`` is intentionally omitted: it
+# skips retrieval entirely, which is meaningless for a knowledge base. ``mix``
+# (knowledge-graph + vector + chunks) is the server's own default and the safest
+# general choice, matching the shared per-KB ``search_mode`` default.
+SUPPORTED_MODES = ("naive", "local", "global", "hybrid", "mix")
+DEFAULT_MODE = "mix"
+
+
+class LightRagServerNotConfiguredError(RuntimeError):
+    """Raised when a KB has no LightRAG server URL bound to it."""
+
+
+@dataclass(frozen=True)
+class LightRagServerConfig:
+    """A KB's resolved connection to an external LightRAG server."""
+
+    base_url: str
+    api_key: str
+
+
+def normalize_base_url(url: str | None) -> str:
+    """Trim and drop any trailing slash so endpoint paths join cleanly."""
+    return (url or "").strip().rstrip("/")
+
+
+def config_from_entry(entry: dict[str, Any]) -> LightRagServerConfig:
+    """Build a :class:`LightRagServerConfig` from a ``kb_config.json`` KB entry.
+
+    Raises :class:`LightRagServerNotConfiguredError` when no ``server_url`` is
+    present, so retrieval fails with a clear message instead of an opaque
+    connection error.
+    """
+    base_url = normalize_base_url(entry.get("server_url"))
+    if not base_url:
+        raise LightRagServerNotConfiguredError(
+            "This knowledge base is not connected to a LightRAG server "
+            "(no server URL configured). Re-create it with a valid server URL."
+        )
+    return LightRagServerConfig(base_url=base_url, api_key=str(entry.get("api_key") or "").strip())
+
+
+__all__ = [
+    "SUPPORTED_MODES",
+    "DEFAULT_MODE",
+    "LightRagServerNotConfiguredError",
+    "LightRagServerConfig",
+    "normalize_base_url",
+    "config_from_entry",
+]

deeptutor/services/rag/pipelines/lightrag_server/pipeline.py

@@ -0,0 +1,120 @@
+"""Retrieval-only pipeline backed by an external LightRAG server.
+
+Implements the same contract as the other pipelines (see ``..base.RAGPipeline``)
+but owns no index: a ``lightrag-server`` KB is a connection pointer (``type:
+lightrag_server`` in ``kb_config.json``) to a standalone LightRAG server the user
+runs and indexed themselves. Only :meth:`search` does real work — it reads the
+KB's endpoint, asks the server for grounded context (no server-side generation),
+and shapes the result for the ``rag`` tool. Indexing is offloaded entirely to the
+server, so :meth:`initialize` / :meth:`add_documents` are not part of this
+engine's job and fail with a clear message; :meth:`delete` is a no-op because
+deleting the KB only drops DeepTutor's pointer (handled by the manager) and must
+never touch the user's server.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from deeptutor.runtime.home import get_runtime_data_root
+from deeptutor.services.rag.provider_binding import load_kb_config_entry
+
+from ..modes import resolve_kb_mode
+from .config import (
+    DEFAULT_MODE,
+    SUPPORTED_MODES,
+    LightRagServerNotConfiguredError,
+    config_from_entry,
+)
+
+logger = logging.getLogger(__name__)
+
+PROVIDER = "lightrag-server"
+DEFAULT_KB_BASE_DIR = str(get_runtime_data_root() / "knowledge_bases")
+
+
+class LightRagServerPipeline:
+    """Query an external LightRAG server on behalf of a connected KB."""
+
+    def __init__(self, kb_base_dir: Optional[str] = None, *, client_factory=None, **_: Any) -> None:
+        self.logger = logging.getLogger(__name__)
+        self.kb_base_dir = kb_base_dir or DEFAULT_KB_BASE_DIR
+        # Injection seam for tests: (config) -> client. None uses the real client.
+        self._client_factory = client_factory
+
+    # ----- helpers --------------------------------------------------------
+
+    def _client(self, config):
+        if self._client_factory is not None:
+            return self._client_factory(config)
+        from .client import LightRagServerClient
+
+        return LightRagServerClient(config)
+
+    def _resolve_mode(self, kb_name: str, kwargs: dict[str, Any]) -> str:
+        return resolve_kb_mode(
+            self.kb_base_dir,
+            kb_name,
+            PROVIDER,
+            explicit=kwargs.get("mode"),
+            supported=SUPPORTED_MODES,
+            default=DEFAULT_MODE,
+        )
+
+    # ----- retrieval ------------------------------------------------------
+
+    async def search(self, query: str, kb_name: str, **kwargs) -> Dict[str, Any]:
+        try:
+            config = config_from_entry(load_kb_config_entry(self.kb_base_dir, kb_name))
+        except LightRagServerNotConfiguredError as exc:
+            return self._error_result(query, exc, error_type="not_configured")
+
+        mode = self._resolve_mode(kb_name, kwargs)
+        try:
+            result = await self._client(config).query_context(query, mode)
+        except Exception as exc:
+            self.logger.error("LightRAG server search failed for '%s': %s", kb_name, exc)
+            return self._error_result(query, exc, error_type="retrieval_error")
+
+        content = result.get("content") or ""
+        return {
+            "query": query,
+            "answer": content,
+            "content": content,
+            "sources": result.get("sources") or [],
+            "provider": PROVIDER,
+            "mode": mode,
+        }
+
+    def _error_result(self, query: str, exc: Exception, *, error_type: str) -> Dict[str, Any]:
+        return {
+            "query": query,
+            "answer": str(exc),
+            "content": "",
+            "sources": [],
+            "provider": PROVIDER,
+            "error_type": error_type,
+        }
+
+    # ----- indexing (not applicable — owned by the external server) -------
+
+    async def initialize(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
+        raise RuntimeError(
+            "LightRAG Server knowledge bases are indexed on the external server; "
+            "DeepTutor does not build or store their index. Add documents on the "
+            "LightRAG server directly."
+        )
+
+    async def add_documents(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
+        return await self.initialize(kb_name, file_paths, **kwargs)
+
+    # ----- lifecycle ------------------------------------------------------
+
+    async def delete(self, kb_name: str, **kwargs) -> bool:
+        # The KB is only a pointer; the manager removes its config entry. Never
+        # touch the user's server. Nothing local to clean up here.
+        return True
+
+
+__all__ = ["LightRagServerPipeline", "PROVIDER"]

deeptutor/services/rag/pipelines/lightrag_server/probe.py

@@ -0,0 +1,110 @@
+"""Probe an external LightRAG server before connecting a KB to it.
+
+Connecting is cheap and reversible, but a typo'd URL or a wrong API key should
+fail loudly at connect time rather than silently at every later query. This
+module answers, in one round-trip pair, the questions the UI needs to confirm:
+
+1. **Reachable, and is it actually a LightRAG server?** ``GET /auth-status`` is
+   whitelisted on the server, so it answers without credentials; its LightRAG-
+   specific keys confirm we're talking to the right kind of server.
+2. **Does it require an API key, and is ours accepted?** When auth is enabled we
+   validate the key against the read-only ``GET /documents/pipeline_status``.
+
+Always returns a :class:`ServerProbe` (never raises); ``ok`` is the single
+boolean the caller gates on, with ``error`` explaining any failure.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any, Optional
+
+from .client import LightRagServerClient
+from .config import LightRagServerConfig, normalize_base_url
+
+
+@dataclass
+class ServerProbe:
+    base_url: str
+    ok: bool = False
+    reachable: bool = False
+    auth_required: bool = False
+    auth_ok: bool = False
+    core_version: Optional[str] = None
+    api_version: Optional[str] = None
+    error: Optional[str] = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+async def probe_server(
+    server_url: str,
+    api_key: str = "",
+    *,
+    client_factory=None,
+) -> ServerProbe:
+    """Inspect ``server_url`` for a reachable, authorised LightRAG server.
+
+    ``client_factory`` (config → client) is an injection seam for tests; in
+    production it defaults to a real :class:`LightRagServerClient`.
+    """
+    base_url = normalize_base_url(server_url)
+    probe = ServerProbe(base_url=base_url)
+    if not base_url:
+        probe.error = "Server URL is required."
+        return probe
+    if not (base_url.startswith("http://") or base_url.startswith("https://")):
+        probe.error = "Server URL must start with http:// or https://."
+        return probe
+
+    config = LightRagServerConfig(base_url=base_url, api_key=(api_key or "").strip())
+    client = client_factory(config) if client_factory else LightRagServerClient(config)
+
+    try:
+        status = await client.auth_status()
+    except Exception as exc:
+        probe.error = f"Could not reach a LightRAG server at {base_url}: {exc}"
+        return probe
+
+    # ``auth_configured`` is LightRAG-specific; its absence means we reached
+    # something that isn't a LightRAG server.
+    if "auth_configured" not in status:
+        probe.error = f"{base_url} responded but does not look like a LightRAG server."
+        return probe
+
+    probe.reachable = True
+    probe.core_version = _opt_str(status.get("core_version"))
+    probe.api_version = _opt_str(status.get("api_version"))
+    probe.auth_required = bool(status.get("auth_configured"))
+
+    if not probe.auth_required:
+        # Open server — nothing to validate. A stray key is simply ignored.
+        probe.auth_ok = True
+        probe.ok = True
+        return probe
+
+    try:
+        probe.auth_ok = await client.verify_key()
+    except Exception as exc:
+        probe.error = f"Reached the server but could not validate the API key: {exc}"
+        return probe
+
+    if not probe.auth_ok:
+        probe.error = (
+            "This server requires an API key and the one provided was rejected."
+            if api_key
+            else "This server requires an API key. Provide one to connect."
+        )
+        return probe
+
+    probe.ok = True
+    return probe
+
+
+def _opt_str(value: Any) -> Optional[str]:
+    text = str(value).strip() if value is not None else ""
+    return text or None
+
+
+__all__ = ["ServerProbe", "probe_server"]

deeptutor/services/rag/pipelines/llamaindex/__init__.py

@@ -0,0 +1,3 @@
+"""LlamaIndex RAG pipeline implementation package."""
+
+__all__: list[str] = []

deeptutor/services/rag/pipelines/llamaindex/config.py

@@ -0,0 +1,199 @@
+"""Configuration helpers for DeepTutor's LlamaIndex RAG pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import os
+import sys
+
+VECTOR_PROFILE = "vector"
+HYBRID_PROFILE = "hybrid"
+SUPPORTED_RETRIEVAL_PROFILES = {VECTOR_PROFILE, HYBRID_PROFILE}
+FLAT_VECTOR_INDEX = "flat"
+HNSW_VECTOR_INDEX = "hnsw"
+SUPPORTED_VECTOR_INDEX_TYPES = {FLAT_VECTOR_INDEX, HNSW_VECTOR_INDEX}
+
+
+def should_show_progress() -> bool:
+    """Whether to emit LlamaIndex ``tqdm`` progress bars.
+
+    tqdm writes carriage-return progress lines to ``sys.stdout``. When
+    DeepTutor runs as a server that stream is a pipe whose read end (the
+    launcher's relay thread) can close mid-indexing, and the next tqdm write
+    then raises :class:`BrokenPipeError`, killing document indexing. DeepTutor
+    reports indexing progress through its own ``ProgressTracker``, so the tqdm
+    output is only wanted in an interactive CLI/REPL session.
+    """
+    return bool(getattr(sys.stdout, "isatty", lambda: False)())
+
+
+@dataclass(frozen=True)
+class RetrievalConfig:
+    """Runtime retrieval knobs for the LlamaIndex pipeline."""
+
+    profile: str = HYBRID_PROFILE
+    vector_top_k_multiplier: int = 2
+    bm25_top_k_multiplier: int = 2
+    fusion_num_queries: int = 1
+    reranker_model: str = ""
+    rerank_top_k: int = 50
+
+    def candidate_top_k(self, top_k: int, multiplier: int) -> int:
+        """Return the number of candidates to ask a child retriever for."""
+        requested = max(1, int(top_k))
+        return max(requested, requested * max(1, int(multiplier)))
+
+    def rerank_candidate_top_k(self, top_k: int) -> int:
+        """Return the first-stage candidate count before optional reranking."""
+        requested = max(1, int(top_k))
+        if not self.reranker_model:
+            return requested
+        return max(requested, min(100, max(1, int(self.rerank_top_k))))
+
+
+@dataclass(frozen=True)
+class VectorIndexConfig:
+    """FAISS construction knobs for the next full index build."""
+
+    type: str = FLAT_VECTOR_INDEX
+    hnsw_m: int = 32
+    hnsw_ef_construction: int = 200
+    hnsw_ef_search: int = 64
+
+
+def normalize_retrieval_profile(value: str | None) -> str:
+    """Return a supported retrieval profile, defaulting to hybrid."""
+    profile = (value or "").strip().lower()
+    if profile in SUPPORTED_RETRIEVAL_PROFILES:
+        return profile
+    return HYBRID_PROFILE
+
+
+def normalize_reranker_model(value: str | None) -> str:
+    """Return a bounded Hugging Face model identifier (empty disables rerank)."""
+    return (value or "").strip()[:200]
+
+
+def normalize_vector_index_type(value: str | None) -> str:
+    """Return a supported vector index type, defaulting to exact flat search."""
+    index_type = (value or "").strip().lower()
+    if index_type in SUPPORTED_VECTOR_INDEX_TYPES:
+        return index_type
+    return FLAT_VECTOR_INDEX
+
+
+def retrieval_config_from_env() -> RetrievalConfig:
+    """Build retrieval config from environment variables.
+
+    The default is intentionally ``hybrid``. If the optional LlamaIndex BM25
+    integration is not installed, the retriever builder transparently falls
+    back to plain vector retrieval.
+    """
+
+    return RetrievalConfig(
+        profile=normalize_retrieval_profile(
+            os.getenv("DEEPTUTOR_RAG_RETRIEVAL_PROFILE") or os.getenv("RAG_RETRIEVAL_PROFILE")
+        )
+    )
+
+
+def _load_runtime_settings() -> dict:
+    """Load the persisted LlamaIndex engine settings (env overrides applied)."""
+    from deeptutor.services.config import load_llamaindex_settings
+
+    return load_llamaindex_settings()
+
+
+def retrieval_config_from_settings() -> RetrievalConfig:
+    """Build retrieval config from persisted engine settings.
+
+    Falls back to defaults on any read error so retrieval never breaks because
+    of a malformed settings file. ``fusion_num_queries`` stays at the dataclass
+    default — query generation needs a real LLM, but the fusion retriever runs
+    on a MockLLM, so it is not user-tunable.
+    """
+    try:
+        settings = _load_runtime_settings()
+    except Exception:
+        return RetrievalConfig()
+    return RetrievalConfig(
+        profile=normalize_retrieval_profile(settings.get("retrieval_profile")),
+        vector_top_k_multiplier=int(settings.get("vector_top_k_multiplier", 2) or 2),
+        bm25_top_k_multiplier=int(settings.get("bm25_top_k_multiplier", 2) or 2),
+        reranker_model=normalize_reranker_model(settings.get("reranker_model")),
+        rerank_top_k=int(settings.get("rerank_top_k", 50) or 50),
+    )
+
+
+def vector_index_config_from_settings() -> VectorIndexConfig:
+    """Build FAISS construction settings, retaining the exact flat default."""
+    try:
+        settings = _load_runtime_settings()
+    except Exception:
+        return VectorIndexConfig()
+    return VectorIndexConfig(
+        type=normalize_vector_index_type(settings.get("vector_index_type")),
+        hnsw_m=int(settings.get("hnsw_m", 32) or 32),
+        hnsw_ef_construction=int(settings.get("hnsw_ef_construction", 200) or 200),
+        hnsw_ef_search=int(settings.get("hnsw_ef_search", 64) or 64),
+    )
+
+
+def default_top_k() -> int:
+    """The configured default number of chunks a retrieval returns."""
+    try:
+        return int(_load_runtime_settings().get("top_k", 5) or 5)
+    except Exception:
+        return 5
+
+
+def chunk_geometry() -> tuple[int, int]:
+    """The configured ``(chunk_size, chunk_overlap)`` for indexing."""
+    try:
+        settings = _load_runtime_settings()
+        chunk_size = settings.get("chunk_size", 512)
+        chunk_overlap = settings.get("chunk_overlap", 50)
+        return int(chunk_size if chunk_size is not None else 512), int(
+            chunk_overlap if chunk_overlap is not None else 50
+        )
+    except Exception:
+        return 512, 50
+
+
+def image_description_batch_size() -> int:
+    """Opt-in images per LlamaIndex caption request; one preserves legacy calls."""
+    try:
+        return min(8, max(1, int(_load_runtime_settings().get("image_description_batch_size", 1))))
+    except (TypeError, ValueError):
+        return 1
+
+
+def image_description_limits() -> tuple[int, float]:
+    """Return the configured vision-call concurrency and per-image timeout."""
+    try:
+        settings = _load_runtime_settings()
+        concurrency = int(settings.get("image_description_concurrency", 4) or 4)
+        timeout_seconds = float(settings.get("image_description_timeout_seconds", 60) or 60)
+        return min(16, max(1, concurrency)), min(600.0, max(5.0, timeout_seconds))
+    except Exception:
+        return 4, 60.0
+
+
+__all__ = [
+    "HYBRID_PROFILE",
+    "RetrievalConfig",
+    "SUPPORTED_RETRIEVAL_PROFILES",
+    "SUPPORTED_VECTOR_INDEX_TYPES",
+    "VECTOR_PROFILE",
+    "FLAT_VECTOR_INDEX",
+    "HNSW_VECTOR_INDEX",
+    "chunk_geometry",
+    "default_top_k",
+    "image_description_limits",
+    "image_description_batch_size",
+    "normalize_retrieval_profile",
+    "normalize_reranker_model",
+    "retrieval_config_from_env",
+    "retrieval_config_from_settings",
+    "vector_index_config_from_settings",
+]