deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/rag/preflight.py

@@ -0,0 +1,274 @@
+"""Per-engine environment preflight checks.
+
+Powers the "check whether this engine can run right now" affordance on each
+engine's detail page. Every check is best-effort and never raises — a failed
+import or missing config becomes a failed/optional check, not an exception.
+
+A check is ``{key, label, ok, detail, optional}``. Overall ``ok`` is true when
+every *required* (non-optional) check passes.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .factory import (
+    DEFAULT_PROVIDER,
+    GRAPHRAG_PROVIDER,
+    IMA_PROVIDER,
+    LIGHTRAG_PROVIDER,
+    LIGHTRAG_SERVER_PROVIDER,
+    PAGEINDEX_OSS_PROVIDER,
+    PAGEINDEX_PROVIDER,
+    normalize_provider_name,
+)
+
+
+def _check(key: str, label: str, ok: bool, detail: str = "", *, optional: bool = False) -> dict:
+    return {"key": key, "label": label, "ok": bool(ok), "detail": detail, "optional": optional}
+
+
+def _active_chat_model() -> tuple[str | None, str]:
+    """Return ``(model, binding)`` for the active chat LLM, or ``(None, "")``."""
+    try:
+        from deeptutor.services.config import resolve_llm_runtime_config
+
+        cfg = resolve_llm_runtime_config()
+        return getattr(cfg, "model", None), str(getattr(cfg, "binding", "") or "")
+    except Exception:
+        return None, ""
+
+
+def _active_embedding() -> tuple[str | None, int]:
+    """Return ``(model, dim)`` for the active embedding model, or ``(None, 0)``."""
+    try:
+        from deeptutor.services.embedding import get_embedding_config
+
+        cfg = get_embedding_config()
+        return getattr(cfg, "model", None), int(getattr(cfg, "dim", 0) or 0)
+    except Exception:
+        return None, 0
+
+
+def _llamaindex_preflight() -> dict:
+    emb_model, emb_dim = _active_embedding()
+    checks = [
+        _check(
+            "embedding",
+            "Active embedding model",
+            bool(emb_model) and emb_dim > 0,
+            f"{emb_model} · {emb_dim}d" if emb_model else "Configure one in the model catalog.",
+        )
+    ]
+    try:
+        from .pipelines.llamaindex.retrievers import _import_bm25_retriever
+
+        bm25_ok = _import_bm25_retriever() is not None
+    except Exception:
+        bm25_ok = False
+    checks.append(
+        _check(
+            "bm25",
+            "BM25 hybrid retrieval",
+            bm25_ok,
+            "Installed." if bm25_ok else "Not installed — hybrid falls back to vector-only.",
+            optional=True,
+        )
+    )
+    return _finalize(checks)
+
+
+def _pageindex_preflight() -> dict:
+    try:
+        from .pipelines.pageindex.config import get_pageindex_config
+
+        cfg = get_pageindex_config(require_key=False)
+        configured = bool(cfg.api_key)
+    except Exception:
+        configured = False
+    return _finalize(
+        [
+            _check(
+                "api_key",
+                "API key configured",
+                configured,
+                "PageIndex Cloud" if configured else "Add a PageIndex API key under Credentials.",
+            )
+        ]
+    )
+
+
+def _pageindex_oss_preflight() -> dict:
+    try:
+        from .pipelines.pageindex.client import resolve_oss_sdk_config
+
+        model, _backend = resolve_oss_sdk_config()
+        llm_ok, detail = bool(model), model
+    except Exception as exc:
+        llm_ok = False
+        detail = str(exc)
+    return _finalize(
+        [
+            _check("chat", "Active LLM for indexing", llm_ok, detail),
+        ]
+    )
+
+
+def _graphrag_preflight() -> dict:
+    try:
+        from .pipelines.graphrag.config import is_graphrag_available
+
+        installed = is_graphrag_available()
+    except Exception:
+        installed = False
+    emb_model, emb_dim = _active_embedding()
+    chat_model, _ = _active_chat_model()
+    return _finalize(
+        [
+            _check(
+                "package",
+                "GraphRAG package installed",
+                installed,
+                "Installed." if installed else "pip install 'deeptutor[graphrag]'",
+            ),
+            _check(
+                "chat",
+                "Active chat model",
+                bool(chat_model),
+                chat_model or "Configure one in the model catalog.",
+            ),
+            _check(
+                "embedding",
+                "Active embedding model",
+                bool(emb_model) and emb_dim > 0,
+                f"{emb_model} · {emb_dim}d" if emb_model else "Configure one in the model catalog.",
+            ),
+        ]
+    )
+
+
+def _lightrag_preflight() -> dict:
+    package_detail = "pip install 'deeptutor[rag-lightrag]'"
+    try:
+        from .pipelines.lightrag.config import is_lightrag_available
+        from .pipelines.lightrag.engine import LIGHTRAG_VERSION, installed_version
+
+        current_version = installed_version() if is_lightrag_available() else ""
+        installed = current_version == LIGHTRAG_VERSION
+        if current_version:
+            package_detail = (
+                f"Installed {current_version}."
+                if installed
+                else f"Found {current_version}; required {LIGHTRAG_VERSION}."
+            )
+    except Exception:
+        installed = False
+    emb_model, emb_dim = _active_embedding()
+    chat_model, binding = _active_chat_model()
+    vision_ok = False
+    if chat_model:
+        try:
+            from deeptutor.services.llm.capabilities import supports_vision
+
+            vision_ok = supports_vision(binding, chat_model)
+        except Exception:
+            vision_ok = False
+    return _finalize(
+        [
+            _check(
+                "package",
+                "LightRAG package installed",
+                installed,
+                package_detail,
+            ),
+            _check(
+                "chat",
+                "Active chat model",
+                bool(chat_model),
+                chat_model or "Configure one in the model catalog.",
+            ),
+            _check(
+                "embedding",
+                "Active embedding model",
+                bool(emb_model) and emb_dim > 0,
+                f"{emb_model} · {emb_dim}d" if emb_model else "Configure one in the model catalog.",
+            ),
+            _check(
+                "vision",
+                "Vision model for multimodal",
+                vision_ok,
+                "Active chat model supports vision."
+                if vision_ok
+                else "Active chat model has no vision — image analysis is disabled.",
+                optional=True,
+            ),
+        ]
+    )
+
+
+def _lightrag_server_preflight() -> dict:
+    try:
+        from deeptutor.services.config import load_lightrag_server_settings
+
+        settings = load_lightrag_server_settings()
+        server_url = str(settings.get("server_url") or "").strip()
+    except Exception:
+        server_url = ""
+    return _finalize(
+        [
+            _check(
+                "server_url",
+                "Default server configured",
+                bool(server_url),
+                server_url or "Add a server URL under Connection defaults.",
+            )
+        ]
+    )
+
+
+def _ima_preflight() -> dict:
+    try:
+        from .pipelines.ima.config import get_account_credentials
+
+        credentials = get_account_credentials()
+    except Exception:
+        from .pipelines.ima.config import ImaCredentials
+
+        credentials = ImaCredentials()
+    return _finalize(
+        [
+            _check(
+                "credentials",
+                "IMA Client ID and API key configured",
+                credentials.complete,
+                credentials.client_id
+                if credentials.complete
+                else "Add them under Credentials, or supply a pair per knowledge base "
+                "when connecting one.",
+            )
+        ]
+    )
+
+
+def _finalize(checks: list[dict]) -> dict:
+    ok = all(c["ok"] for c in checks if not c["optional"])
+    return {"ok": ok, "checks": checks}
+
+
+_PREFLIGHTS = {
+    DEFAULT_PROVIDER: _llamaindex_preflight,
+    PAGEINDEX_PROVIDER: _pageindex_preflight,
+    PAGEINDEX_OSS_PROVIDER: _pageindex_oss_preflight,
+    GRAPHRAG_PROVIDER: _graphrag_preflight,
+    LIGHTRAG_PROVIDER: _lightrag_preflight,
+    LIGHTRAG_SERVER_PROVIDER: _lightrag_server_preflight,
+    IMA_PROVIDER: _ima_preflight,
+}
+
+
+def engine_preflight(provider: str) -> dict[str, Any]:
+    """Run the requirement checks for ``provider`` and return the report."""
+    return _PREFLIGHTS[normalize_provider_name(provider)]()
+
+
+__all__ = ["engine_preflight"]

deeptutor/services/rag/provider_binding.py

@@ -0,0 +1,70 @@
+"""Resolve a knowledge base's bound RAG provider.
+
+DeepTutor owns the knowledge-base lifecycle and stores the authoritative
+provider binding in ``kb_config.json``. Older KBs may only have
+``metadata.json``, so metadata remains a legacy fallback. Retrieval and
+incremental indexing should use this helper instead of hand-reading either file
+directly.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from deeptutor.services.rag.factory import DEFAULT_PROVIDER, normalize_provider_name
+
+
+def load_kb_config_entry(kb_base_dir: str | Path, kb_name: str) -> dict[str, Any]:
+    """Return the raw ``kb_config.json`` entry for ``kb_name``, if present."""
+    config_path = Path(kb_base_dir) / "kb_config.json"
+    if not config_path.exists():
+        return {}
+    try:
+        with open(config_path, encoding="utf-8") as handle:
+            entry = json.load(handle).get("knowledge_bases", {}).get(kb_name, {})
+        return entry if isinstance(entry, dict) else {}
+    except Exception:
+        return {}
+
+
+def load_metadata_provider(kb_base_dir: str | Path, kb_name: str) -> str | None:
+    """Return the provider stored in legacy ``metadata.json``, if present."""
+    metadata_path = Path(kb_base_dir) / kb_name / "metadata.json"
+    if not metadata_path.exists():
+        return None
+    try:
+        with open(metadata_path, encoding="utf-8") as handle:
+            provider = json.load(handle).get("rag_provider")
+    except Exception:
+        return None
+    return normalize_provider_name(provider) if provider else None
+
+
+def resolve_bound_provider(kb_base_dir: str | Path, kb_name: str | None) -> str:
+    """Resolve the provider DeepTutor has bound to a KB.
+
+    Order:
+    1. ``kb_config.json``: authoritative runtime state.
+    2. ``metadata.json``: legacy/import fallback.
+    3. default provider.
+    """
+    if kb_name:
+        entry = load_kb_config_entry(kb_base_dir, kb_name)
+        provider = entry.get("rag_provider")
+        if provider:
+            return normalize_provider_name(provider)
+
+        metadata_provider = load_metadata_provider(kb_base_dir, kb_name)
+        if metadata_provider:
+            return metadata_provider
+
+    return DEFAULT_PROVIDER
+
+
+__all__ = [
+    "load_kb_config_entry",
+    "load_metadata_provider",
+    "resolve_bound_provider",
+]

deeptutor/services/rag/query_rewrite.py

@@ -0,0 +1,186 @@
+"""Single-query rewrite for retrieval.
+
+Rewrites a conversational or anaphoric user query into one self-contained,
+keyword-leaning search query before it reaches the KB pipeline. The layer is
+fail-open by design: any LLM error, timeout, or empty output falls back to
+the original query so retrieval never fails because of a rewrite.
+
+Only the ``standalone`` strategy exists in v1 (one LLM call, one query).
+``multi_query`` is reserved for a follow-up PR; the settings schema already
+carries the ``strategy`` field so no migration is needed later.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+import logging
+import time
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+#: Rewrite is disabled / was forced off.
+STRATEGY_OFF = "off"
+#: Single self-contained query produced by one LLM call.
+STRATEGY_STANDALONE = "standalone"
+
+#: Strategies ``maybe_rewrite`` accepts (``off`` forces the pass-through).
+KNOWN_STRATEGIES = (STRATEGY_OFF, STRATEGY_STANDALONE)
+
+_SYSTEM_PROMPT = (
+    "You rewrite a user question into a single self-contained search query "
+    "for a document retrieval engine. Keep the original language (never "
+    "translate). Resolve pronouns and conversational context into explicit "
+    "keywords. Output ONLY the rewritten query, no quotes, no explanation."
+)
+
+
+@dataclass(frozen=True)
+class RewriteOutcome:
+    """What the rewrite layer did to one query."""
+
+    original: str
+    rewritten: str
+    strategy: str
+    latency_ms: float = 0.0
+    error: str = ""
+
+    @property
+    def applied(self) -> bool:
+        """True when the query sent to the engine differs from the input."""
+        return bool(self.rewritten) and self.rewritten != self.original
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serializable form attached to search results and eval reports."""
+        return {
+            "original": self.original,
+            "rewritten": self.rewritten,
+            "strategy": self.strategy,
+            "latency_ms": round(self.latency_ms, 1),
+            "error": self.error,
+        }
+
+
+def _clean_rewritten(raw: str) -> str:
+    """Take the first non-empty line of LLM output, minus wrapping quotes."""
+    for line in (raw or "").splitlines():
+        text = line.strip().strip("\"'“”‘’")
+        if text:
+            return text
+    return ""
+
+
+async def rewrite_query(
+    query: str,
+    *,
+    settings: dict[str, Any] | None = None,
+    complete_fn: Callable[..., Awaitable[str]] | None = None,
+) -> RewriteOutcome:
+    """Rewrite ``query`` per ``settings``; always fail open to the original."""
+    original = query if isinstance(query, str) else ""
+    started = time.monotonic()
+    strategy = STRATEGY_STANDALONE
+    try:
+        cfg = settings or {}
+        strategy = str(cfg.get("strategy") or STRATEGY_STANDALONE)
+        if not bool(cfg.get("enabled", False)) or strategy == STRATEGY_OFF:
+            return RewriteOutcome(original=original, rewritten=original, strategy=STRATEGY_OFF)
+        min_chars = int(cfg.get("min_query_chars") or 0)
+        if len(original.strip()) < max(0, min_chars):
+            return RewriteOutcome(original=original, rewritten=original, strategy=strategy)
+        timeout = float(cfg.get("timeout_seconds") or 5.0)
+        model = str(cfg.get("model") or "").strip() or None
+        complete = complete_fn
+        if complete is None:
+            from deeptutor.services.llm import complete as _default_complete
+
+            complete = _default_complete
+        if model is not None:
+            raw = await asyncio.wait_for(
+                complete(query, system_prompt=_SYSTEM_PROMPT, model=model),
+                timeout=timeout,
+            )
+        else:
+            raw = await asyncio.wait_for(
+                complete(query, system_prompt=_SYSTEM_PROMPT),
+                timeout=timeout,
+            )
+        rewritten = _clean_rewritten(raw)
+    except Exception as exc:
+        logger.warning("Query rewrite failed (%s); using the original query", type(exc).__name__)
+        return RewriteOutcome(
+            original=original,
+            rewritten=original,
+            strategy=strategy,
+            latency_ms=(time.monotonic() - started) * 1000.0,
+            error=f"{type(exc).__name__}: {exc}",
+        )
+
+    latency_ms = (time.monotonic() - started) * 1000.0
+    if not rewritten:
+        return RewriteOutcome(
+            original=original,
+            rewritten=original,
+            strategy=strategy,
+            latency_ms=latency_ms,
+            error="empty_rewrite",
+        )
+    return RewriteOutcome(
+        original=original,
+        rewritten=rewritten,
+        strategy=strategy,
+        latency_ms=latency_ms,
+    )
+
+
+async def maybe_rewrite(
+    query: str,
+    *,
+    rewrite: bool | None = None,
+    settings: dict[str, Any] | None = None,
+    complete_fn: Callable[..., Awaitable[str]] | None = None,
+) -> tuple[str, Optional[RewriteOutcome]]:
+    """Return ``(effective_query, outcome)`` for :meth:`RAGService.search`.
+
+    ``rewrite`` is the per-call override: True forces the rewrite on, False
+    forces it off, None follows the runtime settings. Returns ``(query, None)``
+    when the layer is off so callers only attach provenance when active.
+    """
+    if rewrite is False:
+        return query, None
+    try:
+        cfg = settings
+        if cfg is None:
+            from deeptutor.services.config import get_runtime_settings_service
+
+            cfg = get_runtime_settings_service().load_query_rewrite()
+        cfg = dict(cfg)
+        if rewrite is not None:
+            cfg["enabled"] = bool(rewrite)
+    except Exception as exc:
+        logger.warning(
+            "Query rewrite settings failed (%s); using the original query", type(exc).__name__
+        )
+        return query, RewriteOutcome(
+            original=query,
+            rewritten=query,
+            strategy=STRATEGY_OFF,
+            error=f"{type(exc).__name__}: {exc}",
+        )
+
+    if not bool(cfg.get("enabled", False)):
+        return query, None
+    outcome = await rewrite_query(query, settings=cfg, complete_fn=complete_fn)
+    return outcome.rewritten, outcome
+
+
+__all__ = [
+    "KNOWN_STRATEGIES",
+    "STRATEGY_OFF",
+    "STRATEGY_STANDALONE",
+    "RewriteOutcome",
+    "maybe_rewrite",
+    "rewrite_query",
+]