deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/rag/pipelines/ima/pipeline.py

@@ -0,0 +1,238 @@
+"""Retrieval-only pipeline backed by a Tencent IMA knowledge base.
+
+Implements the same contract as the other pipelines (see ``..base.RAGPipeline``)
+but owns no index: an ``ima`` KB is a connection pointer (``type: ima`` in
+``kb_config.json``) to a library the user keeps in IMA and curates there. Only
+:meth:`search` does real work — it resolves the KB's credentials, asks IMA for
+matching passages, tops the thin ones up with real source text, and shapes them
+for the ``rag`` tool. Documents are added in IMA (or through the IMA capability's
+own tools), so :meth:`initialize` / :meth:`add_documents` are not part of this
+engine's job and fail with a clear message; :meth:`delete` is a no-op because
+deleting the KB only drops DeepTutor's pointer (handled by the manager) and must
+never touch the user's IMA library.
+
+The retrieval *policy* — which matches deserve a full-text fetch — lives in
+:mod:`.sources`; this module only orchestrates.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any, Dict, List, Optional
+
+import httpx
+
+from deeptutor.runtime.home import get_runtime_data_root
+from deeptutor.services.rag.provider_binding import load_kb_config_entry
+
+from . import media as media_ops
+from . import sources as source_policy
+from .config import ImaNotConfiguredError, resolve_kb_config
+from .envelope import ImaAuthError, ImaRateLimitError
+
+logger = logging.getLogger(__name__)
+
+PROVIDER = "ima"
+DEFAULT_KB_BASE_DIR = str(get_runtime_data_root() / "knowledge_bases")
+
+# How many matched passages one retrieval feeds into the prompt. IMA returns a
+# highlight snippet per item rather than whole documents, so this is a passage
+# budget, not a document budget.
+_DEFAULT_TOP_K = 10
+_MAX_TOP_K = 50
+
+
+class ImaPipeline:
+    """Query a Tencent IMA knowledge base on behalf of a connected KB."""
+
+    def __init__(self, kb_base_dir: Optional[str] = None, *, client_factory=None, **_: Any) -> None:
+        self.logger = logging.getLogger(__name__)
+        self.kb_base_dir = kb_base_dir or DEFAULT_KB_BASE_DIR
+        # Injection seam for tests: (config) -> client. None uses the real client.
+        self._client_factory = client_factory
+
+    # ----- helpers --------------------------------------------------------
+
+    def _client(self, config):
+        if self._client_factory is not None:
+            return self._client_factory(config)
+        from .client import ImaClient
+
+        return ImaClient(config)
+
+    @staticmethod
+    def _top_k(kwargs: dict[str, Any]) -> int:
+        try:
+            requested = int(kwargs.get("top_k") or _DEFAULT_TOP_K)
+        except (TypeError, ValueError):
+            return _DEFAULT_TOP_K
+        return max(1, min(requested, _MAX_TOP_K))
+
+    # ----- retrieval ------------------------------------------------------
+
+    async def search(self, query: str, kb_name: str, **kwargs) -> Dict[str, Any]:
+        try:
+            config = resolve_kb_config(load_kb_config_entry(self.kb_base_dir, kb_name))
+        except ImaNotConfiguredError as exc:
+            return self._error_result(query, exc, error_type="not_configured")
+
+        try:
+            client = self._client(config)
+            page = await client.search_knowledge(query, limit=self._top_k(kwargs))
+        except Exception as exc:
+            self.logger.error("IMA search failed for '%s' (%s)", kb_name, type(exc).__name__)
+            return self._error_result(query, exc, error_type="retrieval_error")
+
+        matched = source_policy.documents_to_sources(page.documents)
+        hydration_failures = await self._hydrate(client, matched)
+        # A title match without a snippet or readable media is useful for
+        # hydration, but it is not evidence that the rag tool may cite (#1500).
+        sources = [source for source in matched if str(source.get("content") or "").strip()]
+        diagnostics = {
+            "knowledge_base": kb_name,
+            "matched_documents": len(matched),
+            "readable_documents": len(sources),
+            "unreadable_documents": len(matched) - len(sources),
+            "unverified_documents": page.unverified_documents + len(page.documents) - len(matched),
+            "hydration_failures": hydration_failures,
+        }
+        if diagnostics["unverified_documents"] and not matched:
+            return {
+                "query": query,
+                "answer": "Tencent IMA returned matches without usable source identifiers. No source passage could be verified.",
+                "content": "",
+                "sources": [],
+                "provider": PROVIDER,
+                "error_type": "unverified_sources",
+                "retrieval_status": "unverified_sources",
+                "diagnostics": diagnostics,
+            }
+        if matched and not sources:
+            return {
+                "query": query,
+                "answer": (
+                    "Tencent IMA matched document titles, but returned no readable text. "
+                    "Check access to the matched media or try again."
+                ),
+                "content": "",
+                "sources": [],
+                "provider": PROVIDER,
+                "error_type": "content_unavailable",
+                "retrieval_status": "content_unavailable",
+                "diagnostics": diagnostics,
+            }
+        content = source_policy.render_context(sources)
+        status = "ready"
+        if not matched:
+            status = "no_matches"
+            content = f"Tencent IMA returned no matching passages for this query in '{kb_name}'."
+        elif (
+            len(sources) < len(matched) or hydration_failures or diagnostics["unverified_documents"]
+        ):
+            status = "partial"
+            content += (
+                "\n\nSome IMA matches could not be read in full. Use only the returned "
+                "passages as evidence; do not invent missing source text or locations."
+            )
+        return {
+            "query": query,
+            "answer": content,
+            "content": content,
+            "sources": sources,
+            "provider": PROVIDER,
+            "retrieval_status": status,
+            "diagnostics": diagnostics,
+        }
+
+    async def _hydrate(self, client, sources: list[dict[str, Any]]) -> list[dict[str, str]]:
+        """Replace thin or missing snippets with real source text, concurrently.
+
+        Each fetch is independent, so they run together — a search that needs
+        four documents costs one round-trip's latency, not four. A document that
+        cannot be loaded keeps its snippet (or stays a title-only reference):
+        one unavailable file must never discard the other matches.
+        """
+        targets = source_policy.hydration_targets(sources)
+        if not targets:
+            return []
+        results = await asyncio.gather(
+            *(self._fetch_text(client, sources[index]) for index in targets),
+            return_exceptions=True,
+        )
+        failures = []
+        for index, result in zip(targets, results):
+            if isinstance(result, BaseException):
+                # HTTP errors may embed a signed COS URL; log only the error
+                # class so short-lived download credentials never reach logs.
+                self.logger.warning(
+                    "Could not load IMA media '%s' (%s)",
+                    sources[index]["chunk_id"],
+                    type(result).__name__,
+                )
+                failures.append(
+                    {"source_id": sources[index]["chunk_id"], "error_type": type(result).__name__}
+                )
+            elif result:
+                sources[index]["content"] = result
+            else:
+                failures.append(
+                    {"source_id": sources[index]["chunk_id"], "error_type": "content_unavailable"}
+                )
+        return failures
+
+    @staticmethod
+    async def _fetch_text(client, source: dict[str, Any]) -> str:
+        media = await client.get_media_content(source["chunk_id"])
+        return await media_ops.extract_text(
+            media,
+            source["title"],
+            max_chars=source_policy.MAX_FULLTEXT_CHARS,
+        )
+
+    def _error_result(self, query: str, exc: Exception, *, error_type: str) -> Dict[str, Any]:
+        # #1500: preserve observable failure classes without exposing signed
+        # media URLs or credentials carried by transport exceptions.
+        message = str(exc)
+        if isinstance(exc, ImaAuthError):
+            error_type = "authentication_error"
+            message = "Tencent IMA rejected the credentials or library access. Check this account's IMA connection and permissions."
+        elif isinstance(exc, ImaRateLimitError):
+            error_type = "rate_limited"
+            message = "Tencent IMA rate limit reached. Wait before retrying this search."
+        elif isinstance(exc, httpx.TimeoutException):
+            error_type = "timeout"
+            message = "Tencent IMA did not respond in time. Retry the search."
+        elif isinstance(exc, httpx.TransportError):
+            error_type = "network_error"
+            message = "Tencent IMA could not be reached. Check the connection and retry."
+        return {
+            "query": query,
+            "answer": message,
+            "content": "",
+            "sources": [],
+            "provider": PROVIDER,
+            "error_type": error_type,
+            "retrieval_status": "failed",
+        }
+
+    # ----- indexing (not applicable — owned by IMA) ------------------------
+
+    async def initialize(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
+        raise RuntimeError(
+            "Tencent IMA knowledge bases are indexed by IMA; DeepTutor does not "
+            "build or store their index. Add documents in IMA directly."
+        )
+
+    async def add_documents(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
+        return await self.initialize(kb_name, file_paths, **kwargs)
+
+    # ----- lifecycle ------------------------------------------------------
+
+    async def delete(self, kb_name: str, **kwargs) -> bool:
+        # The KB is only a pointer; the manager removes its config entry. Never
+        # touch the user's IMA library. Nothing local to clean up here.
+        return True
+
+
+__all__ = ["ImaPipeline", "PROVIDER"]

deeptutor/services/rag/pipelines/ima/probe.py

@@ -0,0 +1,105 @@
+"""Probe a Tencent IMA knowledge base before connecting a KB to it.
+
+Connecting is cheap and reversible, but wrong credentials or a mistyped
+knowledge base id should fail loudly at connect time rather than silently at
+every later query. One ``get_knowledge_base`` round-trip answers both questions
+the UI needs to confirm: are the credentials accepted, and does the id resolve to
+a library we can name back to the user?
+
+Always returns an :class:`ImaProbe` (never raises); ``ok`` is the single boolean
+the caller gates on, with ``error`` explaining any failure.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any, Optional
+
+from .client import ImaClient
+from .config import ImaConfig
+from .envelope import ImaAuthError
+
+
+@dataclass
+class ImaProbe:
+    knowledge_base_id: str
+    ok: bool = False
+    credentials_ok: bool = False
+    knowledge_base_name: Optional[str] = None
+    description: Optional[str] = None
+    error: Optional[str] = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+async def probe_knowledge_base(
+    client_id: str,
+    api_key: str,
+    knowledge_base_id: str,
+    *,
+    client_factory=None,
+) -> ImaProbe:
+    """Check that *knowledge_base_id* is reachable with the given credentials.
+
+    ``client_factory`` (config → client) is an injection seam for tests; in
+    production it defaults to a real :class:`ImaClient`.
+    """
+    client_id = (client_id or "").strip()
+    api_key = (api_key or "").strip()
+    knowledge_base_id = (knowledge_base_id or "").strip()
+
+    probe = ImaProbe(knowledge_base_id=knowledge_base_id)
+    missing = [
+        label
+        for label, value in (
+            ("Client ID", client_id),
+            ("API key", api_key),
+            ("knowledge base ID", knowledge_base_id),
+        )
+        if not value
+    ]
+    if missing:
+        probe.error = f"{', '.join(missing)} is required."
+        return probe
+
+    config = ImaConfig(
+        client_id=client_id,
+        api_key=api_key,
+        knowledge_base_id=knowledge_base_id,
+    )
+    client = client_factory(config) if client_factory else ImaClient(config)
+
+    try:
+        info = await client.get_knowledge_base()
+    except ImaAuthError:
+        probe.error = "IMA rejected these credentials. Check them and try again."
+        return probe
+    except Exception:
+        # Upstream exception strings are not returned because they may contain
+        # request details. The UI only needs a stable, actionable verdict.
+        probe.error = "Could not reach Tencent IMA. Try again shortly."
+        return probe
+
+    # The call is credential-gated, so reaching this point means they were
+    # accepted — an unknown id just yields no entry.
+    probe.credentials_ok = True
+    if not info:
+        probe.error = (
+            "The credentials work, but no knowledge base matches this ID. "
+            "Check the ID in IMA and try again."
+        )
+        return probe
+
+    probe.knowledge_base_name = _opt_str(info.get("name"))
+    probe.description = _opt_str(info.get("description"))
+    probe.ok = True
+    return probe
+
+
+def _opt_str(value: Any) -> Optional[str]:
+    text = str(value).strip() if value is not None else ""
+    return text or None
+
+
+__all__ = ["ImaProbe", "probe_knowledge_base"]

deeptutor/services/rag/pipelines/ima/sources.py

@@ -0,0 +1,105 @@
+"""Shaping IMA search results into the grounded context the ``rag`` tool expects.
+
+IMA answers a search with one *highlight snippet* per matched item — often a
+single sentence, and nothing at all when the match was on the title. Handing
+those straight to the model is what made a connected IMA library feel thin
+compared to an indexed KB, whose chunks are whole passages.
+
+So retrieval has two stages, and this module owns the policy for the second:
+
+1. map matched documents into DeepTutor's ``sources`` shape;
+2. decide which of them are worth spending a full-text fetch on
+   (:func:`hydration_targets`) — items with *no* snippet first, then items whose
+   snippet is too thin to reason from, best-ranked first, bounded by a budget so
+   one search cannot turn into a dozen downloads.
+
+Kept separate from :mod:`.pipeline` (which orchestrates the calls) and
+:mod:`.client` (which makes them) so the retrieval policy can be reasoned about
+and tested without a transport.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Sequence
+
+from .models import ImaDocument
+
+# A snippet shorter than this is treated as a hint that a document matched, not
+# as usable evidence, so it is a candidate for full-text hydration.
+MIN_USEFUL_SNIPPET_CHARS = 240
+
+# How many documents one retrieval may fetch in full. Bounds both the network
+# cost and the prompt footprint even when ``top_k`` is large.
+DEFAULT_HYDRATION_BUDGET = 4
+
+# Per-document character cap for hydrated full text.
+MAX_FULLTEXT_CHARS = 12_000
+
+
+def documents_to_sources(documents: Iterable[ImaDocument]) -> list[dict[str, Any]]:
+    """Map matched IMA documents into DeepTutor's ``sources`` shape.
+
+    Items whose match was on the title alone carry no snippet and are still
+    listed, so the model can see the document exists even before (or without)
+    hydration.
+    """
+    sources: list[dict[str, Any]] = []
+    for document in documents:
+        title = document.title or document.media_id
+        if not title:
+            continue
+        sources.append(
+            {
+                "title": title,
+                "content": document.highlight,
+                "source": title,
+                "chunk_id": document.media_id,
+            }
+        )
+    return sources
+
+
+def hydration_targets(
+    sources: Sequence[dict[str, Any]],
+    *,
+    budget: int = DEFAULT_HYDRATION_BUDGET,
+    min_chars: int = MIN_USEFUL_SNIPPET_CHARS,
+) -> list[int]:
+    """Indices of the sources worth fetching in full, in priority order.
+
+    Snippet-less matches come first (they carry no evidence at all), then thin
+    snippets in rank order. Sources without an item id cannot be fetched and are
+    never returned.
+    """
+    if budget <= 0:
+        return []
+    empty: list[int] = []
+    thin: list[int] = []
+    for index, source in enumerate(sources):
+        if not source.get("chunk_id"):
+            continue
+        content = str(source.get("content") or "").strip()
+        if not content:
+            empty.append(index)
+        elif len(content) < min_chars:
+            thin.append(index)
+    return (empty + thin)[:budget]
+
+
+def render_context(sources: Sequence[dict[str, Any]]) -> str:
+    """Flatten retrieved snippets into the grounded context block."""
+    blocks = [
+        f"[{index}] {source.get('title') or ''}\n{source.get('content') or ''}".rstrip()
+        for index, source in enumerate(sources, start=1)
+    ]
+    return "\n\n".join(blocks)
+
+
+__all__ = [
+    "DEFAULT_HYDRATION_BUDGET",
+    "MAX_FULLTEXT_CHARS",
+    "MIN_USEFUL_SNIPPET_CHARS",
+    "documents_to_sources",
+    "hydration_targets",
+    "render_context",
+]

deeptutor/services/rag/pipelines/ima/transport.py

@@ -0,0 +1,130 @@
+"""HTTP transport shared by every Tencent IMA OpenAPI call.
+
+All IMA methods are ``POST https://ima.qq.com/<prefix>/<method>`` with a JSON
+body and two credential headers, so the wire mechanics are identical across the
+knowledge-base (``wiki``) and notes modules. Concentrating them here leaves the
+client classes as nothing but method tables, and leaves exactly one place that
+attaches credentials.
+
+A fresh ``httpx`` client per call (mirroring ``LightRagServerClient``) keeps the
+object safe to construct once and reuse from anywhere, and the injectable
+``transport`` lets tests stub the wire without a live server.
+
+Both an async and a blocking flavour of the same call are exposed. The async one
+serves retrieval and the tool layer; the blocking one exists for the knowledge-base
+*inventory*, which is read from the deliberately synchronous manifest layer (see
+:mod:`deeptutor.knowledge.manifest`) — giving it a blocking call here is far less
+invasive than making that whole path async, and both share this module's URL,
+header and envelope handling.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+import httpx
+
+from .config import ImaConfig
+from .envelope import ImaAPIError, ImaAuthError, ImaRateLimitError, unwrap
+
+API_BASE_URL = "https://ima.qq.com"
+WIKI_PREFIX = "/openapi/wiki/v1"
+NOTE_PREFIX = "/openapi/note/v1"
+
+DEFAULT_TIMEOUT = 30.0
+
+
+def build_headers(config: ImaConfig) -> dict[str, str]:
+    """The credential headers every IMA request carries."""
+    return {
+        "Content-Type": "application/json",
+        "ima-openapi-clientid": config.client_id,
+        "ima-openapi-apikey": config.api_key,
+    }
+
+
+class ImaTransport:
+    """POST one IMA method and return its unwrapped ``data`` object."""
+
+    def __init__(
+        self,
+        config: ImaConfig,
+        *,
+        timeout: float = DEFAULT_TIMEOUT,
+        transport: Optional[httpx.AsyncBaseTransport] = None,
+    ) -> None:
+        self.config = config
+        self.timeout = timeout
+        self.transport = transport
+
+    async def post(
+        self,
+        method: str,
+        body: dict[str, Any],
+        *,
+        prefix: str = WIKI_PREFIX,
+    ) -> dict[str, Any]:
+        async with httpx.AsyncClient(
+            base_url=API_BASE_URL,
+            headers=build_headers(self.config),
+            timeout=self.timeout,
+            transport=self.transport,
+        ) as client:
+            response = await client.post(f"{prefix}/{method}", json=body)
+        return self._unwrap(response)
+
+    def post_sync(
+        self,
+        method: str,
+        body: dict[str, Any],
+        *,
+        prefix: str = WIKI_PREFIX,
+    ) -> dict[str, Any]:
+        with httpx.Client(
+            base_url=API_BASE_URL,
+            headers=build_headers(self.config),
+            timeout=self.timeout,
+            transport=_sync_transport(self.transport),
+        ) as client:
+            response = client.post(f"{prefix}/{method}", json=body)
+        return self._unwrap(response)
+
+    @staticmethod
+    def _unwrap(response: httpx.Response) -> dict[str, Any]:
+        if response.status_code in {401, 403}:
+            raise ImaAuthError("IMA rejected the credentials or knowledge-base permissions.")
+        # A transport-level 429 never reaches the envelope, so it is mapped here
+        # to the same error the envelope's rate-limit codes raise.
+        if response.status_code == 429:
+            raise ImaRateLimitError("IMA rate limit reached. Try again shortly.")
+        if not response.is_success:
+            # #1500: a failed gateway response must not become a successful
+            # empty retrieval merely because its JSON contains code=0.
+            raise ImaAPIError(
+                f"IMA returned HTTP {response.status_code}. Retry or check the connection."
+            )
+        try:
+            payload = response.json()
+        except Exception:
+            payload = None
+        return unwrap(payload, status_code=response.status_code)
+
+
+def _sync_transport(transport: Optional[httpx.AsyncBaseTransport]) -> Optional[httpx.BaseTransport]:
+    """Reuse an injected test transport for blocking calls when it supports them.
+
+    ``httpx.MockTransport`` implements both the sync and async protocols, so a
+    test's stub works for either flavour; a genuinely async-only transport is
+    ignored rather than passed to a blocking client that cannot drive it.
+    """
+    return transport if isinstance(transport, httpx.BaseTransport) else None
+
+
+__all__ = [
+    "API_BASE_URL",
+    "DEFAULT_TIMEOUT",
+    "NOTE_PREFIX",
+    "WIKI_PREFIX",
+    "ImaTransport",
+    "build_headers",
+]

deeptutor/services/rag/pipelines/kiwix/__init__.py

@@ -0,0 +1 @@
+"""Read-only retrieval from ZIM archives already served by kiwix-serve."""