deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/rag/pipelines/pageindex/storage.py

@@ -0,0 +1,133 @@
+"""On-disk manifest for a PageIndex-backed knowledge base.
+
+PageIndex has no embeddings, so there is nothing to vectorise locally. The only
+local state is a lightweight manifest mapping each ingested file to its hosted
+``doc_id``. It is written into the KB's flat ``version-N`` directory (reusing
+``index_versioning`` with a ``None`` signature) so the Index-versions UI and the
+"is this KB initialised?" checks see a ready version just like LlamaIndex KBs —
+only the file contents differ (a doc-id map instead of a vector store).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from deeptutor.services.file_io import atomic_write_json
+
+logger = logging.getLogger(__name__)
+
+MANIFEST_FILENAME = "pageindex_docs.json"
+META_FILENAME = "meta.json"
+SDK_STORAGE_DIRNAME = "pageindex"
+CLOUD_PROVIDER = "pageindex"
+OSS_PROVIDER = "pageindex-oss"
+
+
+def _empty_manifest(provider: str = CLOUD_PROVIDER) -> dict[str, Any]:
+    return {"provider": provider, "docs": {}}
+
+
+def manifest_path(storage_dir: Path) -> Path:
+    return Path(storage_dir) / MANIFEST_FILENAME
+
+
+def read_manifest(
+    storage_dir: Path | None,
+    *,
+    provider: str = CLOUD_PROVIDER,
+) -> dict[str, Any]:
+    if storage_dir is None:
+        return _empty_manifest(provider)
+    path = manifest_path(storage_dir)
+    if not path.exists():
+        return _empty_manifest(provider)
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except Exception as exc:
+        logger.warning("Failed to read PageIndex manifest %s: %s", path, exc)
+        return _empty_manifest(provider)
+    if not isinstance(data, dict):
+        return _empty_manifest(provider)
+    data.setdefault("provider", provider)
+    if not isinstance(data.get("docs"), dict):
+        data["docs"] = {}
+    return data
+
+
+def write_manifest(storage_dir: Path, manifest: dict[str, Any]) -> None:
+    atomic_write_json(manifest_path(storage_dir), manifest)
+
+
+def write_meta(storage_dir: Path, *, provider: str = CLOUD_PROVIDER) -> None:
+    """Write a flat-layout ``meta.json`` so the version is listed as ready.
+
+    Mirrors ``index_versioning.write_version_meta`` but carries a synthetic
+    ``pageindex`` signature instead of an embedding hash.
+    """
+    target = Path(storage_dir)
+    payload = {
+        "version": target.name,
+        "signature": provider,
+        "provider": provider,
+        "layout": "flat",
+        "created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
+    }
+    atomic_write_json(target / META_FILENAME, payload)
+
+
+def doc_entries(manifest: dict[str, Any]) -> dict[str, Any]:
+    docs = manifest.get("docs")
+    return docs if isinstance(docs, dict) else {}
+
+
+def doc_ids(manifest: dict[str, Any]) -> list[str]:
+    return [
+        str(entry["doc_id"])
+        for entry in doc_entries(manifest).values()
+        if isinstance(entry, dict) and entry.get("doc_id")
+    ]
+
+
+def upsert_doc(
+    manifest: dict[str, Any],
+    file_name: str,
+    doc_id: str,
+    *,
+    size: int | None = None,
+) -> None:
+    docs = manifest.setdefault("docs", {})
+    docs[file_name] = {
+        "doc_id": doc_id,
+        "size": size,
+        "submitted_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+    }
+
+
+def remove_doc(manifest: dict[str, Any], file_name: str) -> dict[str, Any] | None:
+    entry = doc_entries(manifest).pop(file_name, None)
+    return entry if isinstance(entry, dict) else None
+
+
+def sdk_storage_path(storage_dir: Path) -> Path:
+    return Path(storage_dir) / SDK_STORAGE_DIRNAME
+
+
+__all__ = [
+    "MANIFEST_FILENAME",
+    "CLOUD_PROVIDER",
+    "OSS_PROVIDER",
+    "SDK_STORAGE_DIRNAME",
+    "manifest_path",
+    "read_manifest",
+    "write_manifest",
+    "write_meta",
+    "doc_entries",
+    "doc_ids",
+    "upsert_doc",
+    "remove_doc",
+    "sdk_storage_path",
+]

deeptutor/services/rag/pipelines/pageindex/tools.py

@@ -0,0 +1,221 @@
+"""Turn-scoped DeepTutor wrappers for PageIndex Cloud and OSS SDK tools."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from functools import lru_cache
+import json
+from typing import Any
+
+from deeptutor.core.tool_protocol import BaseTool, ToolDefinition, ToolLookup, ToolResult
+from deeptutor.runtime.registry.scoped_registry import ScopedToolRegistry
+
+from .pipeline import PageIndexPipeline
+from .storage import CLOUD_PROVIDER, OSS_PROVIDER
+
+TOOL_PREFIXES = {
+    CLOUD_PROVIDER: "pageindex_cloud_",
+    OSS_PROVIDER: "pageindex_oss_",
+}
+
+
+def pageindex_sources_from_text(
+    text: str,
+    *,
+    provider: str,
+    kb_name: str = "",
+    doc_ids: dict[str, str] | None = None,
+) -> list[dict[str, Any]]:
+    """Normalize real PageIndex page reads into DeepTutor source rows."""
+    try:
+        payload = json.loads(text)
+    except (TypeError, ValueError):
+        return []
+    if not isinstance(payload, dict) or payload.get("success") is False or payload.get("errorCode"):
+        return []
+    document_name = str(payload.get("doc_name") or "").strip()
+    content = payload.get("content")
+    if not document_name or not isinstance(content, list):
+        return []
+
+    rows: list[dict[str, Any]] = []
+    seen: set[tuple[str, str]] = set()
+    for item in content:
+        if not isinstance(item, dict) or item.get("page") in (None, ""):
+            continue
+        page = item["page"]
+        block_id = str(item.get("block_id") or "").strip()
+        key = (str(page), block_id)
+        if key in seen:
+            continue
+        seen.add(key)
+        row: dict[str, Any] = {
+            "type": "pageindex",
+            "provider": provider,
+            "document_name": document_name,
+            "source": document_name,
+            "page": page,
+        }
+        if kb_name:
+            row["kb_name"] = kb_name
+        if doc_ids and document_name in doc_ids:
+            row["doc_id"] = doc_ids[document_name]
+        if block_id:
+            row["block_id"] = block_id
+        rows.append(row)
+    return rows
+
+
+class PageIndexSDKTool(BaseTool):
+    deferred = True
+    provider_kind = "pageindex"
+
+    def __init__(
+        self,
+        sdk_tool: Any,
+        *,
+        provider: str,
+        kb_name: str,
+        doc_ids: dict[str, str],
+    ) -> None:
+        self._sdk_tool = sdk_tool
+        self.provider_id = provider
+        self._kb_name = kb_name
+        self._doc_ids = doc_ids
+
+    def get_definition(self) -> ToolDefinition:
+        label = "Cloud" if self.provider_id == CLOUD_PROVIDER else "OSS"
+        return ToolDefinition(
+            name=f"{TOOL_PREFIXES[self.provider_id]}{self._sdk_tool.name}",
+            description=f"[PageIndex {label}: {self._kb_name}] {self._sdk_tool.description}",
+            raw_parameters=dict(self._sdk_tool.params_json_schema),
+        )
+
+    async def execute(self, **kwargs: Any) -> ToolResult:
+        kwargs.pop("event_sink", None)
+        text = await self._sdk_tool.on_invoke_tool(
+            None,
+            json.dumps(kwargs, ensure_ascii=False),
+        )
+        try:
+            payload = json.loads(text)
+        except (TypeError, ValueError):
+            payload = {}
+        success = not (
+            isinstance(payload, dict)
+            and (payload.get("success") is False or payload.get("errorCode"))
+        )
+        sources = pageindex_sources_from_text(
+            str(text),
+            provider=self.provider_id,
+            kb_name=self._kb_name,
+            doc_ids=self._doc_ids,
+        )
+        return ToolResult(
+            content=str(text),
+            sources=sources,
+            metadata={
+                "pageindex_provider": self.provider_id,
+                "kb_name": self._kb_name,
+                "sources": sources,
+            },
+            success=success,
+        )
+
+
+@dataclass(frozen=True)
+class PageIndexSDKToolBundle:
+    provider: str
+    tools: tuple[PageIndexSDKTool, ...]
+    instructions: str
+    documents: dict[str, str]
+
+
+@dataclass(frozen=True)
+class PageIndexToolContext:
+    provider: str
+    registry: ToolLookup
+    tools: tuple[BaseTool, ...]
+    instructions: str
+    documents: dict[str, str]
+
+
+@lru_cache(maxsize=1)
+def _cloud_read_tools(client: Any) -> tuple[Any, ...]:
+    """Cache live Cloud schemas on the SDK client that owns their MCP session."""
+    return tuple(client.as_openai_tools(include_management=False))
+
+
+async def build_sdk_tool_bundle(
+    kb_name: str,
+    kb_base_dir: str,
+    *,
+    provider: str,
+) -> PageIndexSDKToolBundle:
+    """Build one KB's read-only SDK tools without exposing their transport."""
+
+    def build() -> PageIndexSDKToolBundle:
+        pipeline = PageIndexPipeline(kb_base_dir=kb_base_dir, provider=provider)
+        documents = pipeline.document_map(kb_name)
+        client = pipeline.sdk_client_for_read(kb_name)
+        sdk_tools = (
+            client.as_openai_tools(include_management=False)
+            if provider == OSS_PROVIDER
+            else _cloud_read_tools(client)
+        )
+        instructions = client.agent_instructions()
+        return PageIndexSDKToolBundle(
+            provider=provider,
+            tools=tuple(
+                PageIndexSDKTool(
+                    tool,
+                    provider=provider,
+                    kb_name=kb_name,
+                    doc_ids=documents,
+                )
+                for tool in sdk_tools
+            ),
+            instructions=str(instructions or ""),
+            documents=documents,
+        )
+
+    return await asyncio.to_thread(build)
+
+
+async def build_pageindex_tool_context(
+    kb_name: str | None,
+    *,
+    base_registry: ToolLookup,
+) -> PageIndexToolContext | None:
+    """Resolve one PageIndex KB into tools for an existing workflow loop."""
+    if not kb_name:
+        return None
+    from deeptutor.multi_user.knowledge_access import resolve_kb
+    from deeptutor.services.rag.provider_binding import resolve_bound_provider
+
+    resource = resolve_kb(kb_name, require_write=False)
+    base_dir = str(resource.base_dir)
+    provider = resolve_bound_provider(base_dir, resource.name)
+    if provider in {CLOUD_PROVIDER, OSS_PROVIDER}:
+        bundle = await build_sdk_tool_bundle(resource.name, base_dir, provider=provider)
+        registry = ScopedToolRegistry(base=base_registry, overlay=bundle.tools)
+        return PageIndexToolContext(
+            provider=provider,
+            registry=registry,
+            tools=bundle.tools,
+            instructions=bundle.instructions,
+            documents=bundle.documents,
+        )
+    return None
+
+
+__all__ = [
+    "PageIndexSDKTool",
+    "PageIndexSDKToolBundle",
+    "PageIndexToolContext",
+    "TOOL_PREFIXES",
+    "build_sdk_tool_bundle",
+    "build_pageindex_tool_context",
+    "pageindex_sources_from_text",
+]

deeptutor/services/rag/pipelines/weknora/__init__.py

@@ -0,0 +1,13 @@
+"""Retrieval-only integration with an external Tencent WeKnora server."""
+
+from .client import WeKnoraAPIError, WeKnoraClient
+from .config import WeKnoraConfig, config_from_entry
+from .pipeline import WeKnoraPipeline
+
+__all__ = [
+    "WeKnoraAPIError",
+    "WeKnoraClient",
+    "WeKnoraConfig",
+    "WeKnoraPipeline",
+    "config_from_entry",
+]

deeptutor/services/rag/pipelines/weknora/client.py

@@ -0,0 +1,106 @@
+"""Thin HTTP client for Tencent WeKnora's documented REST API."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Optional
+
+import httpx
+
+from .config import WeKnoraConfig
+
+logger = logging.getLogger(__name__)
+
+# An external service is not allowed to make the DeepTutor process buffer an
+# unbounded body. Four MiB is ample for a page of KB summaries or retrieval
+# chunks while keeping a compromised/misconfigured server cheap to reject.
+MAX_RESPONSE_BYTES = 4 * 1024 * 1024
+
+
+class WeKnoraAPIError(RuntimeError):
+    """Raised when WeKnora returns an error or unexpected payload."""
+
+
+class WeKnoraClient:
+    def __init__(
+        self,
+        config: WeKnoraConfig,
+        *,
+        timeout: float = 60.0,
+        transport: Optional[httpx.AsyncBaseTransport] = None,
+    ) -> None:
+        self._config = config
+        self._timeout = timeout
+        self._transport = transport
+
+    def _open(self) -> httpx.AsyncClient:
+        return httpx.AsyncClient(
+            base_url=self._config.base_url,
+            headers={
+                "Accept": "application/json",
+                "X-API-Key": self._config.api_key,
+            },
+            timeout=self._timeout,
+            transport=self._transport,
+        )
+
+    async def _request_json(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
+        # Re-resolve on every request. Checking only when the connection is
+        # saved would leave retrieval vulnerable to DNS rebinding later.
+        from deeptutor.services.mcp.network import validate_mcp_url_async
+
+        ok, error = await validate_mcp_url_async(self._config.base_url)
+        if not ok:
+            raise WeKnoraAPIError(f"Unsafe WeKnora server URL: {error}")
+
+        async with self._open() as client:
+            async with client.stream(method, path, **kwargs) as resp:
+                declared = resp.headers.get("content-length")
+                if declared:
+                    try:
+                        if int(declared) > MAX_RESPONSE_BYTES:
+                            raise WeKnoraAPIError("WeKnora response exceeds the 4 MiB limit.")
+                    except ValueError:
+                        pass
+                body = bytearray()
+                async for chunk in resp.aiter_bytes():
+                    body.extend(chunk)
+                    if len(body) > MAX_RESPONSE_BYTES:
+                        raise WeKnoraAPIError("WeKnora response exceeds the 4 MiB limit.")
+
+        if resp.status_code >= 400:
+            preview = bytes(body[:300]).decode("utf-8", errors="replace")
+            raise WeKnoraAPIError(f"WeKnora returned {resp.status_code}: {preview}")
+        try:
+            data = json.loads(body)
+        except Exception as exc:
+            raise WeKnoraAPIError(f"WeKnora returned a non-JSON response: {exc}") from exc
+        if not isinstance(data, dict):
+            raise WeKnoraAPIError(f"WeKnora returned unexpected payload: {data!r}")
+        return data
+
+    async def list_knowledge_bases(self) -> list[dict[str, Any]]:
+        data = (await self._request_json("GET", "/api/v1/knowledge-bases")).get("data")
+        if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
+            raise WeKnoraAPIError("WeKnora returned an unexpected knowledge-base list.")
+        return data
+
+    async def search(self, query: str) -> list[dict[str, Any]]:
+        data = (
+            await self._request_json(
+                "POST",
+                "/api/v1/knowledge-search",
+                params={"resource_urls": "handle"},
+                json={
+                    "query": query,
+                    "knowledge_base_id": self._config.knowledge_base_id,
+                },
+            )
+        ).get("data")
+        if not isinstance(data, list) or not all(isinstance(item, dict) for item in data):
+            raise WeKnoraAPIError("WeKnora returned unexpected search results.")
+        return data
+
+
+__all__ = ["MAX_RESPONSE_BYTES", "WeKnoraAPIError", "WeKnoraClient"]

deeptutor/services/rag/pipelines/weknora/config.py

@@ -0,0 +1,56 @@
+"""Per-KB connection configuration for Tencent WeKnora."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlparse
+
+
+class WeKnoraNotConfiguredError(RuntimeError):
+    """Raised when a KB lacks the fields needed to reach WeKnora."""
+
+
+@dataclass(frozen=True)
+class WeKnoraConfig:
+    base_url: str
+    api_key: str
+    knowledge_base_id: str
+
+
+def normalize_base_url(url: str | None) -> str:
+    normalized = (url or "").strip().rstrip("/")
+    if not normalized:
+        return ""
+    parsed = urlparse(normalized)
+    if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+        return ""
+    if parsed.username or parsed.password or parsed.query or parsed.fragment:
+        return ""
+    return normalized
+
+
+def config_from_entry(entry: dict[str, Any]) -> WeKnoraConfig:
+    base_url = normalize_base_url(entry.get("server_url"))
+    api_key = str(entry.get("api_key") or "").strip()
+    knowledge_base_id = str(entry.get("knowledge_base_id") or "").strip()
+    if not base_url or not knowledge_base_id:
+        raise WeKnoraNotConfiguredError(
+            "This knowledge base is not connected to WeKnora "
+            "(server URL and knowledge base ID are required)."
+        )
+    if not api_key:
+        raise WeKnoraNotConfiguredError("A WeKnora API key is required for retrieval.")
+    return WeKnoraConfig(
+        base_url=base_url,
+        api_key=api_key,
+        knowledge_base_id=knowledge_base_id,
+    )
+
+
+__all__ = [
+    "WeKnoraConfig",
+    "WeKnoraNotConfiguredError",
+    "config_from_entry",
+    "normalize_base_url",
+]

deeptutor/services/rag/pipelines/weknora/pipeline.py

@@ -0,0 +1,74 @@
+"""Retrieval-only RAG pipeline backed by an external WeKnora knowledge base."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from deeptutor.runtime.home import get_runtime_data_root
+from deeptutor.services.rag.provider_binding import load_kb_config_entry
+
+from .config import WeKnoraNotConfiguredError, config_from_entry
+
+logger = logging.getLogger(__name__)
+
+PROVIDER = "weknora"
+DEFAULT_KB_BASE_DIR = str(get_runtime_data_root() / "knowledge_bases")
+
+
+class WeKnoraPipeline:
+    def __init__(self, kb_base_dir: Optional[str] = None, *, client_factory=None, **_: Any) -> None:
+        self.kb_base_dir = kb_base_dir or DEFAULT_KB_BASE_DIR
+        self._client_factory = client_factory
+
+    def _client(self, config):
+        if self._client_factory is not None:
+            return self._client_factory(config)
+        from .client import WeKnoraClient
+
+        return WeKnoraClient(config)
+
+    async def search(self, query: str, kb_name: str, **kwargs) -> Dict[str, Any]:
+        try:
+            config = config_from_entry(load_kb_config_entry(self.kb_base_dir, kb_name))
+        except WeKnoraNotConfiguredError as exc:
+            return self._error_result(query, exc, error_type="not_configured")
+
+        try:
+            chunks = await self._client(config).search(query)
+        except Exception as exc:
+            logger.error("WeKnora search failed for '%s': %s", kb_name, exc)
+            return self._error_result(query, exc, error_type="retrieval_error")
+
+        return {
+            "query": query,
+            "answer": "\n\n---\n\n".join(str(item.get("content") or "") for item in chunks),
+            "content": "\n\n---\n\n".join(str(item.get("content") or "") for item in chunks),
+            "sources": chunks,
+            "provider": PROVIDER,
+        }
+
+    def _error_result(self, query: str, exc: Exception, *, error_type: str) -> Dict[str, Any]:
+        return {
+            "query": query,
+            "answer": str(exc),
+            "content": "",
+            "sources": [],
+            "provider": PROVIDER,
+            "error_type": error_type,
+        }
+
+    async def initialize(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
+        raise RuntimeError(
+            "WeKnora knowledge bases are managed in WeKnora; DeepTutor does not "
+            "upload or index their documents."
+        )
+
+    async def add_documents(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
+        return await self.initialize(kb_name, file_paths, **kwargs)
+
+    async def delete(self, kb_name: str, **kwargs) -> bool:
+        return True
+
+
+__all__ = ["WeKnoraPipeline", "PROVIDER"]

deeptutor/services/rag/pipelines/weknora/probe.py

@@ -0,0 +1,80 @@
+"""Validate a WeKnora endpoint, API key, and knowledge-base binding."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any, Optional
+
+from .client import WeKnoraClient
+from .config import WeKnoraConfig, normalize_base_url
+
+
+@dataclass
+class WeKnoraProbe:
+    base_url: str
+    knowledge_base_id: str
+    ok: bool = False
+    reachable: bool = False
+    credentials_ok: bool = False
+    knowledge_base_found: bool = False
+    knowledge_base_name: Optional[str] = None
+    error: Optional[str] = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+async def probe_weknora(
+    server_url: str,
+    api_key: str,
+    knowledge_base_id: str,
+    *,
+    client_factory=None,
+) -> WeKnoraProbe:
+    base_url = normalize_base_url(server_url)
+    knowledge_base_id = (knowledge_base_id or "").strip()
+    api_key = (api_key or "").strip()
+    probe = WeKnoraProbe(base_url=base_url, knowledge_base_id=knowledge_base_id)
+
+    if not base_url or not knowledge_base_id or not api_key:
+        probe.error = "Server URL, API key, and knowledge base ID are required."
+        return probe
+
+    # WeKnora is administrator-configured and often lives on the deployment's
+    # private network, so loopback/LAN remain valid. Link-local, metadata and
+    # malformed targets are still forbidden, and the client repeats this check
+    # at every retrieval request to narrow DNS-rebinding exposure.
+    from deeptutor.services.mcp.network import validate_mcp_url_async
+
+    safe, error = await validate_mcp_url_async(base_url)
+    if not safe:
+        probe.error = f"Unsafe WeKnora server URL: {error}"
+        return probe
+
+    config = WeKnoraConfig(
+        base_url=base_url,
+        api_key=api_key,
+        knowledge_base_id=knowledge_base_id,
+    )
+    client = client_factory(config) if client_factory else WeKnoraClient(config)
+    try:
+        knowledge_bases = await client.list_knowledge_bases()
+    except Exception as exc:
+        probe.error = f"Could not validate the WeKnora connection: {exc}"
+        return probe
+
+    probe.reachable = True
+    probe.credentials_ok = True
+    matches = [item for item in knowledge_bases if str(item.get("id") or "") == knowledge_base_id]
+    if not matches:
+        probe.error = f"Knowledge base {knowledge_base_id} is not visible to this WeKnora API key."
+        return probe
+
+    probe.knowledge_base_found = True
+    name = str(matches[0].get("name") or "").strip()
+    probe.knowledge_base_name = name or None
+    probe.ok = True
+    return probe
+
+
+__all__ = ["WeKnoraProbe", "probe_weknora"]