deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/config/knowledge_base_config.py

@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from deeptutor.services.path_service import get_path_service
+from deeptutor.services.rag.factory import (
+    DEFAULT_PROVIDER,
+    KNOWN_PROVIDERS,
+    has_ready_provider_index,
+    normalize_provider_name,
+)
+
+logger = logging.getLogger(__name__)
+
+# Legacy fallback only — frozen at admin scope at import time. Production code
+# must enter through ``get_kb_config_service()`` (not used directly here, see
+# ``deeptutor/services/config/__init__.py``) which resolves the path lazily.
+DEFAULT_CONFIG_PATH = get_path_service().get_knowledge_bases_root() / "kb_config.json"
+
+
+def _default_payload() -> dict[str, Any]:
+    return {
+        "defaults": {
+            "default_kb": None,
+            "rag_provider": DEFAULT_PROVIDER,
+            "search_mode": "hybrid",
+            # Per-engine default retrieval mode, set from the engine cards
+            # (e.g. {"lightrag": "hybrid", "graphrag": "local"}). A KB's own
+            # ``search_mode`` still wins; this is the fallback for that engine.
+            "provider_modes": {},
+        },
+        "knowledge_bases": {},
+    }
+
+
+class KnowledgeBaseConfigService:
+    _instances: dict[str, "KnowledgeBaseConfigService"] = {}
+
+    def __init__(self, config_path: Path | None = None):
+        self.config_path = config_path or DEFAULT_CONFIG_PATH
+        self._config = self._load_config()
+
+    @classmethod
+    def get_instance(cls, config_path: Path | None = None) -> "KnowledgeBaseConfigService":
+        resolved = (
+            config_path or get_path_service().get_knowledge_bases_root() / "kb_config.json"
+        ).resolve()
+        key = str(resolved)
+        if key not in cls._instances:
+            cls._instances[key] = cls(resolved)
+        return cls._instances[key]
+
+    def _load_config(self) -> dict[str, Any]:
+        payload = _default_payload()
+        if self.config_path.exists():
+            try:
+                with open(self.config_path, "r", encoding="utf-8") as handle:
+                    loaded = json.load(handle) or {}
+                payload.update({k: v for k, v in loaded.items() if k != "defaults"})
+                payload["defaults"].update(loaded.get("defaults", {}))
+            except Exception as exc:
+                logger.warning(f"Failed to load KB config: {exc}")
+        payload.setdefault("knowledge_bases", {})
+        payload.setdefault("defaults", _default_payload()["defaults"])
+        payload = self._normalize_payload(payload)
+        return payload
+
+    def _normalize_payload(self, payload: dict[str, Any]) -> dict[str, Any]:
+        defaults = payload.setdefault("defaults", _default_payload()["defaults"])
+        defaults["rag_provider"] = normalize_provider_name(defaults.get("rag_provider"))
+
+        knowledge_bases = payload.setdefault("knowledge_bases", {})
+        kb_base_dir = self.config_path.parent
+        for kb_name, config in knowledge_bases.items():
+            if not isinstance(config, dict):
+                continue
+
+            raw_provider = config.get("rag_provider")
+            config["rag_provider"] = normalize_provider_name(raw_provider)
+
+            # A KB indexed with an engine that no longer exists collapses to the
+            # default and must be rebuilt.
+            if (
+                isinstance(raw_provider, str)
+                and raw_provider.strip()
+                and raw_provider.strip().lower() not in KNOWN_PROVIDERS
+            ):
+                config["needs_reindex"] = True
+
+            kb_dir = kb_base_dir / kb_name
+            legacy_storage = kb_dir / "rag_storage"
+            has_llamaindex_index = has_ready_provider_index(kb_dir, DEFAULT_PROVIDER)
+            if (
+                config["rag_provider"] == DEFAULT_PROVIDER
+                and legacy_storage.exists()
+                and legacy_storage.is_dir()
+                and not has_llamaindex_index
+            ):
+                config["needs_reindex"] = True
+                if config.get("status") == "ready":
+                    config["status"] = "needs_reindex"
+
+        return payload
+
+    def _save(self) -> None:
+        from deeptutor.services.file_io import atomic_write_json
+
+        self._config = self._normalize_payload(self._config)
+        atomic_write_json(self.config_path, self._config)
+
+    def _refresh(self) -> None:
+        """Re-read kb_config.json so this singleton sees changes made by
+        ``KnowledgeBaseManager`` (orphan pruning, new KB registrations).
+
+        Without this, every mutating call would rewrite the file from the
+        in-memory snapshot taken at process start and undo any external
+        cleanup. Read-modify-write keeps the two writers consistent.
+        """
+        self._config = self._load_config()
+
+    def _ensure_kb(self, kb_name: str) -> dict[str, Any]:
+        knowledge_bases = self._config.setdefault("knowledge_bases", {})
+        if kb_name not in knowledge_bases:
+            knowledge_bases[kb_name] = {
+                "path": kb_name,
+                "description": f"Knowledge base: {kb_name}",
+            }
+        return knowledge_bases[kb_name]
+
+    def get_kb_config(self, kb_name: str) -> dict[str, Any]:
+        self._refresh()
+        defaults = dict(self._config.get("defaults", {}))
+        kb_config = dict(self._config.get("knowledge_bases", {}).get(kb_name, {}))
+        merged = {
+            "default_kb": defaults.get("default_kb"),
+            "rag_provider": defaults.get("rag_provider", DEFAULT_PROVIDER),
+            "search_mode": kb_config.get("search_mode") or defaults.get("search_mode", "hybrid"),
+            "needs_reindex": bool(kb_config.get("needs_reindex", False)),
+            **kb_config,
+        }
+        merged["rag_provider"] = normalize_provider_name(merged.get("rag_provider"))
+        return merged
+
+    def set_kb_config(self, kb_name: str, config: dict[str, Any]) -> None:
+        self._refresh()
+        entry = self._ensure_kb(kb_name)
+        entry.update(config)
+        self._save()
+
+    def get_rag_provider(self, kb_name: str) -> str:
+        return normalize_provider_name(self.get_kb_config(kb_name).get("rag_provider"))
+
+    def set_rag_provider(self, kb_name: str, provider: str) -> None:
+        self.set_kb_config(kb_name, {"rag_provider": normalize_provider_name(provider)})
+
+    def get_search_mode(self, kb_name: str) -> str:
+        return str(self.get_kb_config(kb_name).get("search_mode", "hybrid"))
+
+    def set_search_mode(self, kb_name: str, mode: str) -> None:
+        self.set_kb_config(kb_name, {"search_mode": mode})
+
+    def get_provider_mode(self, provider: str) -> str:
+        """Global default retrieval mode for an engine ("" when unset)."""
+        self._refresh()
+        modes = self._config.get("defaults", {}).get("provider_modes", {})
+        return str(modes.get(provider, "")) if isinstance(modes, dict) else ""
+
+    def set_provider_mode(self, provider: str, mode: str) -> None:
+        self._refresh()
+        defaults = self._config.setdefault("defaults", _default_payload()["defaults"])
+        modes = defaults.setdefault("provider_modes", {})
+        modes[provider] = mode
+        self._save()
+
+    def delete_kb_config(self, kb_name: str) -> None:
+        self._refresh()
+        knowledge_bases = self._config.get("knowledge_bases", {})
+        if kb_name in knowledge_bases:
+            del knowledge_bases[kb_name]
+            self._save()
+
+    def get_all_configs(self) -> dict[str, Any]:
+        self._refresh()
+        return self._config
+
+    def set_global_defaults(self, defaults: dict[str, Any]) -> None:
+        self._refresh()
+        current = self._config.setdefault("defaults", _default_payload()["defaults"])
+        current.update(defaults)
+        self._save()
+
+    def set_default_kb(self, kb_name: str | None) -> None:
+        self._refresh()
+        self._config.setdefault("defaults", _default_payload()["defaults"])["default_kb"] = kb_name
+        self._save()
+
+    def get_default_kb(self) -> str | None:
+        self._refresh()
+        return self._config.get("defaults", {}).get("default_kb")
+
+    def sync_from_metadata(self, kb_name: str, kb_base_dir: Path) -> None:
+        metadata_file = kb_base_dir / kb_name / "metadata.json"
+        if not metadata_file.exists():
+            return
+        try:
+            with open(metadata_file, "r", encoding="utf-8") as handle:
+                metadata = json.load(handle)
+        except Exception as exc:
+            logger.warning(f"Failed to load KB metadata for {kb_name}: {exc}")
+            return
+        config: dict[str, Any] = {}
+        if metadata.get("rag_provider"):
+            raw_provider = metadata["rag_provider"]
+            config["rag_provider"] = normalize_provider_name(raw_provider)
+            if str(raw_provider).strip().lower() not in KNOWN_PROVIDERS:
+                config["needs_reindex"] = True
+        if metadata.get("search_mode"):
+            config["search_mode"] = metadata["search_mode"]
+        if config:
+            self.set_kb_config(kb_name, config)
+
+    def sync_all_from_metadata(self, kb_base_dir: Path) -> None:
+        if not kb_base_dir.exists():
+            return
+        for kb_dir in kb_base_dir.iterdir():
+            if kb_dir.is_dir() and not kb_dir.name.startswith("."):
+                self.sync_from_metadata(kb_dir.name, kb_base_dir)
+
+
+def get_kb_config_service() -> KnowledgeBaseConfigService:
+    return KnowledgeBaseConfigService.get_instance(
+        get_path_service().get_knowledge_bases_root() / "kb_config.json"
+    )
+
+
+__all__ = ["KnowledgeBaseConfigService", "get_kb_config_service"]

deeptutor/services/embedding/__init__.py

@@ -0,0 +1,46 @@
+"""Unified embedding client and adapters for all DeepTutor modules.
+
+Supported bindings are resolved by ``services.config.provider_runtime`` and
+currently include openai, custom, azure_openai, cohere, jina, ollama, vllm,
+siliconflow, aliyun, openrouter, plus legacy custom_openai_sdk configs.
+"""
+
+from importlib import import_module
+
+from .config import EmbeddingConfig, get_embedding_config
+from .validation import validate_embedding_batch
+
+_CLIENT_EXPORTS = {"EmbeddingClient", "get_embedding_client", "reset_embedding_client"}
+
+
+def __getattr__(name: str):
+    # Reading model metadata must not initialize every embedding SDK. Background
+    # KB catalog scans need configuration even when no embedding call will run.
+    if name in _CLIENT_EXPORTS:
+        value = getattr(import_module(".client", __name__), name)
+    elif name in __all__:
+        value = getattr(import_module(".adapters", __name__), name)
+    else:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    globals()[name] = value
+    return value
+
+
+__all__ = [
+    "EmbeddingClient",
+    "EmbeddingConfig",
+    "get_embedding_client",
+    "get_embedding_config",
+    "reset_embedding_client",
+    "validate_embedding_batch",
+    "BaseEmbeddingAdapter",
+    "EmbeddingProviderError",
+    "EmbeddingRequest",
+    "EmbeddingResponse",
+    "OpenAICompatibleEmbeddingAdapter",
+    "OpenAISDKEmbeddingAdapter",
+    "DashScopeMultiModalEmbeddingAdapter",
+    "CohereEmbeddingAdapter",
+    "JinaEmbeddingAdapter",
+    "OllamaEmbeddingAdapter",
+]

deeptutor/services/embedding/adapters/__init__.py

@@ -0,0 +1,40 @@
+"""Embedding adapter implementations and backend registry."""
+
+from .base import (
+    BaseEmbeddingAdapter,
+    EmbeddingProviderError,
+    EmbeddingRequest,
+    EmbeddingResponse,
+)
+from .cohere import CohereEmbeddingAdapter
+from .dashscope_native import DashScopeMultiModalEmbeddingAdapter
+from .gemini import GeminiEmbeddingAdapter
+from .jina import JinaEmbeddingAdapter
+from .ollama import OllamaEmbeddingAdapter
+from .openai_compatible import OpenAICompatibleEmbeddingAdapter
+from .openai_sdk import OpenAISDKEmbeddingAdapter
+
+ADAPTER_BACKENDS: dict[str, type[BaseEmbeddingAdapter]] = {
+    "openai_compat": OpenAICompatibleEmbeddingAdapter,
+    "openai_sdk": OpenAISDKEmbeddingAdapter,
+    "cohere": CohereEmbeddingAdapter,
+    "jina": JinaEmbeddingAdapter,
+    "ollama": OllamaEmbeddingAdapter,
+    "dashscope_native": DashScopeMultiModalEmbeddingAdapter,
+    "gemini": GeminiEmbeddingAdapter,
+}
+
+__all__ = [
+    "ADAPTER_BACKENDS",
+    "BaseEmbeddingAdapter",
+    "EmbeddingProviderError",
+    "EmbeddingRequest",
+    "EmbeddingResponse",
+    "OpenAICompatibleEmbeddingAdapter",
+    "OpenAISDKEmbeddingAdapter",
+    "DashScopeMultiModalEmbeddingAdapter",
+    "GeminiEmbeddingAdapter",
+    "JinaEmbeddingAdapter",
+    "CohereEmbeddingAdapter",
+    "OllamaEmbeddingAdapter",
+]

deeptutor/services/embedding/adapters/base.py

@@ -0,0 +1,196 @@
+"""
+Base Embedding Adapter
+=======================
+
+Abstract base class for all embedding adapters.
+Defines the contract that all embedding providers must implement.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from deeptutor.services.keypool import KeyPool
+
+
+def looks_like_multimodal_embedding_model(model_name: Optional[str]) -> bool:
+    """Best-effort guard for OpenAI-compatible multimodal embedding models."""
+    if not model_name:
+        return False
+    normalized = model_name.lower().replace("_", "-")
+    return any(
+        marker in normalized
+        for marker in (
+            "qwen3-vl-embedding",
+            "multimodal-embedding",
+            "vision-embedding",
+            "vl-embedding",
+            "image-embedding",
+        )
+    )
+
+
+@dataclass
+class EmbeddingRequest:
+    """
+    Standard embedding request structure.
+
+    Provider-agnostic request format. Different providers interpret fields differently:
+
+    Args:
+        texts: List of texts to embed
+        model: Model name to use
+        dimensions: Embedding vector dimensions (optional)
+        input_type: Input type hint for task-aware embeddings (optional)
+            - Cohere: Maps to 'input_type' ("search_document", "search_query", "classification", "clustering")
+            - Jina: Maps to 'task' ("retrieval.passage", "retrieval.query", etc.)
+            - Gemini Embedding 2: Maps to retrieval-specific text instructions
+            - OpenAI/Ollama: Ignored
+        encoding_format: Output format ("float" or "base64"). ``None`` (the
+            default) lets each adapter decide: OpenAI-compatible gateways omit
+            the param entirely (several, e.g. SiliconFlow, return HTTP 400 when
+            it is present), while the official OpenAI SDK path pins "float".
+        truncate: Whether to truncate texts that exceed max tokens (default: True)
+        normalized: Whether to return L2-normalized embeddings (Jina/Ollama only)
+        late_chunking: Enable late chunking for long context (Jina v3 only)
+        contents: Multimodal content list of dicts like
+            ``[{"text": "..."}, {"image": "url|data: URI"}, {"video": "..."}]``.
+            Adapters that support multimodal (DashScope, SiliconFlow Qwen3-VL,
+            Cohere v4) consume this directly; text-only adapters MUST raise
+            ``ValueError`` if it is set so the caller can route differently.
+            When ``contents`` is set, ``texts`` is ignored.
+        enable_fusion: DashScope-specific. ``True`` fuses all multimodal items
+            into one vector; ``False`` (or None) returns one vector per item.
+    """
+
+    texts: List[str]
+    model: str
+    dimensions: Optional[int] = None
+    input_type: Optional[str] = None
+    encoding_format: Optional[str] = None
+    truncate: Optional[bool] = True
+    normalized: Optional[bool] = True
+    late_chunking: Optional[bool] = False
+    contents: Optional[List[Dict[str, Any]]] = None
+    enable_fusion: Optional[bool] = None
+
+
+@dataclass
+class EmbeddingResponse:
+    """Standard embedding response structure."""
+
+    embeddings: List[List[float]]
+    model: str
+    dimensions: int
+    usage: Dict[str, Any]
+
+
+class EmbeddingProviderError(RuntimeError):
+    """Structured error raised by embedding adapters on provider failures.
+
+    Carries the HTTP status, response body excerpt, model name, and request
+    URL so downstream callers (task log streams, UI surfaces) can show
+    actionable diagnostics instead of a bare exception string.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status: Optional[int] = None,
+        body: Optional[str] = None,
+        model: Optional[str] = None,
+        url: Optional[str] = None,
+        provider: Optional[str] = None,
+    ) -> None:
+        super().__init__(message)
+        self.status = status
+        self.body = body
+        self.model = model
+        self.url = url
+        self.provider = provider
+
+    def __str__(self) -> str:  # noqa: D401 - succinct
+        parts = [super().__str__()]
+        if self.provider:
+            parts.append(f"provider={self.provider}")
+        if self.model:
+            parts.append(f"model={self.model}")
+        if self.status is not None:
+            parts.append(f"status={self.status}")
+        if self.url:
+            parts.append(f"url={self.url}")
+        if self.body:
+            snippet = self.body if len(self.body) <= 500 else self.body[:500] + "...(truncated)"
+            parts.append(f"body={snippet}")
+        return " | ".join(parts)
+
+
+class BaseEmbeddingAdapter(ABC):
+    """
+    Base class for all embedding adapters.
+
+    Each adapter implements the specific API interface for a provider
+    (OpenAI, Cohere, Ollama, etc.) while exposing a unified interface.
+    """
+
+    # Whether this adapter turns ``EmbeddingRequest.input_type`` into a
+    # provider parameter. Opt-in, because doing so changes the vectors a
+    # provider returns for the same text: switching it on for a backend that
+    # previously sent no role invalidates every index already built with it.
+    SUPPORTS_INPUT_TYPE: bool = False
+
+    def __init__(self, config: Dict[str, Any]):
+        """
+        Initialize the adapter with configuration.
+
+        Args:
+            config: Dictionary containing:
+                - api_key: API authentication key (optional for local)
+                - base_url: API endpoint URL
+                - model: Model name to use
+                - dimensions: Embedding vector dimensions
+                - send_dimensions: Tri-state opt-in for the `dimensions`
+                  request param. ``True`` always sends, ``False`` never
+                  sends, ``None`` lets the adapter decide based on the
+                  model family (default).
+                - request_timeout: Request timeout in seconds
+        """
+        raw_api_key = config.get("api_key")
+        keys = raw_api_key if isinstance(raw_api_key, list) else [raw_api_key]
+        keys = [str(key).strip() for key in keys if str(key or "").strip()]
+        self._key_pool = KeyPool(keys) if keys else None
+        self.api_key = keys[0] if keys else ""
+        self.base_url = config.get("base_url")
+        self.api_version = config.get("api_version")
+        self.model = config.get("model")
+        self.dimensions = config.get("dimensions")
+        self.send_dimensions: Optional[bool] = config.get("send_dimensions")
+        self.request_timeout = config.get("request_timeout", 60)
+        self.extra_headers = config.get("extra_headers") or {}
+
+    @abstractmethod
+    async def embed(self, request: EmbeddingRequest) -> EmbeddingResponse:
+        """
+        Generate embeddings for a list of texts.
+
+        Args:
+            request: EmbeddingRequest with texts and parameters
+
+        Returns:
+            EmbeddingResponse with embeddings and metadata
+
+        Raises:
+            httpx.HTTPError: If the API request fails
+        """
+        pass
+
+    @abstractmethod
+    def get_model_info(self) -> Dict[str, Any]:
+        """
+        Return information about the configured model.
+
+        Returns:
+            Dictionary with model metadata (name, dimensions, etc.)
+        """
+        pass