deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/knowledge/manager.py

@@ -0,0 +1,2570 @@
+#!/usr/bin/env python
+"""
+Knowledge Base Manager
+
+Manages multiple knowledge bases and provides utilities for accessing them.
+"""
+
+from contextlib import contextmanager
+from copy import deepcopy
+from datetime import datetime, timedelta
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+import shutil
+import stat
+import sys
+from threading import RLock
+import time
+from typing import Any
+from urllib.parse import urlparse
+
+from deeptutor.knowledge.kb_types import (
+    IMA_KB_TYPE,
+    LIGHTRAG_SERVER_KB_TYPE,
+    LINKED_KB_TYPE,
+    MARGINNOTE4_KB_TYPE,
+    OBSIDIAN_KB_TYPE,
+    SUBAGENT_KB_TYPE,
+    WEKNORA_KB_TYPE,
+    external_root_of,
+    is_connected_kb,
+)
+from deeptutor.knowledge.manifest import iter_kb_documents
+from deeptutor.knowledge.naming import validate_knowledge_base_name
+from deeptutor.services.file_io import atomic_write_json
+from deeptutor.services.rag.factory import (
+    DEFAULT_PROVIDER,
+    IMA_PROVIDER,
+    KNOWN_PROVIDERS,
+    LIGHTRAG_PROVIDER,
+    LIGHTRAG_SERVER_PROVIDER,
+    PAGEINDEX_OSS_PROVIDER,
+    PAGEINDEX_PROVIDER,
+    WEKNORA_PROVIDER,
+    has_ready_provider_index,
+    normalize_provider_name,
+    provider_uses_embedding_versions,
+)
+from deeptutor.services.rag.index_probe import (
+    inspect_kb_versions,
+    inspect_provider_version,
+    provider_failure_summary,
+)
+from deeptutor.services.web_source.crawler import MAX_CRAWL_DEPTH, MAX_CRAWL_PAGES
+from deeptutor.utils.file_types import FileTypeRouter
+
+logger = logging.getLogger(__name__)
+
+
+def _assert_move_id_available(base_dir: Path, name: str) -> None:
+    """Prevent a new registration from shadowing a saved moved-KB redirect."""
+    from deeptutor.services.workspace.knowledge import (
+        canonical_kb_id,
+        qualified_kb_id,
+        workspace_id_for_kb_base_dir,
+    )
+
+    workspace_id = workspace_id_for_kb_base_dir(base_dir)
+    if workspace_id is None:
+        # A manager targeting a partner's separate directory has no catalog ID.
+        return
+    resource_id = qualified_kb_id(name, workspace_id)
+    if canonical_kb_id(resource_id) != resource_id:
+        raise ValueError("This knowledge base ID is reserved by an earlier move.")
+
+
+# How long an entry can be missing its KB directory before ``list_knowledge_bases``
+# treats it as a stale orphan. The KB create flow writes the "initializing"
+# config entry before the on-disk folder is created, so a too-short grace would
+# let a list-call mid-creation racy-delete the entry. 60s is comfortably longer
+# than the create handshake while still keeping multi-day zombies out.
+_ORPHAN_PRUNE_GRACE_SECONDS = 60
+
+# External/manual index changes are re-probed within this interval (#1711).
+# Normal indexing writes kb_config.json, which invalidates the snapshot at once.
+_CATALOG_CACHE_SECONDS = 2.0
+
+
+def _entry_updated_after(kb_entry: dict | None, cutoff: datetime) -> bool:
+    """Return True when the entry's ``updated_at`` is strictly after ``cutoff``.
+
+    Entries without a parseable timestamp are treated as old (return False) —
+    a long-stuck orphan that crashed before recording a timestamp should still
+    get pruned.
+    """
+    if not isinstance(kb_entry, dict):
+        return False
+    raw = kb_entry.get("updated_at")
+    if not isinstance(raw, str):
+        return False
+    try:
+        return datetime.fromisoformat(raw) > cutoff
+    except ValueError:
+        return False
+
+
+def _provider_from_version_entry(entry: dict[str, Any]) -> str:
+    provider = str(entry.get("provider") or "").strip().lower()
+    if provider in KNOWN_PROVIDERS:
+        return provider
+    signature = str(entry.get("signature") or "").strip().lower()
+    return signature if signature in KNOWN_PROVIDERS else DEFAULT_PROVIDER
+
+
+def _detect_provider_from_versions(versions: list[dict[str, Any]]) -> str:
+    for entry in versions:
+        provider = _provider_from_version_entry(entry)
+        if provider != DEFAULT_PROVIDER:
+            return provider
+    return DEFAULT_PROVIDER
+
+
+# Cross-platform file locking. Writers no longer take locks — every JSON
+# write in this module goes through ``atomic_write_json`` (temp file +
+# ``os.replace``), so readers always see a complete previous or new file.
+@contextmanager
+def file_lock_shared(file_handle):
+    """Acquire a shared (read) lock on a file - cross-platform."""
+    if sys.platform == "win32":
+        import msvcrt
+
+        msvcrt.locking(file_handle.fileno(), msvcrt.LK_NBLCK, 1)
+        try:
+            yield
+        finally:
+            file_handle.seek(0)
+            msvcrt.locking(file_handle.fileno(), msvcrt.LK_UNLCK, 1)
+    else:
+        import fcntl
+
+        fcntl.flock(file_handle.fileno(), fcntl.LOCK_SH)
+        try:
+            yield
+        finally:
+            fcntl.flock(file_handle.fileno(), fcntl.LOCK_UN)
+
+
+def _get_embedding_fingerprint() -> tuple[str, int] | None:
+    """Return ``(model_name, dimension)`` of the active embedding config."""
+    try:
+        from deeptutor.services.embedding import get_embedding_config
+
+        cfg = get_embedding_config()
+        return (cfg.model, cfg.dim)
+    except Exception:
+        return None
+
+
+def _reconcile_embedding_flags(knowledge_bases: dict, base_dir: Path | None = None) -> bool:
+    """Reconcile per-KB embedding flags against the on-disk index versions.
+
+    For each KB we check the flat ``version-N`` directories (plus legacy
+    layouts) for a version matching the active embedding signature:
+
+    * Match found → clear ``needs_reindex`` and ``embedding_mismatch`` (the
+      user has switched back to a previously-indexed configuration).
+    * No match, but the KB has a stored ``embedding_model`` that differs
+      from the active fingerprint → set both flags so the UI surfaces a
+      "Re-index" CTA.
+
+    Returns ``True`` when any entry changed.
+    """
+    from deeptutor.services.rag.embedding_signature import signature_from_embedding_config
+    from deeptutor.services.rag.index_versioning import (
+        find_matching_version,
+    )
+
+    fp = _get_embedding_fingerprint()
+    signature = signature_from_embedding_config()
+    changed = False
+    if base_dir is not None:
+        from deeptutor.services.rag.embedding_binding import reconcile_bindings
+
+        changed = reconcile_bindings(knowledge_bases, base_dir)
+
+    for kb_name, kb_entry in knowledge_bases.items():
+        if not isinstance(kb_entry, dict):
+            continue
+
+        # Connected KBs (Obsidian vaults, linked indexes) are pointers with no
+        # embedding lifecycle we manage — compatibility is checked once at
+        # connect time, never reconciled here.
+        if is_connected_kb(kb_entry):
+            continue
+
+        provider = normalize_provider_name(kb_entry.get("rag_provider"))
+        if provider == LIGHTRAG_PROVIDER and base_dir is not None:
+            from deeptutor.services.embedding.config import embedding_config_scope
+            from deeptutor.services.rag.embedding_binding import (
+                binding_status,
+                bound_graph_storage_root,
+                entry_signature,
+            )
+            from deeptutor.services.rag.pipelines.lightrag.storage import (
+                embedding_matches,
+                latest_published_root,
+            )
+
+            kb_dir = base_dir / kb_name
+            kb_entry["index_versions"] = inspect_kb_versions(kb_dir, provider)
+            published = latest_published_root(kb_dir)
+            state, bound_config = binding_status(kb_entry)
+            if state in {"missing", "unconfigured"}:
+                continue
+            if bound_config is not None:
+                with embedding_config_scope(bound_config):
+                    try:
+                        published = bound_graph_storage_root(kb_dir, provider, published)
+                    except ValueError:
+                        pass  # Check the unmatched published index below.
+            lightrag_mismatch = published is not None and not embedding_matches(
+                published, entry_signature(kb_entry)
+            )
+            lightrag_mismatch = lightrag_mismatch or state == "changed"
+            if lightrag_mismatch and not kb_entry.get("embedding_mismatch"):
+                kb_entry["embedding_mismatch"] = True
+                kb_entry["needs_reindex"] = True
+                changed = True
+            elif (
+                published is not None
+                and not lightrag_mismatch
+                and kb_entry.pop("embedding_mismatch", None)
+            ):
+                kb_entry["needs_reindex"] = False
+                changed = True
+            continue
+
+        if kb_entry.get("embedding_selection"):
+            continue
+
+        if signature is None and not fp:
+            continue
+
+        if not provider_uses_embedding_versions(provider):
+            kb_dir = (base_dir / kb_name) if base_dir is not None else None
+            if kb_dir is not None:
+                versions = inspect_kb_versions(kb_dir, provider)
+                kb_entry["index_versions"] = versions
+                if has_ready_provider_index(kb_dir, provider):
+                    mutated_local = False
+                    had_embedding_mismatch = bool(kb_entry.get("embedding_mismatch"))
+                    if kb_entry.get("embedding_mismatch"):
+                        kb_entry.pop("embedding_mismatch", None)
+                        mutated_local = True
+                    if had_embedding_mismatch and kb_entry.get("needs_reindex"):
+                        kb_entry["needs_reindex"] = False
+                        mutated_local = True
+                    if mutated_local:
+                        changed = True
+            continue
+
+        kb_dir = (base_dir / kb_name) if base_dir is not None else None
+        matched = False
+        if kb_dir is not None and signature is not None:
+            matched_entry = find_matching_version(kb_dir, signature)
+            matched = (
+                matched_entry is not None
+                and inspect_provider_version(matched_entry, DEFAULT_PROVIDER).ready
+            )
+
+        if matched:
+            mutated_local = False
+            if kb_entry.get("needs_reindex"):
+                kb_entry["needs_reindex"] = False
+                mutated_local = True
+            if kb_entry.get("embedding_mismatch"):
+                kb_entry.pop("embedding_mismatch", None)
+                mutated_local = True
+            if mutated_local:
+                changed = True
+            # Refresh the surfaced version list either way so the UI sees
+            # accurate state.
+            if kb_dir is not None:
+                kb_entry["index_versions"] = inspect_kb_versions(kb_dir, provider)
+            continue
+
+        # No matching ready index version on disk.
+        stored_model = kb_entry.get("embedding_model")
+        # Empty/in-progress version dirs are created before indexing finishes.
+        # They should not mark a brand-new KB as needing re-index.
+        versions = []
+        has_ready_version = False
+        if kb_dir is not None:
+            versions = inspect_kb_versions(kb_dir, provider)
+            has_ready_version = any(bool(version.get("ready")) for version in versions)
+            kb_entry["index_versions"] = versions
+
+        if not has_ready_version and not stored_model:
+            continue
+
+        current_model = fp[0] if fp else ""
+        current_dim = fp[1] if fp else 0
+        stored_dim = kb_entry.get("embedding_dim")
+        mismatch = (stored_model and stored_model != current_model) or (
+            stored_dim is not None and current_dim and stored_dim != current_dim
+        )
+        # If ready versions exist but none match active signature, that's also a mismatch.
+        if has_ready_version:
+            mismatch = True
+
+        if mismatch and not kb_entry.get("embedding_mismatch"):
+            kb_entry["embedding_mismatch"] = True
+            if not kb_entry.get("needs_reindex"):
+                kb_entry["needs_reindex"] = True
+            changed = True
+        elif not mismatch and kb_entry.get("embedding_mismatch"):
+            kb_entry.pop("embedding_mismatch", None)
+            changed = True
+
+    return changed
+
+
+class KnowledgeBaseManager:
+    """Manager for knowledge bases"""
+
+    def __init__(self, base_dir="./data/knowledge_bases"):
+        self.base_dir = Path(base_dir)
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+
+        # Config file to track knowledge bases
+        self.config_file = self.base_dir / "kb_config.json"
+        self._catalog_lock = RLock()
+        self._config_cache = None
+        self._info_cache: dict[tuple, tuple] = {}
+        self.config = self._load_config()
+
+        # PocketBase sync — enabled when integrations.pocketbase_url is set.
+        # The local JSON file stays the source of truth; PocketBase gets a
+        # mirrored copy for admin-panel visibility and future multi-user access.
+        from deeptutor.services.pocketbase_client import is_pocketbase_enabled
+
+        self._pb_enabled = is_pocketbase_enabled()
+
+    def _catalog_revision(self) -> tuple:
+        from deeptutor.multi_user.context import get_current_user_or_none
+        from deeptutor.services.rag.embedding_signature import signature_from_embedding_config
+        from deeptutor.services.workspace.context import current_workspace_id
+
+        try:
+            stat_result = self.config_file.stat()
+            revision = (stat_result.st_ino, stat_result.st_mtime_ns, stat_result.st_size)
+        except FileNotFoundError:
+            revision = None
+        signature = signature_from_embedding_config()
+        user = get_current_user_or_none()
+        return (
+            revision,
+            signature.hash() if signature is not None else None,
+            user.id if user is not None else None,
+            current_workspace_id(),
+        )
+
+    def _load_config(self) -> dict:
+        # The registry is atomic; one stat detects writes by other processes.
+        # Single-flight the expensive reconciliation on slow volumes (#1711).
+        with self._catalog_lock:
+            revision = self._catalog_revision()
+            cached = self._config_cache
+            if cached and cached[0] == revision and time.monotonic() < cached[1]:
+                return deepcopy(cached[2])
+            config = self._read_and_reconcile_config()
+            self._config_cache = (
+                self._catalog_revision(),
+                time.monotonic() + _CATALOG_CACHE_SECONDS,
+                deepcopy(config),
+            )
+            return config
+
+    def _read_and_reconcile_config(self) -> dict:
+        """Load knowledge base configuration from the canonical kb_config.json file."""
+        if self.config_file.exists():
+            try:
+                with open(self.config_file, encoding="utf-8") as f:
+                    with file_lock_shared(f):
+                        content = f.read()
+                        if not content.strip():
+                            # Empty file, return default
+                            return {"knowledge_bases": {}}
+                        config = json.loads(content)
+
+                # Ensure knowledge_bases key exists
+                if "knowledge_bases" not in config:
+                    config["knowledge_bases"] = {}
+
+                # Migration: remove old "default" field if present
+                if "default" in config:
+                    del config["default"]
+                    # Note: Don't save during load to avoid recursion issues
+                    # The next _save_config() call will persist this change
+
+                # Migration: normalize unknown/removed providers to the default
+                # and mark them for rebuild. Known non-default providers are
+                # first-class engines and must be preserved.
+                knowledge_bases = config.get("knowledge_bases", {})
+                config_changed = False
+                for kb_name, kb_entry in knowledge_bases.items():
+                    if not isinstance(kb_entry, dict):
+                        continue
+
+                    # Connected KBs (Obsidian vaults, linked indexes) are
+                    # pointers with no index pipeline — none of the
+                    # provider/embedding normalization below applies. Leave
+                    # their type/external pointer untouched.
+                    if is_connected_kb(kb_entry):
+                        continue
+
+                    raw_provider = kb_entry.get("rag_provider")
+                    provider = normalize_provider_name(raw_provider)
+                    if kb_entry.get("rag_provider") != provider:
+                        kb_entry["rag_provider"] = provider
+                        config_changed = True
+
+                    raw_provider_text = str(raw_provider or "").strip().lower()
+                    if raw_provider_text and raw_provider_text not in KNOWN_PROVIDERS:
+                        if not kb_entry.get("needs_reindex", False):
+                            kb_entry["needs_reindex"] = True
+                            config_changed = True
+
+                    kb_dir = self.base_dir / kb_name
+                    legacy_storage = kb_dir / "rag_storage"
+                    has_llamaindex_index = has_ready_provider_index(kb_dir, DEFAULT_PROVIDER)
+                    if (
+                        provider == DEFAULT_PROVIDER
+                        and legacy_storage.exists()
+                        and legacy_storage.is_dir()
+                        and not has_llamaindex_index
+                    ):
+                        if not kb_entry.get("needs_reindex", False):
+                            kb_entry["needs_reindex"] = True
+                            config_changed = True
+                        if kb_entry.get("status") == "ready":
+                            kb_entry["status"] = "needs_reindex"
+                            config_changed = True
+
+                if _reconcile_embedding_flags(knowledge_bases, self.base_dir):
+                    config_changed = True
+
+                if config_changed:
+                    try:
+                        atomic_write_json(self.config_file, config)
+                    except Exception as save_err:
+                        logger.warning(f"Failed to persist normalized KB config: {save_err}")
+
+                return config
+            except (json.JSONDecodeError, Exception) as e:
+                logger.warning(f"Error loading config: {e}")
+                return {"knowledge_bases": {}}
+        return {"knowledge_bases": {}}
+
+    def _save_config(self):
+        """Save knowledge base configuration.
+
+        Written via temp-file + ``os.replace`` so concurrent readers only
+        ever see the previous or the new file — ``open(..., "w")`` used to
+        truncate the config before the lock was even acquired.
+        """
+        with self._catalog_lock:
+            atomic_write_json(self.config_file, self.config)
+            self._config_cache = None
+            self._info_cache.clear()
+
+    def _sync_kb_to_pb(self, name: str, kb_entry: dict) -> None:
+        """
+        Mirror a KB metadata entry to PocketBase (best-effort, non-blocking).
+        Called after every local config save when PocketBase is enabled.
+        """
+        if not self._pb_enabled:
+            return
+        try:
+            from deeptutor.services.pocketbase_client import get_pb_client
+
+            pb = get_pb_client()
+            records = pb.collection("knowledge_bases").get_full_list(
+                query_params={"filter": f'kb_name="{name}"'}
+            )
+            payload = {
+                "kb_name": name,
+                "description": kb_entry.get("description", f"Knowledge base: {name}"),
+                "rag_provider": kb_entry.get("rag_provider", "llamaindex"),
+                "needs_reindex": bool(kb_entry.get("needs_reindex", False)),
+                "status": kb_entry.get("status", "unknown"),
+                "kb_created_at": kb_entry.get("created_at", ""),
+            }
+            if records:
+                pb.collection("knowledge_bases").update(records[0].id, payload)
+            else:
+                pb.collection("knowledge_bases").create(payload)
+        except Exception as exc:
+            logger.debug(f"PocketBase KB sync failed for '{name}': {exc}")
+
+    def update_kb_status(
+        self,
+        name: str,
+        status: str,
+        progress: dict | None = None,
+    ):
+        """
+        Update knowledge base status and progress in kb_config.json.
+
+        When PocketBase is enabled, the updated entry is also mirrored to the
+        PocketBase knowledge_bases collection (best-effort).
+
+        Args:
+            name: Knowledge base name
+            status: Status string ("initializing", "processing", "ready", "error")
+            progress: Optional progress dict with keys like:
+                - stage: Current stage name
+                - message: Human-readable message
+                - percent: Progress percentage (0-100)
+                - current: Current item number
+                - total: Total items
+                - file_name: Current file being processed
+                - error: Error message (if status is "error")
+        """
+        # Reload config to get latest state
+        self.config = self._load_config()
+
+        if "knowledge_bases" not in self.config:
+            self.config["knowledge_bases"] = {}
+
+        if name not in self.config["knowledge_bases"]:
+            # Auto-register if not exists
+            self.config["knowledge_bases"][name] = {
+                "path": name,
+                "description": f"Knowledge base: {name}",
+            }
+
+        kb_config = self.config["knowledge_bases"][name]
+        kb_config["status"] = status
+        kb_config["updated_at"] = datetime.now().isoformat()
+        index_changed = False
+        indexed_count: int | None = None
+        index_action: str | None = None
+        if isinstance(progress, dict):
+            raw_indexed_count = progress.get("indexed_count")
+            if isinstance(raw_indexed_count, bool):
+                indexed_count = int(raw_indexed_count)
+            elif isinstance(raw_indexed_count, (int, float)):
+                indexed_count = int(raw_indexed_count)
+            elif isinstance(raw_indexed_count, str):
+                try:
+                    indexed_count = int(raw_indexed_count)
+                except ValueError:
+                    indexed_count = None
+
+            index_changed = bool(progress.get("index_changed")) or (
+                indexed_count is not None and indexed_count > 0
+            )
+            raw_index_action = progress.get("index_action")
+            if isinstance(raw_index_action, str) and raw_index_action.strip():
+                index_action = raw_index_action.strip()
+
+        if status == "ready":
+            # Ready KBs should look like stable resources in the UI instead of
+            # permanently carrying a "completed" progress banner.
+            kb_config.pop("progress", None)
+            kb_config.pop("last_error", None)
+            kb_config.pop("last_error_at", None)
+            if progress is not None:
+                kb_config["last_completed_at"] = (
+                    progress.get("timestamp") or datetime.now().isoformat()
+                )
+                if index_changed:
+                    kb_config["last_indexed_at"] = kb_config["last_completed_at"]
+                    if indexed_count is not None:
+                        kb_config["last_indexed_count"] = max(indexed_count, 0)
+                    if index_action:
+                        kb_config["last_indexed_action"] = index_action
+        elif status == "error":
+            if progress is not None:
+                kb_config["progress"] = progress
+                kb_config["last_error"] = progress.get("error") or progress.get("message")
+                kb_config["last_error_at"] = progress.get("timestamp") or datetime.now().isoformat()
+        elif progress is not None:
+            kb_config["progress"] = progress
+
+        if status == "ready":
+            provider = normalize_provider_name(kb_config.get("rag_provider"))
+            pageindex_provider = provider in {PAGEINDEX_PROVIDER, PAGEINDEX_OSS_PROVIDER}
+            if pageindex_provider:
+                for key in (
+                    "embedding_model",
+                    "embedding_dim",
+                    "embedding_signature",
+                    "embedding_mismatch",
+                ):
+                    kb_config.pop(key, None)
+            elif not kb_config.get("embedding_selection"):
+                fp = _get_embedding_fingerprint()
+                if fp:
+                    kb_config["embedding_model"], kb_config["embedding_dim"] = fp
+            # Record the active signature + the on-disk version registry so
+            # the UI can render version chips without recomputing.
+            try:
+                from deeptutor.services.rag.embedding_binding import entry_signature
+
+                sig = None if pageindex_provider else entry_signature(kb_config)
+                if sig is not None and not kb_config.get("embedding_selection"):
+                    kb_config["embedding_signature"] = sig.hash()
+                kb_dir = self.base_dir / name
+                if kb_dir.is_dir():
+                    kb_config["index_versions"] = inspect_kb_versions(kb_dir, provider)
+            except Exception as exc:  # best-effort metadata
+                logger.warning(
+                    f"Failed to refresh embedding/index metadata for KB '{name}' "
+                    f"on ready transition: {exc}"
+                )
+
+        try:
+            self._save_config()
+        except Exception as exc:
+            logger.error(
+                f"Failed to persist KB status update for '{name}' (status='{status}'): {exc}"
+            )
+            raise
+        self._sync_kb_to_pb(name, kb_config)
+
+    def get_kb_entry(self, name: str) -> dict | None:
+        """The KB's raw ``kb_config.json`` record, or ``None`` if unregistered.
+
+        A cheap read for callers that need the registered facts (provider,
+        status, connected-KB pointers) without paying for :meth:`get_info`,
+        which additionally probes every index version on disk — and parses a
+        provider's docstore to do it.
+        """
+        self.config = self._load_config()
+        entry = self.config.get("knowledge_bases", {}).get(name)
+        return dict(entry) if isinstance(entry, dict) else None
+
+    def get_kb_status(self, name: str) -> dict | None:
+        """Get status and progress for a knowledge base."""
+        self.config = self._load_config()
+        kb_config = self.config.get("knowledge_bases", {}).get(name)
+        if not kb_config:
+            return None
+        return {
+            "status": kb_config.get("status", "unknown"),
+            "progress": kb_config.get("progress"),
+            "updated_at": kb_config.get("updated_at"),
+        }
+
+    def list_knowledge_bases(self) -> list[str]:
+        with self._catalog_lock:
+            return self._list_knowledge_bases()
+
+    def _list_knowledge_bases(self) -> list[str]:
+        """List all available knowledge bases.
+
+        This method:
+        1. Loads registered KBs from kb_config.json
+        2. Drops registered entries whose on-disk directory no longer exists
+           (orphans from failed inits or manual ``rm -rf`` of a KB folder).
+        3. Scans the directory for existing KBs not yet registered
+        4. Auto-registers discovered KBs with valid raw/index structure
+        """
+        # Always reload config from file to ensure we have the latest data
+        self.config = self._load_config()
+
+        config_kbs = self.config.get("knowledge_bases", {})
+        kb_list: set[str] = set()
+        config_changed = False
+
+        # Filter out orphan entries whose KB directory is gone. The on-disk
+        # folder is the source of truth for existence — without it the KB
+        # has no documents, no index, and surfacing it in the UI just shows
+        # zombies that the user can't act on.
+        #
+        # Grace period: a freshly-created KB writes its config entry before
+        # ``create_directory_structure`` mkdir-s the folder (so the UI can
+        # render the "initializing" row immediately). If ``list`` races into
+        # that window we'd prune a perfectly healthy in-flight KB. Skip the
+        # prune when ``updated_at`` is recent enough that an init could still
+        # be wiring things up.
+        base_exists = self.base_dir.exists()
+        grace_cutoff = datetime.now() - timedelta(seconds=_ORPHAN_PRUNE_GRACE_SECONDS)
+        for kb_name, kb_entry in list(config_kbs.items()):
+            # Connected KBs (Obsidian vaults, linked indexes) live outside
+            # ``base_dir`` — they have no on-disk KB folder by design, so the
+            # orphan prune below would wrongly delete them. Keep them
+            # unconditionally.
+            if is_connected_kb(kb_entry):
+                kb_list.add(kb_name)
+                continue
+            rel_path = (kb_entry or {}).get("path", kb_name)
+            kb_dir = self.base_dir / rel_path
+            if base_exists and not kb_dir.exists():
+                if _entry_updated_after(kb_entry, grace_cutoff):
+                    kb_list.add(kb_name)
+                    continue
+                logger.warning(
+                    "Pruning orphaned KB entry '%s': directory %s no longer exists.",
+                    kb_name,
+                    kb_dir,
+                )
+                del config_kbs[kb_name]
+                config_changed = True
+                continue
+            kb_list.add(kb_name)
+
+        # Also scan directory for KBs that may not be registered yet
+        # This ensures backward compatibility and auto-discovery
+        if base_exists:
+            for item in self.base_dir.iterdir():
+                if not item.is_dir() or item.name.startswith(("__", ".")):
+                    continue
+
+                # Skip if already in config
+                if item.name in kb_list:
+                    continue
+
+                # Check if this is a valid KB directory (flat versions or legacy stores)
+                rag_storage = item / "rag_storage"
+                from deeptutor.services.rag.index_versioning import list_kb_versions
+
+                versions = list_kb_versions(item)
+                detected_provider = _detect_provider_from_versions(versions)
+                is_valid_kb = has_ready_provider_index(item, detected_provider) or (
+                    rag_storage.exists() and rag_storage.is_dir()
+                )
+
+                if is_valid_kb:
+                    # Auto-register this KB to kb_config.json
+                    kb_list.add(item.name)
+                    self._auto_register_kb(item.name)
+                    config_changed = True
+
+        # Save config if we pruned orphans or registered new KBs
+        if config_changed:
+            self._save_config()
+
+        return sorted(kb_list)
+
+    def _auto_register_kb(self, name: str):
+        """Auto-register an existing KB to kb_config.json.
+
+        Reads info from metadata.json (if exists) for backward compatibility.
+        """
+        kb_dir = self.base_dir / name
+        from deeptutor.services.rag.index_versioning import list_kb_versions
+
+        rag_storage = kb_dir / "rag_storage"
+        versions = list_kb_versions(kb_dir)
+        detected_provider = _detect_provider_from_versions(versions)
+
+        # Default values
+        kb_entry: dict[str, Any] = {
+            "path": name,
+            "description": f"Knowledge base: {name}",
+            "updated_at": datetime.now().isoformat(),
+        }
+
+        # Try to read metadata.json for existing info (backward compatibility)
+        metadata_file = kb_dir / "metadata.json"
+        if metadata_file.exists():
+            try:
+                with open(metadata_file, encoding="utf-8") as f:
+                    metadata = json.load(f)
+                # Migrate relevant fields
+                if metadata.get("description"):
+                    kb_entry["description"] = metadata["description"]
+                if metadata.get("rag_provider"):
+                    raw_provider = str(metadata["rag_provider"]).strip().lower()
+                    kb_entry["rag_provider"] = normalize_provider_name(raw_provider)
+                    if raw_provider and raw_provider not in KNOWN_PROVIDERS:
+                        kb_entry["needs_reindex"] = True
+                if metadata.get("created_at"):
+                    kb_entry["created_at"] = metadata["created_at"]
+                if metadata.get("last_updated"):
+                    kb_entry["updated_at"] = metadata["last_updated"]
+                if metadata.get("last_indexed_at"):
+                    kb_entry["last_indexed_at"] = metadata["last_indexed_at"]
+                elif metadata.get("last_updated"):
+                    kb_entry["last_indexed_at"] = metadata["last_updated"]
+                if metadata.get("last_indexed_count") is not None:
+                    kb_entry["last_indexed_count"] = metadata["last_indexed_count"]
+                if metadata.get("last_indexed_action"):
+                    kb_entry["last_indexed_action"] = metadata["last_indexed_action"]
+            except Exception as e:
+                logger.warning(f"Failed to read metadata.json for '{name}': {e}")
+
+        # Detect rag_provider from storage type if not set
+        if "rag_provider" not in kb_entry:
+            if has_ready_provider_index(kb_dir, detected_provider):
+                kb_entry["rag_provider"] = detected_provider
+            elif rag_storage.exists():
+                kb_entry["rag_provider"] = DEFAULT_PROVIDER
+                kb_entry["needs_reindex"] = True
+
+        provider = normalize_provider_name(kb_entry.get("rag_provider"))
+        if has_ready_provider_index(kb_dir, provider):
+            kb_entry["status"] = "ready"
+        elif rag_storage.exists() and rag_storage.is_dir():
+            kb_entry["status"] = "needs_reindex"
+            kb_entry["needs_reindex"] = True
+        else:
+            kb_entry["status"] = "unknown"
+
+        # Add to config
+        if "knowledge_bases" not in self.config:
+            self.config["knowledge_bases"] = {}
+        self.config["knowledge_bases"][name] = kb_entry
+
+        logger.info(f"Auto-registered KB '{name}' to kb_config.json")
+
+    def register_knowledge_base(self, name: str, description: str = "", set_default: bool = False):
+        """Register a knowledge base"""
+        name = validate_knowledge_base_name(name)
+        _assert_move_id_available(self.base_dir, name)
+        kb_dir = self.base_dir / name
+        if not kb_dir.exists():
+            raise ValueError(f"Knowledge base directory does not exist: {kb_dir}")
+
+        if "knowledge_bases" not in self.config:
+            self.config["knowledge_bases"] = {}
+
+        self.config["knowledge_bases"][name] = {"path": name, "description": description}
+
+        # Only set default if explicitly requested
+        if set_default:
+            self.set_default(name)
+
+        self._save_config()
+
+    def register_connected_entry(self, name: str, entry: dict) -> bool:
+        """Adopt an already-built connected-KB entry under this manager.
+
+        Every other ``register_*`` method builds a pointer entry from user
+        input. This one takes an entry that already exists elsewhere: a
+        connected KB has no ``<kb>/`` tree under ``base_dir``, so handing one
+        to a partner workspace means copying its ``kb_config.json`` row rather
+        than the folder that provisioning copies for an indexed KB.
+
+        Returns ``False`` when the name is already registered here, leaving
+        the existing entry untouched, so callers can provision idempotently.
+        """
+        name = (name or "").strip()
+        if not name:
+            raise ValueError("Knowledge base name is required.")
+        if not is_connected_kb(entry):
+            raise ValueError(f"Not a connected knowledge base entry: {name}")
+        _assert_move_id_available(self.base_dir, name)
+
+        self.config = self._load_config()
+        knowledge_bases = self.config.setdefault("knowledge_bases", {})
+        if name in knowledge_bases:
+            return False
+        knowledge_bases[name] = dict(entry)
+        self._save_config()
+        return True
+
+    def register_obsidian_vault(self, name: str, vault_path: str, description: str = "") -> dict:
+        """Register a connected Obsidian vault as a pointer-type KB.
+
+        Unlike a normal KB this creates no folder under ``base_dir`` and runs no
+        index pipeline: it records a ``type: obsidian`` entry pointing at the
+        user's existing vault directory, which the Obsidian capability reads
+        live. Raises ``ValueError`` on a missing/invalid path or a name clash.
+        """
+        name = validate_knowledge_base_name(name)
+        _assert_move_id_available(self.base_dir, name)
+        vault = Path(vault_path).expanduser()
+        if not vault.is_dir():
+            raise ValueError(f"Vault path is not a directory: {vault_path}")
+
+        self.config = self._load_config()
+        knowledge_bases = self.config.setdefault("knowledge_bases", {})
+        if name in knowledge_bases:
+            raise ValueError(f"A knowledge base named '{name}' already exists.")
+
+        now = datetime.now().isoformat()
+        entry = {
+            "path": name,
+            "type": OBSIDIAN_KB_TYPE,
+            "vault_path": str(vault.resolve()),
+            "description": description or f"Obsidian vault: {name}",
+            "status": "ready",
+            "created_at": now,
+            "updated_at": now,
+        }
+        knowledge_bases[name] = entry
+        self._save_config()
+        return entry
+
+    def register_linked_kb(
+        self,
+        name: str,
+        external_path: str,
+        provider: str,
+        *,
+        description: str = "",
+        stats: dict | None = None,
+    ) -> dict:
+        """Register a pointer to a pre-built engine index as a ``linked`` KB.
+
+        Like :meth:`register_obsidian_vault` this creates no folder under
+        ``base_dir`` and runs no index pipeline: it records an
+        ``external_path`` the bound ``provider`` reads in place, so retrieval
+        skips indexing entirely. ``stats`` (embedding model/dim/signature, doc
+        count) is surfaced read-only in the UI. Callers should validate the
+        folder with the probe helper first; this only guards basic invariants.
+        Raises ``ValueError`` on a missing/invalid path or a name clash.
+        """
+        name = validate_knowledge_base_name(name)
+        _assert_move_id_available(self.base_dir, name)
+        provider = normalize_provider_name(provider)
+        folder = Path(external_path).expanduser()
+        if not folder.is_dir():
+            raise ValueError(f"Folder path is not a directory: {external_path}")
+
+        self.config = self._load_config()
+        knowledge_bases = self.config.setdefault("knowledge_bases", {})
+        if name in knowledge_bases:
+            raise ValueError(f"A knowledge base named '{name}' already exists.")
+
+        now = datetime.now().isoformat()
+        entry: dict[str, Any] = {
+            "path": name,
+            "type": LINKED_KB_TYPE,
+            "external_path": str(folder.resolve()),
+            "rag_provider": provider,
+            "description": description or f"Linked {provider} index: {name}",
+            "status": "ready",
+            "needs_reindex": False,
+            "created_at": now,
+            "updated_at": now,
+        }
+        for key in ("embedding_model", "embedding_dim", "embedding_signature"):
+            if stats and stats.get(key) is not None:
+                entry[key] = stats[key]
+        if stats and stats.get("doc_count") is not None:
+            entry["last_indexed_count"] = stats["doc_count"]
+            entry["last_indexed_action"] = "link"
+        knowledge_bases[name] = entry
+        self._save_config()
+        return entry
+
+    def register_subagent_connection(
+        self,
+        name: str,
+        agent_kind: str,
+        *,
+        cwd: str = "",
+        description: str = "",
+    ) -> dict:
+        """Register a local or remote agent connection without creating an index."""
+        name = validate_knowledge_base_name(name)
+        _assert_move_id_available(self.base_dir, name)
+        agent_kind = (agent_kind or "").strip()
+        if agent_kind == "partner":
+            raise ValueError("Select partners directly through Ask partner instead.")
+        if not agent_kind:
+            raise ValueError("agent_kind is required.")
+        resolved_cwd = ""
+        if cwd:
+            folder = Path(cwd).expanduser()
+            if not folder.is_dir():
+                raise ValueError(f"Working directory is not a directory: {cwd}")
+            resolved_cwd = str(folder.resolve())
+
+        self.config = self._load_config()
+        knowledge_bases = self.config.setdefault("knowledge_bases", {})
+        if name in knowledge_bases:
+            raise ValueError(f"A knowledge base named '{name}' already exists.")
+
+        now = datetime.now().isoformat()
+        entry = {
+            "path": name,
+            "type": SUBAGENT_KB_TYPE,
+            "agent_kind": agent_kind,
+            "cwd": resolved_cwd,
+            "description": description or f"Connected subagent: {name}",
+            "status": "ready",
+            "created_at": now,
+            "updated_at": now,
+        }
+        knowledge_bases[name] = entry
+        self._save_config()
+        return entry
+
+    def register_lightrag_server_kb(
+        self,
+        name: str,
+        server_url: str,
+        *,
+        api_key: str = "",
+        search_mode: str = "",
+        description: str = "",
+    ) -> dict:
+        """Register a pointer to an external LightRAG server as a connected KB.
+
+        Like the other connected types this creates no folder under ``base_dir``
+        and runs no index pipeline: it records a ``type: lightrag_server`` entry
+        whose ``server_url`` (+ optional ``api_key``) the ``lightrag-server``
+        provider queries over HTTP. The server owns indexing entirely. Callers
+        should validate reachability with the probe helper first; this only
+        guards basic invariants. Raises ``ValueError`` on a missing name/URL or a
+        name clash.
+        """
+        name = validate_knowledge_base_name(name)
+        _assert_move_id_available(self.base_dir, name)
+        server_url = (server_url or "").strip().rstrip("/")
+        if not server_url:
+            raise ValueError("LightRAG server URL is required.")
+
+        self.config = self._load_config()
+        knowledge_bases = self.config.setdefault("knowledge_bases", {})
+        if name in knowledge_bases:
+            raise ValueError(f"A knowledge base named '{name}' already exists.")
+
+        now = datetime.now().isoformat()
+        entry: dict[str, Any] = {
+            "path": name,
+            "type": LIGHTRAG_SERVER_KB_TYPE,
+            "rag_provider": LIGHTRAG_SERVER_PROVIDER,
+            "server_url": server_url,
+            "api_key": (api_key or "").strip(),
+            "description": description or f"LightRAG server: {name}",
+            "status": "ready",
+            "needs_reindex": False,
+            "created_at": now,
+            "updated_at": now,
+        }
+        search_mode = (search_mode or "").strip().lower()
+        if search_mode:
+            entry["search_mode"] = search_mode
+        knowledge_bases[name] = entry
+        self._save_config()
+        return entry
+
+    def register_marginnote4_kb(
+        self,
+        name: str,
+        *,
+        db_path: str = "",
+        description: str = "",
+    ) -> dict:
+        """Register a connected MarginNote 4 library as a pointer KB.
+
+        Creates no folder under ``base_dir`` and runs no index pipeline: it
+        records a ``type: marginnote4`` entry whose ``db_path`` (when given)
+        the MarginNote capability binds to. When ``db_path`` is omitted the
+        capability derives a default SQLite path from the KB name, so callers
+        can leave it blank for the simple single-library case. Raises
+        ``ValueError`` on a missing name, a name clash, or a store already
+        claimed by another library.
+        """
+        name = validate_knowledge_base_name(name)
+        _assert_move_id_available(self.base_dir, name)
+
+        self.config = self._load_config()
+        knowledge_bases = self.config.setdefault("knowledge_bases", {})
+        if name in knowledge_bases:
+            raise ValueError(f"A knowledge base named '{name}' already exists.")
+
+        db_path = (db_path or "").strip()
+        claimed_by = self._marginnote4_store_owner(name, db_path, knowledge_bases)
+        if claimed_by:
+            # Distinct names can still derive one store: the default path keeps
+            # only alphanumerics, `-` and `_`, so "My Lib" and "My/Lib" both
+            # land on My_Lib.db. Sharing it would merge two libraries' objects
+            # and let either one's devices sync into the other.
+            raise ValueError(
+                f"Knowledge base '{claimed_by}' already uses that MarginNote store. "
+                "Pick a name that differs by more than punctuation."
+            )
+
+        now = datetime.now().isoformat()
+        entry: dict[str, Any] = {
+            "path": name,
+            "type": MARGINNOTE4_KB_TYPE,
+            "description": description or f"MarginNote 4 library: {name}",
+            "status": "ready",
+            "needs_reindex": False,
+            "created_at": now,
+            "updated_at": now,
+        }
+        if db_path:
+            entry["db_path"] = db_path
+        knowledge_bases[name] = entry
+        self._save_config()
+        return entry
+
+    @staticmethod
+    def _marginnote4_store_owner(
+        name: str,
+        db_path: str,
+        knowledge_bases: dict[str, Any],
+    ) -> str | None:
+        """Name of the MarginNote library already using this store, if any."""
+        from deeptutor.capabilities.marginnote4.store import resolve_db_path
+
+        def _store(kb_name: str, entry: dict) -> Path:
+            return resolve_db_path(kb_name, metadata=entry).expanduser().resolve()
+
+        wanted = _store(name, {"db_path": db_path})
+        for other_name, other in knowledge_bases.items():
+            if not isinstance(other, dict) or other.get("type") != MARGINNOTE4_KB_TYPE:
+                continue
+            if _store(other_name, other) == wanted:
+                return other_name
+        return None
+
+    def register_ima_kb(
+        self,
+        name: str,
+        client_id: str,
+        api_key: str,
+        knowledge_base_id: str,
+        *,
+        description: str = "",
+    ) -> dict:
+        """Register a pointer to a Tencent IMA knowledge base as a connected KB.
+
+        Like the other connected types this creates no folder under ``base_dir``
+        and runs no index pipeline: it records a ``type: ima`` entry whose
+        library id the ``ima`` provider queries over IMA's OpenAPI. IMA owns
+        indexing entirely.
+
+        Credentials are optional here: leave them empty and the KB retrieves
+        with the account-level pair from the engine settings, so rotating that
+        key updates every such KB at once. Passing a pair pins this KB to it —
+        the way to reach a second IMA account. Callers should validate the
+        binding with the probe helper first; this only guards basic invariants.
+        Raises ``ValueError`` on a missing field, a half-filled credential pair,
+        or a name clash.
+        """
+        name = validate_knowledge_base_name(name)
+        _assert_move_id_available(self.base_dir, name)
+        client_id = (client_id or "").strip()
+        api_key = (api_key or "").strip()
+        knowledge_base_id = (knowledge_base_id or "").strip()
+        if bool(client_id) != bool(api_key):
+            raise ValueError("IMA Client ID and API Key must be given together.")
+        if not knowledge_base_id:
+            raise ValueError("IMA knowledge base ID is required.")
+
+        self.config = self._load_config()
+        knowledge_bases = self.config.setdefault("knowledge_bases", {})
+        if name in knowledge_bases:
+            raise ValueError(f"A knowledge base named '{name}' already exists.")
+
+        now = datetime.now().isoformat()
+        entry: dict[str, Any] = {
+            "path": name,
+            "type": IMA_KB_TYPE,
+            "rag_provider": IMA_PROVIDER,
+            # Written only when this KB overrides the account credentials;
+            # absent means "resolve them from the engine settings".
+            **({"client_id": client_id, "api_key": api_key} if client_id else {}),
+            "knowledge_base_id": knowledge_base_id,
+            "description": description or f"Tencent IMA: {name}",
+            "status": "ready",
+            "needs_reindex": False,
+            "created_at": now,
+            "updated_at": now,
+        }
+        knowledge_bases[name] = entry
+        self._save_config()
+        return entry
+
+    def register_weknora_kb(
+        self,
+        name: str,
+        server_url: str,
+        api_key: str,
+        knowledge_base_id: str,
+        *,
+        description: str = "",
+    ) -> dict:
+        """Register a self-hosted WeKnora knowledge base as a pointer KB."""
+        name = validate_knowledge_base_name(name)
+        _assert_move_id_available(self.base_dir, name)
+        server_url = (server_url or "").strip().rstrip("/")
+        api_key = (api_key or "").strip()
+        knowledge_base_id = (knowledge_base_id or "").strip()
+        if not server_url or not knowledge_base_id:
+            raise ValueError("WeKnora server URL and knowledge base ID are required.")
+        if not api_key:
+            raise ValueError("A WeKnora API key is required.")
+
+        self.config = self._load_config()
+        knowledge_bases = self.config.setdefault("knowledge_bases", {})
+        if name in knowledge_bases:
+            raise ValueError(f"A knowledge base named '{name}' already exists.")
+
+        now = datetime.now().isoformat()
+        entry: dict[str, Any] = {
+            "path": name,
+            "type": WEKNORA_KB_TYPE,
+            "rag_provider": WEKNORA_PROVIDER,
+            "server_url": server_url,
+            "api_key": api_key,
+            "knowledge_base_id": knowledge_base_id,
+            "description": description or f"WeKnora knowledge base: {name}",
+            "status": "ready",
+            "needs_reindex": False,
+            "created_at": now,
+            "updated_at": now,
+        }
+        knowledge_bases[name] = entry
+        self._save_config()
+        return entry
+
+    def register_kiwix_kb(
+        self,
+        name: str,
+        server_url: str,
+        zim_name: str,
+        *,
+        zim_title: str = "",
+    ) -> dict:
+        """Register a read-only pointer to one ZIM already served by Kiwix."""
+        from deeptutor.knowledge.kb_types import KIWIX_KB_TYPE
+        from deeptutor.services.rag.pipelines.kiwix.client import (
+            normalize_base_url,
+            validate_zim_name,
+        )
+
+        name = validate_knowledge_base_name(name)
+        server_url = normalize_base_url(server_url)
+        zim_name = validate_zim_name(zim_name)
+        self.config = self._load_config()
+        bases = self.config.setdefault("knowledge_bases", {})
+        if name in bases:
+            raise ValueError(f"A knowledge base named '{name}' already exists.")
+        now = datetime.now().isoformat()
+        entry = {
+            "path": name,
+            "type": KIWIX_KB_TYPE,
+            "rag_provider": KIWIX_KB_TYPE,
+            "server_url": server_url,
+            "zim_name": zim_name,
+            "zim_title": zim_title[:300],
+            "description": f"Kiwix archive: {zim_title or zim_name}",
+            "status": "ready",
+            "needs_reindex": False,
+            "created_at": now,
+            "updated_at": now,
+        }
+        bases[name] = entry
+        self._save_config()
+        return entry
+
+    def get_knowledge_base_path(self, name: str | None = None) -> Path:
+        """Get path to a knowledge base.
+
+        Connected KBs (Obsidian vaults, linked indexes) live outside
+        ``base_dir`` — resolve them to their external pointer so callers that
+        ask for "where is this KB's data" reach the right place.
+        """
+        self.config = self._load_config()
+        if name is None:
+            name = self.config.get("default")
+            if name is None:
+                raise ValueError("No default knowledge base set")
+
+        entry = self.config.get("knowledge_bases", {}).get(name, {})
+        external = external_root_of(entry)
+        if external:
+            folder = Path(external).expanduser()
+            if not folder.is_dir():
+                raise ValueError(f"Linked folder is no longer available: {external}")
+            return folder
+
+        kb_dir = self.base_dir / name
+        if not kb_dir.exists():
+            raise ValueError(f"Knowledge base not found: {name}")
+
+        return kb_dir
+
+    def get_rag_storage_path(self, name: str | None = None) -> Path:
+        """Get active index storage path for a knowledge base."""
+        kb_dir = self.get_knowledge_base_path(name)
+        from deeptutor.services.rag.embedding_binding import binding_status, entry_signature
+        from deeptutor.services.rag.index_versioning import (
+            resolve_storage_dir_for_read,
+        )
+
+        entry = self.config.get("knowledge_bases", {}).get(name or kb_dir.name, {})
+        if entry.get("embedding_selection") and binding_status(entry)[0] != "ready":
+            raise ValueError(
+                "The bound embedding model is unavailable or changed. Check this knowledge base's model settings."
+            )
+        active_storage = resolve_storage_dir_for_read(kb_dir, entry_signature(entry))
+        legacy_storage = kb_dir / "rag_storage"
+        if active_storage is not None:
+            return active_storage
+        if legacy_storage.exists():
+            return legacy_storage
+        raise ValueError(f"Index storage not found for knowledge base: {name or 'default'}")
+
+    def get_images_path(self, name: str | None = None) -> Path:
+        """Get images path for a knowledge base"""
+        kb_dir = self.get_knowledge_base_path(name)
+        return kb_dir / "images"
+
+    def get_content_list_path(self, name: str | None = None) -> Path:
+        """Get content list path for a knowledge base"""
+        kb_dir = self.get_knowledge_base_path(name)
+        return kb_dir / "content_list"
+
+    def get_raw_path(self, name: str | None = None) -> Path:
+        """Get raw documents path for a knowledge base"""
+        kb_dir = self.get_knowledge_base_path(name)
+        return kb_dir / "raw"
+
+    def set_default(self, name: str):
+        """Set default knowledge base using centralized config service."""
+        if name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {name}")
+
+        # Persist default KB selection via the canonical KB config service.
+        try:
+            from deeptutor.services.config import get_kb_config_service
+
+            kb_config_service = get_kb_config_service()
+            kb_config_service.set_default_kb(name)
+        except Exception as e:
+            logger.warning(f"Failed to save default to centralized config: {e}")
+
+    def get_default(self, *, available_names: list[str] | None = None) -> str | None:
+        """
+        Get default knowledge base name.
+
+        Priority:
+        1. Canonical KB config service (`data/knowledge_bases/kb_config.json`)
+        2. First knowledge base in the list (auto-fallback)
+
+        Args:
+            available_names: An already-reconciled knowledge-base list. Passing
+                this avoids rescanning every index when the caller just listed
+                the available knowledge bases.
+        """
+        kb_list = available_names if available_names is not None else self.list_knowledge_bases()
+
+        # Try centralized config first
+        try:
+            from deeptutor.services.config import get_kb_config_service
+
+            kb_config_service = get_kb_config_service()
+            default_kb = kb_config_service.get_default_kb()
+            if default_kb and default_kb in kb_list:
+                return default_kb
+        except Exception:
+            pass
+
+        # Fallback to first knowledge base in sorted list
+        if kb_list:
+            return kb_list[0]
+
+        return None
+
+    @staticmethod
+    def _embedding_fields(kb_config: dict) -> dict:
+        """Extract embedding fingerprint fields from a KB config entry."""
+        fields = {}
+        for key in ("embedding_model", "embedding_dim", "embedding_selection", "embedding_status"):
+            val = kb_config.get(key)
+            if val is not None:
+                fields[key] = val
+        if kb_config.get("embedding_mismatch"):
+            fields["embedding_mismatch"] = True
+        return fields
+
+    def get_metadata(self, name: str | None = None) -> dict:
+        """Get knowledge base metadata.
+
+        Source:
+        1. kb_config.json (authoritative source)
+        """
+        kb_name = name
+        if kb_name is None:
+            kb_name = self.get_default()
+            if kb_name is None:
+                return {}
+
+        # First, try kb_config.json (authoritative source)
+        self.config = self._load_config()
+        kb_config = self.config.get("knowledge_bases", {}).get(kb_name, {})
+
+        if kb_config:
+            # Build metadata from config
+            metadata = {
+                "name": kb_name,
+                "description": kb_config.get("description", f"Knowledge base: {kb_name}"),
+                "rag_provider": normalize_provider_name(kb_config.get("rag_provider")),
+                "needs_reindex": bool(kb_config.get("needs_reindex", False)),
+                "created_at": kb_config.get("created_at"),
+                "last_updated": kb_config.get("updated_at"),
+                "last_indexed_at": kb_config.get("last_indexed_at"),
+                "last_indexed_count": kb_config.get("last_indexed_count"),
+                "last_indexed_action": kb_config.get("last_indexed_action"),
+                # Connected-KB fields (None for ordinary indexed KBs, dropped below).
+                "type": kb_config.get("type"),
+                "vault_path": kb_config.get("vault_path"),
+                "external_path": kb_config.get("external_path"),
+                # MarginNote 4 pointer (SQLite store path for synced data).
+                "db_path": kb_config.get("db_path"),
+                # LightRAG server pointer (the URL is safe to surface; the API
+                # key deliberately is not).
+                "server_url": kb_config.get("server_url"),
+                "zim_name": kb_config.get("zim_name"),
+                "zim_title": kb_config.get("zim_title"),
+                # IMA pointer. The library id identifies which IMA knowledge
+                # base this KB reads; the client id and API key are credentials
+                # and are deliberately absent from this allowlist.
+                "knowledge_base_id": kb_config.get("knowledge_base_id"),
+                # Subagent connection fields (None for non-subagent KBs).
+                "agent_kind": kb_config.get("agent_kind"),
+                "cwd": kb_config.get("cwd"),
+                "partner_id": kb_config.get("partner_id"),
+            }
+            metadata.update(self._embedding_fields(kb_config))
+            # Remove None values
+            metadata = {k: v for k, v in metadata.items() if v is not None}
+            return metadata
+
+        return {}
+
+    def get_info(
+        self,
+        name: str | None = None,
+        *,
+        refresh_config: bool = True,
+        default_name: str | None = None,
+    ) -> dict:
+        if refresh_config:
+            return self._get_info(name, default_name=default_name)
+        with self._catalog_lock:
+            key = (name, default_name)
+            revision = (
+                self._catalog_revision(),
+                json.dumps(
+                    self.config.get("knowledge_bases", {}).get(name or default_name, {}),
+                    sort_keys=True,
+                    default=str,
+                ),
+            )
+            cached = self._info_cache.get(key)
+            if cached and cached[0] == revision and time.monotonic() < cached[1]:
+                return deepcopy(cached[2])
+            info = self._get_info(name, refresh_config=False, default_name=default_name)
+            # Keep only the current catalog's entries, including connected KBs.
+            if len(self._info_cache) >= max(1, len(self.config.get("knowledge_bases", {})) * 2):
+                self._info_cache.clear()
+            self._info_cache[key] = (
+                revision,
+                time.monotonic() + _CATALOG_CACHE_SECONDS,
+                deepcopy(info),
+            )
+            return info
+
+    def _get_info(
+        self,
+        name: str | None = None,
+        *,
+        refresh_config: bool = True,
+        default_name: str | None = None,
+    ) -> dict:
+        """Get detailed information about a knowledge base.
+
+        This method:
+        1. Gets the KB name (from parameter or default)
+        2. Reads all config from kb_config.json (authoritative source)
+        3. Falls back to metadata.json for legacy KBs
+        4. Collects statistics about files and RAG status
+
+        Args:
+            name: Knowledge-base name, or the configured default when omitted.
+            refresh_config: Reload and reconcile the complete configuration.
+                Bulk callers that just invoked :meth:`list_knowledge_bases`
+                can reuse that snapshot by passing ``False``.
+            default_name: A pre-resolved default name for bulk callers. This
+                avoids rescanning the knowledge-base list for every item.
+        """
+        # Reload config to get latest status
+        if refresh_config:
+            self.config = self._load_config()
+
+        resolved_default = default_name
+        if resolved_default is None:
+            resolved_default = self.get_default()
+
+        kb_name = name or resolved_default
+        if kb_name is None:
+            raise ValueError("No knowledge base name provided and no default set")
+
+        # Get config from kb_config.json (authoritative source)
+        kb_config = self.config.get("knowledge_bases", {}).get(kb_name, {})
+
+        # Connected KBs live outside ``base_dir``; resolve to their external
+        # pointer so the on-disk stats/index-version scan below reflect reality.
+        external = external_root_of(kb_config)
+        kb_dir = Path(external).expanduser() if external else self.base_dir / kb_name
+
+        status = kb_config.get("status")
+        progress = kb_config.get("progress")
+        description = kb_config.get("description", f"Knowledge base: {kb_name}")
+        rag_provider = normalize_provider_name(kb_config.get("rag_provider"))
+        needs_reindex = bool(kb_config.get("needs_reindex", False))
+        created_at = kb_config.get("created_at")
+        updated_at = kb_config.get("updated_at")
+
+        live_status = status in {"initializing", "processing"}
+        if live_status and isinstance(progress, dict):
+            live_status = progress.get("stage") not in {"completed", "error"}
+        effective_needs_reindex = needs_reindex and not live_status
+
+        # KB might not have a directory yet if still initializing
+        dir_exists = kb_dir.exists()
+        index_versions: list[dict[str, Any]] = []
+        has_ready_provider = False
+        if dir_exists:
+            index_versions = inspect_kb_versions(kb_dir, rag_provider)
+            has_ready_provider = any(bool(version.get("ready")) for version in index_versions)
+        provider_error_summary = (
+            provider_failure_summary(kb_dir, rag_provider, versions=index_versions)
+            if dir_exists
+            else ""
+        )
+
+        # For old KBs without status field, determine status from rag_storage
+        if effective_needs_reindex:
+            status = "needs_reindex"
+        elif status == "ready" and not has_ready_provider and provider_error_summary:
+            status = "error"
+            progress = {
+                "stage": "error",
+                "message": "Previous indexing failed.",
+                "error": provider_error_summary,
+            }
+        elif (
+            status in {"processing", "initializing"}
+            and has_ready_provider
+            and not (isinstance(progress, dict) and progress.get("stage") == "error")
+        ):
+            # A ready index version exists on disk but the persisted status is
+            # still a "live" sentinel — typically because the progress writer
+            # crashed (or the process was killed) after the index was finalised
+            # but before status was promoted to "ready". Recover the actual
+            # state on read so the UI does not show a perpetual processing
+            # banner. The persistent kb_config.json is left untouched; the
+            # next legitimate update_kb_status() call will clean it up.
+            # See issue #418.
+            status = "ready"
+            progress = None
+        elif not status and dir_exists:
+            rag_storage_dir = kb_dir / "rag_storage"
+            if has_ready_provider:
+                status = "ready"
+            elif rag_storage_dir.exists() and any(rag_storage_dir.iterdir()):
+                status = "needs_reindex"
+                needs_reindex = True
+                effective_needs_reindex = True
+            else:
+                status = "unknown"
+        elif not status:
+            status = "unknown"
+
+        # Build metadata from kb_config.json (authoritative source)
+        metadata = {
+            "name": kb_name,
+            "description": description,
+            "rag_provider": rag_provider,
+            "needs_reindex": effective_needs_reindex,
+        }
+        if created_at:
+            metadata["created_at"] = created_at
+        if updated_at:
+            metadata["last_updated"] = updated_at
+        if kb_config.get("last_indexed_at"):
+            metadata["last_indexed_at"] = kb_config.get("last_indexed_at")
+        if kb_config.get("last_indexed_count") is not None:
+            metadata["last_indexed_count"] = kb_config.get("last_indexed_count")
+        if kb_config.get("last_indexed_action"):
+            metadata["last_indexed_action"] = kb_config.get("last_indexed_action")
+        if kb_config.get("last_error"):
+            metadata["last_error"] = kb_config.get("last_error")
+        if kb_config.get("last_error_at"):
+            metadata["last_error_at"] = kb_config.get("last_error_at")
+        # Connected-KB fields, so the UI can badge it and show the path.
+        if kb_config.get("type"):
+            metadata["type"] = kb_config.get("type")
+        if kb_config.get("vault_path"):
+            metadata["vault_path"] = kb_config.get("vault_path")
+        if kb_config.get("external_path"):
+            metadata["external_path"] = kb_config.get("external_path")
+        if kb_config.get("db_path"):
+            metadata["db_path"] = kb_config.get("db_path")
+        if kb_config.get("agent_kind"):
+            metadata["agent_kind"] = kb_config.get("agent_kind")
+        # The server URL is shown read-only in the UI; the API key never leaves
+        # the backend, so it is deliberately not surfaced here.
+        if kb_config.get("server_url"):
+            metadata["server_url"] = kb_config.get("server_url")
+        # Same split for IMA: the library id is shown, the credentials are not.
+        if kb_config.get("knowledge_base_id"):
+            metadata["knowledge_base_id"] = kb_config.get("knowledge_base_id")
+        if kb_config.get("zim_name"):
+            metadata["zim_name"] = kb_config.get("zim_name")
+            metadata["zim_title"] = kb_config.get("zim_title") or ""
+
+        if rag_provider == LIGHTRAG_PROVIDER:
+            from deeptutor.services.rag.pipelines.lightrag.storage import (
+                published_root_for_embedding,
+                read_published_policy,
+            )
+
+            binding_signature = (
+                kb_config.get("embedding_signature")
+                if kb_config.get("embedding_selection")
+                else None
+            )
+            published_root = (
+                published_root_for_embedding(kb_dir, binding_signature) if dir_exists else None
+            )
+            indexing_policy = read_published_policy(published_root)
+            if published_root is not None:
+                metadata["indexed_version"] = published_root.name
+            if indexing_policy is None:
+                pending = kb_config.get("pending_indexing_policy")
+                indexing_policy = (
+                    {"policy": "defaults"}
+                    if not index_versions
+                    else pending
+                    if isinstance(pending, dict)
+                    else {"policy": "legacy_unpinned"}
+                )
+            from deeptutor.services.rag.pipelines.lightrag.indexing_policy import public_policy
+
+            metadata["indexing_policy"] = public_policy(indexing_policy)
+            if published_root is not None and indexing_policy.get("policy") == "pinned":
+                from deeptutor.services.rag.pipelines.lightrag.indexing_policy import (
+                    IndexingPolicyError,
+                    snapshot_from_persisted,
+                )
+
+                try:
+                    snapshot_from_persisted(indexing_policy)
+                except (IndexingPolicyError, ValueError, PermissionError):
+                    # This is local catalog/access validation, not a model health probe.
+                    # Keep credential/provider exception details out of public metadata.
+                    metadata["indexing_model_unavailable"] = True
+            for version in index_versions:
+                if isinstance(version.get("indexing_policy"), dict):
+                    version["indexing_policy"] = public_policy(version["indexing_policy"])
+            if published_root is not None or kb_config.get("embedding_selection"):
+                from deeptutor.services.rag.embedding_binding import entry_signature
+
+                published = next(
+                    (
+                        v
+                        for v in index_versions
+                        if published_root is not None and v.get("version") == published_root.name
+                    ),
+                    {},
+                )
+                metadata["indexed_embedding_model"] = published.get(
+                    "embedding_model"
+                ) or kb_config.get("embedding_model")
+                metadata["indexed_embedding_dim"] = published.get("embedding_dim") or kb_config.get(
+                    "embedding_dim"
+                )
+                current_embedding = entry_signature(kb_config)
+                if current_embedding is not None:
+                    metadata["current_embedding_model"] = current_embedding.model
+                    metadata["current_embedding_dim"] = current_embedding.dimension
+
+        metadata.update(self._embedding_fields(kb_config))
+
+        # Remove None values
+        metadata = {k: v for k, v in metadata.items() if v is not None}
+
+        info = {
+            "name": kb_name,
+            "path": str(kb_dir),
+            "is_default": kb_name == resolved_default,
+            "metadata": metadata,
+            "status": status,
+            "progress": progress,
+        }
+
+        # Count files - handle errors gracefully
+        raw_dir = kb_dir / "raw" if dir_exists else None
+        images_dir = kb_dir / "images" if dir_exists else None
+        content_list_dir = kb_dir / "content_list" if dir_exists else None
+
+        raw_count = 0
+        images_count = 0
+        content_lists_count = 0
+
+        if dir_exists:
+            try:
+                # One definition of "a document in a KB", shared with the chat
+                # manifest / ``kb_files`` so a user is never told two different
+                # counts for the same KB (see :mod:`deeptutor.knowledge.manifest`).
+                raw_count = sum(1 for _ in iter_kb_documents(raw_dir)) if raw_dir else 0
+            except Exception:
+                pass
+
+            try:
+                images_count = (
+                    len([f for f in images_dir.iterdir() if f.is_file()]) if images_dir else 0
+                )
+            except Exception:
+                pass
+
+            try:
+                content_lists_count = (
+                    len(list(content_list_dir.glob("*.json"))) if content_list_dir else 0
+                )
+            except Exception:
+                pass
+
+        # Check rag_initialized from provider-owned real output, not metadata alone.
+        from deeptutor.services.rag.index_versioning import (
+            find_matching_version,
+        )
+
+        kb_probe_dir = kb_dir if dir_exists else None
+        rag_initialized = has_ready_provider
+
+        pageindex_provider = rag_provider in {PAGEINDEX_PROVIDER, PAGEINDEX_OSS_PROVIDER}
+        from deeptutor.services.rag.embedding_binding import entry_signature
+
+        active_signature = None if pageindex_provider else entry_signature(kb_config)
+        if provider_uses_embedding_versions(rag_provider):
+            matched_entry = (
+                find_matching_version(kb_probe_dir, active_signature)
+                if (kb_probe_dir and active_signature)
+                else None
+            )
+            active_match = False
+            if matched_entry:
+                # Reuse the probe results already computed for ``index_versions``
+                # instead of probing the matched storage a second time — probing
+                # parses provider-owned files (e.g. the multi-MB LlamaIndex
+                # docstore.json) and is the dominant cost of kb list / the
+                # knowledge API (see issue #859).
+                matched_path = matched_entry.get("storage_path")
+                active_match = any(
+                    entry.get("storage_path") == matched_path and entry.get("ready")
+                    for entry in index_versions
+                )
+        else:
+            active_match = rag_initialized
+
+        if kb_config.get("embedding_status") in {"missing", "changed", "unconfigured"}:
+            active_match = False
+
+        info["statistics"] = {
+            "raw_documents": raw_count,
+            "images": images_count,
+            "content_lists": content_lists_count,
+            "rag_initialized": rag_initialized,
+            "rag_provider": rag_provider,
+            "needs_reindex": effective_needs_reindex,
+            "index_versions": index_versions,
+            "active_signature": active_signature.hash() if active_signature else None,
+            "active_match": active_match,
+            # Include status and progress in statistics for backward compatibility
+            "status": status,
+            "progress": progress,
+        }
+
+        return info
+
+    def delete_knowledge_base(self, name: str, confirm: bool = False) -> bool:
+        """
+        Delete a knowledge base
+
+        Args:
+            name: Knowledge base name
+            confirm: If True, skip confirmation (use with caution!)
+
+        Returns:
+            True if deleted successfully
+        """
+        # Look up against the raw config rather than ``list_knowledge_bases``:
+        # the latter prunes orphan entries (dir missing) as a side effect, so
+        # calling it here would race-delete the entry we are about to clean up
+        # and then raise "not found" on the now-empty config.
+        self.config = self._load_config()
+        config_kbs = self.config.get("knowledge_bases", {})
+        if name not in config_kbs and not (self.base_dir / name).exists():
+            raise ValueError(f"Knowledge base not found: {name}")
+
+        # Resolve the directory directly to stay idempotent: if the on-disk
+        # folder was already removed (e.g. manually rm-rf'd) we still want to
+        # purge the orphaned entry from kb_config.json instead of failing.
+        kb_dir = self.base_dir / name
+        dir_exists = kb_dir.exists()
+
+        # Connected KBs (Obsidian vaults, linked indexes, subagent pointers)
+        # reference the user's own external resource — or, for subagents, no
+        # folder at all. Deleting one must only drop our pointer entry; never
+        # touch what it references, and don't warn about the "missing" folder.
+        entry = config_kbs.get(name, {})
+        connected = is_connected_kb(entry)
+        if connected:
+            dir_exists = False
+        # One connected kind does own storage we created: a MarginNote library's
+        # synced objects live in a SQLite file under our own data directory, not
+        # in an external resource the user manages. Leaving it behind would also
+        # resurrect every paired device the moment a library of the same name is
+        # connected again.
+        if not confirm:
+            # Ask for confirmation in CLI
+            print(f"⚠️  Warning: This will permanently delete the knowledge base '{name}'")
+            print(f"   Path: {kb_dir}")
+            response = input("Are you sure? Type 'yes' to confirm: ")
+            if response.lower() != "yes":
+                print("Deletion cancelled.")
+                return False
+
+        if entry.get("type") == MARGINNOTE4_KB_TYPE:
+            self._delete_marginnote4_store(name, entry)
+
+        if dir_exists:
+
+            def _on_rmtree_error(func, path, exc_info):
+                exc = exc_info[1]
+                if isinstance(exc, FileNotFoundError):
+                    # Race: something else removed the entry between walk and unlink.
+                    return
+                # On Windows (and some bind-mounted filesystems) a read-only bit
+                # or a stale handle from a failed RAG init can block removal.
+                # Clear the read-only bit and retry once; if it still fails, log
+                # and continue so the config entry gets cleaned up regardless —
+                # leaving the KB stuck in the list is worse than orphan files on
+                # disk (issue #370).
+                try:
+                    current_mode = os.stat(path).st_mode
+                    writable_mode = current_mode | stat.S_IWRITE
+                    if stat.S_ISDIR(current_mode):
+                        # POSIX directories need execute permission to remain
+                        # traversable. Replacing the whole mode with S_IWRITE
+                        # leaves an orphan that even later cleanup cannot enter.
+                        writable_mode |= stat.S_IXUSR
+                    os.chmod(path, writable_mode)
+                    func(path)
+                except Exception as retry_exc:
+                    logger.warning(
+                        f"Could not remove '{path}' while deleting KB '{name}': "
+                        f"{retry_exc}. Continuing; orphan files may remain on disk."
+                    )
+
+            shutil.rmtree(kb_dir, onerror=_on_rmtree_error)
+        elif not connected:
+            logger.warning(
+                f"KB directory '{kb_dir}' missing on disk; cleaning up orphaned config entry."
+            )
+
+        # Remove from config
+        if name in self.config.get("knowledge_bases", {}):
+            del self.config["knowledge_bases"][name]
+
+        # Update default if this was the default
+        if self.config.get("default") == name:
+            remaining = [n for n in self.config.get("knowledge_bases", {}).keys() if n != name]
+            self.config["default"] = sorted(remaining)[0] if remaining else None
+
+        self._save_config()
+        return True
+
+    def _delete_marginnote4_store(self, name: str, entry: dict) -> None:
+        """Remove a MarginNote library's SQLite store, best-effort.
+
+        A failure here must not strand the config entry: leaving the KB in the
+        list is worse than an orphan file, exactly as for the index directory
+        above.
+        """
+        from deeptutor.capabilities.marginnote4.store import resolve_db_path
+
+        try:
+            db_path = resolve_db_path(name, metadata=entry)
+            db_path.unlink(missing_ok=True)
+            # SQLite's WAL companions, when the last connection left them.
+            for suffix in ("-wal", "-shm"):
+                db_path.with_name(db_path.name + suffix).unlink(missing_ok=True)
+        except Exception as exc:  # noqa: BLE001 - orphan file beats a stuck entry
+            logger.warning(
+                f"Could not remove the MarginNote store for KB '{name}': {exc}. "
+                "Continuing; the config entry is still cleaned up."
+            )
+
+    def clean_rag_storage(self, name: str | None = None, backup: bool = True) -> bool:
+        """
+        Clean (delete) index storage for a knowledge base.
+
+        Args:
+            name: Knowledge base name (default if not specified)
+            backup: If True, backup storage before deleting
+
+        Returns:
+            True if cleaned successfully
+        """
+        kb_name = name or self.get_default()
+        kb_dir = self.get_knowledge_base_path(kb_name)
+        from deeptutor.services.rag.index_versioning import (
+            LEGACY_VERSION_DIRNAME,
+            VERSION_PREFIX,
+        )
+
+        legacy_llamaindex_storage_dir = kb_dir / "llamaindex_storage"
+        legacy_versions_dir = kb_dir / LEGACY_VERSION_DIRNAME
+        legacy_storage_dir = kb_dir / "rag_storage"
+
+        flat_version_dirs = [
+            path
+            for path in kb_dir.iterdir()
+            if path.is_dir() and path.name.startswith(VERSION_PREFIX)
+        ]
+
+        if (
+            not flat_version_dirs
+            and not legacy_versions_dir.exists()
+            and not legacy_llamaindex_storage_dir.exists()
+            and not legacy_storage_dir.exists()
+        ):
+            logger.info(f"Index storage does not exist for '{kb_name}'")
+            return False
+
+        targets = []
+        for version_dir in flat_version_dirs:
+            targets.append((version_dir.name, version_dir))
+        if legacy_versions_dir.exists():
+            targets.append((LEGACY_VERSION_DIRNAME, legacy_versions_dir))
+        if legacy_llamaindex_storage_dir.exists():
+            targets.append(("llamaindex_storage", legacy_llamaindex_storage_dir))
+        if legacy_storage_dir.exists():
+            targets.append(("rag_storage", legacy_storage_dir))
+
+        for label, target in targets:
+            if backup:
+                timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+                backup_dir = kb_dir / f"{label}_backup_{timestamp}"
+                shutil.copytree(target, backup_dir)
+                logger.info(f"Backed up {label} to: {backup_dir}")
+
+            shutil.rmtree(target)
+            logger.info(f"Cleaned {label} for '{kb_name}'")
+
+        return True
+
+    def link_folder(self, kb_name: str, folder_path: str) -> dict:
+        """
+        Link a local folder to a knowledge base.
+
+        Args:
+            kb_name: Knowledge base name
+            folder_path: Path to local folder (supports ~, relative paths)
+
+        Returns:
+            Dict with folder info including id, path, and file count
+
+        Raises:
+            ValueError: If KB not found or folder doesn't exist
+        """
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+
+        # Normalize path (cross-platform: handles ~, relative paths, etc.)
+        folder = Path(folder_path).expanduser().resolve()
+
+        if not folder.exists():
+            raise ValueError(f"Folder does not exist: {folder}")
+        if not folder.is_dir():
+            raise ValueError(f"Path is not a directory: {folder}")
+
+        files = FileTypeRouter.collect_supported_files(folder, recursive=True)
+
+        # Generate folder ID
+
+        folder_id = hashlib.md5(  # noqa: S324
+            str(folder).encode(), usedforsecurity=False
+        ).hexdigest()[:8]
+
+        # Load existing linked folders from metadata
+        kb_dir = self.base_dir / kb_name
+        metadata_file = kb_dir / "metadata.json"
+        metadata: dict = {}
+
+        if metadata_file.exists():
+            try:
+                with open(metadata_file, encoding="utf-8") as fp:
+                    metadata = json.load(fp)
+            except Exception:
+                metadata = {}
+
+        if "linked_folders" not in metadata:
+            metadata["linked_folders"] = []
+
+        # Check if already linked
+        existing_ids = [item["id"] for item in metadata.get("linked_folders", [])]
+        if folder_id in existing_ids:
+            # If already linked, treat as success (idempotent)
+            # Find and return existing info
+            for item in metadata.get("linked_folders", []):
+                if item["id"] == folder_id:
+                    return item
+
+        # Add folder info
+        folder_info = {
+            "id": folder_id,
+            "path": str(folder),
+            "added_at": datetime.now().isoformat(),
+            "file_count": len(files),
+            "last_sync": None,
+        }
+        metadata["linked_folders"].append(folder_info)
+
+        atomic_write_json(metadata_file, metadata)
+
+        return folder_info
+
+    def get_linked_folders(self, kb_name: str) -> list[dict]:
+        """
+        Get list of linked folders for a knowledge base.
+
+        Args:
+            kb_name: Knowledge base name
+
+        Returns:
+            List of linked folder info dicts
+        """
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+
+        kb_dir = self.base_dir / kb_name
+        metadata_file = kb_dir / "metadata.json"
+
+        if not metadata_file.exists():
+            return []
+
+        try:
+            with open(metadata_file, encoding="utf-8") as f:
+                metadata = json.load(f)
+                return metadata.get("linked_folders", [])
+        except Exception:
+            return []
+
+    def unlink_folder(self, kb_name: str, folder_id: str) -> bool:
+        """
+        Unlink a folder from a knowledge base.
+
+        Args:
+            kb_name: Knowledge base name
+            folder_id: Folder ID to unlink
+
+        Returns:
+            True if unlinked successfully, False if not found
+        """
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+
+        kb_dir = self.base_dir / kb_name
+        metadata_file = kb_dir / "metadata.json"
+
+        if not metadata_file.exists():
+            return False
+
+        try:
+            with open(metadata_file, encoding="utf-8") as f:
+                metadata = json.load(f)
+        except Exception:
+            return False
+
+        linked = metadata.get("linked_folders", [])
+        new_linked = [f for f in linked if f["id"] != folder_id]
+
+        if len(new_linked) == len(linked):
+            return False  # Not found
+
+        metadata["linked_folders"] = new_linked
+
+        atomic_write_json(metadata_file, metadata)
+
+        return True
+
+    def scan_linked_folder(self, folder_path: str, provider: str = DEFAULT_PROVIDER) -> list[str]:
+        """
+        Scan a linked folder and return list of supported file paths.
+
+        Args:
+            folder_path: Path to folder
+            provider: RAG provider to determine supported extensions (default: llamaindex)
+
+        Returns:
+            List of file paths (as strings)
+        """
+        folder = Path(folder_path).expanduser().resolve()
+
+        if not folder.exists() or not folder.is_dir():
+            return []
+
+        files = [
+            str(file_path)
+            for file_path in FileTypeRouter.collect_supported_files(folder, recursive=True)
+        ]
+
+        return sorted(files)
+
+    def detect_folder_changes(self, kb_name: str, folder_id: str) -> dict:
+        """
+        Detect new and modified files in a linked folder since last sync.
+
+        This enables automatic sync of changes from local folders that may
+        be synced with cloud services like SharePoint, Google Drive, etc.
+
+        Args:
+            kb_name: Knowledge base name
+            folder_id: Folder ID to check for changes
+
+        Returns:
+            Dict with 'new_files', 'modified_files', and 'has_changes' keys
+        """
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+
+        # Get folder info
+        folders = self.get_linked_folders(kb_name)
+        folder_info = next((f for f in folders if f["id"] == folder_id), None)
+
+        if not folder_info:
+            raise ValueError(f"Linked folder not found: {folder_id}")
+
+        folder_path = Path(folder_info["path"]).expanduser().resolve()
+        synced_files = folder_info.get("synced_files", {})
+
+        new_files = []
+        modified_files = []
+
+        for file_path in FileTypeRouter.collect_supported_files(folder_path, recursive=True):
+            file_str = str(file_path)
+            file_mtime = datetime.fromtimestamp(file_path.stat().st_mtime)
+
+            if file_str in synced_files:
+                # Check if modified since last sync
+                prev_mtime_str = synced_files[file_str]
+                try:
+                    prev_mtime = datetime.fromisoformat(prev_mtime_str)
+                    if file_mtime > prev_mtime:
+                        modified_files.append(file_str)
+                except Exception:
+                    modified_files.append(file_str)
+            else:
+                # New file (not in synced files)
+                new_files.append(file_str)
+
+        return {
+            "new_files": sorted(new_files),
+            "modified_files": sorted(modified_files),
+            "has_changes": len(new_files) > 0 or len(modified_files) > 0,
+            "new_count": len(new_files),
+            "modified_count": len(modified_files),
+        }
+
+    def update_folder_sync_state(
+        self,
+        kb_name: str,
+        folder_id: str,
+        synced_files: list[str],
+        source_mtimes: dict[str, str] | None = None,
+    ):
+        """
+        Update the sync state for a linked folder after successful sync.
+
+        Records which files were synced and their modification times,
+        enabling future change detection.
+
+        Args:
+            kb_name: Knowledge base name
+            folder_id: Folder ID
+            synced_files: List of file paths that were successfully synced
+            source_mtimes: Modification times captured before source staging.
+        """
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+
+        kb_dir = self.base_dir / kb_name
+        metadata_file = kb_dir / "metadata.json"
+
+        if not metadata_file.exists():
+            return
+
+        try:
+            with open(metadata_file, encoding="utf-8") as f:
+                metadata = json.load(f)
+        except Exception:
+            return
+
+        linked = metadata.get("linked_folders", [])
+
+        for folder in linked:
+            if folder["id"] == folder_id:
+                # Record sync timestamp
+                folder["last_sync"] = datetime.now().isoformat()
+
+                # Record file modification times
+                file_states = folder.get("synced_files", {})
+                for file_path in synced_files:
+                    try:
+                        if source_mtimes is not None:
+                            if file_path in source_mtimes:
+                                file_states[file_path] = source_mtimes[file_path]
+                        else:
+                            p = Path(file_path)
+                            if p.exists():
+                                mtime = datetime.fromtimestamp(p.stat().st_mtime)
+                                file_states[file_path] = mtime.isoformat()
+                    except Exception as exc:
+                        # Recording failed: leave the file un-recorded so the
+                        # next scan re-syncs it instead of trusting a missing
+                        # mtime, and make the degradation visible in logs.
+                        logger.warning(
+                            f"Failed to record sync state for '{file_path}' in folder "
+                            f"'{folder_id}' of KB '{kb_name}': {exc}"
+                        )
+
+                folder["synced_files"] = file_states
+                folder["file_count"] = len(file_states)
+                atomic_write_json(metadata_file, metadata)
+                break
+
+    # ------------------------------------------------------------------
+    # GitHub source management
+    # ------------------------------------------------------------------
+
+    def add_github_source(self, kb_name, repo, branch="main", path="", glob="*.md"):
+        """Register a GitHub repo as a document source for a KB."""
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+        repo_clean = repo.strip().rstrip("/")
+        if repo_clean.endswith(".git"):
+            repo_clean = repo_clean[:-4]
+        if "github.com/" in repo_clean:
+            repo_clean = repo_clean.split("github.com/", 1)[-1]
+        repo_clean = repo_clean.strip("/")
+        source_id = hashlib.md5(  # noqa: S324
+            f"{repo_clean}:{branch}:{path}".encode(), usedforsecurity=False
+        ).hexdigest()[:8]
+        kb_dir = self.base_dir / kb_name
+        metadata_file = kb_dir / "metadata.json"
+        metadata = self._read_kb_metadata(metadata_file)
+        sources = metadata.get("github_sources", [])
+        for existing in sources:
+            if existing.get("id") == source_id:
+                return existing
+        source_info = {
+            "id": source_id,
+            "repo": repo_clean,
+            "branch": branch,
+            "path": path,
+            "glob": glob,
+            "enabled": True,
+            "last_synced_sha": "",
+            "last_synced_at": "",
+            "last_sync_status": "pending",
+            "last_sync_error": None,
+            "files_synced": 0,
+            "added_at": datetime.now().isoformat(),
+        }
+        sources.append(source_info)
+        metadata["github_sources"] = sources
+        atomic_write_json(metadata_file, metadata)
+        return source_info
+
+    def remove_github_source(self, kb_name, source_id):
+        """Remove a GitHub source from a KB."""
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+        metadata_file = self.base_dir / kb_name / "metadata.json"
+        metadata = self._read_kb_metadata(metadata_file)
+        sources = metadata.get("github_sources", [])
+        new_sources = [s for s in sources if s.get("id") != source_id]
+        if len(new_sources) == len(sources):
+            return False
+        metadata["github_sources"] = new_sources
+        atomic_write_json(metadata_file, metadata)
+        return True
+
+    def get_github_sources(self, kb_name):
+        """Return all GitHub sources registered for a KB."""
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+        metadata_file = self.base_dir / kb_name / "metadata.json"
+        metadata = self._read_kb_metadata(metadata_file)
+        return metadata.get("github_sources", [])
+
+    def update_github_source_state(self, kb_name, source_id, **fields):
+        """Persist sync state fields into a GitHub source entry."""
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+        metadata_file = self.base_dir / kb_name / "metadata.json"
+        metadata = self._read_kb_metadata(metadata_file)
+        sources = metadata.get("github_sources", [])
+        for src in sources:
+            if src.get("id") == source_id:
+                src.update(fields)
+                atomic_write_json(metadata_file, metadata)
+                return
+
+    def get_all_github_sources(self):
+        """Scan every KB and return (kb_name, source_dict) pairs."""
+        result = []
+        for kb_name in self.list_knowledge_bases():
+            for src in self.get_github_sources(kb_name):
+                result.append((kb_name, src))
+        return result
+
+    # ------------------------------------------------------------------
+    # Web source management
+    # ------------------------------------------------------------------
+
+    def add_web_source(
+        self,
+        kb_name: str,
+        url: str,
+        max_depth: int = 3,
+        max_pages: int = 200,
+    ) -> dict:
+        """Register a documentation site URL as a document source for a KB."""
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+        if not 1 <= max_depth <= MAX_CRAWL_DEPTH:
+            raise ValueError(f"Web source crawl depth must be between 1 and {MAX_CRAWL_DEPTH}")
+        if not 1 <= max_pages <= MAX_CRAWL_PAGES:
+            raise ValueError(f"Web source crawl page count must be between 1 and {MAX_CRAWL_PAGES}")
+        normalized_url = url.strip()
+        parsed_url = urlparse(normalized_url)
+        if parsed_url.scheme.lower() not in ("http", "https") or not parsed_url.hostname:
+            raise ValueError("Web source URL must be an absolute http(s) URL")
+        source_id = hashlib.md5(  # noqa: S324
+            normalized_url.encode(), usedforsecurity=False
+        ).hexdigest()[:8]
+        metadata_file = self.base_dir / kb_name / "metadata.json"
+        metadata = self._read_kb_metadata(metadata_file)
+        sources = metadata.get("web_sources", [])
+        for existing in sources:
+            if existing.get("id") == source_id:
+                return existing
+
+        source_info = {
+            "id": source_id,
+            "url": normalized_url,
+            "max_depth": max_depth,
+            "max_pages": max_pages,
+            "enabled": True,
+            "auto_sync_enabled": True,
+            "sync_interval_hours": 24,
+            "page_hashes": {},
+            "page_count": 0,
+            "last_synced_at": "",
+            "last_sync_status": "pending",
+            "last_sync_error": None,
+            "added_at": datetime.now().isoformat(),
+        }
+        sources.append(source_info)
+        metadata["web_sources"] = sources
+        atomic_write_json(metadata_file, metadata)
+        return source_info
+
+    def remove_web_source(self, kb_name: str, source_id: str) -> bool:
+        """Remove a web source from a KB."""
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+        metadata_file = self.base_dir / kb_name / "metadata.json"
+        metadata = self._read_kb_metadata(metadata_file)
+        sources = metadata.get("web_sources", [])
+        remaining = [source for source in sources if source.get("id") != source_id]
+        if len(remaining) == len(sources):
+            return False
+
+        metadata["web_sources"] = remaining
+        atomic_write_json(metadata_file, metadata)
+        return True
+
+    def get_web_sources(self, kb_name: str) -> list[dict]:
+        """Return all web sources registered for a KB."""
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+        metadata_file = self.base_dir / kb_name / "metadata.json"
+        metadata = self._read_kb_metadata(metadata_file)
+        return metadata.get("web_sources", [])
+
+    def update_web_source_state(self, kb_name: str, source_id: str, **fields: object) -> None:
+        """Persist sync state fields into a web source entry."""
+        if kb_name not in self.list_knowledge_bases():
+            raise ValueError(f"Knowledge base not found: {kb_name}")
+        metadata_file = self.base_dir / kb_name / "metadata.json"
+        metadata = self._read_kb_metadata(metadata_file)
+        for source in metadata.get("web_sources", []):
+            if source.get("id") == source_id:
+                source.update(fields)
+                atomic_write_json(metadata_file, metadata)
+                return
+
+    def update_web_source_schedule(
+        self,
+        kb_name: str,
+        source_id: str,
+        *,
+        auto_sync_enabled: bool,
+        sync_interval_hours: int,
+    ) -> dict:
+        """Persist the reviewable schedule fields for one web source."""
+        source = next(
+            (item for item in self.get_web_sources(kb_name) if item.get("id") == source_id),
+            None,
+        )
+        if source is None:
+            raise ValueError(f"Source '{source_id}' not found")
+        self.update_web_source_state(
+            kb_name,
+            source_id,
+            auto_sync_enabled=auto_sync_enabled,
+            sync_interval_hours=sync_interval_hours,
+        )
+        return next(item for item in self.get_web_sources(kb_name) if item.get("id") == source_id)
+
+    def get_all_web_sources(self) -> list[tuple[str, dict]]:
+        """Scan every KB and return (kb_name, source_dict) pairs."""
+        result = []
+        for kb_name in self.list_knowledge_bases():
+            for source in self.get_web_sources(kb_name):
+                result.append((kb_name, source))
+        return result
+
+    @staticmethod
+    def _read_kb_metadata(metadata_file):
+        """Load metadata.json, returning {} on absence or parse error."""
+        if not metadata_file.exists():
+            return {}
+        try:
+            with open(metadata_file, encoding="utf-8") as f:
+                data = json.load(f)
+        except Exception:
+            return {}
+        return data if isinstance(data, dict) else {}
+
+
+def main():
+    """Command-line interface for knowledge base manager"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Knowledge Base Manager")
+    parser.add_argument(
+        "--base-dir", default="./knowledge_bases", help="Base directory for knowledge bases"
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Commands")
+
+    # List command
+    subparsers.add_parser("list", help="List all knowledge bases")
+
+    # Info command
+    info_parser = subparsers.add_parser("info", help="Show knowledge base information")
+    info_parser.add_argument(
+        "name", nargs="?", help="Knowledge base name (default if not specified)"
+    )
+
+    # Set default command
+    default_parser = subparsers.add_parser("set-default", help="Set default knowledge base")
+    default_parser.add_argument("name", help="Knowledge base name")
+
+    # Delete command
+    delete_parser = subparsers.add_parser("delete", help="Delete a knowledge base")
+    delete_parser.add_argument("name", help="Knowledge base name")
+    delete_parser.add_argument("--force", action="store_true", help="Skip confirmation")
+
+    # Clean RAG command
+    clean_parser = subparsers.add_parser(
+        "clean-rag", help="Clean RAG storage (useful for corrupted data)"
+    )
+    clean_parser.add_argument(
+        "name", nargs="?", help="Knowledge base name (default if not specified)"
+    )
+    clean_parser.add_argument(
+        "--no-backup", action="store_true", help="Don't backup before cleaning"
+    )
+
+    args = parser.parse_args()
+
+    manager = KnowledgeBaseManager(args.base_dir)
+
+    if args.command == "list":
+        kb_list = manager.list_knowledge_bases()
+        default_kb = manager.get_default()
+
+        print("\nAvailable Knowledge Bases:")
+        print("=" * 60)
+        if not kb_list:
+            print("No knowledge bases found")
+        else:
+            for kb_name in kb_list:
+                default_marker = " (default)" if kb_name == default_kb else ""
+                print(f"  • {kb_name}{default_marker}")
+        print()
+
+    elif args.command == "info":
+        try:
+            info = manager.get_info(args.name)
+
+            print("\nKnowledge Base Information:")
+            print("=" * 60)
+            print(f"Name: {info['name']}")
+            print(f"Path: {info['path']}")
+            print(f"Default: {'Yes' if info['is_default'] else 'No'}")
+
+            if info.get("metadata"):
+                print("\nMetadata:")
+                for key, value in info["metadata"].items():
+                    print(f"  {key}: {value}")
+
+            print("\nStatistics:")
+            stats = info["statistics"]
+            print(f"  Raw documents: {stats['raw_documents']}")
+            print(f"  Images: {stats['images']}")
+            print(f"  Content lists: {stats['content_lists']}")
+            print(f"  RAG initialized: {'Yes' if stats['rag_initialized'] else 'No'}")
+
+            if "rag" in stats:
+                print("\n  RAG Statistics:")
+                for key, value in stats["rag"].items():
+                    print(f"    {key}: {value}")
+
+            print()
+        except Exception as e:
+            print(f"Error: {e!s}")
+
+    elif args.command == "set-default":
+        try:
+            manager.set_default(args.name)
+            print(f"✓ Set '{args.name}' as default knowledge base")
+        except Exception as e:
+            print(f"Error: {e!s}")
+
+    elif args.command == "delete":
+        try:
+            success = manager.delete_knowledge_base(args.name, confirm=args.force)
+            if success:
+                print(f"✓ Deleted knowledge base '{args.name}'")
+        except Exception as e:
+            print(f"Error: {e!s}")
+
+    elif args.command == "clean-rag":
+        try:
+            manager.clean_rag_storage(args.name, backup=not args.no_backup)
+        except Exception as e:
+            print(f"Error: {e!s}")
+
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()