deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor_cli/kb.py

@@ -0,0 +1,756 @@
+"""
+CLI Knowledge Base Command
+===========================
+
+Manage knowledge bases from the command line.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+from rich.console import Console
+from rich.table import Table
+import typer
+
+from deeptutor.knowledge.manager import KnowledgeBaseManager
+from deeptutor.knowledge.naming import validate_knowledge_base_name
+from deeptutor.services.path_service import get_path_service
+from deeptutor.services.rag.factory import DEFAULT_PROVIDER
+from deeptutor.services.rag.file_routing import FileTypeRouter
+
+if TYPE_CHECKING:  # pragma: no cover - imported for type annotations only
+    from deeptutor.services.rag.eval import EvalDataset, EvalReport, ProgressFn, QueryEvaluation
+
+console = Console()
+
+
+def _get_kb_manager() -> KnowledgeBaseManager:
+    """Return a KnowledgeBaseManager rooted at the canonical project-level KB directory."""
+    base_dir = get_path_service().project_root / "data" / "knowledge_bases"
+    return KnowledgeBaseManager(base_dir=str(base_dir))
+
+
+def _collect_documents(docs: list[str], docs_dir: Optional[str]) -> list[str]:
+    """Collect and de-duplicate document files from explicit paths and a directory."""
+    candidates: list[Path] = []
+
+    for doc in docs:
+        path = Path(doc).expanduser().resolve()
+        if path.exists() and path.is_file():
+            candidates.append(path)
+
+    if docs_dir:
+        base = Path(docs_dir).expanduser().resolve()
+        if not base.exists() or not base.is_dir():
+            raise typer.BadParameter(f"docs directory does not exist: {base}")
+        candidates.extend(FileTypeRouter.collect_supported_files(base, recursive=True))
+
+    unique: list[str] = []
+    seen: set[str] = set()
+    for path in candidates:
+        key = str(path)
+        if key in seen:
+            continue
+        seen.add(key)
+        unique.append(key)
+
+    return unique
+
+
+def _load_eval_dataset(dataset: str) -> EvalDataset:
+    """Load the QA set for ``kb eval``, exiting with a readable message when it is invalid."""
+    from deeptutor.services.rag.eval import EvalDatasetError, load_dataset
+
+    try:
+        return load_dataset(dataset)
+    except EvalDatasetError as exc:
+        console.print(f"[red]{exc}[/]")
+        raise typer.Exit(code=1) from exc
+
+
+def _resolve_eval_provider(mgr: KnowledgeBaseManager, name: str) -> str:
+    """Return the provider bound to ``name``, rejecting engines that return no ranked chunks."""
+    from deeptutor.services.rag.factory import PAGEINDEX_OSS_PROVIDER, PAGEINDEX_PROVIDER
+    from deeptutor.services.rag.provider_binding import resolve_bound_provider
+
+    provider = resolve_bound_provider(str(mgr.base_dir), name)
+    if provider in {PAGEINDEX_PROVIDER, PAGEINDEX_OSS_PROVIDER}:
+        console.print(
+            "[red]PageIndex answers by reasoning over pages inside an agent loop, so it "
+            "returns no ranked chunks to score. Evaluate a KB bound to llamaindex, "
+            "graphrag, lightrag, a LightRAG server, WeKnora or IMA instead.[/]"
+        )
+        raise typer.Exit(code=1)
+    return provider
+
+
+def _build_eval_progress(top_k: int, fmt: str) -> ProgressFn:
+    """Return a per-case progress printer that stays silent while emitting JSON."""
+    from deeptutor.services.rag.eval import format_metric
+
+    recall_key = f"recall@{top_k}"
+
+    def _report(index: int, total: int, item: QueryEvaluation) -> None:
+        if fmt == "json":
+            return
+        if item.failed:
+            console.print(f"  {index}/{total} [red]failed[/] {item.case_id}: {item.error}")
+            return
+        value = format_metric(item.metrics.values()[recall_key])
+        console.print(f"  {index}/{total} [green]ok[/] {item.case_id} {recall_key}={value}")
+
+    return _report
+
+
+def _save_eval_report(save: str, payload: str) -> None:
+    """Write a serialized evaluation report to ``save``, creating parent directories."""
+    target = Path(save).expanduser()
+    try:
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(payload, encoding="utf-8")
+    except OSError as exc:
+        typer.echo(f"Could not write report: {exc}", err=True)
+        raise typer.Exit(code=1) from exc
+    typer.echo(f"Report written to {target}", err=True)
+
+
+def _render_eval_report(report: EvalReport, name: str, provider: str) -> None:
+    """Print the per-case metric table followed by the aggregate summary lines."""
+    columns = report.metric_columns()
+    table = Table(title=f"Retrieval evaluation - {name} ({provider})")
+    table.add_column("Case", style="bold")
+    table.add_column("Status")
+    table.add_column("Query", overflow="fold", max_width=48)
+    for column in columns:
+        table.add_column(column, justify="right")
+    for item in report.evaluations:
+        cells = report.format_metrics(item)
+        table.add_row(
+            item.case_id,
+            "[red]failed[/]" if item.failed else "[green]ok[/]",
+            item.query,
+            *(cells[column] for column in columns),
+        )
+    console.print(table)
+    for line in report.summary_lines():
+        console.print(line)
+
+
+def _parse_rewrite_flag(value: Optional[str]) -> Optional[bool]:
+    """Map a ``--rewrite`` value to the three-state rewrite flag, or None to defer to settings.
+
+    Args:
+        value: ``None`` (flag omitted, defer to settings), ``"off"`` or ``"standalone"``.
+
+    Returns:
+        ``False`` for ``"off"``, ``True`` for ``"standalone"``, ``None`` when omitted.
+
+    Raises:
+        typer.BadParameter: If ``value`` is not a recognized strategy.
+    """
+    if value is None:
+        return None
+    normalized = value.strip().lower()
+    if normalized == "off":
+        return False
+    if normalized == "standalone":
+        return True
+    raise typer.BadParameter(
+        f"unknown rewrite strategy '{value}'; expected 'off' or 'standalone'",
+        param_hint="--rewrite",
+    )
+
+
+def _load_eval_report(path: str) -> dict:
+    """Load a saved ``kb eval --save`` report for ``kb eval-diff``, exiting readably on error.
+
+    Args:
+        path: Path to a report JSON file written by ``kb eval --save``.
+
+    Returns:
+        The parsed report as a mapping, ready for
+        :func:`deeptutor.services.rag.eval.diff_reports`.
+
+    Raises:
+        typer.Exit: If the file is missing, is not valid JSON, or is not a JSON object.
+    """
+    target = Path(path).expanduser()
+    if not target.exists():
+        console.print(f"[red]Report not found: {target}[/]")
+        raise typer.Exit(code=1)
+    try:
+        loaded = json.loads(target.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        console.print(f"[red]Could not read report '{target}': {exc}[/]")
+        raise typer.Exit(code=1) from exc
+    if not isinstance(loaded, dict):
+        console.print(f"[red]Report '{target}' is not a JSON object.[/]")
+        raise typer.Exit(code=1)
+    return loaded
+
+
+def register(app: typer.Typer) -> None:
+    @app.command("list")
+    def kb_list(
+        fmt: str = typer.Option("rich", "--format", "-f", help="Output format: rich | json."),
+    ) -> None:
+        """List all knowledge bases."""
+        mgr = _get_kb_manager()
+        kb_names = mgr.list_knowledge_bases()
+        if not kb_names:
+            if fmt == "json":
+                console.print_json("[]")
+            else:
+                console.print("[dim]No knowledge bases found.[/]")
+            return
+
+        if fmt == "json":
+            items = []
+            for name in kb_names:
+                info = mgr.get_info(name)
+                stats = info.get("statistics", {})
+                metadata = info.get("metadata", {})
+                items.append(
+                    {
+                        "name": name,
+                        "status": info.get("status", "unknown"),
+                        "documents": stats.get("raw_documents", 0),
+                        "rag_provider": metadata.get(
+                            "rag_provider", stats.get("rag_provider", DEFAULT_PROVIDER)
+                        ),
+                        "is_default": bool(info.get("is_default")),
+                    }
+                )
+            console.print_json(json.dumps(items, ensure_ascii=False, default=str))
+            return
+
+        table = Table(title="Knowledge Bases")
+        table.add_column("Name", style="bold")
+        table.add_column("Status")
+        table.add_column("Documents", justify="right")
+        table.add_column("RAG Provider")
+        table.add_column("Default")
+
+        for name in kb_names:
+            info = mgr.get_info(name)
+            stats = info.get("statistics", {})
+            metadata = info.get("metadata", {})
+            table.add_row(
+                name,
+                str(info.get("status", "unknown")),
+                str(stats.get("raw_documents", 0)),
+                str(metadata.get("rag_provider", stats.get("rag_provider", DEFAULT_PROVIDER))),
+                "yes" if info.get("is_default") else "",
+            )
+
+        console.print(table)
+
+    @app.command("info")
+    def kb_info(name: str = typer.Argument(..., help="Knowledge base name.")) -> None:
+        """Show details of a knowledge base."""
+        mgr = _get_kb_manager()
+        try:
+            info = mgr.get_info(name)
+        except Exception as exc:
+            console.print(f"[red]Knowledge base '{name}' not found: {exc}[/]")
+            raise typer.Exit(code=1) from exc
+        console.print_json(json.dumps(info, indent=2, ensure_ascii=False, default=str))
+
+    @app.command("set-default")
+    def kb_set_default(name: str = typer.Argument(..., help="Knowledge base name.")) -> None:
+        """Set the default knowledge base."""
+        mgr = _get_kb_manager()
+        try:
+            mgr.set_default(name)
+        except Exception as exc:
+            console.print(f"[red]Failed to set default KB '{name}': {exc}[/]")
+            raise typer.Exit(code=1) from exc
+        console.print(f"[green]Set '{name}' as default knowledge base.[/]")
+
+    @app.command("create")
+    def kb_create(
+        name: str = typer.Argument(..., help="New KB name."),
+        docs: list[str] = typer.Option([], "--doc", "-d", help="Document paths."),
+        docs_dir: Optional[str] = typer.Option(None, "--docs-dir", help="Directory of documents."),
+    ) -> None:
+        """Initialize a new knowledge base from documents."""
+        mgr = _get_kb_manager()
+        try:
+            name = validate_knowledge_base_name(name)
+        except ValueError as exc:
+            console.print(f"[red]{exc}[/]")
+            raise typer.Exit(code=1) from exc
+
+        if name in mgr.list_knowledge_bases():
+            console.print(f"[red]Knowledge base '{name}' already exists.[/]")
+            raise typer.Exit(code=1)
+
+        try:
+            doc_paths = _collect_documents(docs, docs_dir)
+        except typer.BadParameter as exc:
+            console.print(f"[red]{exc}[/]")
+            raise typer.Exit(code=1) from exc
+
+        if not doc_paths:
+            console.print("[red]Provide at least one supported document (--doc or --docs-dir).[/]")
+            raise typer.Exit(code=1)
+
+        console.print(
+            f"Creating KB [bold]{name}[/] with {len(doc_paths)} document(s) via [bold]LlamaIndex[/]..."
+        )
+        from deeptutor.knowledge.initializer import initialize_knowledge_base
+
+        try:
+            asyncio.run(
+                initialize_knowledge_base(
+                    kb_name=name,
+                    source_files=doc_paths,
+                    base_dir=str(mgr.base_dir),
+                )
+            )
+        except Exception as exc:
+            console.print(f"[red]KB creation failed: {exc}[/]")
+            raise typer.Exit(code=1) from exc
+        console.print("[green]Knowledge base created successfully.[/]")
+
+    @app.command("connect-kiwix")
+    def kb_connect_kiwix(
+        name: str = typer.Argument(..., help="New KB name."),
+        server_url: str = typer.Option(..., "--server-url", help="kiwix-serve base URL."),
+        zim_name: str = typer.Option(..., "--zim-name", help="One ZIM name served by Kiwix."),
+    ) -> None:
+        """Connect an existing ZIM archive without copying or indexing it."""
+        from deeptutor.services.rag.pipelines.kiwix.client import KiwixClient
+
+        try:
+            client = KiwixClient(server_url, zim_name)
+            title = asyncio.run(client.probe())
+            _get_kb_manager().register_kiwix_kb(
+                name, client.base_url, client.zim_name, zim_title=title
+            )
+        except Exception as exc:
+            console.print(f"[red]Could not connect Kiwix archive: {exc}[/]")
+            raise typer.Exit(code=1) from exc
+        console.print(f"[green]Connected '{name}' to Kiwix archive '{title}'.[/]")
+
+    @app.command("add")
+    def kb_add(
+        name: str = typer.Argument(..., help="KB name."),
+        docs: list[str] = typer.Option([], "--doc", "-d", help="Document paths to add."),
+        docs_dir: Optional[str] = typer.Option(None, "--docs-dir", help="Directory of documents."),
+    ) -> None:
+        """Add documents to an existing knowledge base."""
+        mgr = _get_kb_manager()
+        if name not in mgr.list_knowledge_bases():
+            console.print(f"[red]Knowledge base '{name}' not found.[/]")
+            raise typer.Exit(code=1)
+
+        try:
+            doc_paths = _collect_documents(docs, docs_dir)
+        except typer.BadParameter as exc:
+            console.print(f"[red]{exc}[/]")
+            raise typer.Exit(code=1) from exc
+
+        if not doc_paths:
+            console.print("[red]Provide at least one supported document.[/]")
+            raise typer.Exit(code=1)
+
+        console.print(f"Adding {len(doc_paths)} document(s) to [bold]{name}[/]...")
+        from deeptutor.knowledge.add_documents import add_documents
+
+        try:
+            processed_count = asyncio.run(
+                add_documents(
+                    kb_name=name,
+                    source_files=doc_paths,
+                    base_dir=str(mgr.base_dir),
+                    allow_duplicates=False,
+                )
+            )
+        except Exception as exc:
+            console.print(f"[red]Document upload failed: {exc}[/]")
+            raise typer.Exit(code=1) from exc
+
+        if processed_count:
+            console.print(f"[green]Done. Indexed {processed_count} document(s).[/]")
+        else:
+            console.print("[yellow]No new unique documents were indexed.[/]")
+
+    @app.command("delete")
+    def kb_delete(
+        name: str = typer.Argument(..., help="KB name."),
+        force: bool = typer.Option(False, "--force", "-f", help="Skip confirmation."),
+    ) -> None:
+        """Delete a knowledge base."""
+        if not force:
+            confirm = typer.confirm(f"Delete knowledge base '{name}'?")
+            if not confirm:
+                raise typer.Abort()
+
+        mgr = _get_kb_manager()
+        try:
+            deleted = mgr.delete_knowledge_base(name, confirm=True)
+        except Exception as exc:
+            console.print(f"[red]Failed to delete '{name}': {exc}[/]")
+            raise typer.Exit(code=1) from exc
+
+        if deleted:
+            console.print(f"[green]Deleted '{name}'.[/]")
+        else:
+            console.print(f"[yellow]Knowledge base '{name}' was not deleted.[/]")
+
+    @app.command("search")
+    def kb_search(
+        name: str = typer.Argument(..., help="KB name."),
+        query: str = typer.Argument(..., help="Search query."),
+        mode: str = typer.Option("hybrid", help="Search mode."),
+        fmt: str = typer.Option("rich", "--format", "-f", help="Output format: rich | json."),
+    ) -> None:
+        """Search a knowledge base."""
+        from deeptutor.tools.rag_tool import rag_search
+
+        mgr = _get_kb_manager()
+        if name not in mgr.list_knowledge_bases():
+            console.print(f"[red]Knowledge base '{name}' not found.[/]")
+            raise typer.Exit(code=1)
+
+        try:
+            from deeptutor.services.rag.factory import PAGEINDEX_OSS_PROVIDER, PAGEINDEX_PROVIDER
+            from deeptutor.services.rag.provider_binding import resolve_bound_provider
+
+            provider = resolve_bound_provider(str(mgr.base_dir), name)
+            if provider in {PAGEINDEX_PROVIDER, PAGEINDEX_OSS_PROVIDER}:
+                from deeptutor.services.rag.pipelines.pageindex.reasoning import (
+                    read_pageindex_with_agent,
+                )
+
+                reading = asyncio.run(
+                    read_pageindex_with_agent(
+                        kb_name=name,
+                        system_prompt=(
+                            "Answer the CLI user's question from the selected PageIndex "
+                            "knowledge base. Include page references from tool results."
+                        ),
+                        user_prompt=query,
+                        source="cli_kb_search",
+                        stage="search",
+                    )
+                )
+                result = {
+                    "query": query,
+                    "answer": reading.text,
+                    "content": reading.text,
+                    "sources": reading.sources,
+                    "provider": reading.tool_context.provider,
+                }
+            else:
+                result = asyncio.run(
+                    rag_search(
+                        query=query,
+                        kb_name=name,
+                        mode=mode,
+                        kb_base_dir=str(mgr.base_dir),
+                    )
+                )
+        except Exception as exc:
+            console.print(f"[red]Search failed: {exc}[/]")
+            raise typer.Exit(code=1) from exc
+
+        if fmt == "json":
+            console.print_json(json.dumps(result, indent=2, ensure_ascii=False, default=str))
+            return
+
+        answer = result.get("answer") or result.get("content", "")
+        provider = str(result.get("provider", DEFAULT_PROVIDER))
+        console.print(f"[bold]Provider:[/] {provider}")
+        console.print(f"[bold]Answer:[/]\n{answer}")
+
+    @app.command("eval")
+    def kb_eval(
+        name: str = typer.Argument(..., help="KB name."),
+        dataset: str = typer.Option(..., "--dataset", "-d", help="QA set file (.jsonl or .json)."),
+        top_k: int = typer.Option(5, "--top-k", "-k", help="Rank cutoff for the metrics."),
+        mode: Optional[str] = typer.Option(
+            None, "--mode", help="Search mode for engines that support one (e.g. hybrid)."
+        ),
+        limit: Optional[int] = typer.Option(
+            None, "--limit", help="Evaluate only the first N cases of the set."
+        ),
+        min_ratio: float = typer.Option(
+            0.5, "--min-ratio", help="Token coverage a retrieved chunk needs to count as a match."
+        ),
+        save: Optional[str] = typer.Option(None, "--save", help="Write the JSON report to a file."),
+        fmt: str = typer.Option("rich", "--format", "-f", help="Output format: rich | json."),
+        rewrite: Optional[str] = typer.Option(
+            None,
+            "--rewrite",
+            help="Force the query-rewrite layer: off | standalone. Omit to defer to settings.",
+        ),
+    ) -> None:
+        """Score a knowledge base's retrieval quality against a QA set."""
+        from deeptutor.services.rag.eval import MatchPolicy, RetrievalEvaluator
+
+        rewrite_flag = _parse_rewrite_flag(rewrite)
+
+        mgr = _get_kb_manager()
+        if name not in mgr.list_knowledge_bases():
+            console.print(f"[red]Knowledge base '{name}' not found.[/]")
+            raise typer.Exit(code=1)
+
+        eval_set = _load_eval_dataset(dataset)
+        provider = _resolve_eval_provider(mgr, name)
+
+        try:
+            evaluator = RetrievalEvaluator(
+                kb_name=name,
+                kb_base_dir=str(mgr.base_dir),
+                k=top_k,
+                mode=mode,
+                query_limit=limit,
+                policy=MatchPolicy(min_ratio=min_ratio),
+                provider=provider,
+                rewrite=rewrite_flag,
+            )
+        except ValueError as exc:
+            console.print(f"[red]{exc}[/]")
+            raise typer.Exit(code=1) from exc
+
+        cases = eval_set.limited(limit)
+        if fmt != "json":
+            suffix = ""
+            if rewrite_flag is True:
+                suffix = ", rewrite=standalone"
+            elif rewrite_flag is False:
+                suffix = ", rewrite=off"
+            console.print(
+                f"[bold]Evaluating[/] '{name}' ({provider}) on {len(cases)} case(s), "
+                f"k={top_k}{suffix} ..."
+            )
+
+        progress = _build_eval_progress(top_k, fmt)
+        try:
+            report = asyncio.run(evaluator.evaluate(eval_set, progress=progress))
+        except Exception as exc:
+            console.print(f"[red]Evaluation failed: {exc}[/]")
+            raise typer.Exit(code=1) from exc
+
+        payload = report.to_json()
+        if save:
+            _save_eval_report(save, payload)
+
+        if fmt == "json":
+            console.print_json(payload)
+            return
+
+        _render_eval_report(report, name, provider)
+
+    @app.command("eval-diff")
+    def kb_eval_diff(
+        baseline: str = typer.Argument(..., help="Baseline report JSON (from `kb eval --save`)."),
+        candidate: str = typer.Argument(..., help="Candidate report JSON (from `kb eval --save`)."),
+        fmt: str = typer.Option("rich", "--format", "-f", help="Output format: rich | json."),
+    ) -> None:
+        """Diff two `kb eval` reports to quantify a retrieval change.
+
+        Records a baseline before a settings change (e.g. turning the query-rewrite
+        layer on), a candidate after, and shows the per-metric and per-case movement.
+        """
+        from deeptutor.services.rag.eval.diff import diff_reports
+
+        base_report = _load_eval_report(baseline)
+        cand_report = _load_eval_report(candidate)
+
+        try:
+            diff = diff_reports(
+                base_report,
+                cand_report,
+                baseline_label=Path(baseline).stem,
+                candidate_label=Path(candidate).stem,
+            )
+        except ValueError as exc:
+            console.print(f"[red]{exc}[/]")
+            raise typer.Exit(code=1) from exc
+
+        if fmt == "json":
+            console.print_json(data=diff.to_dict())
+            return
+
+        for line in diff.summary_lines():
+            console.print(line)
+
+        if diff.has_regression:
+            raise typer.Exit(code=2)
+
+    # ── GitHub source commands ──────────────────────────────────────
+
+    @app.command("add-github-source")
+    def kb_add_github_source(
+        name: str = typer.Argument(..., help="KB name."),
+        repo: str = typer.Option(..., "--repo", "-r", help="GitHub repo as owner/name or URL."),
+        branch: str = typer.Option("main", "--branch", "-b", help="Branch to track."),
+        path: str = typer.Option("", "--path", "-p", help="Subdirectory prefix (e.g. docs/)."),
+        glob: str = typer.Option("*.md", "--glob", "-g", help="File glob pattern."),
+    ) -> None:
+        """Add a GitHub repo as a Markdown document source for a KB."""
+        mgr = _get_kb_manager()
+        if name not in mgr.list_knowledge_bases():
+            console.print(f"[red]Knowledge base '{name}' not found.[/]")
+            raise typer.Exit(code=1)
+        try:
+            info = mgr.add_github_source(name, repo, branch, path, glob)
+        except Exception as exc:
+            console.print(f"[red]Failed: {exc}[/]")
+            raise typer.Exit(code=1) from exc
+        console.print("[green]GitHub source added:[/]")
+        console.print_json(json.dumps(info, ensure_ascii=False))
+
+    @app.command("remove-github-source")
+    def kb_remove_github_source(
+        name: str = typer.Argument(..., help="KB name."),
+        source_id: str = typer.Option(..., "--source-id", "-s", help="Source ID to remove."),
+    ) -> None:
+        """Remove a GitHub source from a KB."""
+        mgr = _get_kb_manager()
+        ok = mgr.remove_github_source(name, source_id)
+        if ok:
+            console.print(f"[green]Removed '{source_id}'.[/]")
+        else:
+            console.print(f"[yellow]Source '{source_id}' not found.[/]")
+
+    @app.command("add-web-source")
+    def kb_add_web_source(
+        name: str = typer.Argument(..., help="KB name."),
+        url: str = typer.Option(..., "--url", "-u", help="Documentation site base URL."),
+        max_depth: int = typer.Option(3, "--max-depth", help="Crawl depth."),
+        max_pages: int = typer.Option(200, "--max-pages", help="Max pages to crawl."),
+    ) -> None:
+        """Add a documentation site URL as a source for a KB."""
+        mgr = _get_kb_manager()
+        if name not in mgr.list_knowledge_bases():
+            console.print(f"[red]Knowledge base '{name}' not found.[/]")
+            raise typer.Exit(code=1)
+        try:
+            info = mgr.add_web_source(name, url, max_depth, max_pages)
+        except Exception as exc:
+            console.print(f"[red]Failed: {exc}[/]")
+            raise typer.Exit(code=1) from exc
+        console.print("[green]Web source added:[/]")
+        console.print_json(json.dumps(info, ensure_ascii=False))
+
+    @app.command("remove-web-source")
+    def kb_remove_web_source(
+        name: str = typer.Argument(..., help="KB name."),
+        source_id: str = typer.Option(..., "--source-id", "-s", help="Source ID to remove."),
+    ) -> None:
+        """Remove a web source from a KB."""
+        mgr = _get_kb_manager()
+        removed = mgr.remove_web_source(name, source_id)
+        if removed:
+            console.print(f"[green]Removed '{source_id}'.[/]")
+        else:
+            console.print(f"[yellow]Source '{source_id}' not found.[/]")
+
+    @app.command("list-sources")
+    def kb_list_sources(
+        name: str = typer.Argument(..., help="KB name."),
+    ) -> None:
+        """List GitHub and web sources for a KB."""
+        mgr = _get_kb_manager()
+        if name not in mgr.list_knowledge_bases():
+            console.print(f"[red]Knowledge base '{name}' not found.[/]")
+            raise typer.Exit(code=1)
+
+        gh_sources = mgr.get_github_sources(name)
+        web_sources = mgr.get_web_sources(name)
+
+        if not gh_sources and not web_sources:
+            console.print("[dim]No sources for this KB.[/]")
+            return
+
+        if gh_sources:
+            table = Table(title=f"GitHub Sources - {name}")
+            table.add_column("ID", style="dim")
+            table.add_column("Repo")
+            table.add_column("Branch")
+            table.add_column("Status")
+            table.add_column("Files", justify="right")
+            for source in gh_sources:
+                table.add_row(
+                    source.get("id", ""),
+                    source.get("repo", ""),
+                    source.get("branch", ""),
+                    source.get("last_sync_status", "pending"),
+                    str(source.get("files_synced", 0)),
+                )
+            console.print(table)
+
+        if web_sources:
+            table = Table(title=f"Web Sources - {name}")
+            table.add_column("ID", style="dim")
+            table.add_column("URL")
+            table.add_column("Status")
+            table.add_column("Pages", justify="right")
+            for source in web_sources:
+                table.add_row(
+                    source.get("id", ""),
+                    source.get("url", ""),
+                    source.get("last_sync_status", "pending"),
+                    str(source.get("page_count", 0)),
+                )
+            console.print(table)
+
+    @app.command("sync")
+    def kb_sync(name: str = typer.Argument(..., help="KB name.")) -> None:
+        """Immediately sync all GitHub and web sources for a KB."""
+        mgr = _get_kb_manager()
+        if name not in mgr.list_knowledge_bases():
+            console.print(f"[red]Knowledge base '{name}' not found.[/]")
+            raise typer.Exit(code=1)
+
+        gh_sources = mgr.get_github_sources(name)
+        web_sources = mgr.get_web_sources(name)
+        if not gh_sources and not web_sources:
+            console.print("[yellow]No sources to sync.[/]")
+            raise typer.Exit(code=0)
+
+        from deeptutor.services.github_source.sync import sync_source as gh_sync
+        from deeptutor.services.web_source.sync import sync_source as web_sync
+
+        for src in gh_sources:
+            if not src.get("enabled", True):
+                continue
+            console.print(f"Syncing GitHub [bold]{src.get('repo')}[/] ...")
+            try:
+                r = asyncio.run(gh_sync(kb_name=name, source=src, base_dir=str(mgr.base_dir)))
+            except Exception as exc:
+                console.print(f"  [red]Error: {exc}[/]")
+                continue
+            if r.skipped:
+                console.print("  [dim]Up to date.[/]")
+            elif r.ok:
+                console.print(f"  [green]+{r.files_added} ~{r.files_updated} -{r.files_removed}[/]")
+            else:
+                console.print(f"  [red]Failed: {r.error}[/]")
+
+        for source in web_sources:
+            if not source.get("enabled", True):
+                continue
+            console.print(f"Crawling [bold]{source.get('url')}[/] ...")
+            try:
+                result = asyncio.run(
+                    web_sync(kb_name=name, source=source, base_dir=str(mgr.base_dir))
+                )
+            except Exception as exc:
+                console.print(f"  [red]Error: {exc}[/]")
+                continue
+            if result.ok:
+                console.print(
+                    f"  [green]+{result.pages_added} ~{result.pages_updated} "
+                    f"-{result.pages_removed} ({result.pages_unchanged} unchanged)[/]"
+                )
+            else:
+                console.print(f"  [red]Failed: {result.error}[/]")