deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/knowledge/add_documents.py

@@ -0,0 +1,734 @@
+#!/usr/bin/env python
+"""Incrementally add documents to an existing knowledge base."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime
+import json
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, List, Optional
+
+from deeptutor.knowledge.staging import content_hash, stage_source
+from deeptutor.services.config import resolve_llm_runtime_config
+from deeptutor.services.file_io import atomic_write_json
+from deeptutor.services.rag.factory import (
+    DEFAULT_PROVIDER,
+    LIGHTRAG_PROVIDER,
+    has_ready_provider_index,
+    normalize_provider_name,
+)
+from deeptutor.services.rag.index_probe import provider_failure_summary
+from deeptutor.services.rag.index_versioning import list_kb_versions
+from deeptutor.services.rag.provider_binding import resolve_bound_provider
+from deeptutor.services.rag.service import RAGService
+from deeptutor.services.setup.data_volume import (
+    DataVolumePermissionError,
+    ensure_data_volume_writable,
+    format_data_volume_permission_error,
+)
+from deeptutor.utils.file_types import FileTypeRouter
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_BASE_DIR = "./data/knowledge_bases"
+
+if TYPE_CHECKING:
+    from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+
+@dataclass(frozen=True)
+class DocumentIndexFailure:
+    """One file that could not be added to the provider index."""
+
+    file_path: Path
+    error: str
+
+
+@dataclass(frozen=True)
+class DocumentIndexResult:
+    """Structured incremental-index result.
+
+    A task can no longer infer success from "no exception": every staged file is
+    explicitly accounted for as either processed or failed.
+    """
+
+    processed_files: list[Path]
+    failures: list[DocumentIndexFailure]
+
+    @property
+    def processed_count(self) -> int:
+        return len(self.processed_files)
+
+    @property
+    def failed_count(self) -> int:
+        return len(self.failures)
+
+    @property
+    def has_failures(self) -> bool:
+        return bool(self.failures)
+
+    def failure_summary(self, *, limit: int = 3) -> str:
+        if not self.failures:
+            return ""
+        shown = [f"{failure.file_path.name}: {failure.error}" for failure in self.failures[:limit]]
+        remaining = len(self.failures) - len(shown)
+        if remaining > 0:
+            shown.append(f"... and {remaining} more file(s)")
+        return "; ".join(shown)
+
+
+@dataclass(frozen=True)
+class RawDocumentRemoval:
+    """Outcome of removing one raw document from a knowledge base."""
+
+    rel_path: str
+    was_indexed: bool
+
+
+def _read_metadata(metadata_file: Path) -> dict:
+    """Load a KB's metadata.json, returning {} when absent or unreadable."""
+    if not metadata_file.exists():
+        return {}
+    try:
+        with open(metadata_file, "r", encoding="utf-8") as f:
+            data = json.load(f)
+    except Exception:
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def _write_metadata(metadata_file: Path, metadata: dict) -> None:
+    """Persist a KB's metadata.json atomically (pretty-printed, non-ASCII preserved)."""
+    atomic_write_json(metadata_file, metadata)
+
+
+def _raw_hash_key(file_path: Path, raw_dir: Path) -> str:
+    """Stable ``file_hashes`` key for a staged file.
+
+    The key is the POSIX path relative to ``raw/`` (so folder uploads keep
+    distinct entries), falling back to the basename for anything that resolves
+    outside ``raw/``. Indexing and removal MUST share this rule, otherwise a
+    removed file's hash record would leak and silently skip a later re-add.
+    """
+    try:
+        return file_path.resolve().relative_to(raw_dir.resolve()).as_posix()
+    except ValueError:
+        return file_path.name
+
+
+def hashes_for_indexed_files(file_paths: list[str], raw_dir: Path) -> dict[str, str]:
+    """Hash source files already confirmed in an index, keyed by their raw path."""
+    return {
+        _raw_hash_key(Path(file_path), raw_dir): content_hash(Path(file_path))
+        for file_path in file_paths
+    }
+
+
+def remove_raw_document(kb_dir: Path, file_path: Path) -> RawDocumentRemoval:
+    """Delete one staged raw file and drop its indexed-hash record.
+
+    Deliberately decoupled from :class:`DocumentAdder` (whose constructor
+    requires a ready provider index): a KB stuck in an *error* state often has
+    no index at all, yet its raw/ files must still be removable so the user can
+    drop an unparseable document instead of deleting and rebuilding the whole
+    KB. Vectors are not touched here — the providers index whole KBs, so any
+    vectors from an already-indexed file are cleared by a subsequent re-index,
+    which the returned ``was_indexed`` flag lets the caller surface.
+
+    ``file_path`` must already be sandbox-resolved under the KB's raw/ dir.
+    """
+    raw_dir = kb_dir / "raw"
+    hash_key = _raw_hash_key(file_path, raw_dir)
+
+    target = file_path.resolve()
+    if target.exists():
+        target.unlink()
+
+    metadata_file = kb_dir / "metadata.json"
+    metadata = _read_metadata(metadata_file)
+    hashes = metadata.get("file_hashes")
+    was_indexed = isinstance(hashes, dict) and hash_key in hashes
+    if was_indexed:
+        del hashes[hash_key]
+        _write_metadata(metadata_file, metadata)
+
+    return RawDocumentRemoval(rel_path=hash_key, was_indexed=was_indexed)
+
+
+class DocumentAdder:
+    """Stage and index new files through a KB's bound RAG provider."""
+
+    def __init__(
+        self,
+        kb_name: str,
+        base_dir: str = DEFAULT_BASE_DIR,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        progress_tracker=None,
+        rag_provider: str | None = None,
+        accepted_indexing_snapshot=None,
+    ):
+        self.kb_name = kb_name
+        self.base_dir = Path(base_dir)
+        self.kb_dir = self.base_dir / kb_name
+
+        if not self.kb_dir.exists():
+            raise ValueError(f"Knowledge base does not exist: {kb_name}")
+
+        self.raw_dir = self.kb_dir / "raw"
+        self.llamaindex_storage_dir = self.kb_dir / "llamaindex_storage"
+        self.legacy_rag_storage_dir = self.kb_dir / "rag_storage"
+        self.metadata_file = self.kb_dir / "metadata.json"
+
+        # Fail on UID-mismatched / unwritable volumes before the "not initialized"
+        # check, which otherwise hides Unraid bind-mount permission errors.
+        ensure_data_volume_writable(self.kb_dir)
+
+        # Incremental adds must use the engine DeepTutor has bound to this KB. An
+        # explicit rag_provider (from the API, already matched against the KB)
+        # wins; otherwise use the shared binding resolver.
+        if rag_provider:
+            self.rag_provider = normalize_provider_name(rag_provider)
+        else:
+            self.rag_provider = self._provider_from_binding()
+
+        has_provider_index = has_ready_provider_index(self.kb_dir, self.rag_provider)
+
+        if (
+            self.rag_provider == DEFAULT_PROVIDER
+            and not has_provider_index
+            and self.legacy_rag_storage_dir.exists()
+        ):
+            raise ValueError(
+                f"Knowledge base '{kb_name}' uses legacy index format and requires reindex before incremental add"
+            )
+
+        # Both pipelines create their first index on add; existing broken versions
+        # still require reindex instead of being silently replaced (#1458).
+        versions = list_kb_versions(self.kb_dir)
+        allows_bootstrap = (
+            self.rag_provider
+            in {
+                DEFAULT_PROVIDER,
+                LIGHTRAG_PROVIDER,
+            }
+            and not versions
+        )
+        if not has_provider_index and not allows_bootstrap:
+            if versions:
+                summary = provider_failure_summary(self.kb_dir, self.rag_provider)
+                raise ValueError(
+                    f"Knowledge base has no ready {self.rag_provider} index; reindex required: "
+                    f"{summary or 'stored index version is incomplete'}"
+                )
+            raise ValueError(f"Knowledge base not initialized ({self.rag_provider}): {kb_name}")
+
+        self.accepted_indexing_snapshot = accepted_indexing_snapshot
+        if self.rag_provider == LIGHTRAG_PROVIDER and self.accepted_indexing_snapshot is None:
+            from deeptutor.services.rag.pipelines.lightrag.indexing_policy import (
+                bind_target,
+                resolve_write_snapshot,
+            )
+
+            self.accepted_indexing_snapshot = bind_target(
+                resolve_write_snapshot(
+                    self.kb_dir, base_dir=str(self.base_dir), kb_name=self.kb_name
+                ),
+                self.kb_dir,
+            )
+
+        self.api_key = api_key
+        self.base_url = base_url
+        self.progress_tracker = progress_tracker
+
+        self.raw_dir.mkdir(parents=True, exist_ok=True)
+
+    def _provider_from_binding(self) -> str:
+        return resolve_bound_provider(self.base_dir, self.kb_name)
+
+    def _get_file_hash(self, file_path: Path) -> str:
+        return content_hash(file_path)
+
+    def get_ingested_hashes(self) -> dict[str, str]:
+        if self.metadata_file.exists():
+            try:
+                with open(self.metadata_file, "r", encoding="utf-8") as f:
+                    data = json.load(f)
+                return data.get("file_hashes", {})
+            except Exception:
+                return {}
+        return {}
+
+    def add_documents(
+        self,
+        source_files: List[str],
+        allow_duplicates: bool = False,
+        source_root: str | Path | None = None,
+    ) -> List[Path]:
+        """Validate and stage files into raw/ before indexing.
+
+        ``source_root``, when given, is the root a caller is syncing from
+        (e.g. a linked folder). Any source file found under it is staged at
+        the same path *relative to that root*, instead of being flattened to
+        its bare filename: without this, every externally-sourced subfolder
+        collapses into raw/'s top level and same-named files in different
+        subfolders collide.
+        """
+        logger.info(f"Validating documents for '{self.kb_name}'...")
+
+        ingested_hashes = self.get_ingested_hashes()
+        files_to_process: list[Path] = []
+        resolved_source_root = Path(source_root).resolve() if source_root else None
+
+        for source in source_files:
+            source_path = Path(source)
+            if not source_path.exists() or not source_path.is_file():
+                logger.warning(f"Missing file: {source}")
+                continue
+
+            current_hash = self._get_file_hash(source_path)
+            if current_hash in ingested_hashes.values() and not allow_duplicates:
+                logger.info(f"Skipped (content already indexed): {source_path.name}")
+                continue
+
+            resolved_source = source_path.resolve()
+
+            # Files already saved under raw/ (e.g. by the upload route, possibly
+            # inside a folder) are indexed in place — never flattened to the
+            # basename — so the uploaded folder structure is preserved verbatim.
+            if resolved_source.is_relative_to(self.raw_dir.resolve()):
+                files_to_process.append(source_path)
+                continue
+
+            if resolved_source_root and resolved_source.is_relative_to(resolved_source_root):
+                dest_path = self.raw_dir / resolved_source.relative_to(resolved_source_root)
+            else:
+                dest_path = self.raw_dir / source_path.name
+
+            staged_path, copied = stage_source(source_path, dest_path, source_hash=current_hash)
+            if copied:
+                logger.info(f"Staged to raw: {staged_path.relative_to(self.raw_dir).as_posix()}")
+            else:
+                logger.info(f"Recovering staged file: {source_path.name}")
+            files_to_process.append(staged_path)
+
+        return files_to_process
+
+    async def process_new_documents(self, new_files: List[Path]) -> DocumentIndexResult:
+        """Index staged files via the KB's bound provider."""
+        if not new_files:
+            return DocumentIndexResult(processed_files=[], failures=[])
+
+        if self.rag_provider == LIGHTRAG_PROVIDER:
+            return await self._process_lightrag_batch(new_files)
+
+        rag_service = RAGService(kb_base_dir=str(self.base_dir), provider=self.rag_provider)
+        processed_files: list[Path] = []
+        failures: list[DocumentIndexFailure] = []
+        total_files = len(new_files)
+
+        for idx, doc_file in enumerate(new_files, 1):
+            try:
+                if self.progress_tracker is not None:
+                    from deeptutor.knowledge.progress_tracker import ProgressStage
+
+                    self.progress_tracker.update(
+                        ProgressStage.PROCESSING_FILE,
+                        message_key="Indexing {{name}}",
+                        message_params={"name": doc_file.name},
+                        current=len(processed_files),
+                        total=total_files,
+                    )
+
+                success = await rag_service.add_documents(self.kb_name, [str(doc_file)])
+                if success:
+                    processed_files.append(doc_file)
+                    # Re-hashes the whole file, so it stays off the event loop:
+                    # this runs inside a FastAPI BackgroundTasks entry, which is
+                    # awaited on the loop and would otherwise stall unrelated
+                    # requests once per indexed file (#777).
+                    try:
+                        await asyncio.to_thread(self._record_successful_hash, doc_file)
+                    except Exception:
+                        if self.rag_provider != LIGHTRAG_PROVIDER:
+                            raise
+                        logger.warning(
+                            "LightRAG published '%s' before its duplicate-detection hash "
+                            "was recorded",
+                            doc_file.name,
+                            exc_info=True,
+                        )
+                    logger.info(f"Processed: {doc_file.name}")
+                    if self.progress_tracker is not None:
+                        self.progress_tracker.update(
+                            ProgressStage.PROCESSING_FILE,
+                            message_key="Indexed {{name}}",
+                            message_params={"name": doc_file.name},
+                            current=len(processed_files),
+                            total=total_files,
+                        )
+                else:
+                    error = "Provider returned failure without details."
+                    failures.append(DocumentIndexFailure(doc_file, error))
+                    logger.error(f"Failed to index: {doc_file.name}")
+            except PermissionError as e:
+                logger.exception("Permission denied while indexing %s: %s", doc_file.name, e)
+                raise DataVolumePermissionError(
+                    format_data_volume_permission_error(self.kb_dir, cause=e)
+                ) from e
+            except Exception as e:
+                logger.exception(f"Failed {doc_file.name}: {e}")
+                failures.append(DocumentIndexFailure(doc_file, str(e)))
+
+        return DocumentIndexResult(processed_files=processed_files, failures=failures)
+
+    async def _process_lightrag_batch(self, files: List[Path]) -> DocumentIndexResult:
+        """Hold native write ownership across the entire accepted batch."""
+        from deeptutor.services.rag.pipelines.lightrag.pipeline import LightRagBatchError
+
+        def prepare_publication(root: Path) -> None:
+            from deeptutor.knowledge.progress_tracker import ProgressStage, ProgressTracker
+
+            tracker = self.progress_tracker or ProgressTracker(self.kb_name, self.base_dir)
+            tracker.update(
+                ProgressStage.COMPLETED,
+                message="Document indexing complete",
+                current=len(files),
+                total=len(files),
+                indexed_count=len(files),
+                index_changed=True,
+                index_action="upload",
+                publication_version=root.name,
+            )
+            tracker.verify_terminal(
+                current=len(files),
+                total=len(files),
+                indexed_count=len(files),
+                index_action="upload",
+                publication_version=root.name,
+            )
+
+        service = RAGService(kb_base_dir=str(self.base_dir), provider=LIGHTRAG_PROVIDER)
+        try:
+            success = await service.add_documents(
+                self.kb_name,
+                [str(path) for path in files],
+                accepted_indexing_snapshot=self.accepted_indexing_snapshot,
+                before_publish=prepare_publication,
+            )
+            if not success:
+                raise RuntimeError("LightRAG returned failure without details.")
+            processed = files
+            failures = []
+        except LightRagBatchError as exc:
+            processed = [path for path in files if path.name in exc.outcome.processed]
+            failures = [
+                DocumentIndexFailure(path, str(exc)) for path in files if path not in processed
+            ]
+        except Exception as exc:
+            return DocumentIndexResult([], [DocumentIndexFailure(path, str(exc)) for path in files])
+        for path in processed:
+            try:
+                await asyncio.to_thread(self._record_successful_hash, path)
+            except Exception:
+                logger.warning(
+                    "Indexed document hash could not be recorded: %s", path.name, exc_info=True
+                )
+        return DocumentIndexResult(processed, failures)
+
+    def _record_successful_hash(self, file_path: Path) -> None:
+        file_hash = self._get_file_hash(file_path)
+        metadata = _read_metadata(self.metadata_file)
+        hash_key = _raw_hash_key(file_path, self.raw_dir)
+        metadata.setdefault("file_hashes", {})[hash_key] = file_hash
+        _write_metadata(self.metadata_file, metadata)
+
+    def update_metadata(self, added_count: int) -> None:
+        """Update metadata after incremental add."""
+        metadata: dict = {}
+        if self.metadata_file.exists():
+            try:
+                with open(self.metadata_file, "r", encoding="utf-8") as f:
+                    metadata = json.load(f)
+            except Exception:
+                metadata = {}
+
+        metadata["rag_provider"] = self.rag_provider
+        metadata["needs_reindex"] = False
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        metadata["last_updated"] = timestamp
+        if added_count > 0:
+            metadata["last_indexed_at"] = timestamp
+            metadata["last_indexed_count"] = added_count
+            metadata["last_indexed_action"] = "upload"
+
+        history = metadata.get("update_history", [])
+        history.append(
+            {
+                "timestamp": metadata["last_updated"],
+                "action": "incremental_add",
+                "count": added_count,
+                "provider": self.rag_provider,
+            }
+        )
+        metadata["update_history"] = history
+
+        _write_metadata(self.metadata_file, metadata)
+
+
+async def _bootstrap_index_from_files(
+    kb_name: str,
+    source_files: list[str],
+    base_dir: str,
+    manager: "KnowledgeBaseManager",
+) -> int:
+    """Create a fresh index for an empty KB from the given source files.
+
+    Called when :class:`DocumentAdder` rejects an add because the KB has no
+    existing index (it was created empty, e.g. via the no-files fast path or
+    a web/GitHub source sync before any documents were indexed). Uses
+    :meth:`RAGService.initialize` to build the index in one batch, then
+    records file hashes so subsequent incremental adds can detect duplicates.
+    """
+    rag_service = RAGService(kb_base_dir=base_dir)
+    kb_dir = Path(base_dir) / kb_name
+    raw_dir = kb_dir / "raw"
+    metadata_file = kb_dir / "metadata.json"
+
+    requested = {str(Path(path).resolve()): str(Path(path)) for path in source_files}
+    confirmed: set[str] = set()
+
+    def on_indexed_file(paths: list[str]) -> None:
+        for path in paths:
+            canonical = str(Path(path).resolve())
+            if canonical in requested:
+                confirmed.add(canonical)
+
+    success = await rag_service.initialize(
+        kb_name=kb_name,
+        file_paths=source_files,
+        indexed_file_callback=on_indexed_file,
+    )
+    if not success:
+        raise RuntimeError(
+            f"Failed to initialize index for KB '{kb_name}' from {len(source_files)} file(s)"
+        )
+
+    indexed_files = [requested[path] for path in sorted(confirmed)]
+    indexed = len(indexed_files)
+    provider = rag_service._resolve_provider(kb_name)
+    try:
+        # Record hashes so future syncs detect unchanged files.
+        metadata = _read_metadata(metadata_file)
+        metadata.setdefault("file_hashes", {}).update(
+            hashes_for_indexed_files(indexed_files, raw_dir)
+        )
+        metadata["rag_provider"] = provider
+        metadata["needs_reindex"] = False
+        ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        metadata["last_updated"] = ts
+        metadata["last_indexed_at"] = ts
+        metadata["last_indexed_count"] = indexed
+        metadata["last_indexed_action"] = "create"
+        _write_metadata(metadata_file, metadata)
+
+        manager.update_kb_status(
+            name=kb_name,
+            status="ready",
+            progress={
+                "stage": "completed",
+                "message": f"Initialized index with {indexed} file(s).",
+                "percent": 100,
+                "current": indexed,
+                "total": max(indexed, 1),
+                "file_name": "",
+                "error": None,
+                "timestamp": datetime.now().isoformat(),
+                "indexed_count": indexed,
+                "index_changed": True,
+                "index_action": "create",
+            },
+        )
+    except Exception:
+        if provider != LIGHTRAG_PROVIDER:
+            raise
+        logger.warning(
+            "LightRAG bootstrap for '%s' published before bookkeeping failed",
+            kb_name,
+            exc_info=True,
+        )
+    logger.info("Bootstrapped index for empty KB '%s' with %d file(s)", kb_name, indexed)
+    return indexed
+
+
+async def add_documents(
+    kb_name: str,
+    source_files: list[str],
+    base_dir: str = DEFAULT_BASE_DIR,
+    api_key: Optional[str] = None,
+    base_url: Optional[str] = None,
+    allow_duplicates: bool = False,
+    accepted_indexing_snapshot=None,
+) -> int:
+    """Convenience function used by CLI wrappers."""
+    from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+    manager = KnowledgeBaseManager(base_dir=base_dir)
+    published_count = 0
+    try:
+        manager.update_kb_status(
+            name=kb_name,
+            status="processing",
+            progress={
+                "stage": "processing_documents",
+                "message": "Processing uploaded documents...",
+                "percent": 0,
+                "current": 0,
+                "total": max(len(source_files), 1),
+                "file_name": "",
+                "error": None,
+                "timestamp": datetime.now().isoformat(),
+            },
+        )
+
+        try:
+            adder = DocumentAdder(
+                kb_name=kb_name,
+                base_dir=base_dir,
+                api_key=api_key,
+                base_url=base_url,
+                accepted_indexing_snapshot=accepted_indexing_snapshot,
+            )
+        except ValueError as exc:
+            if "not initialized" not in str(exc).lower() or not source_files:
+                raise
+            # Empty KB (created without initial documents): bootstrap the
+            # index from these files instead of erroring.
+            return await _bootstrap_index_from_files(
+                kb_name=kb_name,
+                source_files=source_files,
+                base_dir=base_dir,
+                manager=manager,
+            )
+
+        new_files = adder.add_documents(source_files, allow_duplicates=allow_duplicates)
+        if not new_files:
+            manager.update_kb_status(
+                name=kb_name,
+                status="ready",
+                progress={
+                    "stage": "completed",
+                    "message": "No new unique documents to process.",
+                    "percent": 100,
+                    "current": 1,
+                    "total": 1,
+                    "file_name": "",
+                    "error": None,
+                    "timestamp": datetime.now().isoformat(),
+                },
+            )
+            return 0
+        result = await adder.process_new_documents(new_files)
+        if result.has_failures:
+            raise RuntimeError(
+                f"Failed to index {result.failed_count}/{len(new_files)} file(s): "
+                f"{result.failure_summary()}"
+            )
+        if adder.rag_provider == LIGHTRAG_PROVIDER:
+            published_count = result.processed_count
+        adder.update_metadata(result.processed_count)
+
+        manager.update_kb_status(
+            name=kb_name,
+            status="ready",
+            progress={
+                "stage": "completed",
+                "message": f"Successfully processed {result.processed_count} files!",
+                "percent": 100,
+                "current": result.processed_count,
+                "total": max(len(new_files), 1),
+                "file_name": "",
+                "error": None,
+                "timestamp": datetime.now().isoformat(),
+                "indexed_count": result.processed_count,
+                "index_changed": result.processed_count > 0,
+                "index_action": "upload",
+            },
+        )
+        return result.processed_count
+    except Exception as exc:
+        if published_count:
+            logger.warning(
+                "LightRAG append for '%s' published before bookkeeping failed",
+                kb_name,
+                exc_info=True,
+            )
+            return published_count
+        manager.update_kb_status(
+            name=kb_name,
+            status="error",
+            progress={
+                "stage": "error",
+                "message": "Document upload failed",
+                "percent": 0,
+                "current": 0,
+                "total": max(len(source_files), 1),
+                "file_name": "",
+                "error": str(exc),
+                "timestamp": datetime.now().isoformat(),
+            },
+        )
+        raise
+
+
+async def main() -> None:
+    try:
+        llm_config = resolve_llm_runtime_config()
+        default_api_key = llm_config.api_key
+        default_base_url = llm_config.effective_url
+    except Exception:
+        default_api_key = ""
+        default_base_url = ""
+
+    parser = argparse.ArgumentParser(description="Incrementally add documents to a KB")
+    parser.add_argument("kb_name", help="KB Name")
+    parser.add_argument("--docs", nargs="+", help="Files")
+    parser.add_argument("--docs-dir", help="Directory")
+    parser.add_argument("--base-dir", default=DEFAULT_BASE_DIR)
+    parser.add_argument("--api-key", default=default_api_key)
+    parser.add_argument("--base-url", default=default_base_url)
+    parser.add_argument("--allow-duplicates", action="store_true")
+
+    args = parser.parse_args()
+    doc_files: list[str] = []
+    if args.docs:
+        doc_files.extend(args.docs)
+    if args.docs_dir:
+        p = Path(args.docs_dir)
+        doc_files.extend(str(f) for f in FileTypeRouter.collect_supported_files(p))
+
+    if not doc_files:
+        logger.error("No documents provided.")
+        return
+
+    processed_count = await add_documents(
+        kb_name=args.kb_name,
+        source_files=doc_files,
+        base_dir=args.base_dir,
+        api_key=args.api_key,
+        base_url=args.base_url,
+        allow_duplicates=args.allow_duplicates,
+    )
+
+    if processed_count:
+        logger.info(f"Done! Successfully added {processed_count} documents.")
+    else:
+        logger.info("No new unique documents to add.")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())