deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/knowledge/module_service.py

@@ -0,0 +1,29 @@
+"""Background services contributed by this installed module."""
+
+
+async def start_github_sync(application_container) -> None:
+    from deeptutor.services.github_source.sync_service import get_sync_service
+
+    await get_sync_service().start()
+
+
+async def stop_github_sync(application_container) -> None:
+    from deeptutor.services.github_source.sync_service import get_sync_service
+
+    await get_sync_service().stop()
+
+
+async def start_web_source_sync(application_container) -> None:
+    from deeptutor.services.web_source.scheduler import (
+        start_web_source_sync_scheduler,
+    )
+
+    await start_web_source_sync_scheduler()
+
+
+async def stop_web_source_sync(application_container) -> None:
+    from deeptutor.services.web_source.scheduler import (
+        stop_web_source_sync_scheduler,
+    )
+
+    await stop_web_source_sync_scheduler()

deeptutor/knowledge/naming.py

@@ -0,0 +1,40 @@
+"""Knowledge-base name validation helpers."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]")
+_FORBIDDEN_CHARS = set('<>:"/\\|?*#%')
+_MAX_KB_NAME_LENGTH = 120
+
+
+def validate_knowledge_base_name(name: str) -> str:
+    """Validate and normalize a user-facing knowledge-base name.
+
+    Names may contain Unicode letters, spaces, dots, hyphens, underscores, and
+    common punctuation, but must not contain filesystem or URL-reserved
+    separators that would break KB directories or API route paths.
+    """
+    normalized = unicodedata.normalize("NFC", str(name or "")).strip()
+    if not normalized:
+        raise ValueError("Knowledge base name is required")
+    if normalized in {".", ".."}:
+        raise ValueError("Knowledge base name cannot be '.' or '..'")
+    if len(normalized) > _MAX_KB_NAME_LENGTH:
+        raise ValueError(
+            f"Knowledge base name is too long; maximum length is {_MAX_KB_NAME_LENGTH}"
+        )
+    if _CONTROL_CHARS.search(normalized):
+        raise ValueError("Knowledge base name cannot contain control characters")
+
+    forbidden = sorted(ch for ch in _FORBIDDEN_CHARS if ch in normalized)
+    if forbidden:
+        joined = " ".join(forbidden)
+        raise ValueError(
+            "Knowledge base name contains reserved characters: "
+            f"{joined}. Avoid path or URL separators such as /, \\, ?, #, and %."
+        )
+
+    return normalized

deeptutor/knowledge/progress_events.py

@@ -0,0 +1,45 @@
+"""Application ports used by knowledge indexing to report live progress.
+
+The knowledge domain owns progress facts but knows nothing about FastAPI,
+WebSockets, or the API task-stream implementation.  API adapters install the
+two callables for the lifetime of a server process; CLI/SDK use the no-op
+defaults.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+ProgressBroadcast = Callable[[str, dict[str, Any]], Awaitable[None]]
+TaskEventEmitter = Callable[[str, str, dict[str, Any]], None]
+
+_broadcast: ProgressBroadcast | None = None
+_emit_task_event: TaskEventEmitter | None = None
+
+
+def install_progress_ports(
+    *,
+    broadcast: ProgressBroadcast | None,
+    emit_task_event: TaskEventEmitter | None,
+) -> None:
+    global _broadcast, _emit_task_event
+    _broadcast = broadcast
+    _emit_task_event = emit_task_event
+
+
+async def broadcast_progress(kb_name: str, progress: dict[str, Any]) -> None:
+    if _broadcast is not None:
+        await _broadcast(kb_name, progress)
+
+
+def emit_task_progress(task_id: str, progress: dict[str, Any]) -> None:
+    if _emit_task_event is not None:
+        _emit_task_event(task_id, "progress", progress)
+
+
+__all__ = [
+    "broadcast_progress",
+    "emit_task_progress",
+    "install_progress_ports",
+]

deeptutor/knowledge/progress_tracker.py

@@ -0,0 +1,355 @@
+"""
+Progress Tracker - Tracks knowledge base initialization progress
+"""
+
+import asyncio
+from collections.abc import Callable
+from datetime import datetime
+from enum import Enum
+import json
+import logging
+from pathlib import Path
+
+from deeptutor.services.file_io import atomic_write_json
+
+# Use unified logging system
+
+_logger = logging.getLogger(__name__)
+
+
+def _logger_instance():
+    return _logger
+
+
+def render_message_template(template: str, params: dict[str, object]) -> str:
+    """Fill an i18next-style ``{{name}}`` template with *params*.
+
+    Progress lines are shown verbatim in the web log box, so they have to be
+    translatable — but the backend has no viewer language (indexing runs as a
+    detached task, and the language of record lives in the browser). So the
+    wire carries the English template plus its values and the frontend renders
+    it with ``t()``; this produces the English fallback for every other
+    consumer, from the same single string.
+    """
+    text = template
+    for name, value in params.items():
+        text = text.replace("{{" + name + "}}", str(value))
+    return text
+
+
+class ProgressStage(Enum):
+    """Initialization stage"""
+
+    INITIALIZING = "initializing"  # Initializing
+    PROCESSING_DOCUMENTS = "processing_documents"  # Processing documents
+    PROCESSING_FILE = "processing_file"  # Processing single file
+    COMPLETED = "completed"  # Completed
+    ERROR = "error"  # Error
+
+
+def visible_progress(progress: dict, kb_dir: Path) -> dict:
+    """A prepared native terminal record is visible only after publication."""
+    version = progress.get("publication_version")
+    if progress.get("stage") != "completed" or not version:
+        return progress
+    from deeptutor.services.rag.pipelines.lightrag.storage import meta_is_native_published
+
+    if isinstance(version, str) and Path(version).name == version and version not in {".", ".."}:
+        if meta_is_native_published(kb_dir / version):
+            return progress
+    return {
+        **progress,
+        "stage": "processing_documents",
+        "message": "Finalizing index publication",
+        "message_key": "Finalizing index publication",
+        "message_params": {},
+        "progress_percent": 99,
+        "index_changed": False,
+    }
+
+
+class ProgressTracker:
+    """Progress tracker"""
+
+    def __init__(self, kb_name: str, base_dir: Path):
+        self.kb_name = kb_name
+        self.base_dir = base_dir
+        self.kb_dir = base_dir / kb_name
+        self.progress_file = self.kb_dir / ".progress.json"
+        self._callbacks: list = []  # Support multiple callbacks
+        self.task_id: str | None = None  # Task ID (for log identification)
+
+    def set_callback(self, callback: Callable[[dict], None]):
+        """Set progress callback function (can be called multiple times to add multiple callbacks)"""
+        if callback not in self._callbacks:
+            self._callbacks.append(callback)
+
+    def remove_callback(self, callback: Callable[[dict], None]):
+        """Remove progress callback function"""
+        if callback in self._callbacks:
+            self._callbacks.remove(callback)
+
+    def _notify(self, progress: dict):
+        """Notify progress update (call all callbacks)"""
+        from deeptutor.runtime.mode import is_server
+
+        if is_server():
+            try:
+                from deeptutor.knowledge.progress_events import broadcast_progress
+
+                try:
+                    loop = asyncio.get_running_loop()
+                    loop.create_task(broadcast_progress(self.kb_name, progress))
+                except RuntimeError:
+                    pass
+            except (ImportError, Exception):
+                pass
+
+        for callback in self._callbacks:
+            try:
+                callback(progress)
+            except Exception as e:
+                _logger_instance().debug("Progress callback error: %s", e)
+
+    def _save_progress(self, progress: dict):
+        """Save progress to kb_config.json and local .progress.json file"""
+        # Save to kb_config.json (centralized config)
+        stored_progress = progress
+        try:
+            from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+            manager = KnowledgeBaseManager(base_dir=str(self.base_dir))
+
+            # A prepared completion remains processing until its version is published.
+            progress = visible_progress(progress, self.kb_dir)
+            stage = progress.get("stage", "")
+            if stage == "completed":
+                status = "ready"
+            elif stage == "error":
+                status = "error"
+            elif stage in [
+                "initializing",
+                "processing_documents",
+                "processing_file",
+            ]:
+                status = "processing"
+            else:
+                status = "initializing"
+
+            # Update kb_config.json with status and progress
+            manager.update_kb_status(
+                name=self.kb_name,
+                status=status,
+                progress={
+                    "stage": progress.get("stage"),
+                    "message": progress.get("message"),
+                    "percent": progress.get("progress_percent", 0),
+                    "current": progress.get("current", 0),
+                    "total": progress.get("total", 0),
+                    "file_name": progress.get("file_name"),
+                    "error": progress.get("error"),
+                    "error_code": progress.get("error_code"),
+                    "retryable": progress.get("retryable"),
+                    "timestamp": progress.get("timestamp"),
+                    "task_id": progress.get("task_id"),
+                    "indexed_count": progress.get("indexed_count"),
+                    "index_changed": progress.get("index_changed"),
+                    "index_action": progress.get("index_action"),
+                    # Carried alongside the rendered English so a page reload
+                    # can still translate the last line it shows.
+                    "message_key": progress.get("message_key"),
+                    "message_params": progress.get("message_params"),
+                },
+            )
+        except Exception as e:
+            _logger_instance().warning("Failed to save progress to kb_config.json: %s", e)
+
+        # Persist the last seen progress snapshot so websocket subscribers and
+        # page reloads can recover the live state without relying on in-memory callbacks.
+        try:
+            atomic_write_json(self.progress_file, stored_progress)
+        except Exception as e:
+            _logger_instance().warning(
+                "Failed to persist progress snapshot for '%s': %s", self.kb_name, e
+            )
+
+    def update(
+        self,
+        stage: ProgressStage,
+        message: str = "",
+        current: int = 0,
+        total: int = 0,
+        file_name: str = "",
+        error: str | None = None,
+        error_code: str | None = None,
+        retryable: bool | None = None,
+        indexed_count: int | None = None,
+        index_changed: bool | None = None,
+        index_action: str | None = None,
+        message_key: str | None = None,
+        message_params: dict[str, object] | None = None,
+        publication_version: str | None = None,
+    ):
+        """Update progress.
+
+        Pass ``message_key`` (an English ``{{name}}`` template) plus
+        ``message_params`` instead of a pre-formatted ``message`` so the web log
+        box can translate the line; ``message`` is then rendered from the same
+        template for every consumer that has no i18n of its own.
+        """
+        params = message_params or {}
+        if message_key and not message:
+            message = render_message_template(message_key, params)
+        progress: dict[str, object] = {
+            "kb_name": self.kb_name,
+            "task_id": self.task_id,
+            "stage": stage.value,
+            "message": message,
+            "current": current,
+            "total": total,
+            "file_name": file_name,
+            "progress_percent": int(current / total * 100) if total > 0 else 0,
+            "timestamp": datetime.now().isoformat(),
+        }
+        if indexed_count is not None:
+            progress["indexed_count"] = indexed_count
+        if index_changed is not None:
+            progress["index_changed"] = index_changed
+        if index_action:
+            progress["index_action"] = index_action
+        if message_key:
+            progress["message_key"] = message_key
+            progress["message_params"] = params
+
+        if error:
+            progress["error"] = error
+            progress["stage"] = ProgressStage.ERROR.value
+        if error_code:
+            progress["error_code"] = error_code
+        if retryable is not None:
+            progress["retryable"] = retryable
+
+        if publication_version is not None:
+            progress["publication_version"] = publication_version
+        persisted_progress = progress
+        progress = visible_progress(progress, self.kb_dir)
+        message = str(progress["message"])
+
+        # Output to logger (terminal and log file)
+        try:
+            logger = _logger_instance()
+            prefix = f"[{self.task_id}]" if self.task_id else ""
+
+            if total > 0:
+                percent = progress["progress_percent"]
+                progress_msg = f"{prefix} {message} ({current}/{total}, {percent}%)"
+                if file_name:
+                    progress_msg += f" - File: {file_name}"
+            else:
+                progress_msg = f"{prefix} {message}"
+                if file_name:
+                    progress_msg += f" - File: {file_name}"
+
+            if error:
+                logger.error(f"{progress_msg} - Error: {error}")
+            else:
+                logger.info(progress_msg)
+        except Exception:
+            # If unified logging fails unexpectedly, use stdlib logger as fallback.
+            fallback_logger = logging.getLogger("deeptutor.ProgressTracker")
+            prefix = f"[{self.task_id}]" if self.task_id else ""
+            fallback_logger.warning(
+                "%s [ProgressTracker] %s (%s/%s)",
+                prefix,
+                message,
+                current,
+                total if total > 0 else "?",
+            )
+            if error:
+                fallback_logger.error("%s [ProgressTracker] Error: %s", prefix, error)
+
+        self._save_progress(persisted_progress)
+
+        if self.task_id:
+            try:
+                from deeptutor.knowledge.progress_events import emit_task_progress
+
+                emit_task_progress(self.task_id, progress)
+            except Exception as e:
+                _logger_instance().debug("Failed to emit task progress event: %s", e)
+
+        self._notify(progress)
+
+    def verify_terminal(
+        self,
+        *,
+        current: int,
+        total: int,
+        indexed_count: int,
+        index_action: str,
+        publication_version: str | None = None,
+    ) -> None:
+        """Fail publication if either required terminal persistence write was lost."""
+        expected = {
+            "stage": "completed",
+            "task_id": self.task_id,
+            "current": current,
+            "total": total,
+            "indexed_count": indexed_count,
+            "index_changed": True,
+            "index_action": index_action,
+            "progress_percent": 100,
+        }
+        if publication_version is not None:
+            expected["publication_version"] = publication_version
+        try:
+            with self.progress_file.open(encoding="utf-8") as handle:
+                persisted = json.load(handle)
+            if not isinstance(persisted, dict) or any(
+                persisted.get(key) != value for key, value in expected.items()
+            ):
+                raise ValueError("Terminal progress does not match this task.")
+            from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+            manager = KnowledgeBaseManager(base_dir=str(self.base_dir))
+            entry = manager.get_kb_entry(self.kb_name) or {}
+            expected_status = "processing" if publication_version else "ready"
+            if entry.get("status") != expected_status:
+                raise ValueError("Knowledge-base status was not persisted.")
+            if publication_version and (entry.get("progress") or {}).get("task_id") != self.task_id:
+                raise ValueError("Knowledge-base progress belongs to another task.")
+        except Exception as exc:
+            raise RuntimeError("Index terminal state was not persisted.") from exc
+
+    def get_progress(self) -> dict | None:
+        """Get current progress"""
+        if self.progress_file.exists():
+            try:
+                with open(self.progress_file, encoding="utf-8") as f:
+                    return visible_progress(json.load(f), self.kb_dir)
+            except Exception as e:
+                _logger_instance().debug(f"Failed to read progress file for '{self.kb_name}': {e}")
+
+        try:
+            from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+            manager = KnowledgeBaseManager(base_dir=str(self.base_dir))
+            status = manager.get_kb_status(self.kb_name)
+            if status and status.get("progress"):
+                return status.get("progress")
+        except Exception as e:
+            _logger_instance().debug(
+                "Failed to recover progress snapshot from kb_config for '%s': %s",
+                self.kb_name,
+                e,
+            )
+
+        return None
+
+    def clear(self):
+        """Clear progress file"""
+        if self.progress_file.exists():
+            try:
+                self.progress_file.unlink()
+            except Exception as e:
+                _logger_instance().debug(f"Failed to clear progress file for '{self.kb_name}': {e}")

deeptutor/knowledge/staging.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+"""Shared staging rule for putting source documents into a KB's ``raw/`` tree.
+
+Both ways of filling a knowledge base — creating one from a docs directory, and
+adding documents to an existing one — flatten sources to their basename, so two
+documents that share a name want the same path. Only one place should answer
+what happens then, or the two entry points drift apart: the incremental path
+kept both files while the create path copied straight onto the taken name and
+indexed only the last one, silently losing the rest (#2041).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import itertools
+from pathlib import Path
+import shutil
+
+
+def content_hash(file_path: Path) -> str:
+    """SHA-256 of a file, streamed in blocks so large documents stay off memory."""
+    sha256_hash = hashlib.sha256()
+    with open(file_path, "rb") as handle:
+        for block in iter(lambda: handle.read(65536), b""):
+            sha256_hash.update(block)
+    return sha256_hash.hexdigest()
+
+
+def non_colliding(dest_path: Path) -> Path:
+    """A free sibling name for a *different* document that wants a taken one.
+
+    Two files can legitimately share a name and hold different content — a
+    docs tree with a README.md per folder is the ordinary case, and a book
+    series saved with one ``full.md`` per volume is the same shape. Dropping
+    the later one loses a document the user asked to index, silently.
+    """
+    if not dest_path.exists():
+        return dest_path
+    for index in itertools.count(2):
+        candidate = dest_path.with_name(f"{dest_path.stem} ({index}){dest_path.suffix}")
+        if not candidate.exists():
+            return candidate
+    raise AssertionError("unreachable")  # pragma: no cover
+
+
+def stage_source(
+    source_path: Path,
+    dest_path: Path,
+    *,
+    source_hash: str | None = None,
+) -> tuple[Path, bool]:
+    """Copy *source_path* to *dest_path* without replacing a different document.
+
+    Returns the path the document actually landed on, and whether a copy was
+    made. A name already holding the *same* content is reused rather than
+    written twice; a name holding a different document is left alone and the
+    newcomer takes a free sibling name, so both get indexed.
+    """
+    if dest_path.exists():
+        current = source_hash or content_hash(source_path)
+        if current == content_hash(dest_path):
+            return dest_path, False
+    staged = non_colliding(dest_path)
+    staged.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(source_path, staged)
+    return staged, True
+
+
+__all__ = ["content_hash", "non_colliding", "stage_source"]