deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/rag/pipelines/lightrag/storage.py

@@ -0,0 +1,338 @@
+"""On-disk layout for a LightRAG-backed knowledge base.
+
+Like the GraphRAG/PageIndex pipelines, a LightRAG KB keeps a self-contained
+store inside the KB's flat ``version-N`` directory (reused from
+``index_versioning`` with a ``None`` signature). That dir is LightRAG's
+``working_dir``: LightRAG writes its KV stores, vector DBs and the knowledge
+graph there::
+
+    <kb_dir>/version-N/
+        kv_store_*.json
+        vdb_*.json
+        graph_chunk_entity_relation.graphml
+        meta.json            # synthetic "ready" marker (see write_meta)
+
+The synthetic ``meta.json`` makes the existing "is this KB initialised?" and
+index-versions UI checks treat a LightRAG KB as ready without teaching the
+manager about LightRAG internals.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from deeptutor.services.embedding.config import EmbeddingConfig
+    from deeptutor.services.rag.index_versioning import EmbeddingSignature
+
+from deeptutor.services.file_io import atomic_write_json
+
+logger = logging.getLogger(__name__)
+
+META_FILENAME = "meta.json"
+PROVIDER = "lightrag"
+ADAPTER_SCHEMA = 2
+PUBLISHED_STATE = "published"
+
+# Glob patterns LightRAG writes once it has actually built chunk/vector data.
+# A graphml file alone is not enough: LightRAG creates an empty graph at startup
+# before any document is successfully processed.
+_OUTPUT_GLOBS = ("vdb_*.json", "kv_store_text_chunks.json")
+_DOC_STATUS_FILENAME = "kv_store_doc_status.json"
+_SUCCESS_STATUSES = {"processed", "completed", "done", "success", "indexed"}
+_FAILED_STATUSES = {"failed", "error"}
+
+
+def working_dir(root_dir: Path) -> Path:
+    """LightRAG's working dir == the version-N root."""
+    return Path(root_dir)
+
+
+def _store_root(root_dir: Path) -> Path:
+    """Resolve native workspace storage while retaining legacy flat reads."""
+    root = Path(root_dir)
+    if (root / _DOC_STATUS_FILENAME).exists() or any(root.glob("vdb_*.json")):
+        return root
+    meta = _read_meta(root)
+    workspace = str((meta or {}).get("workspace") or "").strip()
+    if not workspace:
+        from .engine import workspace_for
+
+        workspace = workspace_for(root)
+    candidate = root / workspace
+    return candidate if candidate.is_dir() else root
+
+
+def has_output(root_dir: Path | None) -> bool:
+    """True when LightRAG has at least one successfully indexed document."""
+    if root_dir is None:
+        return False
+    root = _store_root(Path(root_dir))
+    if not root.is_dir():
+        return False
+
+    status_signal = _doc_status_has_success(root)
+    if status_signal is not None:
+        return status_signal
+
+    for pattern in _OUTPUT_GLOBS:
+        for path in root.glob(pattern):
+            try:
+                if path.is_file() and path.stat().st_size > 2:
+                    return True
+            except OSError:
+                continue
+    return False
+
+
+def _doc_status_has_success(root_dir: Path) -> bool | None:
+    payload = _read_doc_status(root_dir)
+    if not payload:
+        return None
+
+    saw_failure = False
+    for item in payload.values():
+        if not isinstance(item, dict):
+            continue
+        chunks = item.get("chunks_list")
+        if isinstance(chunks, list) and len(chunks) > 0:
+            return True
+        status = str(item.get("status") or "").lower()
+        if status in _SUCCESS_STATUSES:
+            return True
+        if status in _FAILED_STATUSES:
+            saw_failure = True
+
+    return False if saw_failure else None
+
+
+def failure_summary(root_dir: Path | None, *, limit: int = 3) -> str:
+    """Return a short human-readable summary of failed LightRAG documents."""
+    if root_dir is None:
+        return ""
+    payload = _read_doc_status(_store_root(Path(root_dir)))
+    if not payload:
+        return ""
+
+    failures: list[str] = []
+    for item in payload.values():
+        if not isinstance(item, dict):
+            continue
+        status = str(item.get("status") or "").lower()
+        error = str(item.get("error_msg") or "").strip()
+        if status not in _FAILED_STATUSES and not error:
+            continue
+        name = str(item.get("file_path") or "document").strip()
+        failures.append(f"{name}: {error or status}")
+        if len(failures) >= limit:
+            break
+    return "; ".join(failures)
+
+
+def document_error(root_dir: Path | None, doc_id: str) -> str:
+    """Return the stored LightRAG error for one document, if present."""
+    if root_dir is None or not doc_id:
+        return ""
+    payload = _read_doc_status(_store_root(Path(root_dir)))
+    if not payload:
+        return ""
+    item = payload.get(doc_id)
+    if not isinstance(item, dict):
+        return ""
+    status = str(item.get("status") or "").lower()
+    error = str(item.get("error_msg") or "").strip()
+    if status in _FAILED_STATUSES or error:
+        return error or status
+    return ""
+
+
+def _read_doc_status(root_dir: Path) -> dict[str, Any] | None:
+    path = root_dir / _DOC_STATUS_FILENAME
+    if not path.exists():
+        return None
+    try:
+        with open(path, encoding="utf-8") as handle:
+            payload = json.load(handle)
+        return payload if isinstance(payload, dict) else None
+    except Exception as exc:
+        logger.warning("Failed to read LightRAG doc status %s: %s", path, exc)
+        return None
+
+
+def has_any_doc_status(root_dir: Path | None) -> bool:
+    if root_dir is None:
+        return False
+    payload = _read_doc_status(_store_root(Path(root_dir)))
+    return bool(payload)
+
+
+def _read_meta(root_dir: Path) -> dict[str, Any] | None:
+    path = Path(root_dir) / META_FILENAME
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+    return payload if isinstance(payload, dict) else None
+
+
+def meta_is_native_published(root_dir: Path | None) -> bool:
+    if root_dir is None:
+        return False
+    meta = _read_meta(Path(root_dir))
+    return bool(
+        meta
+        and meta.get("provider") == PROVIDER
+        and meta.get("signature") == PROVIDER
+        and meta.get("lightrag_adapter_schema") == ADAPTER_SCHEMA
+        and meta.get("parser_bridge_schema") == 1
+        and meta.get("state") == PUBLISHED_STATE
+        and has_output(Path(root_dir))
+    )
+
+
+def _parser_inputs(root_dir: Path) -> list[dict[str, str]]:
+    from .ingress import bundles_root, load_verified_bundle
+
+    records: set[tuple[str, str]] = set()
+    for bundle in sorted(bundles_root(root_dir).glob("*.bundle")):
+        canonical_name = bundle.name.removesuffix(".bundle")
+        manifest, _ = load_verified_bundle(root_dir, canonical_name)
+        parser = manifest.get("parser")
+        if not isinstance(parser, dict):
+            continue
+        records.add(
+            (
+                str(parser.get("engine") or ""),
+                str(parser.get("parser_signature") or ""),
+            )
+        )
+    return [
+        {"engine": engine, "parser_signature": signature} for engine, signature in sorted(records)
+    ]
+
+
+def read_published_policy(root_dir: Path | None) -> dict[str, Any] | None:
+    if root_dir is None or not meta_is_native_published(root_dir):
+        return None
+    meta = _read_meta(Path(root_dir)) or {}
+    policy = meta.get("indexing_policy")
+    if isinstance(policy, dict):
+        return policy
+    return {"policy": "legacy_unpinned"}
+
+
+def latest_published_root(kb_dir: Path) -> Path | None:
+    """Return the newest native published version, ignoring failed candidates."""
+    from deeptutor.services.rag.index_versioning import list_kb_versions
+
+    for entry in list_kb_versions(Path(kb_dir)):
+        root = Path(str(entry.get("storage_path") or ""))
+        if meta_is_native_published(root):
+            return root
+    return None
+
+
+def published_root_for_embedding(kb_dir: Path, signature: str | None) -> Path | None:
+    """Find provenance for the recorded binding, even after catalog identity drift."""
+    if signature is None:
+        return latest_published_root(kb_dir)
+    from deeptutor.services.rag.index_versioning import list_kb_versions
+
+    for entry in list_kb_versions(kb_dir):
+        root = Path(entry["storage_path"])
+        if entry.get("embedding_signature") == signature and meta_is_native_published(root):
+            return root
+    return None
+
+
+def embedding_matches(root_dir: Path, signature: EmbeddingSignature | None) -> bool:
+    """Compare the published vector identity, never just its dimension."""
+    meta = _read_meta(root_dir) or {}
+    return signature is not None and meta.get("embedding_signature") == signature.hash()
+
+
+def require_compatible_embedding(root_dir: Path, config: EmbeddingConfig) -> None:
+    """Reject reads and appends when the published embedding identity differs."""
+    from deeptutor.services.rag.embedding_signature import signature_from_config
+
+    from .indexing_policy import EmbeddingMismatchError
+
+    if not embedding_matches(root_dir, signature_from_config(config)):
+        raise EmbeddingMismatchError(
+            "The current embedding configuration does not match this LightRAG index. "
+            "Restore the original embedding configuration or rebuild with the current embedding "
+            "before querying or adding documents."
+        )
+
+
+def write_meta(
+    root_dir: Path,
+    *,
+    indexing_policy: dict[str, Any] | None = None,
+    embedding_config: EmbeddingConfig | None = None,
+) -> None:
+    """Write a flat-layout ``meta.json`` so the version lists as ready.
+
+    Mirrors ``index_versioning.write_version_meta`` but carries a synthetic
+    ``lightrag`` signature instead of an embedding hash. The embedding identity
+    is stamped alongside so an externally-linked index can be checked for
+    embedding compatibility at connect time (LightRAG otherwise fails retrieval
+    silently on a dimension mismatch).
+    """
+    from deeptutor.services.rag.embedding_signature import (
+        embedding_meta_fields,
+        signature_from_config,
+    )
+
+    from .engine import installed_version, workspace_for
+
+    target = Path(root_dir)
+    previous = _read_meta(target) or {}
+    now = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z"
+    if embedding_config is None:
+        embedding_fields = embedding_meta_fields()
+    else:
+        signature = signature_from_config(embedding_config)
+        embedding_fields = {
+            "embedding_signature": signature.hash(),
+            "embedding_model": signature.model,
+            "embedding_dim": signature.dimension,
+        }
+    payload = {
+        "version": target.name,
+        "signature": PROVIDER,
+        "provider": PROVIDER,
+        "state": PUBLISHED_STATE,
+        "lightrag_adapter_schema": ADAPTER_SCHEMA,
+        "lightrag_package_version": installed_version(),
+        "parser_bridge_schema": 1,
+        "parser_inputs": _parser_inputs(target),
+        "workspace": workspace_for(target),
+        "layout": "flat",
+        "created_at": str(previous.get("created_at") or now),
+        "updated_at": now,
+        "indexing_policy": indexing_policy
+        or previous.get("indexing_policy")
+        or {"policy": "legacy_unpinned"},
+        **embedding_fields,
+    }
+    atomic_write_json(target / META_FILENAME, payload)
+
+
+__all__ = [
+    "META_FILENAME",
+    "PROVIDER",
+    "document_error",
+    "failure_summary",
+    "has_any_doc_status",
+    "working_dir",
+    "has_output",
+    "meta_is_native_published",
+    "latest_published_root",
+    "read_published_policy",
+    "write_meta",
+]

deeptutor/services/rag/pipelines/lightrag/worker.py

@@ -0,0 +1,259 @@
+"""Event-loop isolation helpers for local LightRAG indexing.
+
+LightRAG's local storage backends perform synchronous graph merging and
+JSON serialization from inside async methods.  Running those methods on the
+service event loop therefore stalls unrelated API and LLM work.  This module
+provides one narrow boundary: run local LightRAG coroutines on a process-wide
+worker event loop, while explicitly forwarding network I/O and callbacks to
+the event loop that owns each request. LightRAG's storage locks are process-wide.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+import concurrent.futures
+import contextvars
+import inspect
+import logging
+import os
+import threading
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_WORKER_CANCEL_GRACE_SECONDS = 10.0
+_WORKER_LOOP_LOCK = threading.Lock()
+_WORKER_LOOP: asyncio.AbstractEventLoop | None = None
+_WORKER_THREAD: threading.Thread | None = None
+_WORKER_PID: int | None = None
+
+
+def _worker_loop() -> asyncio.AbstractEventLoop:
+    """Return the one loop that owns all local LightRAG asyncio locks (#1578)."""
+    global _WORKER_LOOP, _WORKER_THREAD, _WORKER_PID
+    with _WORKER_LOOP_LOCK:
+        pid = os.getpid()
+        if _WORKER_PID != pid:
+            # A fork inherits Python globals, but not the worker's thread.
+            _WORKER_LOOP = None
+            _WORKER_THREAD = None
+            _WORKER_PID = pid
+        if _WORKER_LOOP is None:
+            loop = asyncio.new_event_loop()
+            thread = threading.Thread(
+                target=loop.run_forever,
+                name="lightrag-worker-loop",
+                daemon=True,
+            )
+            thread.start()
+            _WORKER_LOOP = loop
+            _WORKER_THREAD = thread
+        elif _WORKER_THREAD is None or not _WORKER_THREAD.is_alive():
+            # A new loop in this process would reuse locks bound to the old one.
+            raise RuntimeError("LightRAG worker loop stopped; restart DeepTutor.")
+        return _WORKER_LOOP
+
+
+class _WorkerLoopController:
+    """Thread-safe cancellation handle for the worker loop's top-level task."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._cancel_requested = threading.Event()
+        self._loop: asyncio.AbstractEventLoop | None = None
+        self._task: asyncio.Task[Any] | None = None
+
+    def bind_current_task(self) -> None:
+        """Bind from inside the worker loop and honor an earlier cancellation."""
+        loop = asyncio.get_running_loop()
+        task = asyncio.current_task()
+        if task is None:  # pragma: no cover - asyncio always owns this coroutine
+            raise RuntimeError("Worker loop has no current task")
+        with self._lock:
+            self._loop = loop
+            self._task = task
+            cancel_requested = self._cancel_requested.is_set()
+        if cancel_requested:
+            task.cancel()
+
+    def clear(self) -> None:
+        """Drop references once the worker job reaches a terminal state."""
+        with self._lock:
+            self._loop = None
+            self._task = None
+
+    def cancel(self) -> None:
+        """Cancel the worker's actual top-level task, including before bind."""
+        if self._cancel_requested.is_set():
+            return
+        self._cancel_requested.set()
+        with self._lock:
+            loop = self._loop
+            task = self._task
+        if loop is None or task is None:
+            return
+        try:
+            loop.call_soon_threadsafe(task.cancel)
+        except RuntimeError:
+            # The worker loop failed between the snapshot and the signal.
+            pass
+
+
+class OwnerLoopBridge:
+    """Run selected awaitables and callbacks on the service event loop.
+
+    The worker receives a copy of the caller's :mod:`contextvars` context.
+    ``run`` schedules from that copied context, so request-local model and user
+    configuration remains visible on the owner loop.
+    """
+
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        self._loop = loop
+        self._cancelled = threading.Event()
+        self._pending_lock = threading.Lock()
+        self._pending: set[concurrent.futures.Future[Any]] = set()
+
+    def cancel(self) -> None:
+        """Reject new owner-loop work and cancel requests already in flight."""
+        self._cancelled.set()
+        with self._pending_lock:
+            pending = tuple(self._pending)
+        for future in pending:
+            future.cancel()
+
+    def raise_if_cancelled(self) -> None:
+        """Cooperatively stop the worker at a safe async boundary."""
+        if self._cancelled.is_set():
+            raise asyncio.CancelledError
+
+    async def run(self, factory: Callable[[], Awaitable[T]]) -> T:
+        """Await ``factory`` on the owner loop and propagate its result/error."""
+        self.raise_if_cancelled()
+        if asyncio.get_running_loop() is self._loop:
+            return await factory()
+
+        async def invoke() -> T:
+            return await factory()
+
+        coroutine = invoke()
+        try:
+            future = asyncio.run_coroutine_threadsafe(coroutine, self._loop)
+        except BaseException:
+            coroutine.close()
+            raise
+        with self._pending_lock:
+            if self._cancelled.is_set():
+                future.cancel()
+            else:
+                self._pending.add(future)
+        try:
+            return await asyncio.wrap_future(future)
+        except asyncio.CancelledError:
+            future.cancel()
+            raise
+        finally:
+            with self._pending_lock:
+                self._pending.discard(future)
+
+    async def call(self, callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
+        """Invoke a sync or async callback on the owner loop."""
+
+        async def invoke() -> Any:
+            result = callback(*args, **kwargs)
+            if inspect.isawaitable(result):
+                return await result
+            return result
+
+        return await self.run(invoke)
+
+
+async def run_in_worker_loop(
+    job: Callable[[OwnerLoopBridge], Awaitable[T]],
+    *,
+    cancel_grace_seconds: float = DEFAULT_WORKER_CANCEL_GRACE_SECONDS,
+) -> T:
+    """Run one local LightRAG job on the process-wide worker event loop.
+
+    ``job`` and every object it creates should remain confined to that worker.
+    The supplied bridge is the only supported route back to the owner loop.
+    Worker exceptions are re-raised in the awaiting task.
+    """
+    owner_loop = asyncio.get_running_loop()
+    bridge = OwnerLoopBridge(owner_loop)
+    controller = _WorkerLoopController()
+    caller_context = contextvars.copy_context()
+
+    finished: concurrent.futures.Future[T] = concurrent.futures.Future()
+
+    async def run_bound_job() -> None:
+        try:
+            controller.bind_current_task()
+            result = await job(bridge)
+        except BaseException as exc:
+            finished.set_exception(exc)
+        else:
+            finished.set_result(result)
+        finally:
+            controller.clear()
+
+    worker_loop = _worker_loop()
+
+    def submit() -> None:
+        try:
+            worker_loop.create_task(run_bound_job(), context=caller_context)
+        except BaseException as exc:
+            finished.set_exception(exc)
+
+    worker_loop.call_soon_threadsafe(submit)
+    # run_coroutine_threadsafe's Future becomes cancelled before its coroutine
+    # finishes cleanup. This completion Future resolves only after job exits.
+    worker = asyncio.wrap_future(finished)
+    try:
+        # Shielding keeps request cancellation from cancelling the completion
+        # signal. The owner waits for the worker's cleanup below.
+        return await asyncio.shield(worker)
+    except asyncio.CancelledError:
+        bridge.cancel()
+        controller.cancel()
+        grace = max(float(cancel_grace_seconds), 0.0)
+        deadline = owner_loop.time() + grace
+        while not worker.done() and owner_loop.time() < deadline:
+            try:
+                remaining = max(deadline - owner_loop.time(), 0.0)
+                await asyncio.wait({worker}, timeout=remaining)
+            except asyncio.CancelledError:
+                # Repeated cancellation still must not strand the worker on a
+                # request scheduled back to this owner loop.
+                bridge.cancel()
+                controller.cancel()
+        if not worker.done():
+            # Stopping this loop would strand unrelated jobs and leave locks
+            # bound to a stopped loop. Wait for confirmed cleanup instead.
+            logger.error(
+                "LightRAG worker did not stop within %.1fs; waiting for confirmed cleanup",
+                grace,
+            )
+            while not worker.done():
+                try:
+                    await asyncio.wait({worker})
+                except asyncio.CancelledError:
+                    bridge.cancel()
+                    controller.cancel()
+        # Retrieve the terminal exception so the completion Future never emits
+        # an "exception was never retrieved" warning.  The caller's
+        # cancellation remains authoritative.
+        try:
+            worker.result()
+        except BaseException:
+            pass
+        raise
+
+
+__all__ = [
+    "DEFAULT_WORKER_CANCEL_GRACE_SECONDS",
+    "OwnerLoopBridge",
+    "run_in_worker_loop",
+]

deeptutor/services/rag/pipelines/lightrag/write_lock.py

@@ -0,0 +1,55 @@
+"""One nonblocking writer per knowledge base, shared across worker processes."""
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+import hashlib
+import os
+from pathlib import Path
+
+from .indexing_policy import IndexingPolicyError
+
+
+@contextmanager
+def write_ownership(kb_dir: Path) -> Iterator[None]:
+    """Hold exclusive nonblocking write ownership for a knowledge base.
+
+    Yields:
+        None while the cross-process indexing lock is held.
+
+    Raises:
+        IndexingPolicyError: The target uses a symlink or another writer owns it."""
+    kb_dir = Path(kb_dir)
+    if kb_dir.is_symlink():
+        raise IndexingPolicyError("A writable knowledge-base directory cannot be a symbolic link.")
+    lock_dir = kb_dir.parent / ".lightrag-locks"
+    if lock_dir.is_symlink():
+        raise IndexingPolicyError("The indexing lock directory cannot be a symbolic link.")
+    lock_dir.mkdir(parents=True, exist_ok=True)
+    key = hashlib.sha256(str(kb_dir.resolve()).encode("utf-8")).hexdigest()
+    lock_path = lock_dir / f"{key}.lock"
+    if lock_path.is_symlink():
+        raise IndexingPolicyError("The indexing lock cannot be a symbolic link.")
+    descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600)
+    with os.fdopen(descriptor, "r+b") as handle:
+        try:
+            if os.name == "nt":
+                import msvcrt
+
+                handle.seek(0)
+                msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
+            else:
+                import fcntl
+
+                fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError as exc:
+            raise IndexingPolicyError(
+                "Another indexing operation is active for this knowledge base; resubmit after it finishes."
+            ) from exc
+        try:
+            yield
+        finally:
+            if os.name == "nt":
+                handle.seek(0)
+                msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+            else:
+                fcntl.flock(handle.fileno(), fcntl.LOCK_UN)

deeptutor/services/rag/pipelines/lightrag_server/__init__.py

@@ -0,0 +1,15 @@
+"""Retrieval-only RAG pipeline backed by an external LightRAG server.
+
+A KB bound to the ``lightrag-server`` provider is a connection pointer to a
+standalone LightRAG server the user runs and indexed themselves. DeepTutor never
+indexes or stores anything locally: retrieval is offloaded to the server's
+``/query`` endpoint (context only — DeepTutor's chat LLM still writes the
+answer), and the endpoint is configured per-KB. See ``client`` for the wire and
+``probe`` for the connect-time health check.
+"""
+
+from __future__ import annotations
+
+from .pipeline import LightRagServerPipeline
+
+__all__ = ["LightRagServerPipeline"]