deeptutor-parsing 1.7.0__py3-none-any.whl

deeptutor/services/parsing/__init__.py

@@ -0,0 +1,34 @@
+"""Shared, engine-pluggable document-parsing layer (the "bridge").
+
+Sits between input material and its consumers (question extraction, RAG
+indexing, future LightRAG): one canonical IR (:class:`ParsedDocument`), one
+content-addressed cache, a registry of pluggable engines (text-only, MinerU,
+Docling, markitdown). Parsing is upstream of and independent from retrieval.
+
+``ParseService`` / ``get_parse_service`` are the public entry; they are imported
+lazily here to keep this package importable before the service module lands and
+to avoid pulling engine deps at import time.
+"""
+
+from __future__ import annotations
+
+from .base import Parser, ReadinessReport
+from .signature import ParserSignature
+from .types import ParsedDocument, ParserError
+
+
+def get_parse_service():
+    """Return the process-wide :class:`ParseService` singleton."""
+    from .service import get_parse_service as _get
+
+    return _get()
+
+
+__all__ = [
+    "ParsedDocument",
+    "ParserError",
+    "Parser",
+    "ReadinessReport",
+    "ParserSignature",
+    "get_parse_service",
+]

deeptutor/services/parsing/base.py

@@ -0,0 +1,84 @@
+"""Structural contract every document-parsing engine implements.
+
+Parsing is deliberately kept OFF the ``RAGPipeline`` protocol
+(``services/rag/pipelines/base.py``): it is an upstream, shared, *optional*
+stage. An engine turns bytes/path into a :class:`~deeptutor.services.parsing.types.ParsedDocument`
+and declares its availability + model readiness so the UI can gate local model
+downloads (default: no silent multi-GB pull).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable, Optional, Protocol, runtime_checkable
+
+from .signature import ParserSignature
+
+
+@dataclass(frozen=True)
+class ReadinessReport:
+    """Whether an engine can run a parse right now."""
+
+    ready: bool
+    # Machine code: "ready" | "models_missing" | "cli_missing" | "not_configured"
+    reason: str = "ready"
+    # User-facing guidance shown when not ready (which escape hatch to take).
+    message: str = ""
+
+
+@runtime_checkable
+class Parser(Protocol):
+    """A document-parsing engine: ``source_path`` in → ``ParsedDocument`` out.
+
+    Concrete engines also expose a ``classmethod is_available() -> bool`` (probed
+    by the factory before instantiation); it is intentionally not part of the
+    instance Protocol.
+    """
+
+    name: str
+    # True when the engine downloads/needs local model weights (MinerU local,
+    # Docling). text-only, markitdown, and cloud backends are False.
+    needs_local_models: bool
+
+    def resolve_config(self) -> Any:
+        """Load this engine's effective config slice from runtime settings."""
+        ...
+
+    def supported_formats(self) -> frozenset[str]:
+        """Lower-case file suffixes (including the dot) this engine can parse.
+
+        Compound suffixes such as ``.dclg.xml`` and ``.tar.gz`` are allowed.
+        An empty set means the engine performs its own content-based/runtime
+        detection (for example a remote Tika server with custom parsers).
+        """
+        ...
+
+    def signature(self, config: Any) -> ParserSignature:
+        """Stable identity of ``(engine, version, output-affecting config)``."""
+        ...
+
+    def is_ready(self, config: Any) -> ReadinessReport:
+        """Whether a parse can run now (models present / cloud configured)."""
+        ...
+
+    def parse(
+        self,
+        source_path: Path,
+        workdir: Path,
+        *,
+        config: Any,
+        on_output: Optional[Callable[[str], None]] = None,
+    ) -> None:
+        """Write canonical artifacts for ``source_path`` into ``workdir``.
+
+        Engines emit ``<stem>.md`` (always) and may add
+        ``<stem>_content_list.json`` and an ``images/`` dir. The caller
+        (``ParseService``) assembles the :class:`ParsedDocument` from the
+        workdir via :func:`deeptutor.services.parsing.cache.load_ir` and writes
+        the cache manifest. Raises :class:`ParserError` on failure.
+        """
+        ...
+
+
+__all__ = ["Parser", "ReadinessReport"]

deeptutor/services/parsing/cache.py

@@ -0,0 +1,203 @@
+"""Content-addressed parse cache + canonical IR loader.
+
+The cache is keyed by ``(source_hash, parser_signature)`` and shared across all
+consumers (question extraction, RAG indexing). Re-parsing the same bytes with
+the same engine config is a directory lookup; a different engine/version/knob
+lands in a different signature dir and re-parses. Layout::
+
+    parse_cache/<hash[:2]>/<source_hash>/<signature>/
+        manifest.json              # atomically written last → completed
+        <stem>.md
+        <stem>_content_list.json   # optional (engines that emit structure)
+        images/                    # optional
+
+``load_ir`` turns such a dir back into ``(markdown, blocks, asset_dir)``,
+mirroring the question extractor's loader but free of stdout side effects so
+both it and the engines share one loader.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import hashlib
+import json
+import logging
+from pathlib import Path
+from typing import Any, Optional
+from uuid import uuid4
+
+from deeptutor.services.file_io import atomic_write_json
+
+logger = logging.getLogger(__name__)
+
+MANIFEST_FILENAME = "manifest.json"
+_HASH_PREFIX = "sha256"
+_READ_CHUNK = 1 << 20  # 1 MiB
+
+
+def source_hash_from_path(path: Path) -> str:
+    """Hash the *bytes* of ``path`` (not its name) so re-uploads of the same
+    document under a random temp name still hit cache."""
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(_READ_CHUNK), b""):
+            digest.update(chunk)
+    return digest.hexdigest()[:16]
+
+
+def signature_dir(cache_root: Path, source_hash: str, sig_hash: str) -> Path:
+    return cache_root / source_hash[:2] / source_hash / sig_hash
+
+
+def is_ready(workdir: Optional[Path]) -> bool:
+    if workdir is None:
+        return False
+    try:
+        manifest = json.loads((workdir / MANIFEST_FILENAME).read_text(encoding="utf-8"))
+        return isinstance(manifest, dict) and bool(manifest.get("created_at"))
+    except (OSError, ValueError):
+        return False
+
+
+def lookup(cache_root: Path, source_hash: str, sig_hash: str) -> Optional[Path]:
+    """Return a ready cache dir for the key, or ``None`` on miss."""
+    target = signature_dir(cache_root, source_hash, sig_hash)
+    return target if is_ready(target) else None
+
+
+def reserve(cache_root: Path, source_hash: str, sig_hash: str) -> Path:
+    """Create (or reuse) the signature dir the engine writes its artifacts into.
+
+    Incomplete attempts are retained beside the cache entry for diagnosis, but
+    never reused as completed parser output (#1612).
+    """
+    target = signature_dir(cache_root, source_hash, sig_hash)
+    if target.exists() and not is_ready(target):
+        cleanup_failed(target)
+        if target.exists():
+            raise OSError(f"Could not retain incomplete parse output: {target}")
+    target.mkdir(parents=True, exist_ok=True)
+    return target
+
+
+def write_manifest(workdir: Path, meta: dict[str, Any]) -> None:
+    """Stamp the cache dir ready. Written last so a half-written dir never reads
+    as a cache hit."""
+    payload = {
+        **meta,
+        "created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
+    }
+    atomic_write_json(workdir / MANIFEST_FILENAME, payload)
+
+
+def cleanup_failed(workdir: Path) -> None:
+    """Retain unfinished work outside the active signature directory."""
+    try:
+        if workdir.is_dir() and not is_ready(workdir):
+            retained = workdir.with_name(f".{workdir.name}.failed-{uuid4().hex}")
+            workdir.rename(retained)
+            logger.warning("Retained incomplete parse output at %s", retained)
+    except Exception as exc:  # pragma: no cover - best-effort
+        logger.warning("Could not clean up failed parse dir %s: %s", workdir, exc)
+
+
+def find_content_dir(workdir: Path) -> Path:
+    """Locate the dir holding parsed markdown artifacts.
+
+    Engines (MinerU especially) may nest output under ``auto/``/``hybrid_auto/``
+    or a per-document subdir. Mirrors the question extractor's resolver.
+    """
+    candidate_dirs: list[Path] = []
+
+    for preferred_name in ("auto", "hybrid_auto"):
+        preferred = workdir / preferred_name
+        if preferred.is_dir():
+            candidate_dirs.append(preferred)
+
+    for child in sorted(workdir.iterdir()) if workdir.is_dir() else []:
+        if child.is_dir() and child not in candidate_dirs:
+            candidate_dirs.append(child)
+
+    nested = {
+        artifact.parent
+        for pattern in ("*.md", "*_content_list.json")
+        for artifact in (workdir.rglob(pattern) if workdir.is_dir() else [])
+    }
+    for artifact_dir in sorted(nested):
+        if artifact_dir not in candidate_dirs:
+            candidate_dirs.append(artifact_dir)
+
+    for candidate in candidate_dirs:
+        if list(candidate.glob("*.md")):
+            return candidate
+
+    return candidate_dirs[0] if candidate_dirs else workdir
+
+
+def _absolutize_img_paths(blocks: list[dict], content_dir: Path) -> list[dict]:
+    """Anchor relative ``img_path`` entries at ``content_dir``.
+
+    Engines (MinerU especially) emit ``img_path`` values like ``images/x.png``
+    relative to the content-list file. Consumers such as LightRAG resolve
+    them against their own working directory instead, so the images are never
+    found (issue #624). The cached JSON stays relative — paths are rewritten
+    only on load, keeping the cache dir relocatable.
+    """
+    for block in blocks:
+        if not isinstance(block, dict):
+            continue
+        img_path = block.get("img_path")
+        if not isinstance(img_path, str) or not img_path:
+            continue
+        p = Path(img_path)
+        if p.is_absolute():
+            continue
+        if ".." in p.parts:
+            logger.warning("Skipping traversal img_path in content_list: %s", img_path)
+            continue
+        block["img_path"] = str(content_dir / img_path)
+    return blocks
+
+
+def load_ir(workdir: Path) -> tuple[str, Optional[list[dict]], Optional[Path]]:
+    """Load ``(markdown, blocks, asset_dir)`` from a parsed/cached dir.
+
+    ``markdown`` is "" if no ``.md`` exists; ``blocks`` is the parsed
+    ``*_content_list.json`` or ``None``; ``asset_dir`` is the ``images/`` dir if
+    present.
+    """
+    content_dir = find_content_dir(workdir)
+
+    markdown = ""
+    md_files = list(content_dir.glob("*.md"))
+    if md_files:
+        markdown = md_files[0].read_text(encoding="utf-8")
+
+    blocks: Optional[list[dict]] = None
+    json_files = list(content_dir.glob("*_content_list.json"))
+    if json_files:
+        try:
+            loaded = json.loads(json_files[0].read_text(encoding="utf-8"))
+            if isinstance(loaded, list):
+                blocks = _absolutize_img_paths(loaded, content_dir)
+        except Exception as exc:
+            logger.warning("Failed to read content_list %s: %s", json_files[0], exc)
+
+    images_dir = content_dir / "images"
+    asset_dir = images_dir if images_dir.is_dir() else None
+
+    return markdown, blocks, asset_dir
+
+
+__all__ = [
+    "MANIFEST_FILENAME",
+    "source_hash_from_path",
+    "signature_dir",
+    "is_ready",
+    "lookup",
+    "reserve",
+    "write_manifest",
+    "cleanup_failed",
+    "find_content_dir",
+    "load_ir",
+]

deeptutor/services/parsing/engines/__init__.py

@@ -0,0 +1,7 @@
+"""Pluggable document-parsing engines.
+
+Each engine implements the :class:`~deeptutor.services.parsing.base.Parser`
+contract and is selected by name through :mod:`deeptutor.services.parsing.engines.factory`.
+Third-party imports are lazy so an engine whose dependency is absent simply
+reports ``is_available() is False`` instead of breaking import.
+"""

deeptutor/services/parsing/engines/_install.py

@@ -0,0 +1,281 @@
+"""One-click background jobs for optional parser engines.
+
+The Document Parsing settings page runs two kinds of background job without the
+user dropping to a shell:
+
+* **install** — ``pip install`` an engine's optional PyPI package.
+* **models**  — download an engine's model weights (e.g. Docling's layout/table
+  models via ``docling-tools models download``).
+
+Both mirror the MinerU model-download job (``mineru/models.py``): a single
+background subprocess with a cursor-based line log the UI starts, polls, and
+cancels. Commands come from fixed allow-lists (pip specs / downloader argv),
+never user input, so the subprocess argv can't be injected. Package installs use
+the bare package specs (the same strings as the ``[parse-*]`` extras) so pip
+never re-resolves DeepTutor itself; after a successful install we invalidate the
+import caches so the engine reports available in the same process (no restart).
+"""
+
+from __future__ import annotations
+
+import importlib
+import logging
+import os
+from pathlib import Path
+import shutil
+import subprocess  # nosec B404 - argv from fixed allow-lists, never user input
+import sys
+import threading
+import time
+from typing import Any, Callable, Optional
+
+from ._versions import package_version
+
+logger = logging.getLogger(__name__)
+
+# Engine id -> pip requirement specifiers. Mirrors the optional extras in
+# pyproject.toml. Engines absent here (text_only built-in, mineru external
+# CLI/API) have no one-click install.
+from deeptutor.modules.catalog import manifests
+
+ENGINE_PIP_SPECS: dict[str, list[str]] = {
+    row["parser_engine"]: list(row["requirements"])
+    for row in manifests().values()
+    if "parser_engine" in row
+}
+
+# Engine id -> console-script argv that downloads its model weights. The script
+# is resolved next to the server's python first (same env), then on PATH.
+ENGINE_MODEL_DOWNLOADERS: dict[str, list[str]] = {
+    "docling": ["docling-tools", "models", "download"],
+}
+
+# Buffered log lines kept in memory; older lines are dropped (the cursor
+# protocol keeps clients consistent across trims).
+_MAX_LINES = 2000
+_LINE_MIN_INTERVAL = 0.2
+
+
+def installable_engines() -> frozenset[str]:
+    """Engine ids that support one-click pip install."""
+    return frozenset(ENGINE_PIP_SPECS)
+
+
+def model_downloadable_engines() -> frozenset[str]:
+    """Engine ids that support one-click model-weight download."""
+    return frozenset(ENGINE_MODEL_DOWNLOADERS)
+
+
+def resolve_model_downloader(engine: str) -> Optional[list[str]]:
+    """Resolve the model-download argv for ``engine``.
+
+    Locates the console script next to the server's python first (same env so it
+    matches the installed engine), then falls back to PATH. Returns ``None`` if
+    the engine has no downloader or the script isn't found.
+    """
+    parts = ENGINE_MODEL_DOWNLOADERS.get(engine)
+    if not parts:
+        return None
+    name = parts[0]
+    # Let the platform resolver search the active Python environment too.  On
+    # Windows pip installs console scripts as ``<name>.exe`` and ``which``
+    # applies PATHEXT; probing the extensionless Path directly misses it.
+    sibling = shutil.which(name, path=str(Path(sys.executable).parent))
+    if sibling:
+        return [sibling, *parts[1:]]
+    found = shutil.which(name)
+    if found:
+        return [found, *parts[1:]]
+    return None
+
+
+def _invalidate_import_caches() -> None:
+    """Make a freshly installed package importable in this process and bust the
+    cached "" version so readiness flips without a server restart."""
+    try:
+        importlib.invalidate_caches()
+        package_version.cache_clear()
+        from .markitdown.formats import markitdown_supported_formats
+
+        # Docling format discovery is intentionally static and has no cache to
+        # invalidate: importing its runtime would pull PyTorch's OpenMP runtime
+        # into the FAISS backend process on macOS.
+        markitdown_supported_formats.cache_clear()
+    except Exception:  # noqa: BLE001 - best effort
+        logger.exception("Post-install cache invalidation failed")
+
+
+class BackgroundJobManager:
+    """At most one background subprocess (install or model download), with a
+    cursor-based line log.
+
+    States: ``idle`` → ``running`` → ``done`` / ``failed`` / ``cancelled``.
+    ``status(cursor)`` returns lines after ``cursor`` plus ``next_cursor`` so the
+    UI can poll incrementally; ``kind`` (``install`` | ``models``) and ``engine``
+    tell the UI which job/card the log belongs to.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._state = "idle"
+        self._kind = ""
+        self._engine = ""
+        self._lines: list[str] = []
+        self._base = 0
+        self._message = ""
+        self._process: subprocess.Popen | None = None
+        self._cancel_requested = False
+
+    def start_install(self, *, engine: str, specs: list[str]) -> dict[str, Any]:
+        # Host-package changes cannot be safe while turns/API workers run.
+        # Model downloads remain background jobs because they do not mutate code.
+        if engine not in ENGINE_PIP_SPECS or specs != ENGINE_PIP_SPECS[engine]:
+            return {"ok": False, "message": "Unknown parser installation request."}
+        return {
+            "ok": False,
+            "message": (
+                "Stop DeepTutor, then run `deeptutor module install parser-"
+                + engine
+                + "` in this environment and restart. User data is preserved."
+            ),
+        }
+
+    def start_model_download(self, *, engine: str, cmd: list[str]) -> dict[str, Any]:
+        return self._launch(kind="models", engine=engine, cmd=cmd)
+
+    def status(self, cursor: int = 0) -> dict[str, Any]:
+        with self._lock:
+            start = max(int(cursor) - self._base, 0)
+            return {
+                "state": self._state,
+                "kind": self._kind,
+                "engine": self._engine,
+                "lines": list(self._lines[start:]),
+                "next_cursor": self._base + len(self._lines),
+                "message": self._message,
+            }
+
+    def cancel(self) -> dict[str, Any]:
+        with self._lock:
+            process = self._process
+            running = self._state == "running"
+            if running:
+                self._cancel_requested = True
+        if not (running and process):
+            return {"ok": False, "message": "No job is running."}
+        if process.poll() is None:
+            try:
+                process.terminate()
+            except Exception as exc:  # noqa: BLE001
+                return {"ok": False, "message": f"Failed to cancel: {exc}"}
+        return {"ok": True, "message": ""}
+
+    # ------------------------------------------------------------------
+
+    def _launch(
+        self,
+        *,
+        kind: str,
+        engine: str,
+        cmd: list[str],
+        env: Optional[dict[str, str]] = None,
+        on_success: Optional[Callable[[], None]] = None,
+    ) -> dict[str, Any]:
+        with self._lock:
+            if self._state == "running":
+                return {
+                    "ok": False,
+                    "message": f"A {self._kind or 'job'} is already running ({self._engine}).",
+                }
+            try:
+                process = subprocess.Popen(  # nosec B603 - argv from fixed allow-lists
+                    cmd,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                    shell=False,
+                    env={**os.environ, **(env or {})},
+                )
+            except Exception as exc:  # noqa: BLE001
+                self._state = "failed"
+                self._kind = kind
+                self._engine = engine
+                self._message = f"Failed to launch: {exc}"
+                return {"ok": False, "message": self._message}
+            self._state = "running"
+            self._kind = kind
+            self._engine = engine
+            self._lines = []
+            self._base = 0
+            self._message = ""
+            self._process = process
+            self._cancel_requested = False
+            thread = threading.Thread(
+                target=self._pump, args=(process, kind, engine, on_success), daemon=True
+            )
+            thread.start()
+            logger.info("Parser %s job started (%s): %s", kind, engine, " ".join(cmd))
+            return {"ok": True, "message": ""}
+
+    def _pump(
+        self,
+        process: subprocess.Popen,
+        kind: str,
+        engine: str,
+        on_success: Optional[Callable[[], None]],
+    ) -> None:
+        last_emit = 0.0
+        try:
+            assert process.stdout is not None
+            for raw_line in process.stdout:
+                line = raw_line.strip()
+                if not line:
+                    continue
+                now = time.monotonic()
+                if now - last_emit < _LINE_MIN_INTERVAL:
+                    continue
+                last_emit = now
+                self._append(line[:300])
+        except Exception:
+            logger.exception("Background job output pump failed")
+        returncode = process.wait()
+        with self._lock:
+            if self._cancel_requested:
+                self._state = "cancelled"
+                self._message = "Cancelled."
+            elif returncode == 0:
+                self._state = "done"
+                self._message = "Finished."
+            else:
+                self._state = "failed"
+                self._message = f"Exited with code {returncode}."
+            self._process = None
+        if returncode == 0 and not self._cancel_requested and on_success is not None:
+            on_success()
+        logger.info("Parser %s job finished (%s): %s", kind, engine, self._state)
+
+    def _append(self, line: str) -> None:
+        with self._lock:
+            self._lines.append(line)
+            overflow = len(self._lines) - _MAX_LINES
+            if overflow > 0:
+                del self._lines[:overflow]
+                self._base += overflow
+
+
+_manager = BackgroundJobManager()
+
+
+def get_background_job_manager() -> BackgroundJobManager:
+    return _manager
+
+
+__all__ = [
+    "ENGINE_MODEL_DOWNLOADERS",
+    "ENGINE_PIP_SPECS",
+    "BackgroundJobManager",
+    "get_background_job_manager",
+    "installable_engines",
+    "model_downloadable_engines",
+    "resolve_model_downloader",
+]

deeptutor/services/parsing/engines/_versions.py

@@ -0,0 +1,38 @@
+"""Cheap, cached package-version lookup for parser signatures.
+
+Using the installed distribution version (not a CLI ``--version`` subprocess)
+keeps signature computation cheap while still invalidating the parse cache when
+an engine is upgraded.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+import importlib.metadata
+import re
+
+
+@lru_cache(maxsize=None)
+def package_version(name: str) -> str:
+    try:
+        return importlib.metadata.version(name)
+    except Exception:
+        return ""
+
+
+def version_at_least(version: str | None, minimum: str) -> bool:
+    """Compare numeric release components without importing packaging tooling."""
+
+    def release(value: str | None) -> tuple[int, ...]:
+        match = re.match(r"\d+(?:\.\d+)*", str(value or "").strip())
+        return tuple(int(part) for part in match.group(0).split(".")) if match else ()
+
+    current = release(version)
+    required = release(minimum)
+    width = max(len(current), len(required))
+    return bool(current and required) and current + (0,) * (width - len(current)) >= required + (
+        0,
+    ) * (width - len(required))
+
+
+__all__ = ["package_version", "version_at_least"]

deeptutor/services/parsing/engines/docling/__init__.py

@@ -0,0 +1,6 @@
+"""Docling engine — structured document conversion (downloads local models).
+
+Converts every input format advertised by the supported Docling release into
+Markdown. Downloads layout/table models on first run, so it shares the same
+``allow_local_model_download`` gate as MinerU local.
+"""