deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/services/rag/pipelines/lightrag/block_policy.py

@@ -0,0 +1,266 @@
+"""Versioned MinerU block policy for LightRAG indexing.
+
+MinerU's legacy ``content_list.json`` mixes semantic document content with
+page-layout helpers. DeepTutor retains the raw parser cache for audit and
+derives an independent, deep-copied list before the native LightRAG Sidecar
+bridge decides which parser blocks are semantic.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from copy import deepcopy
+from dataclasses import dataclass
+import hashlib
+from pathlib import Path
+import re
+from typing import Any
+import uuid
+
+from deeptutor.services.file_io import atomic_write_json
+
+POLICY_ID = "mineru-legacy-content-list-v1"
+LEDGER_DIRNAME = "deeptutor_ingestion_audit"
+ATTEMPT_LEDGER_DIRNAME = "deeptutor_ingestion_attempts"
+
+FILTERED_LAYOUT_TYPES = frozenset({"footer", "header", "page_number"})
+PRESERVED_AUXILIARY_TYPES = frozenset({"aside_text", "page_footnote"})
+PRESERVED_SEMANTIC_TYPES = frozenset(
+    {"chart", "code", "equation", "image", "list", "table", "text"}
+)
+PRESERVED_TYPES = PRESERVED_AUXILIARY_TYPES | PRESERVED_SEMANTIC_TYPES
+MULTIMODAL_TYPES = PRESERVED_TYPES - {"text"}
+
+# MinerU renamed its auxiliary block types between output schemas: what the
+# legacy ``content_list.json`` calls ``header`` a newer parser emits as
+# ``page_header``. Map the newer names back onto the vocabulary the policy is
+# written in, so upgrading MinerU keeps classifying blocks instead of
+# presenting every layout block as an unrecognized type.
+V2_AUXILIARY_EQUIVALENTS = {
+    "aside_text": "page_aside_text",
+    "footer": "page_footer",
+    "header": "page_header",
+    "page_footnote": "page_footnote",
+    "page_number": "page_number",
+}
+_V2_TO_LEGACY_TYPE = {
+    v2_name: legacy_name
+    for legacy_name, v2_name in V2_AUXILIARY_EQUIVALENTS.items()
+    if v2_name != legacy_name
+}
+
+_SAFE_TYPE = re.compile(r"[a-z][a-z0-9_]{0,63}\Z")
+_INVALID_TYPE = "<invalid>"
+
+
+@dataclass(frozen=True)
+class BlockPolicyDecision:
+    """Sanitized policy outcome plus an independent list for indexing."""
+
+    content_list: list[dict[str, Any]]
+    ledger: dict[str, Any] | None
+    unknown_type_counts: tuple[tuple[str, int], ...] = ()
+
+    def unknown_summary(self) -> str:
+        """``type=count`` rundown of block types the policy does not know."""
+        return ", ".join(
+            f"{content_type}={count}" for content_type, count in self.unknown_type_counts
+        )
+
+
+def _sanitized_type(block: dict[str, Any]) -> str:
+    value = block.get("type")
+    if isinstance(value, str) and _SAFE_TYPE.fullmatch(value):
+        return value
+    return _INVALID_TYPE
+
+
+def _sorted_counts(counter: Counter[str]) -> dict[str, int]:
+    return dict(sorted(counter.items()))
+
+
+def _page_type_key(content_type: str, block: dict[str, Any]) -> str | None:
+    page_idx = block.get("page_idx")
+    if isinstance(page_idx, int) and page_idx >= 0:
+        return f"{content_type}:{page_idx}"
+    return None
+
+
+def prepare_content_list(
+    blocks: list[dict[str, Any]],
+    *,
+    engine: str,
+    source_hash: str,
+    parser_signature: str,
+) -> BlockPolicyDecision:
+    """Derive the indexable content list without mutating parser-owned blocks.
+
+    Only MinerU's legacy structured output is classified. Other parse engines
+    retain their existing behavior because their block vocabularies have
+    independent contracts.
+    """
+    if engine != "mineru":
+        return BlockPolicyDecision(content_list=blocks, ledger=None)
+
+    raw_counts: Counter[str] = Counter()
+    filtered_counts: Counter[str] = Counter()
+    eligible_counts: Counter[str] = Counter()
+    eligible_multimodal_page_counts: Counter[str] = Counter()
+    unknown_counts: Counter[str] = Counter()
+    indexable: list[dict[str, Any]] = []
+
+    for raw_block in blocks:
+        if not isinstance(raw_block, dict):
+            raw_counts[_INVALID_TYPE] += 1
+            unknown_counts[_INVALID_TYPE] += 1
+            continue
+        block = raw_block
+        content_type = _sanitized_type(block)
+        content_type = _V2_TO_LEGACY_TYPE.get(content_type, content_type)
+        raw_counts[content_type] += 1
+        if content_type in FILTERED_LAYOUT_TYPES:
+            filtered_counts[content_type] += 1
+            continue
+        if content_type not in PRESERVED_TYPES:
+            # Filter only what we positively recognize as page chrome; index
+            # anything else. A MinerU release that adds a block type must not
+            # silently drop its content, nor abort the whole ingest — before
+            # this policy existed every block was indexed, and an unrecognized
+            # type is far more likely to be new content than new chrome.
+            unknown_counts[content_type] += 1
+
+        eligible_counts[content_type] += 1
+        if content_type in MULTIMODAL_TYPES:
+            page_type_key = _page_type_key(content_type, block)
+            if page_type_key is not None:
+                eligible_multimodal_page_counts[page_type_key] += 1
+        indexable.append(deepcopy(block))
+
+    ledger: dict[str, Any] = {
+        "schema_version": 1,
+        "policy": {
+            "id": POLICY_ID,
+            "input_schema": "legacy-content-list",
+            "filtered_layout_types": sorted(FILTERED_LAYOUT_TYPES),
+            "preserved_auxiliary_types": sorted(PRESERVED_AUXILIARY_TYPES),
+            "preserved_semantic_types": sorted(PRESERVED_SEMANTIC_TYPES),
+            "v2_auxiliary_equivalents": dict(sorted(V2_AUXILIARY_EQUIVALENTS.items())),
+            "v2_auxiliary_normalized": True,
+        },
+        "parser": {
+            "engine": engine,
+            "source_hash": source_hash,
+            "parser_signature": parser_signature,
+        },
+        "counts": {
+            "raw_total": len(blocks),
+            "raw_by_type": _sorted_counts(raw_counts),
+            "filtered_total": sum(filtered_counts.values()),
+            "filtered_by_type": _sorted_counts(filtered_counts),
+            "eligible_total": len(indexable),
+            "eligible_by_type": _sorted_counts(eligible_counts),
+            "eligible_multimodal_total": sum(
+                count
+                for content_type, count in eligible_counts.items()
+                if content_type in MULTIMODAL_TYPES
+            ),
+            "eligible_multimodal_by_type_and_page": _sorted_counts(eligible_multimodal_page_counts),
+            "unknown_total": sum(unknown_counts.values()),
+            "unknown_by_type": _sorted_counts(unknown_counts),
+        },
+        "invariants": {
+            "input_blocks_mutated": False,
+            "eligible_order_preserved": True,
+            "unknown_types_indexed": True,
+            "raw_parser_artifacts_mutated": False,
+        },
+    }
+    return BlockPolicyDecision(
+        content_list=indexable,
+        ledger=ledger,
+        unknown_type_counts=tuple(sorted(unknown_counts.items())),
+    )
+
+
+def write_decision_ledger(
+    working_dir: Path,
+    document_id: str,
+    ledger: dict[str, Any],
+    *,
+    attempt_id: str | None = None,
+) -> Path:
+    """Persist the accepted decision that corresponds to the current index."""
+    document_id_sha256 = hashlib.sha256(document_id.encode()).hexdigest()
+    path = Path(working_dir) / LEDGER_DIRNAME / f"{document_id_sha256[:16]}.json"
+    decision = {
+        "ledger_role": "current-index",
+        "policy_outcome": "accepted",
+    }
+    if attempt_id is not None:
+        decision["attempt_id"] = attempt_id
+    atomic_write_json(
+        path,
+        {
+            **ledger,
+            "document_id_sha256": document_id_sha256,
+            "decision": decision,
+        },
+    )
+    return path
+
+
+def write_attempt_ledger(
+    kb_dir: Path,
+    document_id: str,
+    ledger: dict[str, Any],
+    *,
+    outcome: str,
+) -> tuple[Path, str]:
+    """Persist one immutable policy attempt outside a candidate version.
+
+    ``unknown_types`` means the document indexed, but carried block types the
+    policy has no entry for — the signal to extend ``PRESERVED_TYPES`` or
+    ``FILTERED_LAYOUT_TYPES``, not an ingest failure.
+    """
+    if outcome not in {"accepted", "unknown_types"}:
+        raise ValueError(f"Unsupported policy outcome: {outcome}")
+
+    document_id_sha256 = hashlib.sha256(document_id.encode()).hexdigest()
+    attempt_id = uuid.uuid4().hex
+    policy = ledger.get("policy")
+    parser = ledger.get("parser")
+    policy_id = str(policy.get("id") or "") if isinstance(policy, dict) else ""
+    parser_signature = str(parser.get("parser_signature") or "") if isinstance(parser, dict) else ""
+    key_material = "\0".join((document_id_sha256, policy_id, parser_signature, outcome, attempt_id))
+    attempt_key_sha256 = hashlib.sha256(key_material.encode()).hexdigest()
+    path = Path(kb_dir) / ATTEMPT_LEDGER_DIRNAME / f"{attempt_key_sha256}.json"
+    atomic_write_json(
+        path,
+        {
+            **ledger,
+            "document_id_sha256": document_id_sha256,
+            "decision": {
+                "attempt_id": attempt_id,
+                "attempt_key_sha256": attempt_key_sha256,
+                "ledger_role": "attempt",
+                "policy_outcome": outcome,
+            },
+        },
+    )
+    return path, attempt_id
+
+
+__all__ = [
+    "ATTEMPT_LEDGER_DIRNAME",
+    "BlockPolicyDecision",
+    "FILTERED_LAYOUT_TYPES",
+    "LEDGER_DIRNAME",
+    "MULTIMODAL_TYPES",
+    "POLICY_ID",
+    "PRESERVED_AUXILIARY_TYPES",
+    "PRESERVED_SEMANTIC_TYPES",
+    "PRESERVED_TYPES",
+    "prepare_content_list",
+    "write_attempt_ledger",
+    "write_decision_ledger",
+]

deeptutor/services/rag/pipelines/lightrag/cache_reuse.py

@@ -0,0 +1,137 @@
+"""Seed a rebuild with content-bound LightRAG indexing responses (#1577)."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+import re
+from typing import Any
+
+from deeptutor.services.file_io import atomic_write_json
+
+from .engine import workspace_for
+
+logger = logging.getLogger(__name__)
+
+_CACHE_FILE = "kv_store_llm_response_cache.json"
+# Query answers and keyword extraction depend on the current graph and must
+# never cross a full rebuild. These types are produced from source content by
+# the pinned LightRAG SDK and include the model identity in their hash.
+_INDEX_TYPES = frozenset({"extract", "summary", "analysis", "smartheading"})
+_CACHE_KEY = re.compile(r"^[^:]+:([a-z]+):[0-9a-f]{32,64}$")
+
+
+def _policy_identity(policy: Any) -> tuple[str, str, str] | None:
+    """Read both current nested role fingerprints and older flat policies."""
+    if not isinstance(policy, dict):
+        return None
+    extract = policy.get("extract")
+    if isinstance(extract, dict):
+        vlm = policy.get("vlm") if isinstance(policy.get("vlm"), dict) else {}
+        vlm_snapshot = vlm.get("snapshot") if isinstance(vlm.get("snapshot"), dict) else {}
+        fingerprint = extract.get("fingerprint")
+        if isinstance(fingerprint, str) and fingerprint:
+            return (
+                fingerprint,
+                str(vlm.get("mode") or "disabled"),
+                str(vlm_snapshot.get("fingerprint") or ""),
+            )
+    fingerprint = policy.get("fingerprint")
+    if isinstance(fingerprint, str) and fingerprint:
+        return (fingerprint, "", "")
+    return None
+
+
+def _read_policy_identity(root: Path) -> tuple[str, str, str] | None:
+    try:
+        meta = json.loads((root / "meta.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        # Interrupted versions have no publication marker but may already
+        # have a durable cache worth preserving.
+        return None
+    return _policy_identity(meta.get("indexing_policy")) if isinstance(meta, dict) else None
+
+
+def _index_entries(path: Path) -> dict[str, dict[str, Any]]:
+    if path.is_symlink() or not path.is_file():
+        return {}
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        logger.warning("Ignoring unreadable LightRAG cache %s: %s", path, type(exc).__name__)
+        return {}
+    if not isinstance(payload, dict):
+        return {}
+    entries: dict[str, dict[str, Any]] = {}
+    for key, value in payload.items():
+        match = _CACHE_KEY.fullmatch(key) if isinstance(key, str) else None
+        if (
+            match
+            and match.group(1) in _INDEX_TYPES
+            and isinstance(value, dict)
+            and value.get("cache_type") == match.group(1)
+            and isinstance(value.get("return"), str)
+            and value["return"]
+        ):
+            entries[key] = value
+    return entries
+
+
+def inherit_index_cache(kb_dir: Path, target_root: Path, policy: dict[str, Any]) -> bool:
+    """Carry only verified indexing responses into a fresh version workspace.
+
+    Merge matching published versions with the immediately preceding
+    interrupted version, which may have no ``meta.json``. A partial interrupted
+    cache must not hide the rest of an older published cache. Newer entries win
+    on duplicate content keys; query answers never enter the merge.
+    """
+    target = target_root / workspace_for(target_root) / _CACHE_FILE
+    if target.exists():
+        return False
+    target_identity = _policy_identity(policy)
+    target_version = int(target_root.name.removeprefix("version-"))
+    candidates: list[tuple[int, int, int, Path]] = []
+    for root in kb_dir.glob("version-*"):
+        if not root.is_dir() or root == target_root:
+            continue
+        try:
+            version = int(root.name.removeprefix("version-"))
+        except ValueError:
+            continue
+        if version >= target_version:
+            continue
+        donor_identity = _read_policy_identity(root)
+        # An unpublished predecessor may already have useful content-addressed
+        # extraction calls. Published versions must match the current roles.
+        if donor_identity is None and version == target_version - 1:
+            identity_rank = 3
+        elif donor_identity is not None and (
+            target_identity is None or donor_identity == target_identity
+        ):
+            identity_rank = 2
+        else:
+            continue
+        paths = [root / _CACHE_FILE, *root.glob(f"deeptutor_*/{_CACHE_FILE}")]
+        for path in paths:
+            if path.is_file() and not path.is_symlink():
+                candidates.append((identity_rank, version, int(path.parent != root), path))
+    if not candidates:
+        return False
+
+    merged: dict[str, dict[str, Any]] = {}
+    used = 0
+    for _, _, _, donor in sorted(candidates):
+        entries = _index_entries(donor)
+        if not entries:
+            continue
+        merged.update(entries)
+        used += 1
+    if not merged:
+        return False
+    atomic_write_json(target, merged)
+    logger.info("Inherited %d LightRAG indexing cache entries from %d donors", len(merged), used)
+    return True
+
+
+__all__ = ["inherit_index_cache"]