graph-knowledge-doc-parser 0.1.0__py3-none-any.whl

kg_doc_parser/workflow_ingest/__init__.py

@@ -0,0 +1,187 @@
+from .adapters import (
+    build_authoritative_source_map,
+    build_parser_input_dict,
+    build_parser_source_map,
+    normalize_ocr_pages,
+)
+from .clients import (
+    CanonicalGraphPersistenceClient,
+    DocumentTreeApiPersistenceClient,
+    DirectRuntimeIngestClient,
+    IngestExecutionClient,
+    ServerCanonicalKgClient,
+    UnsupportedClientOperation,
+)
+from .cache import WorkflowLLMCallCache
+from .design import DEFAULT_WORKFLOW_ID, build_ingest_workflow_design, ensure_ingest_workflow_design
+from .demo_harness import DemoHarnessArtifacts, DemoHarnessConfig, run_demo_harness
+from .handlers import build_ingest_step_resolver
+from .parsing import (
+    OCRParseRequest,
+    PageIndexParseRequest,
+    ParseMode,
+    TreeParseRequest,
+    parse_document,
+    parse_ocr_document,
+    parse_page_index_document,
+    parse_tree_document,
+)
+from .ocr_pipeline import (
+    OCRImagePayload,
+    OCRWorkflowArtifacts,
+    OCRWorkflowStateStore,
+    prepare_ocr_workflow_input,
+    run_ocr_ingest_workflow,
+)
+from .runners import (
+    LayerwiseWorkflowCommandResult,
+    OcrWorkflowCommandResult,
+    PageIndexWorkflowCommandResult,
+    WorkflowCommandResult,
+    discover_input_files,
+    run_demo_harness_workflow,
+    run_layerwise_batch_workflow,
+    run_layerwise_source_workflow,
+    run_ocr_batch_workflow,
+    run_ocr_source_workflow,
+    run_page_index_batch_workflow,
+    run_page_index_source_workflow,
+)
+from .smoke_assets import generate_ocr_smoke_assets
+from .page_index import PageIndexBlockSpec, PageIndexParseResult, build_page_index_workflow_input
+from .models import (
+    BoundingBox,
+    CanonicalGraphWriteResult,
+    CurrentLayerContext,
+    CurrentLayerReview,
+    CurrentLayerResult,
+    LayerCoverageGap,
+    LayerDuplicateChildNote,
+    GroundedSourceRecord,
+    IngestRunHandle,
+    IngestRunResult,
+    LayerChildCandidate,
+    LayerFrontierItem,
+    LayerSpanConflict,
+    NormalizedPage,
+    NormalizedSourceCollection,
+    ParseSessionState,
+    SourceUnit,
+    ValidationReport,
+    WorkflowExportBundle,
+    WorkflowIngestInput,
+)
+from .providers import (
+    EmbeddingProviderConfig,
+    FakeChatModel,
+    ProviderEndpointConfig,
+    WorkflowProviderSettings,
+    build_chat_model,
+    build_chat_model_for_role,
+    build_embedding_function,
+)
+from .probe import WorkflowProbe, emit_probe_event
+from .parser_core import (
+    apply_cud_update,
+    check_layer_coverage,
+    default_parse_semantic_fn,
+    finalize_semantic_tree,
+    initialize_parse_session,
+    prepare_layer_frontier,
+    propose_layer_breakdown,
+    review_layer,
+)
+from .service import build_default_engines, build_runtime, run_ingest_workflow
+from .semantics import HydratedTextPointer, SemanticNode
+
+__all__ = [
+    "BoundingBox",
+    "CanonicalGraphPersistenceClient",
+    "CanonicalGraphWriteResult",
+    "CurrentLayerContext",
+    "CurrentLayerReview",
+    "CurrentLayerResult",
+    "LayerCoverageGap",
+    "LayerDuplicateChildNote",
+    "DemoHarnessArtifacts",
+    "DemoHarnessConfig",
+    "DocumentTreeApiPersistenceClient",
+    "DirectRuntimeIngestClient",
+    "EmbeddingProviderConfig",
+    "GroundedSourceRecord",
+    "HydratedTextPointer",
+    "FakeChatModel",
+    "IngestExecutionClient",
+    "IngestRunHandle",
+    "IngestRunResult",
+    "LayerChildCandidate",
+    "LayerFrontierItem",
+    "LayerSpanConflict",
+    "NormalizedPage",
+    "NormalizedSourceCollection",
+    "OCRImagePayload",
+    "OCRWorkflowArtifacts",
+    "OCRWorkflowStateStore",
+    "OCRParseRequest",
+    "ParseSessionState",
+    "ProviderEndpointConfig",
+    "PageIndexParseRequest",
+    "PageIndexBlockSpec",
+    "PageIndexParseResult",
+    "ParseMode",
+    "TreeParseRequest",
+    "SemanticNode",
+    "ServerCanonicalKgClient",
+    "SourceUnit",
+    "UnsupportedClientOperation",
+    "ValidationReport",
+    "WorkflowLLMCallCache",
+    "WorkflowProbe",
+    "WorkflowProviderSettings",
+    "WorkflowExportBundle",
+    "WorkflowIngestInput",
+    "WorkflowCommandResult",
+    "OcrWorkflowCommandResult",
+    "PageIndexWorkflowCommandResult",
+    "LayerwiseWorkflowCommandResult",
+    "DEFAULT_WORKFLOW_ID",
+    "build_chat_model",
+    "build_chat_model_for_role",
+    "build_embedding_function",
+    "apply_cud_update",
+    "check_layer_coverage",
+    "default_parse_semantic_fn",
+    "finalize_semantic_tree",
+    "initialize_parse_session",
+    "prepare_layer_frontier",
+    "propose_layer_breakdown",
+    "review_layer",
+    "build_authoritative_source_map",
+    "build_default_engines",
+    "build_ingest_workflow_design",
+    "build_ingest_step_resolver",
+    "build_parser_input_dict",
+    "build_page_index_workflow_input",
+    "build_parser_source_map",
+    "build_runtime",
+    "ensure_ingest_workflow_design",
+    "emit_probe_event",
+    "normalize_ocr_pages",
+    "parse_document",
+    "parse_ocr_document",
+    "parse_page_index_document",
+    "parse_tree_document",
+    "prepare_ocr_workflow_input",
+    "run_demo_harness",
+    "run_demo_harness_workflow",
+    "run_ingest_workflow",
+    "discover_input_files",
+    "run_layerwise_batch_workflow",
+    "run_layerwise_source_workflow",
+    "run_ocr_batch_workflow",
+    "run_ocr_source_workflow",
+    "run_page_index_batch_workflow",
+    "run_page_index_source_workflow",
+    "generate_ocr_smoke_assets",
+    "run_ocr_ingest_workflow",
+]

kg_doc_parser/workflow_ingest/_kogwistar.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from importlib import import_module
+
+
+def ensure_kogwistar_on_path() -> None:
+    """Compatibility shim: require a normal kogwistar installation/import.
+
+    We intentionally do not mutate ``sys.path`` here. The repo may contain a sibling
+    checkout for inspection, but runtime imports should resolve through the installed
+    package environment, including editable installs.
+    """
+    import_module("kogwistar")

kg_doc_parser/workflow_ingest/adapters.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+from typing import Any, TypedDict
+
+from .models import (
+    BoundingBox,
+    GroundedSourceRecord,
+    NormalizedPage,
+    NormalizedSourceCollection,
+    SourceUnit,
+    WorkflowIngestInput,
+)
+
+
+class OCRPageJSON(TypedDict, total=False):
+    """Serialized OCR page payload produced by the OCR preparation layer."""
+
+    pdf_page_num: int
+    printed_page_number: str
+    contains_table: bool
+    OCR_text_clusters: list[dict[str, Any]]
+    non_text_objects: list[dict[str, Any]]
+    text: str
+
+
+def normalize_ocr_pages(
+    *,
+    document_id: str,
+    title: str,
+    pages: list[OCRPageJSON],
+) -> WorkflowIngestInput:
+    """Normalize raw OCR JSON into workflow ingest models.
+
+    OCR text clusters and non-text regions are both preserved with page and
+    cluster metadata. `embedding_space="image"` is an intent label here; it
+    does not imply a separate image embedder is already wired at runtime.
+    """
+    normalized_pages: list[NormalizedPage] = []
+    for raw_page in pages:
+        page_number = int(raw_page["pdf_page_num"])
+        units: list[SourceUnit] = []
+        for cluster in raw_page.get("OCR_text_clusters", []):
+            bbox = BoundingBox(
+                y_min=float(cluster.get("bb_y_min", 0.0)),
+                x_min=float(cluster.get("bb_x_min", 0.0)),
+                y_max=float(cluster.get("bb_y_max", 0.0)),
+                x_max=float(cluster.get("bb_x_max", 0.0)),
+            )
+            units.append(
+                SourceUnit(
+                    modality="ocr_text",
+                    page_number=page_number,
+                    cluster_number=cluster.get("cluster_number"),
+                    text=cluster.get("text"),
+                    bbox=bbox,
+                    metadata={},
+                )
+            )
+        for obj in raw_page.get("non_text_objects", []):
+            bbox = BoundingBox(
+                y_min=float(obj.get("bb_y_min", 0.0)),
+                x_min=float(obj.get("bb_x_min", 0.0)),
+                y_max=float(obj.get("bb_y_max", 0.0)),
+                x_max=float(obj.get("bb_x_max", 0.0)),
+            )
+            units.append(
+                SourceUnit(
+                    modality="image_region",
+                    page_number=page_number,
+                    cluster_number=obj.get("cluster_number"),
+                    description=obj.get("description"),
+                    bbox=bbox,
+                    # Keep the "image" space label for future routing; the
+                    # current engine still embeds through a single function.
+                    embedding_space="image",
+                    metadata={"participates_in_semantic_text": False},
+                )
+            )
+        normalized_pages.append(
+            NormalizedPage(
+                page_number=page_number,
+                units=units,
+                metadata={
+                    "printed_page_number": raw_page.get("printed_page_number"),
+                    "contains_table": raw_page.get("contains_table"),
+                },
+            )
+        )
+    embedding_spaces = ["default_text"]
+    if any(
+        unit.embedding_space == "image"
+        for page in normalized_pages
+        for unit in page.units
+    ):
+        embedding_spaces.append("image")
+    return WorkflowIngestInput(
+        request_id=document_id,
+        collections=[
+            NormalizedSourceCollection(
+                collection_id=document_id,
+                title=title,
+                modality="ocr",
+                pages=normalized_pages,
+                embedding_spaces=embedding_spaces,
+            )
+        ],
+    )
+
+
+def build_authoritative_source_map(
+    inp: WorkflowIngestInput,
+) -> dict[str, GroundedSourceRecord]:
+    source_map: dict[str, GroundedSourceRecord] = {}
+    for collection in inp.collections:
+        for page in collection.pages:
+            ordinal_by_modality: dict[str, int] = {}
+            for unit in page.units:
+                modality_prefix = "t" if unit.modality in {"text", "ocr_text"} else "i"
+                original_cluster = unit.cluster_number
+                if original_cluster is None:
+                    original_cluster = ordinal_by_modality.get(modality_prefix, 0)
+                ordinal_by_modality[modality_prefix] = ordinal_by_modality.get(modality_prefix, 0) + 1
+                unit_id = unit.unit_id or f"{collection.collection_id}|p{page.page_number}_{modality_prefix}{original_cluster}"
+                record = GroundedSourceRecord(
+                    unit_id=unit_id,
+                    collection_id=collection.collection_id,
+                    modality=unit.modality,
+                    page_number=page.page_number,
+                    cluster_number=unit.cluster_number,
+                    text=unit.text or unit.description or "",
+                    parser_text=unit.parser_text,
+                    source_uri=unit.source_uri,
+                    embedding_space=unit.embedding_space,
+                    participates_in_semantic_text=unit.modality in {"text", "ocr_text"},
+                    bbox=unit.bbox,
+                    metadata={
+                        **unit.metadata,
+                        "original_cluster_number": unit.cluster_number,
+                    },
+                )
+                if unit_id in source_map:
+                    raise ValueError(f"source map collision detected for {unit_id}")
+                source_map[unit_id] = record
+    return source_map
+
+
+def select_primary_collection(inp: WorkflowIngestInput) -> NormalizedSourceCollection:
+    return inp.collections[0]
+
+
+def build_parser_input_dict(
+    collection: NormalizedSourceCollection,
+) -> dict[str, Any]:
+    pages: list[dict[str, Any]] = []
+    for page in collection.pages:
+        text_clusters = []
+        non_text_objects = []
+        next_cluster = 0
+        for unit in page.units:
+            bbox = unit.bbox
+            cluster_number = unit.cluster_number if unit.cluster_number is not None else next_cluster
+            if unit.modality in {"text", "ocr_text"}:
+                text_clusters.append(
+                    {
+                        "text": unit.text or "",
+                        "bb_x_min": bbox.x_min if bbox else 0.0,
+                        "bb_x_max": bbox.x_max if bbox else 0.0,
+                        "bb_y_min": bbox.y_min if bbox else 0.0,
+                        "bb_y_max": bbox.y_max if bbox else 0.0,
+                        "cluster_number": cluster_number,
+                    }
+                )
+            else:
+                non_text_objects.append(
+                    {
+                        "description": unit.description or unit.source_uri or "image-region",
+                        "bb_x_min": bbox.x_min if bbox else 0.0,
+                        "bb_x_max": bbox.x_max if bbox else 0.0,
+                        "bb_y_min": bbox.y_min if bbox else 0.0,
+                        "bb_y_max": bbox.y_max if bbox else 0.0,
+                        "cluster_number": cluster_number,
+                    }
+                )
+            next_cluster += 1
+        pages.append(
+            {
+                "pdf_page_num": page.page_number,
+                "printed_page_number": page.metadata.get("printed_page_number"),
+                "contains_table": page.metadata.get("contains_table", False),
+                "OCR_text_clusters": text_clusters,
+                "non_text_objects": non_text_objects,
+            }
+        )
+    return {"document_filename": collection.title, "pages": pages}
+
+
+def build_parser_source_map(
+    source_map: dict[str, GroundedSourceRecord],
+) -> dict[str, dict[str, Any]]:
+    return {
+        unit_id: {
+            "id": record.unit_id,
+            "text": record.parser_text,
+            "modality": record.modality,
+            "page_number": record.page_number,
+            "cluster_number": record.cluster_number,
+            "embedding_space": record.embedding_space,
+            "participates_in_semantic_text": record.participates_in_semantic_text,
+            "metadata": record.metadata,
+        }
+        for unit_id, record in source_map.items()
+    }

kg_doc_parser/workflow_ingest/cache.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Callable, TypeVar
+
+from kogwistar.id_provider import stable_id
+
+from .probe import emit_probe_event
+
+T = TypeVar("T")
+
+
+def _jsonable(value: Any) -> Any:
+    if hasattr(value, "model_dump"):
+        try:
+            return value.model_dump(field_mode="backend", dump_format="json")
+        except TypeError:
+            return value.model_dump()
+    if isinstance(value, dict):
+        return {str(k): _jsonable(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_jsonable(v) for v in value]
+    return value
+
+
+class WorkflowLLMCallCache:
+    def __init__(self, cache_dir: str | Path, *, probe=None) -> None:
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.probe = probe
+
+    def _cache_path(self, operation: str, fingerprint: dict[str, Any]) -> Path:
+        payload = json.dumps(_jsonable(fingerprint), sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+        cache_id = stable_id("workflow_ingest.llm_call", operation, payload)
+        return self.cache_dir / f"{cache_id}.json"
+
+    def cached_call(
+        self,
+        *,
+        operation: str,
+        fingerprint: dict[str, Any],
+        fn: Callable[[], T],
+    ) -> Any:
+        path = self._cache_path(operation, fingerprint)
+        if path.exists():
+            emit_probe_event(
+                self.probe,
+                "workflow.llm_cache_hit",
+                operation=operation,
+                cache_path=str(path),
+            )
+            return json.loads(path.read_text(encoding="utf-8"))
+        result = fn()
+        payload = _jsonable(result)
+        path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+        emit_probe_event(
+            self.probe,
+            "workflow.llm_cache_miss",
+            operation=operation,
+            cache_path=str(path),
+        )
+        return payload