graph-knowledge-doc-parser 0.1.0__py3-none-any.whl

kg_doc_parser/workflow_ingest/semantics.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from pydantic import BaseModel, Field
+from pydantic import model_validator
+
+from kogwistar.id_provider import stable_id
+
+
+def _normalize_text(text: str) -> str:
+    return re.sub(r"\s+", "", text or "")
+
+
+class HydratedTextPointer(BaseModel):
+    source_cluster_id: str
+    start_char: int
+    end_char: int
+    verbatim_text: str
+
+
+class SemanticNode(BaseModel):
+    node_id: str | None = None
+    parent_id: str | None = None
+    node_type: str = "TEXT_FLOW"
+    title: str
+    total_content_pointers: list[HydratedTextPointer] = Field(default_factory=list)
+    child_nodes: list["SemanticNode"] = Field(default_factory=list)
+    level_from_root: int = 0
+
+    @model_validator(mode="after")
+    def _ensure_stable_node_id(self) -> "SemanticNode":
+        if self.node_id:
+            return self
+        pointer_fp = "|".join(
+            f"{ptr.source_cluster_id}:{ptr.start_char}:{ptr.end_char}:{ptr.verbatim_text}"
+            for ptr in self.total_content_pointers
+        )
+        self.node_id = str(
+            stable_id(
+                "workflow_ingest.semantic_node",
+                str(self.parent_id or "root"),
+                str(self.node_type),
+                str(self.title),
+                str(self.level_from_root),
+                pointer_fp,
+            )
+        )
+        return self
+
+
+SemanticNode.model_rebuild()
+
+
+def correct_and_validate_pointer(
+    pointer: HydratedTextPointer,
+    source_map: dict[str, dict[str, Any]],
+) -> HydratedTextPointer | None:
+    source = source_map.get(pointer.source_cluster_id)
+    if source is None:
+        return None
+    text = source.get("text", "")
+    end_exclusive = len(text) if pointer.end_char == -1 else pointer.end_char + 1
+    actual = text[max(pointer.start_char, 0):max(end_exclusive, 0)]
+    if _normalize_text(actual) == _normalize_text(pointer.verbatim_text):
+        return pointer
+    if actual and (
+        _normalize_text(actual) in _normalize_text(pointer.verbatim_text)
+        or _normalize_text(pointer.verbatim_text) in _normalize_text(actual)
+    ):
+        return HydratedTextPointer(
+            source_cluster_id=pointer.source_cluster_id,
+            start_char=max(pointer.start_char, 0),
+            end_char=max(end_exclusive - 1, -1),
+            verbatim_text=actual,
+        )
+
+    occurrences = []
+    start = 0
+    needle = pointer.verbatim_text or ""
+    while needle:
+        idx = text.find(needle, start)
+        if idx == -1:
+            break
+        occurrences.append((idx, idx + len(needle) - 1))
+        start = idx + max(1, len(needle))
+    if not occurrences:
+        return None
+    best_start, best_end = min(occurrences, key=lambda item: abs(item[0] - pointer.start_char))
+    return HydratedTextPointer(
+        source_cluster_id=pointer.source_cluster_id,
+        start_char=best_start,
+        end_char=best_end,
+        verbatim_text=text[best_start:best_end + 1],
+    )
+
+
+def compute_pointer_coverage(
+    root_node: SemanticNode,
+    source_map: dict[str, dict[str, Any]],
+) -> dict[str, Any]:
+    ranges: dict[str, list[tuple[int, int]]] = {}
+
+    def walk(node: SemanticNode) -> None:
+        if node.node_type != "DOCUMENT_ROOT":
+            for ptr in node.total_content_pointers:
+                end = ptr.end_char
+                if end == -1:
+                    end = max(0, len(source_map.get(ptr.source_cluster_id, {}).get("text", "")) - 1)
+                ranges.setdefault(ptr.source_cluster_id, []).append((ptr.start_char, end))
+        for child in node.child_nodes:
+            walk(child)
+
+    walk(root_node)
+    per_cluster: dict[str, float] = {}
+    total_len = 0
+    total_covered = 0
+    for cluster_id, cluster_ranges in ranges.items():
+        text = source_map.get(cluster_id, {}).get("text", "")
+        if not text:
+            continue
+        cluster_ranges.sort()
+        merged: list[tuple[int, int]] = []
+        cur_s, cur_e = cluster_ranges[0]
+        for s, e in cluster_ranges[1:]:
+            if s <= cur_e + 1:
+                cur_e = max(cur_e, e)
+            else:
+                merged.append((cur_s, cur_e))
+                cur_s, cur_e = s, e
+        merged.append((cur_s, cur_e))
+        covered = sum((e - s + 1) for s, e in merged)
+        per_cluster[cluster_id] = covered / len(text)
+        total_len += len(text)
+        total_covered += covered
+    overall = total_covered / total_len if total_len else 1.0
+    return {"per_cluster": per_cluster, "overall": overall}
+
+
+def semantic_tree_to_kge_payload(root: SemanticNode, *, doc_id: str) -> dict[str, Any]:
+    nodes: list[dict[str, Any]] = []
+    edges: list[dict[str, Any]] = []
+
+    def spans(ptrs: list[HydratedTextPointer]) -> list[dict[str, Any]]:
+        if not ptrs:
+            return [
+                {
+                    "doc_id": doc_id,
+                    "collection_page_url": f"doc://{doc_id}",
+                    "document_page_url": f"doc://{doc_id}#synthetic",
+                    "insertion_method": "workflow_ingest",
+                    "page_number": 1,
+                    "start_char": 0,
+                    "end_char": 1,
+                    "excerpt": " ",
+                    "context_before": "",
+                    "context_after": "",
+                    "chunk_id": None,
+                    "source_cluster_id": None,
+                    "verification": None,
+                }
+            ]
+        return [
+            {
+                "doc_id": doc_id,
+                "collection_page_url": f"doc://{doc_id}",
+                "document_page_url": f"doc://{doc_id}#{p.source_cluster_id}",
+                "insertion_method": "workflow_ingest",
+                "page_number": 1,
+                "start_char": p.start_char,
+                "end_char": max(p.end_char + 1, p.start_char + max(len(p.verbatim_text), 1)),
+                "excerpt": p.verbatim_text,
+                "context_before": "",
+                "context_after": "",
+                "source_cluster_id": p.source_cluster_id,
+                "verification": None,
+            }
+            for p in ptrs
+        ]
+
+    def walk(node: SemanticNode) -> None:
+        nodes.append(
+            {
+                "id": node.node_id,
+                "label": node.title,
+                "type": "entity",
+                "summary": node.title,
+                "metadata": {
+                    "semantic_node_type": node.node_type,
+                    "doc_id": doc_id,
+                    "parent_id": node.parent_id,
+                    "level_from_root": node.level_from_root,
+                },
+                "mentions": [{"spans": spans(node.total_content_pointers)}],
+            }
+        )
+        for child in node.child_nodes:
+            edges.append(
+                {
+                    "id": str(
+                        stable_id(
+                            "workflow_ingest.edge",
+                            "HAS_CHILD",
+                            str(node.node_id),
+                            str(child.node_id),
+                            str(doc_id),
+                        )
+                    ),
+                    "label": "parent-child",
+                    "type": "relationship",
+                    "summary": f"{node.node_id}->{child.node_id}",
+                    "relation": "HAS_CHILD",
+                    "source_ids": [node.node_id],
+                    "target_ids": [child.node_id],
+                    "source_edge_ids": [],
+                    "target_edge_ids": [],
+                    "mentions": [
+                        {
+                            "spans": spans(
+                                child.total_content_pointers or node.total_content_pointers
+                            )
+                        }
+                    ],
+                    "metadata": {"doc_id": doc_id, "insertion_method": "workflow_ingest"},
+                }
+            )
+            walk(child)
+
+    walk(root)
+    return {"doc_id": doc_id, "insertion_method": "workflow_ingest", "nodes": nodes, "edges": edges}

kg_doc_parser/workflow_ingest/service.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from kogwistar.engine_core.engine import GraphKnowledgeEngine
+from kogwistar.runtime.runtime import WorkflowRuntime
+
+from .clients import DirectRuntimeIngestClient
+from .design import DEFAULT_WORKFLOW_ID
+from .handlers import build_ingest_step_resolver
+from .models import IngestRunResult, WorkflowIngestInput
+from .providers import WorkflowProviderSettings, build_embedding_function
+
+
+class _TinyEmbeddingFunction:
+    _name = "kg-doc-parser-workflow-embedding-v1"
+
+    def name(self):
+        return self._name
+
+    def __call__(self, input):
+        vectors = []
+        for value in input:
+            text = str(value or "")
+            checksum = float((sum(ord(ch) for ch in text) % 97) + 1)
+            vectors.append([float(len(text) + 1), checksum])
+        return vectors
+
+
+def build_default_engines(
+    base_dir: str | Path,
+    *,
+    embedding_function=None,
+    backend_factory=None,
+    provider_settings: WorkflowProviderSettings | None = None,
+):
+    base_dir = Path(base_dir)
+    provider_settings = provider_settings or WorkflowProviderSettings.from_env()
+    # One embedding function is still wired per engine instance. The workflow
+    # can carry embedding-space metadata, but engine-level routing is a future
+    # Kogwistar concern.
+    embedding = embedding_function or build_embedding_function(provider_settings.embedding)
+    workflow_engine = GraphKnowledgeEngine(
+        persist_directory=str(base_dir / "workflow"),
+        kg_graph_type="workflow",
+        embedding_function=embedding,
+        backend_factory=backend_factory,
+    )
+    conversation_engine = GraphKnowledgeEngine(
+        persist_directory=str(base_dir / "conversation"),
+        kg_graph_type="conversation",
+        embedding_function=embedding,
+        backend_factory=backend_factory,
+    )
+    knowledge_engine = GraphKnowledgeEngine(
+        persist_directory=str(base_dir / "knowledge"),
+        kg_graph_type="knowledge",
+        embedding_function=embedding,
+        backend_factory=backend_factory,
+    )
+    return workflow_engine, conversation_engine, knowledge_engine
+
+
+def build_runtime(*, workflow_engine, conversation_engine, deps: dict[str, Any] | None = None):
+    resolver = build_ingest_step_resolver(deps=deps)
+    return WorkflowRuntime(
+        workflow_engine=workflow_engine,
+        conversation_engine=conversation_engine,
+        step_resolver=resolver.resolve,
+        predicate_registry={},
+        trace=False,
+    )
+
+
+def run_ingest_workflow(
+    *,
+    inp: WorkflowIngestInput,
+    workflow_engine,
+    conversation_engine,
+    knowledge_engine=None,
+    workflow_id: str = DEFAULT_WORKFLOW_ID,
+    deps: dict[str, Any] | None = None,
+):
+    client = DirectRuntimeIngestClient(
+        workflow_engine=workflow_engine,
+        conversation_engine=conversation_engine,
+        knowledge_engine=knowledge_engine,
+    )
+    result = client.run_ingest(
+        inp=inp,
+        workflow_id=workflow_id,
+        deps=deps,
+    )
+    return _legacy_run_result(result)
+
+
+def _legacy_run_result(result: IngestRunResult):
+    class _RunCompat:
+        def __init__(self, *, run_id: str, final_state: dict[str, Any], status: str) -> None:
+            self.run_id = run_id
+            self.final_state = final_state
+            self.status = status
+
+    return (
+        _RunCompat(
+            run_id=result.handle.run_id,
+            final_state=result.final_state,
+            status=result.status,
+        ),
+        result.bundle,
+    )

kg_doc_parser/workflow_ingest/smoke_assets.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+"""Reusable OCR smoke asset generation for manual and CLI workflows."""
+
+from pathlib import Path
+
+from PIL import Image, ImageDraw
+
+
+def _draw_page(path: Path, *, title: str, lines: list[str]) -> None:
+    image = Image.new("RGB", (1400, 1000), "white")
+    draw = ImageDraw.Draw(image)
+    draw.text((80, 70), title, fill="black")
+    y = 160
+    for line in lines:
+        draw.text((80, y), line, fill="black")
+        y += 90
+    path.parent.mkdir(parents=True, exist_ok=True)
+    image.save(path, "PNG")
+
+
+def _build_pdf(image_paths: list[Path], pdf_path: Path) -> None:
+    images = [Image.open(path).convert("RGB") for path in image_paths]
+    first, rest = images[0], images[1:]
+    pdf_path.parent.mkdir(parents=True, exist_ok=True)
+    first.save(pdf_path, "PDF", save_all=True, append_images=rest)
+    for image in images:
+        image.close()
+
+
+def generate_ocr_smoke_assets(output_dir: Path) -> dict[str, str]:
+    """Generate inspectable OCR smoke assets and return their file paths."""
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    page_1 = output_dir / "ocr_smoke_page_1.png"
+    page_2 = output_dir / "ocr_smoke_page_2.png"
+    pdf_path = output_dir / "ocr_smoke_document.pdf"
+
+    _draw_page(
+        page_1,
+        title="OCR Workflow Smoke Page 1",
+        lines=[
+            "Section 1. Overview",
+            "Alpha clause: the workflow should preserve page order.",
+            "Beta clause: per-page OCR artifacts should remain inspectable.",
+        ],
+    )
+    _draw_page(
+        page_2,
+        title="OCR Workflow Smoke Page 2",
+        lines=[
+            "Section 2. Review",
+            "Gamma clause: downstream source maps should include OCR text.",
+            "Delta clause: resume state should survive reruns.",
+        ],
+    )
+    _build_pdf([page_1, page_2], pdf_path)
+    return {
+        "page_1_png": str(page_1),
+        "page_2_png": str(page_2),
+        "pdf": str(pdf_path),
+    }