langparse 0.1.0__py3-none-any.whl

langparse/workbooks/blocks.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from openpyxl.utils import get_column_letter, range_boundaries
+
+from langparse.workbooks.classification import BlockClassification
+from langparse.workbooks.types import (
+    CandidateRegion,
+    FormBlock,
+    FormField,
+    MatrixBlock,
+    MatrixHeader,
+    SheetSnapshot,
+    SourceRef,
+    TextBlock,
+    TextLine,
+    stable_id,
+)
+
+
+def interpret_form_block(
+    sheet: SheetSnapshot,
+    candidate: CandidateRegion,
+    classification: BlockClassification,
+) -> FormBlock:
+    """Interpret adjacent label/value pairs without inventing missing fields."""
+
+    min_col, min_row, max_col, max_row = range_boundaries(candidate.source_ref.range)
+    title = ""
+    fields: list[FormField] = []
+    free_text: list[TextLine] = []
+    for row_number in range(min_row, max_row + 1):
+        entries = _row_entries(sheet, row_number, min_col, max_col)
+        if row_number == min_row and len(entries) == 1:
+            title = entries[0][1]
+            continue
+        pairs = _adjacent_pairs(entries)
+        if pairs is not None:
+            for (label_coordinate, label), (value_coordinate, value) in pairs:
+                label_ref = SourceRef(sheet_name=sheet.name, range=label_coordinate)
+                value_ref = SourceRef(sheet_name=sheet.name, range=value_coordinate)
+                fields.append(
+                    FormField(
+                        field_id=stable_id(
+                            "field",
+                            candidate.source_ref.key,
+                            label_ref.key,
+                            value_ref.key,
+                        ),
+                        label=label,
+                        value=value,
+                        label_source_refs=[label_ref],
+                        value_source_refs=[value_ref],
+                        confidence=classification.confidence,
+                    )
+                )
+            continue
+        if entries:
+            free_text.append(
+                TextLine(
+                    text=" ".join(value for _, value in entries),
+                    source_refs=[
+                        SourceRef(sheet_name=sheet.name, range=coordinate)
+                        for coordinate, _ in entries
+                    ],
+                )
+            )
+    return FormBlock(
+        form_id=stable_id("form", candidate.source_ref.key),
+        title=title,
+        fields=fields,
+        free_text=free_text,
+        source_refs=[candidate.source_ref],
+        confidence=classification.confidence,
+        diagnostics=[{"reason_codes": list(classification.reason_codes)}],
+    )
+
+
+def interpret_matrix_block(
+    sheet: SheetSnapshot,
+    candidate: CandidateRegion,
+    classification: BlockClassification,
+) -> MatrixBlock:
+    """Preserve a two-axis matrix and its physical value grid."""
+
+    min_col, min_row, max_col, max_row = range_boundaries(candidate.source_ref.range)
+    if max_col - min_col < 2 or max_row - min_row < 2:
+        raise ValueError("matrix requires at least two row and column dimensions")
+
+    title = _display_value(sheet, min_row, min_col)
+    column_headers = [
+        MatrixHeader(
+            value=_display_value(sheet, min_row, column),
+            source_refs=[_source_ref(sheet, min_row, column)],
+        )
+        for column in range(min_col + 1, max_col + 1)
+    ]
+    row_headers = [
+        MatrixHeader(
+            value=_display_value(sheet, row, min_col),
+            source_refs=[_source_ref(sheet, row, min_col)],
+        )
+        for row in range(min_row + 1, max_row + 1)
+    ]
+    if any(not header.value for header in [*column_headers, *row_headers]):
+        raise ValueError("matrix axes must be complete")
+
+    values = []
+    value_source_refs = []
+    for row in range(min_row + 1, max_row + 1):
+        value_row = []
+        ref_row = []
+        for column in range(min_col + 1, max_col + 1):
+            value_row.append(_display_value(sheet, row, column))
+            coordinate = f"{get_column_letter(column)}{row}"
+            ref_row.append(_source_ref(sheet, row, column) if coordinate in sheet.cells else None)
+        values.append(value_row)
+        value_source_refs.append(ref_row)
+
+    return MatrixBlock(
+        matrix_id=stable_id("matrix", candidate.source_ref.key),
+        title=title,
+        row_headers=row_headers,
+        column_headers=column_headers,
+        values=values,
+        source_refs=[candidate.source_ref],
+        value_source_refs=value_source_refs,
+        confidence=classification.confidence,
+        diagnostics=[{"reason_codes": list(classification.reason_codes)}],
+    )
+
+
+def interpret_text_block(
+    sheet: SheetSnapshot,
+    candidate: CandidateRegion,
+    classification: BlockClassification,
+) -> TextBlock:
+    """Render source-ordered display text without merged-cell duplication."""
+
+    min_col, min_row, max_col, max_row = range_boundaries(candidate.source_ref.range)
+    lines = []
+    for row_number in range(min_row, max_row + 1):
+        entries = _row_entries(sheet, row_number, min_col, max_col)
+        if entries:
+            lines.append(
+                TextLine(
+                    text=" ".join(value for _, value in entries),
+                    source_refs=[
+                        SourceRef(sheet_name=sheet.name, range=coordinate)
+                        for coordinate, _ in entries
+                    ],
+                )
+            )
+    return TextBlock(
+        text_id=stable_id("text", candidate.source_ref.key),
+        lines=lines,
+        source_refs=[candidate.source_ref],
+        confidence=classification.confidence,
+        diagnostics=[{"reason_codes": list(classification.reason_codes)}],
+    )
+
+
+def _row_entries(
+    sheet: SheetSnapshot,
+    row_number: int,
+    min_col: int,
+    max_col: int,
+) -> list[tuple[str, str]]:
+    entries = []
+    for column in range(min_col, max_col + 1):
+        coordinate = f"{get_column_letter(column)}{row_number}"
+        cell = sheet.cells.get(coordinate)
+        if cell is None or cell.merge_anchor is not None or not cell.display_value.strip():
+            continue
+        entries.append((coordinate, cell.display_value))
+    return entries
+
+
+def _adjacent_pairs(
+    entries: list[tuple[str, str]],
+) -> list[tuple[tuple[str, str], tuple[str, str]]] | None:
+    if len(entries) < 2 or len(entries) % 2:
+        return None
+    pairs = []
+    for offset in range(0, len(entries), 2):
+        label, value = entries[offset : offset + 2]
+        label_column = _column_number(label[0])
+        value_column = _column_number(value[0])
+        if value_column != label_column + 1:
+            return None
+        pairs.append((label, value))
+    return pairs
+
+
+def _column_number(coordinate: str) -> int:
+    from openpyxl.utils.cell import coordinate_to_tuple
+
+    return coordinate_to_tuple(coordinate)[1]
+
+
+def _display_value(sheet: SheetSnapshot, row: int, column: int) -> str:
+    coordinate = f"{get_column_letter(column)}{row}"
+    cell = sheet.cells.get(coordinate)
+    if cell is None or cell.merge_anchor is not None:
+        return ""
+    return cell.display_value
+
+
+def _source_ref(sheet: SheetSnapshot, row: int, column: int) -> SourceRef:
+    return SourceRef(sheet_name=sheet.name, range=f"{get_column_letter(column)}{row}")

langparse/workbooks/bundle-v1.schema.json

@@ -0,0 +1,71 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/syw2014/langparse/blob/main/langparse/workbooks/bundle-v1.schema.json",
+  "title": "LangParse Workbook Bundle v1",
+  "type": "object",
+  "required": ["schema_version", "facts", "structures", "relations", "diagnostics", "provenance"],
+  "properties": {
+    "schema_version": {"const": 1},
+    "facts": {
+      "type": "object",
+      "required": ["sheets", "references"],
+      "properties": {
+        "sheets": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["name", "visibility", "used_range", "cell_count", "cells", "objects"],
+            "properties": {
+              "name": {"type": "string", "minLength": 1},
+              "visibility": {"enum": ["visible", "hidden", "veryHidden"]},
+              "used_range": {"type": ["string", "null"]},
+              "cell_count": {"type": "integer", "minimum": 0},
+              "cells": {"type": ["array", "null"], "items": {"type": "object", "required": ["coordinate", "source_ref"], "properties": {"source_ref": {"$ref": "#/$defs/sourceRef"}}}},
+              "objects": {"type": "array", "items": {"type": "object"}}
+            }
+          }
+        },
+        "references": {"type": "object", "required": ["formulas", "defined_names", "tables", "external_references", "diagnostics"]}
+      }
+    },
+    "structures": {
+      "type": "object", "required": ["blocks"],
+      "properties": {
+        "blocks": {"type": "array", "items": {
+          "type": "object", "required": ["id", "sheet", "kind", "source_refs", "confidence", "diagnostics"],
+          "properties": {
+            "id": {"type": "string"}, "sheet": {"type": "string"},
+            "kind": {"enum": ["logical_table", "form", "matrix", "text", "unclassified", "chart", "image"]},
+            "source_refs": {"type": "array", "items": {"$ref": "#/$defs/sourceRef"}},
+            "confidence": {"type": "number"}, "diagnostics": {"type": "array"},
+            "table": {"type": "object", "required": ["table_id", "columns", "rows", "row_count", "omitted_rows", "source_refs"], "properties": {"rows": {"type": "array"}, "row_count": {"type": "integer", "minimum": 0}, "omitted_rows": {"type": "integer", "minimum": 0}}}
+          }
+        }}
+      }
+    },
+    "relations": {
+      "type": "object", "required": ["dependencies", "continuations"],
+      "properties": {
+        "dependencies": {"type": "array", "items": {
+          "type": "object", "required": ["dependent", "target", "status", "reference", "source_refs"],
+          "properties": {
+            "dependent": {"$ref": "#/$defs/sourceRef"},
+            "target": {"anyOf": [{"$ref": "#/$defs/sourceRef"}, {"type": "null"}]},
+            "status": {"enum": ["resolved", "external", "unresolved", "unsupported", "dynamic", "circular"]},
+            "reference": {"type": "string"}, "source_refs": {"type": "array", "items": {"$ref": "#/$defs/sourceRef"}}
+          }
+        }},
+        "continuations": {"type": "array", "items": {"type": "object", "required": ["id", "member_table_ids", "source_refs"]}}
+      }
+    },
+    "diagnostics": {"type": "object"},
+    "provenance": {
+      "type": "object", "required": ["source", "filename", "parser_version", "engine", "omissions"],
+      "properties": {"omissions": {"type": "object", "required": ["cells", "media_binary", "source_binary", "max_rows_per_block"], "properties": {
+        "cells": {"type": "boolean"}, "media_binary": {"const": true}, "source_binary": {"const": true},
+        "max_rows_per_block": {"type": ["integer", "null"], "minimum": 1}
+      }}}
+    }
+  },
+  "$defs": {"sourceRef": {"type": "string", "pattern": "^.+![A-Z]+[1-9][0-9]*(?::[A-Z]+[1-9][0-9]*)?$"}}
+}

langparse/workbooks/bundle.py

@@ -0,0 +1,341 @@
+"""Versioned workbook consumption interface with explicit export omissions."""
+
+from __future__ import annotations
+
+import json
+from copy import deepcopy
+from dataclasses import fields, is_dataclass
+from typing import Any
+
+from langparse.types import ParsedDocumentResult
+from langparse.workbooks.lineage import overlaps
+from langparse.workbooks.types import SourceRef, WorkbookIR
+
+BUNDLE_SCHEMA_VERSION = 1
+
+
+def _value(item: Any) -> Any:
+    if isinstance(item, SourceRef):
+        return item.key
+    if is_dataclass(item):
+        return {field.name: _value(getattr(item, field.name)) for field in fields(item)}
+    if isinstance(item, dict):
+        if set(item) == {"sheet_name", "range"}:
+            return f"{item['sheet_name']}!{item['range']}"
+        return {str(key): _value(value) for key, value in item.items()}
+    if isinstance(item, (list, tuple)):
+        return [_value(value) for value in item]
+    if hasattr(item, "isoformat"):
+        return item.isoformat()
+    if item is None or isinstance(item, (str, bool, int, float)):
+        return item
+    return str(item)
+
+
+def _cell(sheet, cell) -> dict:
+    return {"source_ref": f"{sheet.name}!{cell.coordinate}", **_value(cell)}
+
+
+def _edge(edge) -> dict:
+    return _value(edge)
+
+
+class WorkbookBundle:
+    """Stable JSON partitions and queries over a parsed or serialized workbook."""
+
+    def __init__(self, payload: dict, parsed: ParsedDocumentResult | None = None):
+        if (
+            type(payload.get("schema_version")) is not int
+            or payload["schema_version"] != BUNDLE_SCHEMA_VERSION
+        ):
+            raise ValueError("Unsupported workbook bundle schema version")
+        for key in ("facts", "structures", "relations", "diagnostics", "provenance"):
+            if not isinstance(payload.get(key), dict):
+                raise ValueError(f"Workbook bundle {key} must be an object")
+        for group, key in (
+            ("facts", "sheets"),
+            ("structures", "blocks"),
+            ("relations", "dependencies"),
+        ):
+            if not isinstance(payload[group].get(key), list):
+                raise ValueError(f"Workbook bundle {group}.{key} must be an array")
+
+        def query_fields(record, required):
+            if not isinstance(record, dict) or not required <= record.keys():
+                raise ValueError("Workbook bundle record is missing a query field")
+
+        for block in payload["structures"]["blocks"]:
+            query_fields(block, {"sheet", "kind"})
+            if not all(isinstance(block[key], str) for key in ("sheet", "kind")):
+                raise ValueError("Workbook bundle block query fields must be strings")
+        for edge in payload["relations"]["dependencies"]:
+            query_fields(edge, {"dependent", "target"})
+
+        sheet_names = set()
+        for sheet in payload["facts"]["sheets"]:
+            if not isinstance(sheet, dict) or not isinstance(sheet.get("name"), str):
+                raise ValueError("Workbook bundle sheet must have a name")
+            if sheet["name"] in sheet_names:
+                raise ValueError("Duplicate workbook bundle sheet")
+            sheet_names.add(sheet["name"])
+            query_fields(sheet, {"cells"})
+            if sheet["cells"] is not None and not isinstance(sheet["cells"], list):
+                raise ValueError("Workbook bundle cells must be an array or null")
+            for cell in sheet["cells"] or []:
+                query_fields(cell, {"coordinate"})
+                if not isinstance(cell["coordinate"], str):
+                    raise ValueError("Workbook bundle cell coordinate must be a string")
+
+        def source(value):
+            if not isinstance(value, str) or "!" not in value:
+                raise ValueError("Invalid workbook bundle source ref")
+            name, cell_range = value.rsplit("!", 1)
+            ref = SourceRef(name, cell_range)
+            if name not in sheet_names:
+                raise ValueError("Workbook bundle source ref uses unknown sheet")
+            try:
+                if not overlaps(ref, ref):
+                    raise ValueError("reversed range")
+            except ValueError as exc:
+                raise ValueError("Invalid workbook bundle source range") from exc
+
+        def sources(value, *, nullable=False):
+            if isinstance(value, list):
+                for item in value:
+                    sources(item, nullable=nullable)
+            elif value is not None or not nullable:
+                source(value)
+
+        containers = {
+            "facts",
+            "structures",
+            "relations",
+            "diagnostics",
+            "blocks",
+            "sheets",
+            "references",
+            "formulas",
+            "defined_names",
+            "tables",
+            "external_references",
+            "reference_diagnostics",
+            "dependencies",
+            "continuations",
+            "object",
+            "objects",
+            "series",
+            "cells",
+            "table",
+            "form",
+            "matrix",
+            "text",
+            "columns",
+            "rows",
+            "fragments",
+            "sections",
+            "fields",
+            "free_text",
+            "lines",
+            "row_headers",
+            "column_headers",
+        }
+
+        def refs(value):
+            if isinstance(value, dict):
+                for key, item in value.items():
+                    if key in {"source_ref", "dependent", "target"} and item is not None:
+                        source(item)
+                    elif key.endswith("source_refs"):
+                        if not isinstance(item, list):
+                            raise ValueError("Workbook bundle source refs must be an array")
+                        sources(item, nullable=key == "value_source_refs")
+                    elif key in containers:
+                        refs(item)
+            elif isinstance(value, list):
+                for item in value:
+                    refs(item)
+
+        refs(payload)
+        self._payload = deepcopy(payload)
+        self._parsed = parsed
+
+    @classmethod
+    def from_result(
+        cls,
+        parsed: ParsedDocumentResult,
+        *,
+        include_cells: bool = False,
+        max_rows: int | None = 200,
+    ) -> WorkbookBundle:
+        if not isinstance(parsed.structure, WorkbookIR) or parsed.structure.snapshot is None:
+            raise TypeError("Workbook Bundle requires a rich OOXML workbook result")
+        if type(include_cells) is not bool:
+            raise ValueError("include_cells must be boolean")
+        if max_rows is not None and (type(max_rows) is not int or max_rows < 1):
+            raise ValueError("max_rows must be a positive integer or None")
+        workbook = parsed.structure
+        snapshot = workbook.snapshot
+        blocks = []
+        for sheet in workbook.sheets:
+            for block in sheet.blocks:
+                item = {
+                    "id": block.block_id,
+                    "sheet": sheet.name,
+                    "kind": block.kind,
+                    "source_refs": [ref.key for ref in block.source_refs],
+                    "confidence": block.confidence,
+                    "diagnostics": _value(block.diagnostics),
+                }
+                for name, content in (
+                    ("table", block.logical_table),
+                    ("form", block.form),
+                    ("matrix", block.matrix),
+                    ("text", block.text),
+                ):
+                    if content is not None:
+                        data = _value(content)
+                        if name == "table":
+                            data["row_count"] = len(content.rows)
+                            data["rows"] = data["rows"][:max_rows]
+                            data["omitted_rows"] = len(content.rows) - len(data["rows"])
+                        elif name == "matrix":
+                            data["row_count"] = len(content.values)
+                            for key in ("values", "row_headers", "value_source_refs"):
+                                data[key] = data[key][:max_rows]
+                            data["omitted_rows"] = len(content.values) - len(data["values"])
+                        item[name] = data
+                if block.kind in {"chart", "image"}:
+                    item["object"] = _value(block.metadata["object"])
+                blocks.append(item)
+        from langparse import __version__
+
+        return cls(
+            {
+                "schema_version": BUNDLE_SCHEMA_VERSION,
+                "facts": {
+                    "sheets": [
+                        {
+                            "name": sheet.name,
+                            "visibility": sheet.visibility,
+                            "used_range": sheet.used_range,
+                            "cell_count": len(sheet.cells),
+                            "merged_ranges": list(sheet.merged_ranges),
+                            "cells": [_cell(sheet, cell) for cell in sheet.cells.values()]
+                            if include_cells
+                            else None,
+                            "objects": _value(sheet.objects),
+                        }
+                        for sheet in snapshot.sheets
+                    ],
+                    "references": _value(snapshot.reference_facts),
+                },
+                "structures": {"blocks": blocks},
+                "relations": {
+                    "dependencies": [_edge(edge) for edge in workbook.lineage.edges]
+                    if workbook.lineage
+                    else [],
+                    "continuations": [
+                        {
+                            "id": item.continuation_id,
+                            "member_table_ids": item.member_table_ids,
+                            "source_refs": [ref.key for ref in item.source_refs],
+                        }
+                        for item in workbook.table_continuations
+                    ],
+                },
+                "diagnostics": _value(parsed.diagnostics) if parsed.diagnostics else {},
+                "provenance": {
+                    "source": parsed.source,
+                    "filename": parsed.filename,
+                    "parser_version": __version__,
+                    "engine": parsed.engine,
+                    "omissions": {
+                        "cells": not include_cells,
+                        "media_binary": True,
+                        "source_binary": True,
+                        "max_rows_per_block": max_rows,
+                    },
+                },
+            },
+            parsed,
+        )
+
+    @classmethod
+    def from_json(cls, text: str) -> WorkbookBundle:
+        def unique(pairs):
+            result = {}
+            for key, value in pairs:
+                if key in result:
+                    raise ValueError(f"Duplicate bundle field: {key}")
+                result[key] = value
+            return result
+
+        payload = json.loads(text, object_pairs_hook=unique)
+        if not isinstance(payload, dict):
+            raise ValueError("Workbook bundle root must be an object")
+        return cls(payload)
+
+    def to_dict(self) -> dict:
+        return deepcopy(self._payload)
+
+    def to_json(self) -> str:
+        return json.dumps(self._payload, ensure_ascii=False, sort_keys=True, indent=2)
+
+    def blocks(self, *, sheet: str | None = None, kind: str | None = None) -> list[dict]:
+        return deepcopy(
+            [
+                block
+                for block in self._payload["structures"]["blocks"]
+                if (sheet is None or block["sheet"] == sheet)
+                and (kind is None or block["kind"] == kind)
+            ]
+        )
+
+    def tables(self, *, sheet: str | None = None) -> list[dict]:
+        return [block["table"] for block in self.blocks(sheet=sheet) if "table" in block]
+
+    def cells(self, *, sheet: str, source_range: str | None = None) -> list[dict]:
+        if self._parsed is not None:
+            snapshot = self._parsed.structure.snapshot
+            selected = next((item for item in snapshot.sheets if item.name == sheet), None)
+            if selected is None:
+                raise ValueError(f"Unknown workbook sheet: {sheet}")
+            cells = [_cell(selected, cell) for cell in selected.cells.values()]
+        else:
+            selected = next(
+                (item for item in self._payload["facts"]["sheets"] if item["name"] == sheet), None
+            )
+            if selected is None:
+                raise ValueError(f"Unknown workbook sheet: {sheet}")
+            cells = selected["cells"]
+            if cells is None:
+                raise ValueError(
+                    "Cells were omitted; export with include_cells=True to query a loaded bundle"
+                )
+        if source_range is not None:
+            region = SourceRef(sheet, source_range)
+            cells = [
+                cell for cell in cells if overlaps(region, SourceRef(sheet, cell["coordinate"]))
+            ]
+        return deepcopy(cells)
+
+    def relations(
+        self, *, source_ref: str | None = None, direction: str = "dependencies"
+    ) -> list[dict]:
+        if direction not in {"dependencies", "dependents"}:
+            raise ValueError("direction must be dependencies or dependents")
+        edges = self._payload["relations"]["dependencies"]
+        if source_ref is not None:
+            name, cell_range = source_ref.rsplit("!", 1)
+            selected = SourceRef(name, cell_range)
+            key = "dependent" if direction == "dependencies" else "target"
+            edges = [
+                edge
+                for edge in edges
+                if edge[key] and overlaps(selected, SourceRef(*edge[key].rsplit("!", 1)))
+            ]
+        return deepcopy(edges)
+
+    @property
+    def diagnostics(self) -> dict:
+        return deepcopy(self._payload["diagnostics"])