parse-bench 1.0.0__py3-none-any.whl

parse_bench/schemas/layout_ontology.py

@@ -0,0 +1,315 @@
+"""Layout detection ontology definitions.
+
+This module defines:
+- CanonicalLabel: Canonical17 labels (the standardized dataset label set)
+- CORE_LABELS: Core11 subset (DocLayNet-compatible labels)
+- CANONICAL_TO_CORE: Mapping from Canonical17 to Core11
+- DOCLAYNET_ID_TO_LABEL: DocLayNet ground truth category ID mapping
+- Ontology classes for dataset-side label management
+
+Reference: layout_detection_class_label_canonicalization_proposal.md
+"""
+
+from abc import ABC, abstractmethod
+from enum import StrEnum
+
+# =============================================================================
+# Canonical17 Label Enum (Standardized Dataset Labels)
+# =============================================================================
+
+
+class CanonicalLabel(StrEnum):
+    """Canonical17 layout detection labels.
+
+    This is the standardized label set based on Docling Heron's 17-class schema.
+    It serves as the canonical representation for all dataset ground truth.
+
+    Core11 (DocLayNet-compatible) is a subset: use CORE_LABELS to check membership.
+    """
+
+    # Core11 labels (DocLayNet-compatible) - these 11 are the minimum required
+    CAPTION = "Caption"
+    FOOTNOTE = "Footnote"
+    FORMULA = "Formula"
+    LIST_ITEM = "List-item"
+    PAGE_FOOTER = "Page-footer"
+    PAGE_HEADER = "Page-header"
+    PICTURE = "Picture"
+    SECTION_HEADER = "Section-header"
+    TABLE = "Table"
+    TEXT = "Text"
+    TITLE = "Title"
+
+    # Extended Canonical17 labels (not in Core11)
+    CHECKBOX_SELECTED = "Checkbox-Selected"
+    CHECKBOX_UNSELECTED = "Checkbox-Unselected"
+    CODE = "Code"
+    DOCUMENT_INDEX = "Document Index"
+    FORM = "Form"
+    KEY_VALUE_REGION = "Key-Value Region"
+
+
+# =============================================================================
+# Core11 Subset (DocLayNet-compatible labels)
+# =============================================================================
+
+CORE_LABELS: frozenset[CanonicalLabel] = frozenset(
+    {
+        CanonicalLabel.CAPTION,
+        CanonicalLabel.FOOTNOTE,
+        CanonicalLabel.FORMULA,
+        CanonicalLabel.LIST_ITEM,
+        CanonicalLabel.PAGE_FOOTER,
+        CanonicalLabel.PAGE_HEADER,
+        CanonicalLabel.PICTURE,
+        CanonicalLabel.SECTION_HEADER,
+        CanonicalLabel.TABLE,
+        CanonicalLabel.TEXT,
+        CanonicalLabel.TITLE,
+    }
+)
+
+
+# =============================================================================
+# Canonical17 to Core11 Mapping
+# =============================================================================
+
+# Labels in Core11 map to themselves, others map to closest Core11 equivalent or None
+CANONICAL_TO_CORE: dict[CanonicalLabel, CanonicalLabel | None] = {
+    # Core11 labels map to themselves
+    CanonicalLabel.CAPTION: CanonicalLabel.CAPTION,
+    CanonicalLabel.FOOTNOTE: CanonicalLabel.FOOTNOTE,
+    CanonicalLabel.FORMULA: CanonicalLabel.FORMULA,
+    CanonicalLabel.LIST_ITEM: CanonicalLabel.LIST_ITEM,
+    CanonicalLabel.PAGE_FOOTER: CanonicalLabel.PAGE_FOOTER,
+    CanonicalLabel.PAGE_HEADER: CanonicalLabel.PAGE_HEADER,
+    CanonicalLabel.PICTURE: CanonicalLabel.PICTURE,
+    CanonicalLabel.SECTION_HEADER: CanonicalLabel.SECTION_HEADER,
+    CanonicalLabel.TABLE: CanonicalLabel.TABLE,
+    CanonicalLabel.TEXT: CanonicalLabel.TEXT,
+    CanonicalLabel.TITLE: CanonicalLabel.TITLE,
+    # Extended labels map to closest Core11 equivalent or None
+    CanonicalLabel.CHECKBOX_SELECTED: None,  # No Core11 equivalent
+    CanonicalLabel.CHECKBOX_UNSELECTED: None,  # No Core11 equivalent
+    CanonicalLabel.CODE: CanonicalLabel.TEXT,  # Code is text content
+    CanonicalLabel.DOCUMENT_INDEX: CanonicalLabel.TEXT,  # Index is text content
+    CanonicalLabel.FORM: None,  # No Core11 equivalent
+    CanonicalLabel.KEY_VALUE_REGION: CanonicalLabel.TEXT,  # KV regions contain text
+}
+
+
+# =============================================================================
+# Basic7 Label Enum (Simplified Ontology)
+# =============================================================================
+
+
+class BasicLabel(StrEnum):
+    """Basic7 layout detection labels (simplified ontology).
+
+    Merges Title + Section-header → Section
+    Merges List-item → Text (with text_role attribute)
+    Merges Caption + Footnote + Formula → Text (with text_role attribute)
+
+    This ontology is useful for evaluation when finer distinctions
+    between Title/Section-header or Text/List-item are not needed.
+    """
+
+    # Kept as a legacy accepted label so older on-disk "basic" datasets remain loadable.
+    FORMULA = "Formula"
+    PAGE_FOOTER = "Page-footer"
+    PAGE_HEADER = "Page-header"
+    PICTURE = "Picture"
+    SECTION = "Section"  # Title + Section-header
+    TABLE = "Table"
+    TEXT = "Text"  # Includes List-item, Caption, Footnote
+
+
+BASIC_LABELS: frozenset[BasicLabel] = frozenset(BasicLabel)
+
+# Evaluation ontology defaults/choices (used by evaluation-time adaptation).
+DEFAULT_LAYOUT_EVALUATION_ONTOLOGY = "basic"
+SUPPORTED_LAYOUT_EVALUATION_ONTOLOGIES = frozenset({"basic", "canonical", "core"})
+
+
+# =============================================================================
+# Canonical17 to Basic7 Mapping
+# =============================================================================
+
+
+# Canonical17 → Basic7 mapping with attributes for reversibility
+CANONICAL_TO_BASIC: dict[CanonicalLabel, tuple[BasicLabel, dict[str, str]]] = {
+    # Merged into Section
+    CanonicalLabel.TITLE: (BasicLabel.SECTION, {"title_level": "title"}),
+    CanonicalLabel.SECTION_HEADER: (BasicLabel.SECTION, {"title_level": "section-header"}),
+    # Merged into Text
+    CanonicalLabel.TEXT: (BasicLabel.TEXT, {}),
+    CanonicalLabel.LIST_ITEM: (BasicLabel.TEXT, {"text_role": "list-item"}),
+    CanonicalLabel.CAPTION: (BasicLabel.TEXT, {"text_role": "caption"}),
+    CanonicalLabel.FOOTNOTE: (BasicLabel.TEXT, {"text_role": "footnote"}),
+    CanonicalLabel.FORMULA: (BasicLabel.TEXT, {"text_role": "formula"}),
+    # Identity mappings (unchanged)
+    CanonicalLabel.PAGE_FOOTER: (BasicLabel.PAGE_FOOTER, {}),
+    CanonicalLabel.PAGE_HEADER: (BasicLabel.PAGE_HEADER, {}),
+    CanonicalLabel.PICTURE: (BasicLabel.PICTURE, {}),
+    CanonicalLabel.TABLE: (BasicLabel.TABLE, {}),
+    # Extended Canonical17 labels → Text (not present in current datasets)
+    CanonicalLabel.CODE: (BasicLabel.TEXT, {"text_role": "code"}),
+    CanonicalLabel.DOCUMENT_INDEX: (BasicLabel.TEXT, {"text_role": "document-index"}),
+    CanonicalLabel.KEY_VALUE_REGION: (BasicLabel.TEXT, {"text_role": "key-value"}),
+    CanonicalLabel.FORM: (BasicLabel.TEXT, {"text_role": "form"}),
+    CanonicalLabel.CHECKBOX_SELECTED: (BasicLabel.TEXT, {"text_role": "checkbox-selected"}),
+    CanonicalLabel.CHECKBOX_UNSELECTED: (BasicLabel.TEXT, {"text_role": "checkbox-unselected"}),
+}
+
+
+# =============================================================================
+# DocLayNet Ground Truth Mapping (1-indexed category IDs)
+# =============================================================================
+
+DOCLAYNET_ID_TO_LABEL: dict[int, CanonicalLabel] = {
+    1: CanonicalLabel.CAPTION,
+    2: CanonicalLabel.FOOTNOTE,
+    3: CanonicalLabel.FORMULA,
+    4: CanonicalLabel.LIST_ITEM,
+    5: CanonicalLabel.PAGE_FOOTER,
+    6: CanonicalLabel.PAGE_HEADER,
+    7: CanonicalLabel.PICTURE,
+    8: CanonicalLabel.SECTION_HEADER,
+    9: CanonicalLabel.TABLE,
+    10: CanonicalLabel.TEXT,
+    11: CanonicalLabel.TITLE,
+}
+
+
+# =============================================================================
+# Ontology Base Class
+# =============================================================================
+
+
+class LayoutDetectionOntology(ABC):
+    """Abstract base class for layout detection ontologies.
+
+    Ontologies define which labels are valid for a given dataset evaluation view.
+    """
+
+    def __init__(self) -> None:
+        self._labels: frozenset[CanonicalLabel] = frozenset()
+        self._setup()
+
+    @abstractmethod
+    def _setup(self) -> None:
+        """Initialize the label set."""
+        pass
+
+    @property
+    def labels(self) -> frozenset[CanonicalLabel]:
+        """Return all labels in this ontology."""
+        return self._labels
+
+    @property
+    def label_names(self) -> list[str]:
+        """Return sorted list of label names (string values)."""
+        return sorted(label.value for label in self._labels)
+
+    def has_label(self, label: CanonicalLabel | str) -> bool:
+        """Check if a label exists in this ontology.
+
+        :param label: CanonicalLabel enum or string value
+        :return: True if label exists
+        """
+        if isinstance(label, str):
+            return label in {lbl.value for lbl in self._labels}
+        return label in self._labels
+
+    def map_label(
+        self,
+        source_label: str,
+        source_schema: str,
+    ) -> tuple[str, dict[str, str]]:
+        """Map a label from a source schema to canonical class and attributes.
+
+        :param source_label: The label in the source schema
+        :param source_schema: Name of the source schema (e.g., "doclaynet")
+        :return: Tuple of (canonical_class_name, attributes_dict)
+        """
+        # For DocLayNet, it's an identity mapping (Core11 labels)
+        if source_schema == "doclaynet":
+            if self.has_label(source_label):
+                return (source_label, {})
+            # Fallback to Text if label not found
+            return ("Text", {})
+
+        # Unknown schema, fallback to Text
+        return ("Text", {})
+
+
+# =============================================================================
+# Core11 Ontology (DocLayNet compatible)
+# =============================================================================
+
+
+class CoreLayoutDetectionOntology(LayoutDetectionOntology):
+    """Core11 layout detection ontology.
+
+    This ontology contains the 11 classes from DocLayNet, which is the
+    intersection view for compatibility with DocLayNet and many other benchmarks.
+
+    Labels: Caption, Footnote, Formula, List-item, Page-footer, Page-header,
+            Picture, Section-header, Table, Text, Title
+    """
+
+    def _setup(self) -> None:
+        """Initialize Core11 labels."""
+        self._labels = CORE_LABELS
+
+
+# =============================================================================
+# Canonical17 Ontology (Full canonical schema)
+# =============================================================================
+
+
+class CanonicalLayoutDetectionOntology(LayoutDetectionOntology):
+    """Canonical17 layout detection ontology.
+
+    This ontology adopts Docling Heron's 17-class schema as the primary
+    canonical class set, because it is a superset of DocLayNet's 11 labels
+    and covers several additional document elements commonly needed in
+    real pipelines.
+
+    Additional labels beyond Core11: Checkbox-Selected, Checkbox-Unselected,
+                                     Code, Document Index, Form, Key-Value Region
+    """
+
+    def _setup(self) -> None:
+        """Initialize Canonical17 labels."""
+        self._labels = frozenset(CanonicalLabel)
+
+
+# =============================================================================
+# Ontology Type Enum and Factory
+# =============================================================================
+
+
+class OntologyType(StrEnum):
+    """Supported ontology types."""
+
+    CORE = "core"
+    CANONICAL = "canonical"
+
+
+def get_ontology(ontology_type: OntologyType | str) -> LayoutDetectionOntology:
+    """Factory function to get an ontology instance.
+
+    :param ontology_type: Either OntologyType enum or string ("core" or "canonical")
+    :return: LayoutDetectionOntology instance
+    :raises ValueError: If unknown ontology type
+    """
+    if isinstance(ontology_type, str):
+        ontology_type = OntologyType(ontology_type.lower())
+
+    if ontology_type == OntologyType.CORE:
+        return CoreLayoutDetectionOntology()
+    elif ontology_type == OntologyType.CANONICAL:
+        return CanonicalLayoutDetectionOntology()
+    else:
+        raise ValueError(f"Unknown ontology type: {ontology_type}")

parse_bench/schemas/metrics.py

@@ -0,0 +1,69 @@
+"""Schemas for evaluation metrics and confusion matrix data."""
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class ConfusionMatrixCell(BaseModel):
+    """Single cell in confusion matrix.
+
+    Represents a specific (GT class, Pred class) pair with count,
+    percentage, and the test case IDs that contributed to this cell.
+    """
+
+    gt_class: str = Field(..., description="Ground truth class name")
+    pred_class: str = Field(..., description="Predicted class name")
+    count: int = Field(..., description="Number of instances in this cell")
+    percentage: float = Field(
+        ...,
+        description="Percentage of total GT instances for this gt_class (count / gt_total * 100)",
+    )
+    example_ids: list[str] = Field(
+        default_factory=list,
+        description="Test case IDs (test_id) that contributed to this cell",
+    )
+
+
+class ConfusionMatrixMetrics(BaseModel):
+    """Confusion matrix computed during evaluation.
+
+    Contains full confusion matrix data with metadata to enable
+    interactive filtering in the HTML report.
+    """
+
+    iou_threshold: float = Field(default=0.5, description="IoU threshold used for matching predictions to GT")
+    evaluation_view: Literal["core", "canonical"] = Field(
+        default="core",
+        description="Evaluation view: 'core' for Core11 labels, 'canonical' for Canonical17",
+    )
+
+    cells: list[ConfusionMatrixCell] = Field(
+        default_factory=list,
+        description="All confusion matrix cells (including diagonal for correct predictions)",
+    )
+
+    false_negatives: dict[str, list[str]] = Field(
+        default_factory=dict,
+        description="Unmatched GT boxes by class. Maps class name → list of test_ids",
+    )
+
+    false_positives: dict[str, list[str]] = Field(
+        default_factory=dict,
+        description="Unmatched predictions by class. Maps class name → list of test_ids",
+    )
+
+    gt_totals: dict[str, int] = Field(
+        default_factory=dict,
+        description="Total GT instances per class. Maps class name → count",
+    )
+
+    pred_totals: dict[str, int] = Field(
+        default_factory=dict,
+        description="Total predictions per class. Maps class name → count",
+    )
+
+    all_classes: list[str] = Field(
+        default_factory=list,
+        description="Sorted list of all unique class names in GT and predictions",
+    )

parse_bench/schemas/parse_output.py

@@ -0,0 +1,152 @@
+"""
+Normalized output schema for parse tasks.
+"""
+
+from typing import Literal
+
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, field_validator
+
+
+class PageIR(BaseModel):
+    """Intermediate representation of a single page."""
+
+    page_index: int = Field(ge=0, description="0-indexed page number")
+    markdown: str = Field(description="Markdown content of the page")
+
+
+class LayoutSegmentIR(BaseModel):
+    """Normalized layout segment coordinates and attribution span metadata."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="ignore")
+
+    x: float
+    y: float
+    w: float
+    h: float
+    confidence: float | None = None
+    label: str | None = None
+    start_index: int | None = Field(
+        default=None,
+        validation_alias=AliasChoices("start_index", "startIndex"),
+    )
+    end_index: int | None = Field(
+        default=None,
+        validation_alias=AliasChoices("end_index", "endIndex"),
+    )
+
+
+class LayoutItemIR(BaseModel):
+    """Normalized layout item used for attribution/layout reconstruction."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="ignore")
+
+    type: str = "text"
+    md: str = Field(
+        default="",
+        validation_alias=AliasChoices("md", "markdown"),
+    )
+    html: str = ""
+    value: str = ""
+    bbox: LayoutSegmentIR | None = Field(
+        default=None,
+        validation_alias=AliasChoices("bbox", "bBox"),
+    )
+    layout_segments: list[LayoutSegmentIR] = Field(
+        default_factory=list,
+        validation_alias=AliasChoices("layout_segments", "layoutAwareBbox"),
+    )
+    # Populated by layout-detection providers; left default for parse providers.
+    score: float | None = Field(
+        default=None,
+        ge=0.0,
+        le=1.0,
+        description="Detector confidence in [0,1]. None for parse-pipeline items.",
+    )
+    attributes: dict[str, str] = Field(
+        default_factory=dict,
+        description="Semantic attributes (e.g. scope=mark, picture_type=chart).",
+    )
+
+    @field_validator("type", mode="before")
+    @classmethod
+    def _normalize_type(cls, value: object) -> str:
+        if value is None:
+            return "text"
+        return str(value)
+
+    @field_validator("md", "html", "value", mode="before")
+    @classmethod
+    def _normalize_value(cls, value: object) -> str:
+        if value is None:
+            return ""
+        return str(value)
+
+
+class ParseLayoutPageIR(BaseModel):
+    """Normalized per-page layout payload embedded in ParseOutput."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="ignore")
+
+    page_number: int = Field(
+        ge=1,
+        validation_alias=AliasChoices("page_number", "page"),
+    )
+    width: float | None = Field(
+        default=None,
+        validation_alias=AliasChoices("width", "page_width"),
+    )
+    height: float | None = Field(
+        default=None,
+        validation_alias=AliasChoices("height", "page_height"),
+    )
+    md: str = ""
+    text: str = ""
+    page_header_markdown: str = Field(
+        default="",
+        validation_alias=AliasChoices("page_header_markdown", "pageHeaderMarkdown"),
+    )
+    page_footer_markdown: str = Field(
+        default="",
+        validation_alias=AliasChoices("page_footer_markdown", "pageFooterMarkdown"),
+    )
+    printed_page_number: str = Field(
+        default="",
+        validation_alias=AliasChoices("printed_page_number", "printedPageNumber"),
+    )
+    original_orientation_angle: int | None = Field(
+        default=None,
+        validation_alias=AliasChoices(
+            "original_orientation_angle",
+            "originalOrientationAngle",
+        ),
+    )
+    items: list[LayoutItemIR] = Field(default_factory=list)
+
+    @field_validator(
+        "md",
+        "text",
+        "page_header_markdown",
+        "page_footer_markdown",
+        "printed_page_number",
+        mode="before",
+    )
+    @classmethod
+    def _normalize_optional_text(cls, value: object) -> str:
+        if value is None:
+            return ""
+        return str(value)
+
+
+class ParseOutput(BaseModel):
+    """Normalized output for parse tasks."""
+
+    task_type: Literal["parse"] = Field(default="parse", frozen=True, description="Task type discriminator")
+    example_id: str = Field(description="Unique identifier for the example")
+    pipeline_name: str = Field(description="Name of the pipeline that produced this output")
+    pages: list[PageIR] = Field(default_factory=list, description="List of parsed pages")
+    layout_pages: list[ParseLayoutPageIR] = Field(
+        default_factory=list,
+        description=("Normalized page/item/segment layout payload used by layout attribution and overlays"),
+    )
+    markdown: str = Field(description="Markdown content of the entire document")
+    job_id: str | None = Field(default=None, description="Optional job ID from the provider (e.g., LlamaParse job ID)")

parse_bench/schemas/pipeline.py

@@ -0,0 +1,22 @@
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from parse_bench.schemas.product import ProductTypeName
+
+
+class PipelineSpec(BaseModel):
+    """Specification for a pipeline configuration."""
+
+    pipeline_name: str = Field(description="Name of this pipeline")
+    provider_name: str = Field(description="Name of the provider (e.g., 'llama', 'openai')")
+    product_type: ProductTypeName = Field(description="Type of product task (parse or extract)")
+    config: dict[str, Any] = Field(default_factory=dict, description="Configuration dictionary to pass to the provider")
+    per_file_timeout: float | None = Field(
+        default=None,
+        description=(
+            "Optional per-pipeline override (seconds) for the runner's per-file inference timeout. "
+            "The run-level --per_file_timeout takes precedence when set. None → use the run-level "
+            "value, then the global default. Kept off `config` so providers never receive it."
+        ),
+    )

parse_bench/schemas/pipeline_io.py

@@ -0,0 +1,106 @@
+from datetime import datetime
+from typing import Annotated, Any
+
+from pydantic import BaseModel, BeforeValidator, Discriminator, Field, SerializeAsAny
+
+from parse_bench.schemas.extract_output import ExtractOutput
+from parse_bench.schemas.layout_detection_output import LayoutOutput
+from parse_bench.schemas.parse_output import ParseOutput
+from parse_bench.schemas.pipeline import PipelineSpec
+from parse_bench.schemas.product import ProductTypeName
+
+
+class InferenceRequest(BaseModel):
+    """Request for running inference on a document."""
+
+    example_id: str = Field(description="Unique identifier for the example")
+    source_file_path: str = Field(description="Path to the source file (PDF, etc.)")
+    product_type: ProductTypeName = Field(description="Type of product task to run")
+    schema_override: dict[str, Any] | None = Field(
+        default=None,
+        description="Optional schema override",
+    )
+    config_override: dict[str, Any] | None = Field(
+        default=None,
+        description=("Optional configuration override to merge with pipeline config"),
+    )
+
+
+PipelineOutputType = Annotated[
+    ParseOutput | LayoutOutput | ExtractOutput,
+    Discriminator("task_type"),
+]
+"""Union of the built-in normalized output models (for type annotations)."""
+
+_OUTPUT_MODEL_REGISTRY: dict[str, type[BaseModel]] = {
+    "parse": ParseOutput,
+    "layout_detection": LayoutOutput,
+    "extract": ExtractOutput,
+}
+
+
+def register_output_model(task_type: str, model: type[BaseModel]) -> None:
+    """Register a normalized output model for ``task_type``.
+
+    ``InferenceResult.output`` dispatches on the ``task_type`` discriminator, so
+    extensions that add a product (e.g. a QA task) register the model their
+    provider returns here and saved results round-trip through JSON unchanged.
+    Re-registering a built-in task type raises ``ValueError``.
+    """
+    if not isinstance(task_type, str) or not task_type:
+        raise ValueError("task_type must be a non-empty string")
+    if task_type in {"parse", "layout_detection", "extract"} and _OUTPUT_MODEL_REGISTRY[task_type] is not model:
+        raise ValueError(f"{task_type!r} is a built-in output model and cannot be replaced")
+    _OUTPUT_MODEL_REGISTRY[task_type] = model
+
+
+def registered_output_models() -> dict[str, type[BaseModel]]:
+    """Snapshot of ``task_type`` -> model for every registered output."""
+    return dict(_OUTPUT_MODEL_REGISTRY)
+
+
+def coerce_pipeline_output(value: Any) -> BaseModel:
+    """Validate a normalized output payload against the registered model for its ``task_type``."""
+    if isinstance(value, BaseModel):
+        return value
+    if not isinstance(value, dict):
+        raise TypeError(f"Pipeline output must be a dict or model, got {type(value).__name__}")
+    task_type = value.get("task_type")
+    model = _OUTPUT_MODEL_REGISTRY.get(task_type) if isinstance(task_type, str) else None
+    if model is None:
+        raise ValueError(f"Unknown output task_type {task_type!r}; registered: {sorted(_OUTPUT_MODEL_REGISTRY)}")
+    return model.model_validate(value)
+
+
+PipelineOutput = Annotated[SerializeAsAny[BaseModel], BeforeValidator(coerce_pipeline_output)]
+"""Field annotation for a normalized output: dispatches on ``task_type`` via the registry."""
+
+
+class RawInferenceResult(BaseModel):
+    """Raw result from provider before normalization."""
+
+    request: InferenceRequest = Field(description="Original inference request")
+    pipeline: PipelineSpec = Field(description="Pipeline used")
+    pipeline_name: str = Field(description="Name of the pipeline used")
+    product_type: ProductTypeName = Field(description="Type of product task that was run")
+    raw_output: dict = Field(description="Raw output from the provider API")
+    started_at: datetime = Field(description="Timestamp when inference started")
+    completed_at: datetime = Field(description="Timestamp when inference completed")
+    latency_in_ms: int = Field(ge=0, description="Latency in milliseconds")
+
+
+class InferenceResult(BaseModel):
+    """Result of running inference on a document with both raw and normalized outputs."""
+
+    request: InferenceRequest = Field(description="Original inference request")
+    pipeline_name: str = Field(description="Name of the pipeline used")
+    product_type: ProductTypeName = Field(description="Type of product task that was run")
+
+    # Both outputs stored here
+    raw_output: dict = Field(description="Raw output from the provider (for debugging/re-normalization)")
+    output: PipelineOutput = Field(description="Normalized output from the pipeline")
+
+    # metadata
+    started_at: datetime = Field(description="Timestamp when inference started")
+    completed_at: datetime = Field(description="Timestamp when inference completed")
+    latency_in_ms: int = Field(ge=0, description="Latency in milliseconds")