modelwright 0.1.0a1__py3-none-any.whl

modelwright/evaluation.py

@@ -0,0 +1,173 @@
+"""Validation orchestration over generated outputs, cached values, and oracles."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from modelwright.execution import GeneratedExecutionResult, execute_generated_model
+from modelwright.extraction import WorkbookRecord
+from modelwright.generation import GeneratedModuleContract
+from modelwright.oracle_validation import build_oracle_validation_report
+from modelwright.oracles import OracleResult
+from modelwright.validation import (
+    Diagnostic,
+    JsonValue,
+    OracleConfig,
+    ValidationReport,
+    ValidationScenario,
+    build_validation_report,
+)
+
+
+@dataclass(frozen=True)
+class ValidationEvaluationResult:
+    """Generated-model execution plus validation reports for one scenario."""
+
+    scenario_id: str
+    generated_execution: GeneratedExecutionResult
+    cached_validation_report: ValidationReport | None = None
+    oracle_validation_report: ValidationReport | None = None
+    diagnostics: tuple[Diagnostic, ...] = field(default_factory=tuple)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ValidationEvaluationResult":
+        cached_report_data = data.get("cached_validation_report")
+        oracle_report_data = data.get("oracle_validation_report")
+        return cls(
+            scenario_id=data["scenario_id"],
+            generated_execution=GeneratedExecutionResult.from_dict(data["generated_execution"]),
+            cached_validation_report=ValidationReport.from_dict(cached_report_data)
+            if cached_report_data is not None
+            else None,
+            oracle_validation_report=ValidationReport.from_dict(oracle_report_data)
+            if oracle_report_data is not None
+            else None,
+            diagnostics=tuple(Diagnostic.from_dict(item) for item in data.get("diagnostics", [])),
+        )
+
+    def to_dict(self) -> dict[str, JsonValue]:
+        return {
+            "scenario_id": self.scenario_id,
+            "generated_execution": self.generated_execution.to_dict(),
+            "cached_validation_report": self.cached_validation_report.to_dict()
+            if self.cached_validation_report is not None
+            else None,
+            "oracle_validation_report": self.oracle_validation_report.to_dict()
+            if self.oracle_validation_report is not None
+            else None,
+            "diagnostics": [diagnostic.to_dict() for diagnostic in self.diagnostics],
+        }
+
+
+def evaluate_generated_model(
+    *,
+    contract: GeneratedModuleContract,
+    module_path: str | Path,
+    scenario: ValidationScenario,
+    workbook: WorkbookRecord | None = None,
+    oracle_result: OracleResult | None = None,
+) -> ValidationEvaluationResult:
+    """Execute a generated model and build available validation reports."""
+
+    generated_execution = execute_generated_model(
+        contract=contract,
+        module_path=module_path,
+        inputs={scenario_input.cell_ref: scenario_input.value for scenario_input in scenario.inputs},
+    )
+    diagnostics = _execution_diagnostics(generated_execution)
+
+    cached_validation_report = None
+    if workbook is not None:
+        cached_values, cached_diagnostics = _cached_output_values(workbook=workbook, scenario=scenario)
+        diagnostics.extend(cached_diagnostics)
+        cached_report = build_validation_report(
+            scenario=_scenario_with_oracle_backend(scenario, "cached_workbook"),
+            generated_values=generated_execution.output_values,
+            oracle_values=cached_values,
+        )
+        cached_validation_report = _report_with_diagnostics(
+            cached_report,
+            tuple(diagnostics),
+        )
+
+    oracle_validation_report = None
+    if oracle_result is not None:
+        oracle_report = build_oracle_validation_report(
+            scenario=scenario,
+            generated_values=generated_execution.output_values,
+            oracle_result=oracle_result,
+        )
+        oracle_validation_report = _report_with_diagnostics(
+            oracle_report,
+            tuple(_execution_diagnostics(generated_execution)),
+        )
+
+    return ValidationEvaluationResult(
+        scenario_id=scenario.scenario_id,
+        generated_execution=generated_execution,
+        cached_validation_report=cached_validation_report,
+        oracle_validation_report=oracle_validation_report,
+        diagnostics=tuple(diagnostics),
+    )
+
+
+def _cached_output_values(
+    *,
+    workbook: WorkbookRecord,
+    scenario: ValidationScenario,
+) -> tuple[dict[str, JsonValue], tuple[Diagnostic, ...]]:
+    cells_by_ref = {cell.cell_ref: cell for cell in workbook.cells}
+    values: dict[str, JsonValue] = {}
+    diagnostics: list[Diagnostic] = []
+    for output in scenario.outputs:
+        cell = cells_by_ref.get(output.cell_ref)
+        if cell is None or cell.cached_value is None:
+            diagnostics.append(
+                Diagnostic(
+                    diagnostic_code="missing_cached_formula_value",
+                    message="cached workbook value is unavailable for validation output",
+                    location=output.cell_ref,
+                )
+            )
+            continue
+        values[output.cell_ref] = cell.cached_value
+    return values, tuple(diagnostics)
+
+
+def _execution_diagnostics(generated_execution: GeneratedExecutionResult) -> list[Diagnostic]:
+    return [
+        Diagnostic(
+            diagnostic_code=diagnostic.code,
+            message=diagnostic.message,
+            severity=diagnostic.severity,
+            location=diagnostic.location,
+        )
+        for diagnostic in generated_execution.diagnostics
+    ]
+
+
+def _scenario_with_oracle_backend(scenario: ValidationScenario, backend: str) -> ValidationScenario:
+    return ValidationScenario(
+        scenario_id=scenario.scenario_id,
+        description=scenario.description,
+        source_workbook=scenario.source_workbook,
+        generated_model=scenario.generated_model,
+        oracle=OracleConfig(backend=backend),
+        inputs=scenario.inputs,
+        outputs=scenario.outputs,
+        comparison=scenario.comparison,
+    )
+
+
+def _report_with_diagnostics(
+    report: ValidationReport,
+    diagnostics: tuple[Diagnostic, ...],
+) -> ValidationReport:
+    return ValidationReport(
+        scenario_id=report.scenario_id,
+        oracle_backend=report.oracle_backend,
+        comparisons=report.comparisons,
+        diagnostics=(*report.diagnostics, *diagnostics),
+    )

modelwright/execution.py

@@ -0,0 +1,239 @@
+"""Execution helpers for generated Modelwright Python models."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+import uuid
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import ModuleType
+from typing import Any, Literal
+
+from modelwright.generation import GeneratedModuleContract
+
+
+JsonValue = str | int | float | bool | None | list[Any] | dict[str, Any]
+DiagnosticSeverity = Literal["info", "warning", "error"]
+
+
+@dataclass(frozen=True)
+class ExecutionDiagnostic:
+    """Execution concern tied to a generated model run."""
+
+    code: str
+    message: str
+    severity: DiagnosticSeverity = "warning"
+    location: str | None = None
+    raw_value: JsonValue = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ExecutionDiagnostic":
+        return cls(
+            code=data["code"],
+            message=data["message"],
+            severity=data.get("severity", "warning"),
+            location=data.get("location"),
+            raw_value=data.get("raw_value"),
+        )
+
+    def to_dict(self) -> dict[str, JsonValue]:
+        return {
+            "code": self.code,
+            "message": self.message,
+            "severity": self.severity,
+            "location": self.location,
+            "raw_value": self.raw_value,
+        }
+
+
+@dataclass(frozen=True)
+class GeneratedExecutionResult:
+    """Observed outputs from one generated Python model execution."""
+
+    contract: GeneratedModuleContract
+    module_path: str
+    entrypoint: str
+    output_values: dict[str, JsonValue] = field(default_factory=dict)
+    diagnostics: tuple[ExecutionDiagnostic, ...] = field(default_factory=tuple)
+
+    @property
+    def executed(self) -> bool:
+        return not any(diagnostic.severity == "error" for diagnostic in self.diagnostics)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "GeneratedExecutionResult":
+        return cls(
+            contract=GeneratedModuleContract.from_dict(data["contract"]),
+            module_path=data["module_path"],
+            entrypoint=data["entrypoint"],
+            output_values=dict(data.get("output_values", {})),
+            diagnostics=tuple(ExecutionDiagnostic.from_dict(item) for item in data.get("diagnostics", [])),
+        )
+
+    def to_dict(self) -> dict[str, JsonValue]:
+        return {
+            "contract": self.contract.to_dict(),
+            "module_path": self.module_path,
+            "entrypoint": self.entrypoint,
+            "executed": self.executed,
+            "output_values": self.output_values,
+            "diagnostics": [diagnostic.to_dict() for diagnostic in self.diagnostics],
+        }
+
+
+def execute_generated_model(
+    *,
+    contract: GeneratedModuleContract,
+    module_path: str | Path,
+    inputs: Mapping[str, JsonValue] | None = None,
+) -> GeneratedExecutionResult:
+    """Execute a generated Python module and return observed declared outputs."""
+
+    path = Path(module_path)
+    diagnostics: list[ExecutionDiagnostic] = []
+    if not path.exists():
+        return _result(
+            contract=contract,
+            module_path=path,
+            diagnostics=(
+                ExecutionDiagnostic(
+                    code="generated_model_not_found",
+                    message="generated Python model file does not exist",
+                    severity="error",
+                    location=str(path),
+                ),
+            ),
+        )
+
+    module = _load_generated_module(path)
+    if isinstance(module, ExecutionDiagnostic):
+        return _result(contract=contract, module_path=path, diagnostics=(module,))
+
+    entrypoint = getattr(module, contract.entrypoint, None)
+    if not callable(entrypoint):
+        return _result(
+            contract=contract,
+            module_path=path,
+            diagnostics=(
+                ExecutionDiagnostic(
+                    code="generated_model_entrypoint_missing",
+                    message="generated Python model entrypoint is missing or not callable",
+                    severity="error",
+                    location=contract.entrypoint,
+                ),
+            ),
+        )
+
+    try:
+        raw_outputs = entrypoint(dict(inputs or {}))
+    except Exception as error:  # noqa: BLE001
+        return _result(
+            contract=contract,
+            module_path=path,
+            diagnostics=(
+                ExecutionDiagnostic(
+                    code="generated_model_execution_failed",
+                    message="generated Python model raised during execution",
+                    severity="error",
+                    location=contract.entrypoint,
+                    raw_value=repr(error),
+                ),
+            ),
+        )
+
+    if not isinstance(raw_outputs, Mapping):
+        return _result(
+            contract=contract,
+            module_path=path,
+            diagnostics=(
+                ExecutionDiagnostic(
+                    code="generated_model_outputs_not_mapping",
+                    message="generated Python model entrypoint must return a mapping of cell refs to values",
+                    severity="error",
+                    location=contract.entrypoint,
+                    raw_value=repr(raw_outputs),
+                ),
+            ),
+        )
+
+    output_values: dict[str, JsonValue] = {}
+    output_refs = contract.output_refs or tuple(str(key) for key in raw_outputs)
+    for output_ref in output_refs:
+        if output_ref not in raw_outputs:
+            diagnostics.append(
+                ExecutionDiagnostic(
+                    code="missing_generated_output",
+                    message="generated Python model did not return a declared output",
+                    location=output_ref,
+                )
+            )
+            continue
+        value = raw_outputs[output_ref]
+        if not _is_json_value(value):
+            diagnostics.append(
+                ExecutionDiagnostic(
+                    code="non_json_generated_output",
+                    message="generated output value is not JSON-serializable",
+                    severity="error",
+                    location=output_ref,
+                    raw_value=repr(value),
+                )
+            )
+            continue
+        output_values[output_ref] = value
+
+    return _result(contract=contract, module_path=path, output_values=output_values, diagnostics=tuple(diagnostics))
+
+
+def _load_generated_module(path: Path) -> ModuleType | ExecutionDiagnostic:
+    module_name = f"_modelwright_generated_{path.stem}_{uuid.uuid4().hex}"
+    try:
+        spec = importlib.util.spec_from_file_location(module_name, path)
+        if spec is None or spec.loader is None:
+            return ExecutionDiagnostic(
+                code="generated_model_import_failed",
+                message="could not build an import specification for generated Python model",
+                severity="error",
+                location=str(path),
+            )
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+    except Exception as error:  # noqa: BLE001
+        return ExecutionDiagnostic(
+            code="generated_model_import_failed",
+            message="generated Python model could not be imported",
+            severity="error",
+            location=str(path),
+            raw_value=repr(error),
+        )
+    finally:
+        sys.modules.pop(module_name, None)
+    return module
+
+
+def _result(
+    *,
+    contract: GeneratedModuleContract,
+    module_path: Path,
+    output_values: dict[str, JsonValue] | None = None,
+    diagnostics: tuple[ExecutionDiagnostic, ...] = (),
+) -> GeneratedExecutionResult:
+    return GeneratedExecutionResult(
+        contract=contract,
+        module_path=str(module_path),
+        entrypoint=contract.entrypoint,
+        output_values=output_values or {},
+        diagnostics=diagnostics,
+    )
+
+
+def _is_json_value(value: Any) -> bool:
+    try:
+        json.dumps(value)
+    except (TypeError, ValueError):
+        return False
+    return True