ummaya 0.2.4 → 0.2.5

package/src/ummaya/evidence/gates.py

@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Evidence Fabric gate construction."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from ummaya.evidence.dataset_contract import REQUIRED_DOMAINS, ScenarioDataset
+from ummaya.evidence.models import EvidenceGate
+
+
+def build_gates(dataset: ScenarioDataset) -> tuple[EvidenceGate, ...]:
+    """Build deterministic Evidence Fabric gate summaries."""
+    covered_domains = set(dataset.coverage_domains)
+    missing_domains = tuple(sorted(REQUIRED_DOMAINS - covered_domains))
+    scenario_domains = {scenario.lifecycle_domain for scenario in dataset.scenarios}
+    uncovered = tuple(sorted(scenario_domains - covered_domains))
+    scenario_status: Literal["pass", "fail"] = (
+        "pass" if not missing_domains and not uncovered else "fail"
+    )
+    scenario_summary = (
+        "all required citizen infrastructure domains are covered"
+        if scenario_status == "pass"
+        else "missing coverage: " + ", ".join(missing_domains + uncovered)
+    )
+    return (
+        _gate(
+            "contract",
+            "pass",
+            "dataset is versioned, typed, and free of model-visible route-cheat keys",
+            ("dataset-schema", "task-registry", "no-adapter-leakage", "no-route-cheat-keys"),
+        ),
+        _gate(
+            "scenario", scenario_status, scenario_summary, ("coverage-domains", "scenario-shape")
+        ),
+        _gate(
+            "observability",
+            "pass",
+            "RunEvidence carries route traces, route-selection assertions, and join keys",
+            ("trace-join-keys", "route-selection-assertions"),
+        ),
+        _gate(
+            "adversarial",
+            "pass",
+            "adapter IDs, fixture references, expected tool IDs, and route assertion "
+            "cheats are rejected before scoring",
+            ("reward-hack-surface", "hidden-implementation-leakage", "route-assertion-cheats"),
+        ),
+        _gate(
+            "ux",
+            "skip",
+            "UX frame artifacts are attached by interactive runners, not by dataset validation",
+            ("ux-artifact-slot",),
+        ),
+        _gate(
+            "live_canary",
+            "skip",
+            "live public-service checks are manual-only and excluded from CI",
+            ("no-live-ci",),
+        ),
+    )
+
+
+def _gate(
+    name: Literal["contract", "scenario", "observability", "adversarial", "ux", "live_canary"],
+    status: Literal["pass", "fail", "skip"],
+    summary: str,
+    check_ids: tuple[str, ...],
+) -> EvidenceGate:
+    return EvidenceGate(name=name, status=status, summary=summary, check_ids=check_ids)

package/src/ummaya/evidence/json_types.py

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+"""JSON-compatible type aliases for Evidence Fabric payload assembly."""
+
+from __future__ import annotations
+
+from typing import Final
+
+from pydantic import TypeAdapter
+
+type JsonScalar = str | int | float | bool | None
+type JsonValue = JsonScalar | list[JsonValue] | dict[str, JsonValue]
+type JsonArray = list[JsonValue]
+type JsonObject = dict[str, JsonValue]
+
+_JSON_OBJECT_ADAPTER: Final[TypeAdapter[JsonObject]] = TypeAdapter(JsonObject)
+
+
+def parse_json_object(value: JsonValue) -> JsonObject:
+    """Parse a JSON-compatible value into a JSON mapping."""
+    return _JSON_OBJECT_ADAPTER.validate_python(value)

package/src/ummaya/evidence/models.py

@@ -7,9 +7,33 @@ from datetime import UTC, datetime
 from typing import Literal
 from uuid import uuid4
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from ummaya.evidence.tool_layer_models import ToolLayerEvidenceEvent
+from ummaya.tools.routing.decision_types import RouteStopReason
 
 EvidenceStatus = Literal["pass", "fail", "skip"]
+RouteTraceKind = Literal["scenario_route", "negative_control"]
+RouteTraceSource = Literal["expected_route_contract", "route_decision"]
+RouteAssertionStatus = Literal["pass", "fail"]
+RouteAdapterFamily = Literal[
+    "public_service_channel",
+    "location_channel",
+    "weather_channel",
+    "safety_channel",
+    "procurement_channel",
+    "public_data_channel",
+    "document_harness",
+    "no_tool",
+]
+RouteArgumentFeasibility = Literal["sufficient", "needs_clarification", "blocked"]
+RouteFailureRecovery = Literal[
+    "not_required",
+    "permission_gate",
+    "clarification",
+    "handoff",
+    "blocked",
+]
 EvidenceGateName = Literal[
     "contract",
     "scenario",
@@ -31,6 +55,66 @@ class EvidenceGate(BaseModel):
     check_ids: tuple[str, ...] = Field(default_factory=tuple)
 
 
+class RouteTraceRecord(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    trace_kind: RouteTraceKind = "scenario_route"
+    route_source: RouteTraceSource = "expected_route_contract"
+    scenario_id: str = Field(min_length=1)
+    trace_id: str = Field(min_length=1)
+    correlation_id: str = Field(min_length=1)
+    query_hash: str = Field(min_length=64, max_length=64)
+    manifest_hash: str = Field(min_length=64, max_length=64)
+    prompt_manifest_hash: str = Field(min_length=64, max_length=64)
+    tool_catalog_hash: str = Field(min_length=64, max_length=64)
+    selected_domain: str = Field(min_length=1)
+    selected_primitives: tuple[str, ...] = Field(default_factory=tuple)
+    selected_tools: tuple[str, ...] = Field(default_factory=tuple)
+    clarification_reason: str | None = None
+    stop_reason: RouteStopReason
+    evidence_events: tuple[str, ...] = Field(default_factory=tuple)
+
+    @field_validator("query_hash", "manifest_hash", "prompt_manifest_hash", "tool_catalog_hash")
+    @classmethod
+    def _validate_hash(cls, value: str) -> str:
+        if len(value) != 64 or any(ch not in "0123456789abcdef" for ch in value):
+            raise ValueError("hash fields must be lowercase SHA-256 hex")
+        return value
+
+
+class RouteSelectionAssertion(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    assertion_kind: RouteTraceKind = "scenario_route"
+    route_source: RouteTraceSource = "expected_route_contract"
+    status: RouteAssertionStatus
+    scenario_id: str = Field(min_length=1)
+    trace_id: str = Field(min_length=1)
+    correlation_id: str = Field(min_length=1)
+    prompt_manifest_hash: str = Field(min_length=64, max_length=64)
+    tool_catalog_hash: str = Field(min_length=64, max_length=64)
+    expected_domain: str = Field(min_length=1)
+    selected_domain: str = Field(min_length=1)
+    expected_primitives: tuple[str, ...] = Field(default_factory=tuple)
+    selected_primitives: tuple[str, ...] = Field(default_factory=tuple)
+    adapter_family: RouteAdapterFamily
+    argument_feasibility: RouteArgumentFeasibility
+    clarification_expected: bool
+    clarification_reason: str | None
+    stop_reason: RouteStopReason
+    failure_recovery: RouteFailureRecovery
+    coverage_tags: tuple[str, ...] = Field(default_factory=tuple)
+    selected_tool_ids: tuple[str, ...] = Field(default_factory=tuple)
+    assertion_events: tuple[str, ...] = Field(default_factory=tuple)
+
+    @field_validator("prompt_manifest_hash", "tool_catalog_hash")
+    @classmethod
+    def _validate_hash(cls, value: str) -> str:
+        if len(value) != 64 or any(ch not in "0123456789abcdef" for ch in value):
+            raise ValueError("hash fields must be lowercase SHA-256 hex")
+        return value
+
+
 class RunEvidence(BaseModel):
     """Top-level immutable evidence document emitted by the v2 runner."""
 
@@ -47,6 +131,9 @@ class RunEvidence(BaseModel):
     task_ids: tuple[str, ...] = Field(default_factory=tuple)
     scenario_count: int
     scenario_ids: tuple[str, ...]
+    route_trace_records: tuple[RouteTraceRecord, ...] = Field(default_factory=tuple)
+    route_selection_assertions: tuple[RouteSelectionAssertion, ...] = Field(default_factory=tuple)
+    tool_layer_events: tuple[ToolLayerEvidenceEvent, ...] = Field(default_factory=tuple)
     gates: tuple[EvidenceGate, ...]
     trace_join_keys: tuple[str, ...] = (
         "scenario_id",

package/src/ummaya/evidence/output_payload.py

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: Apache-2.0
+"""JSON payload assembly for Evidence Fabric v2."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import assert_never
+
+from ummaya.evidence.document_viewer_ux import DocumentViewerUxArtifact
+from ummaya.evidence.json_types import JsonArray, JsonObject, JsonValue, parse_json_object
+from ummaya.evidence.models import RunEvidence
+from ummaya.evidence.payload_documents import append_document_harness_payload
+
+
+def build_evidence_output_payload(
+    evidence: RunEvidence,
+    *,
+    include_document_harness: bool = True,
+    document_viewer_ux_artifacts: Sequence[DocumentViewerUxArtifact] = (),
+    hwp_bridge_probe_env: Mapping[str, str] | None = None,
+    hwp_bridge_probe_search_path: Sequence[str] | None = None,
+    odf_probe_env: Mapping[str, str] | None = None,
+    odf_probe_search_path: Sequence[str] | None = None,
+    odf_probe_importable_modules: frozenset[str] | None = None,
+    pdfa_probe_env: Mapping[str, str] | None = None,
+    pdfa_probe_search_path: Sequence[str] | None = None,
+    archive_probe_env: Mapping[str, str] | None = None,
+    archive_probe_search_path: Sequence[str] | None = None,
+    passive_probe_env: Mapping[str, str] | None = None,
+    passive_probe_search_path: Sequence[str] | None = None,
+    passive_probe_importable_modules: frozenset[str] | None = None,
+    legacy_office_probe_env: Mapping[str, str] | None = None,
+    legacy_office_probe_search_path: Sequence[str] | None = None,
+) -> JsonObject:
+    """Build the JSON payload emitted by the Evidence Fabric CLI."""
+    payload = parse_json_object(evidence.model_dump(mode="json"))
+    if document_viewer_ux_artifacts:
+        payload["gates"] = _with_passed_ux_gate(payload["gates"])
+        payload["ux_artifacts"] = [
+            parse_json_object(artifact.model_dump(mode="json"))
+            for artifact in document_viewer_ux_artifacts
+        ]
+    if include_document_harness:
+        append_document_harness_payload(
+            payload,
+            document_viewer_ux_artifacts=document_viewer_ux_artifacts,
+            hwp_bridge_probe_env=hwp_bridge_probe_env,
+            hwp_bridge_probe_search_path=hwp_bridge_probe_search_path,
+            odf_probe_env=odf_probe_env,
+            odf_probe_search_path=odf_probe_search_path,
+            odf_probe_importable_modules=odf_probe_importable_modules,
+            pdfa_probe_env=pdfa_probe_env,
+            pdfa_probe_search_path=pdfa_probe_search_path,
+            archive_probe_env=archive_probe_env,
+            archive_probe_search_path=archive_probe_search_path,
+            passive_probe_env=passive_probe_env,
+            passive_probe_search_path=passive_probe_search_path,
+            passive_probe_importable_modules=passive_probe_importable_modules,
+            legacy_office_probe_env=legacy_office_probe_env,
+            legacy_office_probe_search_path=legacy_office_probe_search_path,
+        )
+    return payload
+
+
+def _with_passed_ux_gate(gates: JsonValue) -> JsonArray:
+    match gates:
+        case list():
+            promoted: JsonArray = []
+            for gate in gates:
+                match gate:
+                    case dict() if gate.get("name") == "ux":
+                        check_ids: JsonArray = ["document-viewer-playwright-png", "frame-hash"]
+                        promoted.append(
+                            {
+                                **gate,
+                                "status": "pass",
+                                "summary": "Playwright document viewer UX artifacts are attached",
+                                "check_ids": check_ids,
+                            }
+                        )
+                    case dict() | list() | str() | int() | float() | bool() | None:
+                        promoted.append(gate)
+                    case unreachable:
+                        assert_never(unreachable)
+            return promoted
+        case dict() | str() | int() | float() | bool() | None:
+            return []
+        case unreachable:
+            assert_never(unreachable)

package/src/ummaya/evidence/payload_documents.py

@@ -0,0 +1,233 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Document harness payload enrichment for Evidence Fabric output."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+
+from ummaya.evidence.dataset_contract import _REPO_ROOT
+from ummaya.evidence.document_harness import (
+    DocumentEvidenceRecord,
+    DocumentHarnessEvidenceError,
+    authoring_cases_from_scenario,
+    beta_cases_from_scenario,
+    lifecycle_records_from_scenario,
+    load_document_harness_scenario,
+    negative_cases_from_scenario,
+    records_from_scenario,
+)
+from ummaya.evidence.document_viewer_ux import DocumentViewerUxArtifact
+from ummaya.evidence.json_types import JsonObject, parse_json_object
+from ummaya.tools.documents.hwp_conversion_probe import (
+    HWPXJS_CANDIDATE_ID,
+    HwpToHwpxBridgeProbeReport,
+    probe_hwp_to_hwpx_bridge,
+)
+from ummaya.tools.documents.legacy_office_promotion_probe import (
+    LegacyOfficePromotionProbeReport,
+    probe_legacy_office_promotion,
+)
+from ummaya.tools.documents.models import KnownDocumentFormat
+
+
+def append_document_harness_payload(
+    payload: JsonObject,
+    *,
+    document_viewer_ux_artifacts: Sequence[DocumentViewerUxArtifact],
+    hwp_bridge_probe_env: Mapping[str, str] | None,
+    hwp_bridge_probe_search_path: Sequence[str] | None,
+    odf_probe_env: Mapping[str, str] | None,
+    odf_probe_search_path: Sequence[str] | None,
+    odf_probe_importable_modules: frozenset[str] | None,
+    pdfa_probe_env: Mapping[str, str] | None,
+    pdfa_probe_search_path: Sequence[str] | None,
+    archive_probe_env: Mapping[str, str] | None,
+    archive_probe_search_path: Sequence[str] | None,
+    passive_probe_env: Mapping[str, str] | None,
+    passive_probe_search_path: Sequence[str] | None,
+    passive_probe_importable_modules: frozenset[str] | None,
+    legacy_office_probe_env: Mapping[str, str] | None,
+    legacy_office_probe_search_path: Sequence[str] | None,
+) -> None:
+    """Attach document harness records and local probe outputs to a payload."""
+    scenario = load_document_harness_scenario()
+    document_records = records_from_scenario(scenario)
+    _validate_document_viewer_ux_joins(document_records, document_viewer_ux_artifacts)
+    payload["document_evidence_records"] = [
+        parse_json_object(record.model_dump(mode="json")) for record in document_records
+    ]
+    payload["document_lifecycle_records"] = [
+        parse_json_object(record.model_dump(mode="json"))
+        for record in lifecycle_records_from_scenario(scenario)
+    ]
+    payload["document_beta_cases"] = [
+        parse_json_object(case.model_dump(mode="json"))
+        for case in beta_cases_from_scenario(scenario)
+    ]
+    payload["document_negative_cases"] = [
+        parse_json_object(case.model_dump(mode="json"))
+        for case in negative_cases_from_scenario(scenario)
+    ]
+    payload["document_authoring_cases"] = [
+        parse_json_object(case.model_dump(mode="json"))
+        for case in authoring_cases_from_scenario(scenario)
+    ]
+    bridge_probe = probe_hwp_to_hwpx_bridge(
+        env=hwp_bridge_probe_env,
+        search_path=_default_hwp_bridge_probe_search_path(
+            env=hwp_bridge_probe_env,
+            explicit_search_path=hwp_bridge_probe_search_path,
+        ),
+    )
+    payload["document_bridge_probe_records"] = [
+        parse_json_object(bridge_probe.model_dump(mode="json"))
+    ]
+    _append_document_promotion_payload(
+        payload,
+        bridge_probe=bridge_probe,
+        odf_probe_env=odf_probe_env,
+        odf_probe_search_path=odf_probe_search_path,
+        odf_probe_importable_modules=odf_probe_importable_modules,
+        pdfa_probe_env=pdfa_probe_env,
+        pdfa_probe_search_path=pdfa_probe_search_path,
+        archive_probe_env=archive_probe_env,
+        archive_probe_search_path=archive_probe_search_path,
+        passive_probe_env=passive_probe_env,
+        passive_probe_search_path=passive_probe_search_path,
+        passive_probe_importable_modules=passive_probe_importable_modules,
+        legacy_office_probe_env=legacy_office_probe_env,
+        legacy_office_probe_search_path=legacy_office_probe_search_path,
+    )
+
+
+def _append_document_promotion_payload(
+    payload: JsonObject,
+    *,
+    bridge_probe: HwpToHwpxBridgeProbeReport,
+    odf_probe_env: Mapping[str, str] | None,
+    odf_probe_search_path: Sequence[str] | None,
+    odf_probe_importable_modules: frozenset[str] | None,
+    pdfa_probe_env: Mapping[str, str] | None,
+    pdfa_probe_search_path: Sequence[str] | None,
+    archive_probe_env: Mapping[str, str] | None,
+    archive_probe_search_path: Sequence[str] | None,
+    passive_probe_env: Mapping[str, str] | None,
+    passive_probe_search_path: Sequence[str] | None,
+    passive_probe_importable_modules: frozenset[str] | None,
+    legacy_office_probe_env: Mapping[str, str] | None,
+    legacy_office_probe_search_path: Sequence[str] | None,
+) -> None:
+    from ummaya.tools.documents.archive_container_probe import probe_archive_container_promotion
+    from ummaya.tools.documents.format_completion_audit import audit_document_format_completion
+    from ummaya.tools.documents.odf_promotion_probe import probe_odf_promotion
+    from ummaya.tools.documents.passive_capability_probe import probe_passive_capabilities
+    from ummaya.tools.documents.pdfa_promotion_probe import probe_pdfa_promotion
+
+    odf_probe_records = probe_odf_promotion(
+        env=odf_probe_env,
+        search_path=odf_probe_search_path,
+        importable_modules=odf_probe_importable_modules,
+    )
+    pdfa_probe_record = probe_pdfa_promotion(
+        env=pdfa_probe_env,
+        search_path=pdfa_probe_search_path,
+    )
+    archive_probe_records = probe_archive_container_promotion(
+        env=archive_probe_env,
+        search_path=archive_probe_search_path,
+    )
+    passive_probe_records = probe_passive_capabilities(
+        env=passive_probe_env,
+        search_path=passive_probe_search_path,
+        importable_modules=passive_probe_importable_modules,
+    )
+    legacy_office_probe_records = probe_legacy_office_promotion(
+        env=legacy_office_probe_env,
+        search_path=legacy_office_probe_search_path,
+    )
+    payload["document_odf_probe_records"] = [
+        parse_json_object(record.model_dump(mode="json")) for record in odf_probe_records
+    ]
+    payload["document_pdfa_probe_records"] = [
+        parse_json_object(pdfa_probe_record.model_dump(mode="json"))
+    ]
+    payload["document_archive_probe_records"] = [
+        parse_json_object(record.model_dump(mode="json")) for record in archive_probe_records
+    ]
+    payload["document_passive_probe_records"] = [
+        parse_json_object(record.model_dump(mode="json")) for record in passive_probe_records
+    ]
+    payload["document_legacy_office_probe_records"] = [
+        parse_json_object(record.model_dump(mode="json")) for record in legacy_office_probe_records
+    ]
+    format_completion_audit = audit_document_format_completion(
+        derivative_promoted_formats=_derivative_promoted_formats_from_probe_records(
+            bridge_probe=bridge_probe,
+            legacy_office_probe_records=legacy_office_probe_records,
+        ),
+        pdfa_conformance_promoted=pdfa_probe_record.status == "candidate_available",
+    )
+    payload["document_format_completion_audit"] = parse_json_object(
+        format_completion_audit.model_dump(mode="json")
+    )
+
+
+def _derivative_promoted_formats_from_probe_records(
+    *,
+    bridge_probe: HwpToHwpxBridgeProbeReport,
+    legacy_office_probe_records: Sequence[LegacyOfficePromotionProbeReport],
+) -> frozenset[KnownDocumentFormat]:
+    promoted: set[KnownDocumentFormat] = set()
+    if bridge_probe.status == "configured" or (
+        bridge_probe.status == "available" and bridge_probe.candidate_id == HWPXJS_CANDIDATE_ID
+    ):
+        promoted.add(KnownDocumentFormat.hwp)
+    for record in legacy_office_probe_records:
+        if record.status == "candidate_available":
+            promoted.add(record.known_format)
+    return frozenset(promoted)
+
+
+def _default_hwp_bridge_probe_search_path(
+    *,
+    env: Mapping[str, str] | None,
+    explicit_search_path: Sequence[str] | None,
+) -> Sequence[str] | None:
+    if explicit_search_path is not None:
+        return explicit_search_path
+    if env is not None:
+        return None
+    paths: list[str] = []
+    for root in (Path.cwd(), _REPO_ROOT):
+        node_bin = root / "node_modules" / ".bin"
+        node_bin_str = str(node_bin)
+        if node_bin_str not in paths:
+            paths.append(node_bin_str)
+    process_path = os.environ.get("PATH", "")
+    paths.extend(part for part in process_path.split(os.pathsep) if part)
+    return tuple(paths) if paths else None
+
+
+def _validate_document_viewer_ux_joins(
+    document_records: Sequence[DocumentEvidenceRecord],
+    document_viewer_ux_artifacts: Sequence[DocumentViewerUxArtifact],
+) -> None:
+    if not document_viewer_ux_artifacts:
+        return
+    valid_joins = {
+        (record.structured_diff_id, record.correlation_id) for record in document_records
+    }
+    for artifact in document_viewer_ux_artifacts:
+        if artifact.document_diff_id is None:
+            raise DocumentHarnessEvidenceError(
+                f"document viewer UX artifact does not carry a document_diff_id: "
+                f"{artifact.artifact_id}"
+            )
+        join_key = (artifact.document_diff_id, artifact.correlation_id)
+        if join_key not in valid_joins:
+            raise DocumentHarnessEvidenceError(
+                "document viewer UX artifact does not join a backend document diff "
+                f"record by document_diff_id and correlation_id: {artifact.artifact_id}"
+            )