ummaya 0.2.3 → 0.2.5

package/src/ummaya/evidence/route_contracts.py

@@ -0,0 +1,224 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Route trace and route-selection contract builders."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Literal
+
+from ummaya.evidence.dataset_contract import Scenario, ScenarioDataset
+from ummaya.evidence.models import RouteSelectionAssertion, RouteTraceRecord
+from ummaya.evidence.route_helpers import (
+    adapter_family_for_scenario,
+    argument_feasibility_for_scenario,
+    correlation_id,
+    coverage_tags_for_scenario,
+    failure_recovery_for_stop_reason,
+    prompt_manifest_hash,
+    stop_reason_for_scenario,
+    tool_catalog_hash,
+)
+from ummaya.tools.routing.decision_types import RouteStopReason
+from ummaya.tools.routing.schema import sha256
+
+
+def route_trace_records(dataset: ScenarioDataset) -> tuple[RouteTraceRecord, ...]:
+    """Build route traces from scenario contracts without selecting concrete tools."""
+    records: list[RouteTraceRecord] = []
+    prompt_hash = prompt_manifest_hash(dataset)
+    catalog_hash = tool_catalog_hash(dataset)
+    for scenario in dataset.scenarios:
+        records.append(_scenario_route_trace(scenario, prompt_hash, catalog_hash))
+    records.extend(_negative_control_route_records(prompt_hash, catalog_hash))
+    return tuple(records)
+
+
+def route_selection_assertions(
+    dataset: ScenarioDataset,
+    records: Sequence[RouteTraceRecord],
+) -> tuple[RouteSelectionAssertion, ...]:
+    """Build route-selection assertions from trace records."""
+    scenarios_by_id = {scenario.id: scenario for scenario in dataset.scenarios}
+    assertions: list[RouteSelectionAssertion] = []
+    for record in records:
+        if record.trace_kind == "negative_control":
+            assertions.append(_negative_control_assertion(record))
+            continue
+        scenario = scenarios_by_id[record.scenario_id]
+        expected_primitives = tuple(step.primitive for step in scenario.expected_ax_chain)
+        status: Literal["pass", "fail"] = (
+            "pass"
+            if scenario.lifecycle_domain == record.selected_domain
+            and expected_primitives == record.selected_primitives
+            and not record.selected_tools
+            else "fail"
+        )
+        assertions.append(_route_assertion(scenario, record, expected_primitives, status))
+    return tuple(assertions)
+
+
+def _scenario_route_trace(
+    scenario: Scenario,
+    prompt_hash: str,
+    catalog_hash: str,
+) -> RouteTraceRecord:
+    selected_primitives = tuple(step.primitive for step in scenario.expected_ax_chain)
+    stop_reason = stop_reason_for_scenario(scenario)
+    trace_id = f"route-{scenario.id.lower()}"
+    corr_id = correlation_id(trace_id)
+    return RouteTraceRecord(
+        trace_kind="scenario_route",
+        route_source="expected_route_contract",
+        scenario_id=scenario.id,
+        trace_id=trace_id,
+        correlation_id=corr_id,
+        query_hash=sha256(scenario.request_ko),
+        manifest_hash=sha256(
+            {
+                "correlation_id": corr_id,
+                "scenario_id": scenario.id,
+                "selected_domain": scenario.lifecycle_domain,
+                "selected_primitives": selected_primitives,
+                "stop_reason": stop_reason,
+                "trace_kind": "scenario_route",
+                "route_source": "expected_route_contract",
+            }
+        ),
+        prompt_manifest_hash=prompt_hash,
+        tool_catalog_hash=catalog_hash,
+        selected_domain=scenario.lifecycle_domain,
+        selected_primitives=selected_primitives,
+        clarification_reason=None,
+        stop_reason=stop_reason,
+        evidence_events=("evidence_fabric.route_trace.contract", f"route_stop:{stop_reason}"),
+    )
+
+
+def _negative_control_route_records(
+    prompt_hash: str,
+    catalog_hash: str,
+) -> tuple[RouteTraceRecord, ...]:
+    controls: tuple[tuple[str, str, RouteStopReason, str], ...] = (
+        (
+            "NEG-DIRECT-ANSWER-001",
+            "Explain the difference between a public notice and a civil petition "
+            "without taking action.",
+            "answerable",
+            "negative_control:direct_answer",
+        ),
+        (
+            "NEG-NO-ADAPTER-001",
+            "Change a closed internal agency review score that has no public callable channel.",
+            "blocked_no_adapter",
+            "negative_control:no_adapter",
+        ),
+    )
+    return tuple(
+        _negative_control_route_trace(
+            scenario_id, query, stop_reason, event, prompt_hash, catalog_hash
+        )
+        for scenario_id, query, stop_reason, event in controls
+    )
+
+
+def _negative_control_route_trace(
+    scenario_id: str,
+    query: str,
+    stop_reason: RouteStopReason,
+    event: str,
+    prompt_hash: str,
+    catalog_hash: str,
+) -> RouteTraceRecord:
+    trace_id = f"route-{scenario_id.lower()}"
+    corr_id = correlation_id(trace_id)
+    return RouteTraceRecord(
+        trace_kind="negative_control",
+        route_source="expected_route_contract",
+        scenario_id=scenario_id,
+        trace_id=trace_id,
+        correlation_id=corr_id,
+        query_hash=sha256(query),
+        manifest_hash=sha256(
+            {
+                "correlation_id": corr_id,
+                "scenario_id": scenario_id,
+                "selected_domain": "general_information",
+                "selected_primitives": (),
+                "stop_reason": stop_reason,
+                "trace_kind": "negative_control",
+                "route_source": "expected_route_contract",
+            }
+        ),
+        prompt_manifest_hash=prompt_hash,
+        tool_catalog_hash=catalog_hash,
+        selected_domain="general_information",
+        selected_primitives=(),
+        selected_tools=(),
+        clarification_reason=None,
+        stop_reason=stop_reason,
+        evidence_events=(event, f"route_stop:{stop_reason}"),
+    )
+
+
+def _negative_control_assertion(record: RouteTraceRecord) -> RouteSelectionAssertion:
+    return RouteSelectionAssertion(
+        assertion_kind="negative_control",
+        route_source=record.route_source,
+        status="pass",
+        scenario_id=record.scenario_id,
+        trace_id=record.trace_id,
+        correlation_id=record.correlation_id,
+        prompt_manifest_hash=record.prompt_manifest_hash,
+        tool_catalog_hash=record.tool_catalog_hash,
+        expected_domain="general_information",
+        selected_domain="general_information",
+        expected_primitives=(),
+        selected_primitives=(),
+        adapter_family="no_tool",
+        argument_feasibility="sufficient" if record.stop_reason == "answerable" else "blocked",
+        clarification_expected=False,
+        clarification_reason=None,
+        stop_reason=record.stop_reason,
+        failure_recovery=failure_recovery_for_stop_reason(record.stop_reason),
+        coverage_tags=("negative_control",),
+        selected_tool_ids=(),
+        assertion_events=(
+            "route_assertion:negative_control",
+            "route_assertion:no_internal_tool_id",
+        ),
+    )
+
+
+def _route_assertion(
+    scenario: Scenario,
+    record: RouteTraceRecord,
+    expected_primitives: tuple[str, ...],
+    status: Literal["pass", "fail"],
+) -> RouteSelectionAssertion:
+    return RouteSelectionAssertion(
+        assertion_kind="scenario_route",
+        route_source=record.route_source,
+        status=status,
+        scenario_id=scenario.id,
+        trace_id=record.trace_id,
+        correlation_id=record.correlation_id,
+        prompt_manifest_hash=record.prompt_manifest_hash,
+        tool_catalog_hash=record.tool_catalog_hash,
+        expected_domain=scenario.lifecycle_domain,
+        selected_domain=record.selected_domain,
+        expected_primitives=expected_primitives,
+        selected_primitives=record.selected_primitives,
+        adapter_family=adapter_family_for_scenario(scenario, record.selected_primitives),
+        argument_feasibility=argument_feasibility_for_scenario(scenario),
+        clarification_expected=record.stop_reason == "needs_input",
+        clarification_reason=record.clarification_reason,
+        stop_reason=record.stop_reason,
+        failure_recovery=failure_recovery_for_stop_reason(record.stop_reason),
+        coverage_tags=coverage_tags_for_scenario(scenario, record.selected_primitives),
+        selected_tool_ids=(),
+        assertion_events=(
+            "route_assertion:domain_match",
+            "route_assertion:primitive_match",
+            "route_assertion:no_internal_tool_id",
+        ),
+    )

package/src/ummaya/evidence/route_helpers.py

@@ -0,0 +1,150 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Small route-classification helpers for Evidence Fabric contracts."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from ummaya.evidence.dataset_contract import Scenario, ScenarioDataset
+from ummaya.evidence.models import (
+    RouteAdapterFamily,
+    RouteArgumentFeasibility,
+    RouteFailureRecovery,
+)
+from ummaya.tools.routing.decision_types import RouteStopReason
+from ummaya.tools.routing.schema import sha256, unique
+
+
+def stop_reason_for_scenario(scenario: Scenario) -> RouteStopReason:
+    """Return the expected stop reason for a static scenario contract."""
+    return (
+        "permission_required"
+        if scenario.permission_requirements.user_confirmations
+        else "answerable"
+    )
+
+
+def argument_feasibility_for_scenario(scenario: Scenario) -> RouteArgumentFeasibility:
+    """Classify whether the scenario carries enough arguments for dispatch."""
+    if not scenario.request_ko.strip() or not scenario.expected_ax_chain:
+        return "blocked"
+    return "sufficient"
+
+
+def failure_recovery_for_stop_reason(stop_reason: RouteStopReason) -> RouteFailureRecovery:
+    """Map a route stop reason to the Evidence Fabric recovery family."""
+    if stop_reason == "permission_required":
+        return "permission_gate"
+    if stop_reason == "needs_input":
+        return "clarification"
+    if stop_reason == "handoff_required":
+        return "handoff"
+    if stop_reason.startswith("blocked_") or stop_reason in {
+        "max_turns",
+        "repeated_tool_mismatch",
+        "no_new_evidence",
+        "runtime_tool_failure_unrecovered",
+    }:
+        return "blocked"
+    return "not_required"
+
+
+def adapter_family_for_scenario(
+    scenario: Scenario,
+    selected_primitives: Sequence[str],
+) -> RouteAdapterFamily:
+    """Classify the expected adapter family without exposing concrete tool IDs."""
+    text = scenario_text(scenario)
+    if "procurement" in text or "bid" in text or "나라장터" in text:
+        return "procurement_channel"
+    if scenario.lifecycle_domain == "public_data" or "public data" in text or "공공데이터" in text:
+        return "public_data_channel"
+    if "aed" in text or "defibrillator" in text or "automated external" in text:
+        return "safety_channel"
+    if "aviation" in text or "airport" in text or "flight" in text or "weather" in text:
+        return "weather_channel"
+    if scenario.lifecycle_domain == "safety":
+        return "safety_channel"
+    if "resolve_location" in selected_primitives:
+        return "location_channel"
+    return "public_service_channel"
+
+
+def coverage_tags_for_scenario(
+    scenario: Scenario,
+    selected_primitives: Sequence[str],
+) -> tuple[str, ...]:
+    """Build Evidence Fabric coverage tags from scenario text and primitives."""
+    text = scenario_text(scenario)
+    tags = [f"domain:{scenario.lifecycle_domain}"]
+    if "document" in text:
+        tags.append("document_harness")
+    if "aviation" in text or "airport" in text or "flight" in text:
+        tags.append("aviation_weather")
+    if "aed" in text or "defibrillator" in text or "automated external" in text:
+        tags.append("aed_safety")
+    if "procurement" in text or "bid" in text or "나라장터" in text:
+        tags.append("procurement")
+    if scenario.lifecycle_domain == "public_data" or "public data" in text or "공공데이터" in text:
+        tags.append("public_data_search")
+    if "resolve_location" in selected_primitives:
+        tags.append("location")
+    if {"verify", "submit", "subscribe"} & set(selected_primitives):
+        tags.append("side_effecting_government_request")
+    return tuple(unique(tags))
+
+
+def scenario_text(scenario: Scenario) -> str:
+    """Flatten scenario text fields for deterministic route-family classification."""
+    parts = [
+        scenario.lifecycle_domain,
+        scenario.request_ko,
+        scenario.request_en or "",
+        *scenario.agencies_or_infrastructure,
+        *scenario.citizen_intent_verbs,
+        *scenario.expected_system_behavior,
+        *scenario.evaluation_focus,
+        *(step.primitive for step in scenario.expected_ax_chain),
+        *(step.purpose for step in scenario.expected_ax_chain),
+    ]
+    return " ".join(parts).lower()
+
+
+def prompt_manifest_hash(dataset: ScenarioDataset) -> str:
+    """Hash prompt-facing dataset metadata."""
+    return sha256(
+        {
+            "allowed_primitives": dataset.allowed_primitives,
+            "dataset_id": dataset.dataset_id,
+            "source_basis": dataset.source_basis,
+            "target_system": dataset.target_system,
+            "version": dataset.version,
+        }
+    )
+
+
+def tool_catalog_hash(dataset: ScenarioDataset) -> str:
+    """Hash the abstract adapter-family catalog implied by the dataset."""
+    return sha256(
+        {
+            "adapter_families": tuple(
+                sorted(
+                    {
+                        adapter_family_for_scenario(
+                            scenario,
+                            tuple(step.primitive for step in scenario.expected_ax_chain),
+                        )
+                        for scenario in dataset.scenarios
+                    }
+                )
+            ),
+            "coverage_domains": dataset.coverage_domains,
+            "dataset_id": dataset.dataset_id,
+            "scenario_count": len(dataset.scenarios),
+        }
+    )
+
+
+def correlation_id(trace_id: str) -> str:
+    """Build the deterministic route correlation id used by Evidence Fabric."""
+    return f"corr-route-{sha256(trace_id)[:16]}"

package/src/ummaya/evidence/runner.py

@@ -0,0 +1,177 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Evidence Fabric v2 dataset runner.
+
+The runner is intentionally local and deterministic. It validates scenario
+contracts and emits a typed RunEvidence document without calling live public
+service channels, LLM providers, or observability backends.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from ummaya.evidence.dataset_contract import (
+    DEFAULT_DATASET_REF,
+    DEFAULT_SCENARIO_PATH,
+    DEFAULT_TASK_REGISTRY_PATH,
+    EvidenceContractError,
+    resolve_repo_path,
+)
+from ummaya.evidence.dataset_contract import (
+    parse_dataset as _parse_dataset,
+)
+from ummaya.evidence.document_viewer_ux import DocumentViewerUxArtifact
+from ummaya.evidence.gates import build_gates
+from ummaya.evidence.models import RunEvidence
+from ummaya.evidence.output_payload import build_evidence_output_payload
+from ummaya.evidence.route_contracts import (
+    route_selection_assertions as _route_selection_assertions,
+)
+from ummaya.evidence.route_contracts import (
+    route_trace_records as _route_trace_records,
+)
+from ummaya.evidence.task_registry import EvidenceDatasetRef, load_task_registry
+from ummaya.evidence.tool_layer import build_tool_layer_events
+
+
+def _resolve_task_dataset(
+    *,
+    dataset_id: str,
+    scenario_path: Path,
+    task_registry_path: Path | None,
+    dataset_ref: str,
+) -> tuple[str | None, EvidenceDatasetRef | None]:
+    if task_registry_path is None:
+        return None, None
+    registry = load_task_registry(task_registry_path)
+    task_dataset = registry.resolve_dataset(dataset_ref)
+    if task_dataset.dataset_id != dataset_id:
+        raise EvidenceContractError(
+            f"task registry dataset_id {task_dataset.dataset_id!r} does not match "
+            f"scenario dataset_id {dataset_id!r}"
+        )
+    if resolve_repo_path(task_dataset.scenario_path) != resolve_repo_path(scenario_path):
+        raise EvidenceContractError(
+            f"task registry scenario_path {task_dataset.scenario_path} does not match "
+            f"run scenario_path {scenario_path}"
+        )
+    return registry.registry_id, task_dataset
+
+
+def run_dataset(
+    *,
+    scenario_path: Path = DEFAULT_SCENARIO_PATH,
+    source_ref: str = "local",
+    task_registry_path: Path | None = DEFAULT_TASK_REGISTRY_PATH,
+    dataset_ref: str = DEFAULT_DATASET_REF,
+) -> RunEvidence:
+    """Validate a scenario dataset and return a typed evidence document."""
+
+    dataset = _parse_dataset(scenario_path)
+    task_registry_id, task_dataset = _resolve_task_dataset(
+        dataset_id=dataset.dataset_id,
+        scenario_path=scenario_path,
+        task_registry_path=task_registry_path,
+        dataset_ref=dataset_ref,
+    )
+    traces = _route_trace_records(dataset)
+    return RunEvidence(
+        source_ref=source_ref,
+        dataset_id=dataset.dataset_id,
+        task_registry_id=task_registry_id,
+        dataset_ref=task_dataset.ref if task_dataset else None,
+        task_count=len(task_dataset.tasks) if task_dataset else 0,
+        task_ids=tuple(task.task_id for task in task_dataset.tasks) if task_dataset else (),
+        scenario_count=len(dataset.scenarios),
+        scenario_ids=tuple(scenario.id for scenario in dataset.scenarios),
+        route_trace_records=traces,
+        route_selection_assertions=_route_selection_assertions(dataset, traces),
+        tool_layer_events=build_tool_layer_events(traces),
+        gates=build_gates(dataset),
+    )
+
+
+def main() -> None:
+    """CLI entrypoint for `python -m ummaya.evidence`."""
+
+    parser = argparse.ArgumentParser(prog="python -m ummaya.evidence")
+    parser.add_argument(
+        "--scenarios",
+        type=Path,
+        default=DEFAULT_SCENARIO_PATH,
+        help="Path to the Evidence Fabric scenario dataset.",
+    )
+    parser.add_argument(
+        "--source-ref",
+        default="local",
+        help="Source revision or label recorded in the evidence document.",
+    )
+    parser.add_argument(
+        "--task-registry",
+        type=Path,
+        default=DEFAULT_TASK_REGISTRY_PATH,
+        help="Path to the Harbor-style Evidence Fabric task registry.",
+    )
+    parser.add_argument(
+        "--dataset-ref",
+        default=DEFAULT_DATASET_REF,
+        help="Harbor-style dataset ref to resolve from the task registry.",
+    )
+    parser.add_argument(
+        "--out", type=Path, default=Path(".evidence/run.json"), help="Output JSON path."
+    )
+    parser.add_argument(
+        "--document-viewer-html",
+        type=Path,
+        action="append",
+        default=None,
+        help="Local document viewer HTML path to capture as a Playwright UX artifact.",
+    )
+    parser.add_argument(
+        "--document-viewer-ux-out-dir",
+        type=Path,
+        default=None,
+        help="Directory for Playwright document viewer PNG artifacts.",
+    )
+    parser.add_argument(
+        "--document-viewer-correlation-id",
+        default=None,
+        help="Correlation ID to attach when the viewer manifest does not carry one.",
+    )
+    parser.add_argument(
+        "--document-viewer-diff-id",
+        default=None,
+        help="Document diff ID to attach when the viewer manifest does not carry one.",
+    )
+    args = parser.parse_args()
+
+    evidence = run_dataset(
+        scenario_path=args.scenarios,
+        source_ref=args.source_ref,
+        task_registry_path=args.task_registry,
+        dataset_ref=args.dataset_ref,
+    )
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    ux_out_dir = args.document_viewer_ux_out_dir or args.out.parent / "ux-artifacts"
+    document_viewer_ux_artifacts: list[DocumentViewerUxArtifact] = []
+    document_viewer_html_paths = tuple(args.document_viewer_html or ())
+    if document_viewer_html_paths:
+        from ummaya.evidence.document_viewer_ux import capture_document_viewer_ux_artifact
+
+        for viewer_html_path in document_viewer_html_paths:
+            document_viewer_ux_artifacts.append(
+                capture_document_viewer_ux_artifact(
+                    viewer_html_path=viewer_html_path,
+                    output_dir=ux_out_dir,
+                    source_ref=args.source_ref,
+                    correlation_id=args.document_viewer_correlation_id,
+                    document_diff_id=args.document_viewer_diff_id,
+                )
+            )
+    payload = build_evidence_output_payload(
+        evidence,
+        document_viewer_ux_artifacts=tuple(document_viewer_ux_artifacts),
+    )
+    args.out.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")