ummaya 0.2.4 → 0.2.5

package/src/ummaya/evidence/runner.py

@@ -9,211 +9,36 @@ service channels, LLM providers, or observability backends.
 from __future__ import annotations
 
 import argparse
-from collections.abc import Mapping, Sequence
+import json
 from pathlib import Path
-from typing import Literal, cast
 
-import yaml
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
-
-from ummaya.evidence.models import EvidenceGate, RunEvidence
-from ummaya.evidence.task_registry import EvidenceDatasetRef, load_task_registry
-
-_REPO_ROOT = Path(__file__).resolve().parents[3]
-_DEFAULT_SCENARIO_PATH = _REPO_ROOT / "evidence/scenarios/national_ax_citizen_requests_v1.yaml"
-_DEFAULT_TASK_REGISTRY_PATH = _REPO_ROOT / "evidence/registry.yaml"
-_DEFAULT_DATASET_REF = "ummaya/national-ax-core@local"
-_BANNED_MODEL_VISIBLE_KEYS = frozenset(
-    {
-        "adapter_id",
-        "tool_id",
-        "expected_tool_id",
-        "fixture_refs",
-        "fixture_ref",
-        "current_adapter_id",
-    }
+from ummaya.evidence.dataset_contract import (
+    DEFAULT_DATASET_REF,
+    DEFAULT_SCENARIO_PATH,
+    DEFAULT_TASK_REGISTRY_PATH,
+    EvidenceContractError,
+    resolve_repo_path,
 )
-_REQUIRED_DOMAINS = frozenset(
-    {
-        "tax",
-        "civil_affairs",
-        "payments",
-        "utilities",
-        "identity",
-        "welfare",
-        "healthcare",
-        "housing",
-        "mobility",
-        "business",
-        "labor",
-        "education",
-        "safety",
-        "immigration",
-        "legal",
-        "personal_data",
-    }
+from ummaya.evidence.dataset_contract import (
+    parse_dataset as _parse_dataset,
 )
-
-
-class EvidenceContractError(ValueError):
-    """Raised when a scenario dataset violates the Evidence Fabric contract."""
-
-
-class ExpectedStep(BaseModel):
-    """One expected public-service loop step in a scenario."""
-
-    model_config = ConfigDict(frozen=True, extra="forbid")
-
-    primitive: str
-    purpose: str
-
-
-class PermissionRequirements(BaseModel):
-    """Permission requirements attached to a citizen scenario."""
-
-    model_config = ConfigDict(frozen=True, extra="allow")
-
-    identity_assurance: str
-    user_confirmations: tuple[str, ...] = Field(default_factory=tuple)
-    sensitive_data: tuple[str, ...] = Field(default_factory=tuple)
-
-
-class Scenario(BaseModel):
-    """Minimum scenario shape needed by Evidence Fabric v2."""
-
-    model_config = ConfigDict(frozen=True, extra="allow")
-
-    id: str
-    priority: str = "P2"
-    lifecycle_domain: str
-    request_ko: str
-    expected_ax_chain: tuple[ExpectedStep, ...]
-    permission_requirements: PermissionRequirements
-
-
-class ScenarioDataset(BaseModel):
-    """Versioned citizen-demand scenario dataset."""
-
-    model_config = ConfigDict(frozen=True, extra="allow")
-
-    version: int
-    dataset_id: str
-    coverage_domains: tuple[str, ...]
-    scenarios: tuple[Scenario, ...]
-
-
-def _load_yaml_mapping(path: Path) -> Mapping[str, object]:
-    if not path.exists():
-        raise EvidenceContractError(f"scenario dataset not found: {path}")
-    loaded = yaml.safe_load(path.read_text(encoding="utf-8"))
-    if not isinstance(loaded, Mapping):
-        raise EvidenceContractError(f"scenario dataset must be a mapping: {path}")
-    return cast(Mapping[str, object], loaded)
-
-
-def _find_banned_keys(value: object, path: str = "$") -> tuple[str, ...]:
-    if isinstance(value, Mapping):
-        hits: list[str] = []
-        for key, nested in value.items():
-            key_text = str(key)
-            nested_path = f"{path}.{key_text}"
-            if key_text in _BANNED_MODEL_VISIBLE_KEYS:
-                hits.append(nested_path)
-            hits.extend(_find_banned_keys(nested, nested_path))
-        return tuple(hits)
-    if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
-        hits = []
-        for index, nested in enumerate(value):
-            hits.extend(_find_banned_keys(nested, f"{path}[{index}]"))
-        return tuple(hits)
-    return ()
-
-
-def _parse_dataset(path: Path) -> ScenarioDataset:
-    raw = _load_yaml_mapping(path)
-    banned = _find_banned_keys(raw)
-    if banned:
-        raise EvidenceContractError(
-            "model-visible scenario dataset contains banned implementation keys: "
-            + ", ".join(banned)
-        )
-    try:
-        return ScenarioDataset.model_validate(raw)
-    except ValidationError as exc:
-        raise EvidenceContractError(str(exc)) from exc
-
-
-def _gate(
-    name: Literal["contract", "scenario", "observability", "adversarial", "ux", "live_canary"],
-    status: Literal["pass", "fail", "skip"],
-    summary: str,
-    check_ids: tuple[str, ...],
-) -> EvidenceGate:
-    return EvidenceGate(name=name, status=status, summary=summary, check_ids=check_ids)
-
-
-def _build_gates(dataset: ScenarioDataset) -> tuple[EvidenceGate, ...]:
-    covered_domains = set(dataset.coverage_domains)
-    missing_domains = tuple(sorted(_REQUIRED_DOMAINS - covered_domains))
-    scenario_domains = {scenario.lifecycle_domain for scenario in dataset.scenarios}
-    uncovered_scenario_domains = tuple(sorted(scenario_domains - covered_domains))
-
-    scenario_status: Literal["pass", "fail"] = (
-        "pass" if not missing_domains and not uncovered_scenario_domains else "fail"
-    )
-    scenario_summary = (
-        "all required citizen infrastructure domains are covered"
-        if scenario_status == "pass"
-        else "missing coverage: " + ", ".join(missing_domains + uncovered_scenario_domains)
-    )
-
-    return (
-        _gate(
-            "contract",
-            "pass",
-            "dataset is versioned, typed, and free of model-visible implementation keys",
-            ("dataset-schema", "task-registry", "no-adapter-leakage"),
-        ),
-        _gate(
-            "scenario",
-            scenario_status,
-            scenario_summary,
-            ("coverage-domains", "scenario-shape"),
-        ),
-        _gate(
-            "observability",
-            "pass",
-            "RunEvidence carries trace join keys for OTEL/Langfuse correlation",
-            ("trace-join-keys",),
-        ),
-        _gate(
-            "adversarial",
-            "pass",
-            "adapter IDs, fixture references, and expected tool IDs are rejected before scoring",
-            ("reward-hack-surface", "hidden-implementation-leakage"),
-        ),
-        _gate(
-            "ux",
-            "skip",
-            "UX frame artifacts are attached by interactive runners, not by dataset validation",
-            ("ux-artifact-slot",),
-        ),
-        _gate(
-            "live_canary",
-            "skip",
-            "live public-service checks are manual-only and excluded from CI",
-            ("no-live-ci",),
-        ),
-    )
-
-
-def _resolve_repo_path(path: Path) -> Path:
-    return path if path.is_absolute() else _REPO_ROOT / path
+from ummaya.evidence.document_viewer_ux import DocumentViewerUxArtifact
+from ummaya.evidence.gates import build_gates
+from ummaya.evidence.models import RunEvidence
+from ummaya.evidence.output_payload import build_evidence_output_payload
+from ummaya.evidence.route_contracts import (
+    route_selection_assertions as _route_selection_assertions,
+)
+from ummaya.evidence.route_contracts import (
+    route_trace_records as _route_trace_records,
+)
+from ummaya.evidence.task_registry import EvidenceDatasetRef, load_task_registry
+from ummaya.evidence.tool_layer import build_tool_layer_events
 
 
 def _resolve_task_dataset(
     *,
-    dataset: ScenarioDataset,
+    dataset_id: str,
     scenario_path: Path,
     task_registry_path: Path | None,
     dataset_ref: str,
@@ -222,12 +47,12 @@ def _resolve_task_dataset(
         return None, None
     registry = load_task_registry(task_registry_path)
     task_dataset = registry.resolve_dataset(dataset_ref)
-    if task_dataset.dataset_id != dataset.dataset_id:
+    if task_dataset.dataset_id != dataset_id:
         raise EvidenceContractError(
             f"task registry dataset_id {task_dataset.dataset_id!r} does not match "
-            f"scenario dataset_id {dataset.dataset_id!r}"
+            f"scenario dataset_id {dataset_id!r}"
         )
-    if _resolve_repo_path(task_dataset.scenario_path) != _resolve_repo_path(scenario_path):
+    if resolve_repo_path(task_dataset.scenario_path) != resolve_repo_path(scenario_path):
         raise EvidenceContractError(
             f"task registry scenario_path {task_dataset.scenario_path} does not match "
             f"run scenario_path {scenario_path}"
@@ -237,20 +62,21 @@ def _resolve_task_dataset(
 
 def run_dataset(
     *,
-    scenario_path: Path = _DEFAULT_SCENARIO_PATH,
+    scenario_path: Path = DEFAULT_SCENARIO_PATH,
     source_ref: str = "local",
-    task_registry_path: Path | None = _DEFAULT_TASK_REGISTRY_PATH,
-    dataset_ref: str = _DEFAULT_DATASET_REF,
+    task_registry_path: Path | None = DEFAULT_TASK_REGISTRY_PATH,
+    dataset_ref: str = DEFAULT_DATASET_REF,
 ) -> RunEvidence:
     """Validate a scenario dataset and return a typed evidence document."""
 
     dataset = _parse_dataset(scenario_path)
     task_registry_id, task_dataset = _resolve_task_dataset(
-        dataset=dataset,
+        dataset_id=dataset.dataset_id,
         scenario_path=scenario_path,
         task_registry_path=task_registry_path,
         dataset_ref=dataset_ref,
     )
+    traces = _route_trace_records(dataset)
     return RunEvidence(
         source_ref=source_ref,
         dataset_id=dataset.dataset_id,
@@ -260,7 +86,10 @@ def run_dataset(
         task_ids=tuple(task.task_id for task in task_dataset.tasks) if task_dataset else (),
         scenario_count=len(dataset.scenarios),
         scenario_ids=tuple(scenario.id for scenario in dataset.scenarios),
-        gates=_build_gates(dataset),
+        route_trace_records=traces,
+        route_selection_assertions=_route_selection_assertions(dataset, traces),
+        tool_layer_events=build_tool_layer_events(traces),
+        gates=build_gates(dataset),
     )
 
 
@@ -271,7 +100,7 @@ def main() -> None:
     parser.add_argument(
         "--scenarios",
         type=Path,
-        default=_DEFAULT_SCENARIO_PATH,
+        default=DEFAULT_SCENARIO_PATH,
         help="Path to the Evidence Fabric scenario dataset.",
     )
     parser.add_argument(
@@ -282,19 +111,39 @@ def main() -> None:
     parser.add_argument(
         "--task-registry",
         type=Path,
-        default=_DEFAULT_TASK_REGISTRY_PATH,
+        default=DEFAULT_TASK_REGISTRY_PATH,
         help="Path to the Harbor-style Evidence Fabric task registry.",
     )
     parser.add_argument(
         "--dataset-ref",
-        default=_DEFAULT_DATASET_REF,
+        default=DEFAULT_DATASET_REF,
         help="Harbor-style dataset ref to resolve from the task registry.",
     )
     parser.add_argument(
-        "--out",
+        "--out", type=Path, default=Path(".evidence/run.json"), help="Output JSON path."
+    )
+    parser.add_argument(
+        "--document-viewer-html",
         type=Path,
-        default=Path(".evidence/run.json"),
-        help="Output JSON path.",
+        action="append",
+        default=None,
+        help="Local document viewer HTML path to capture as a Playwright UX artifact.",
+    )
+    parser.add_argument(
+        "--document-viewer-ux-out-dir",
+        type=Path,
+        default=None,
+        help="Directory for Playwright document viewer PNG artifacts.",
+    )
+    parser.add_argument(
+        "--document-viewer-correlation-id",
+        default=None,
+        help="Correlation ID to attach when the viewer manifest does not carry one.",
+    )
+    parser.add_argument(
+        "--document-viewer-diff-id",
+        default=None,
+        help="Document diff ID to attach when the viewer manifest does not carry one.",
     )
     args = parser.parse_args()
 
@@ -305,4 +154,24 @@ def main() -> None:
         dataset_ref=args.dataset_ref,
     )
     args.out.parent.mkdir(parents=True, exist_ok=True)
-    args.out.write_text(evidence.model_dump_json(indent=2), encoding="utf-8")
+    ux_out_dir = args.document_viewer_ux_out_dir or args.out.parent / "ux-artifacts"
+    document_viewer_ux_artifacts: list[DocumentViewerUxArtifact] = []
+    document_viewer_html_paths = tuple(args.document_viewer_html or ())
+    if document_viewer_html_paths:
+        from ummaya.evidence.document_viewer_ux import capture_document_viewer_ux_artifact
+
+        for viewer_html_path in document_viewer_html_paths:
+            document_viewer_ux_artifacts.append(
+                capture_document_viewer_ux_artifact(
+                    viewer_html_path=viewer_html_path,
+                    output_dir=ux_out_dir,
+                    source_ref=args.source_ref,
+                    correlation_id=args.document_viewer_correlation_id,
+                    document_diff_id=args.document_viewer_diff_id,
+                )
+            )
+    payload = build_evidence_output_payload(
+        evidence,
+        document_viewer_ux_artifacts=tuple(document_viewer_ux_artifacts),
+    )
+    args.out.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")

package/src/ummaya/evidence/source_provenance.py

@@ -0,0 +1,246 @@
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal, Self
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from pydantic_core import PydanticCustomError
+
+from ummaya.evidence.source_provenance_redaction import (
+    PromptInjectionState,
+    RedactionCategory,
+    detect_prompt_injection,
+    ordered_redaction_categories,
+    redact_source_text,
+    redact_source_url,
+    redaction_categories_for_text,
+)
+from ummaya.tools.routing.schema import sha256, unique
+
+SourceKind = Literal["web", "mcp", "agent", "file"]
+SourceUseState = Literal["used", "blocked"]
+SourceTrust = Literal["trusted", "untrusted"]
+SynthesisState = Literal[
+    "eligible_for_synthesis",
+    "blocked_pending_user_approval",
+    "blocked_untrusted",
+    "blocked_missing_source",
+]
+DocumentAuthoringState = Literal[
+    "approved_for_document",
+    "blocked_pending_user_approval",
+    "blocked_untrusted",
+    "blocked_missing_source",
+]
+
+
+class SourceRedactionMetadata(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    redacted: bool
+    categories: tuple[RedactionCategory, ...] = Field(default_factory=tuple)
+    raw_private_document_stored: Literal[False] = False
+    secret_values_stored: Literal[False] = False
+    pii_values_stored: Literal[False] = False
+
+    @model_validator(mode="after")
+    def _categories_match_redacted_flag(self) -> Self:
+        if self.redacted and not self.categories:
+            raise PydanticCustomError(
+                "source_provenance_missing_redaction_categories",
+                "redacted provenance requires redaction categories",
+            )
+        if not self.redacted and self.categories:
+            raise PydanticCustomError(
+                "source_provenance_unexpected_redaction_categories",
+                "unredacted provenance cannot carry redaction categories",
+            )
+        return self
+
+
+class SourceProvenanceRecord(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    source_kind: SourceKind
+    source_url: str | None
+    local_evidence_handle: str | None
+    title: str | None
+    description: str | None
+    tool_id: str = Field(min_length=1)
+    observed_at: datetime
+    state: SourceUseState
+    citation_id: str = Field(min_length=1)
+    provenance_id: str = Field(min_length=1)
+    trust: SourceTrust
+    prompt_injection: PromptInjectionState = "not_detected"
+    redaction: SourceRedactionMetadata
+
+    @field_validator(
+        "source_url",
+        "local_evidence_handle",
+        "title",
+        "description",
+        "tool_id",
+        "citation_id",
+        "provenance_id",
+    )
+    @classmethod
+    def _reject_unredacted_sensitive_value(cls, value: str | None) -> str | None:
+        if value is not None and redaction_categories_for_text(value):
+            raise PydanticCustomError(
+                "source_provenance_unredacted_sensitive_value",
+                "source provenance contains unredacted sensitive value",
+            )
+        return value
+
+    @model_validator(mode="after")
+    def _requires_source_handle_and_label(self) -> Self:
+        if self.source_url is None and self.local_evidence_handle is None:
+            raise PydanticCustomError(
+                "source_provenance_missing_source",
+                "source provenance requires source_url or local_evidence_handle",
+            )
+        if self.title is None and self.description is None:
+            raise PydanticCustomError(
+                "source_provenance_missing_label",
+                "source provenance requires title or description when available",
+            )
+        return self
+
+
+class SourceProvenanceDecision(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    decision_id: str = Field(min_length=1)
+    provenance_ids: tuple[str, ...] = Field(min_length=1)
+    synthesis_state: SynthesisState
+    document_authoring_state: DocumentAuthoringState
+    requires_user_approval: bool
+    approved_by_user: bool
+    rationale: str = Field(min_length=1)
+
+    @model_validator(mode="after")
+    def _document_approval_requires_user_approval(self) -> Self:
+        if self.document_authoring_state == "approved_for_document" and (
+            self.requires_user_approval or not self.approved_by_user
+        ):
+            raise PydanticCustomError(
+                "source_provenance_document_approval_missing_user_approval",
+                "approved document authoring requires completed user approval",
+            )
+        return self
+
+
+class SourceProvenanceLedger(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    schema_version: Literal["source_provenance_ledger.v1"] = "source_provenance_ledger.v1"
+    ledger_id: str = Field(min_length=1)
+    records: tuple[SourceProvenanceRecord, ...] = Field(min_length=1)
+    decisions: tuple[SourceProvenanceDecision, ...] = Field(default_factory=tuple)
+
+    @model_validator(mode="after")
+    def _decisions_reference_known_provenance_ids(self) -> Self:
+        known_ids = {record.provenance_id for record in self.records}
+        if len(known_ids) != len(self.records):
+            raise PydanticCustomError(
+                "source_provenance_duplicate_ids",
+                "source provenance ledger contains duplicate provenance ids",
+            )
+        unknown_ids = tuple(
+            unique(
+                provenance_id
+                for decision in self.decisions
+                for provenance_id in decision.provenance_ids
+                if provenance_id not in known_ids
+            )
+        )
+        if unknown_ids:
+            raise PydanticCustomError(
+                "source_provenance_unknown_ids",
+                "unknown provenance ids: " + ", ".join(unknown_ids),
+            )
+        return self
+
+
+def build_source_provenance_record(
+    *,
+    source_kind: SourceKind,
+    tool_id: str,
+    source_url: str | None,
+    local_evidence_handle: str | None,
+    title: str | None,
+    description: str | None,
+    observed_at: datetime,
+    state: SourceUseState,
+    trust: SourceTrust,
+    citation_id: str,
+) -> SourceProvenanceRecord:
+    redacted_url, url_categories = redact_source_url(source_url)
+    redacted_handle, handle_categories = redact_source_text(local_evidence_handle)
+    redacted_title, title_categories = redact_source_text(title)
+    redacted_description, description_categories = redact_source_text(description)
+    categories = ordered_redaction_categories(
+        (
+            *url_categories,
+            *handle_categories,
+            *title_categories,
+            *description_categories,
+        )
+    )
+    prompt_injection_text_parts = (
+        title or "",
+        description or "",
+        source_url or "",
+        local_evidence_handle or "",
+    )
+    return SourceProvenanceRecord(
+        source_kind=source_kind,
+        source_url=redacted_url,
+        local_evidence_handle=redacted_handle,
+        title=redacted_title,
+        description=redacted_description,
+        tool_id=tool_id,
+        observed_at=observed_at,
+        state=state,
+        citation_id=citation_id,
+        provenance_id=stable_source_provenance_id(
+            source_kind=source_kind,
+            tool_id=tool_id,
+            source_url=redacted_url,
+            local_evidence_handle=redacted_handle,
+            citation_id=citation_id,
+        ),
+        trust=trust,
+        prompt_injection=detect_prompt_injection(
+            "\n".join(part for part in prompt_injection_text_parts if part)
+        ),
+        redaction=SourceRedactionMetadata(
+            redacted=bool(categories),
+            categories=categories,
+            raw_private_document_stored=False,
+            secret_values_stored=False,
+            pii_values_stored=False,
+        ),
+    )
+
+
+def stable_source_provenance_id(
+    *,
+    source_kind: SourceKind,
+    tool_id: str,
+    source_url: str | None,
+    local_evidence_handle: str | None,
+    citation_id: str,
+) -> str:
+    digest = sha256(
+        {
+            "citation_id": citation_id,
+            "local_evidence_handle": local_evidence_handle,
+            "source_kind": source_kind,
+            "source_url": source_url,
+            "tool_id": tool_id,
+        }
+    )
+    return f"prov-{digest[:24]}"