ummaya 0.2.2 → 0.2.4

package/src/ummaya/evidence/runner.py

@@ -0,0 +1,308 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Evidence Fabric v2 dataset runner.
+
+The runner is intentionally local and deterministic. It validates scenario
+contracts and emits a typed RunEvidence document without calling live public
+service channels, LLM providers, or observability backends.
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Literal, cast
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from ummaya.evidence.models import EvidenceGate, RunEvidence
+from ummaya.evidence.task_registry import EvidenceDatasetRef, load_task_registry
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_DEFAULT_SCENARIO_PATH = _REPO_ROOT / "evidence/scenarios/national_ax_citizen_requests_v1.yaml"
+_DEFAULT_TASK_REGISTRY_PATH = _REPO_ROOT / "evidence/registry.yaml"
+_DEFAULT_DATASET_REF = "ummaya/national-ax-core@local"
+_BANNED_MODEL_VISIBLE_KEYS = frozenset(
+    {
+        "adapter_id",
+        "tool_id",
+        "expected_tool_id",
+        "fixture_refs",
+        "fixture_ref",
+        "current_adapter_id",
+    }
+)
+_REQUIRED_DOMAINS = frozenset(
+    {
+        "tax",
+        "civil_affairs",
+        "payments",
+        "utilities",
+        "identity",
+        "welfare",
+        "healthcare",
+        "housing",
+        "mobility",
+        "business",
+        "labor",
+        "education",
+        "safety",
+        "immigration",
+        "legal",
+        "personal_data",
+    }
+)
+
+
+class EvidenceContractError(ValueError):
+    """Raised when a scenario dataset violates the Evidence Fabric contract."""
+
+
+class ExpectedStep(BaseModel):
+    """One expected public-service loop step in a scenario."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    primitive: str
+    purpose: str
+
+
+class PermissionRequirements(BaseModel):
+    """Permission requirements attached to a citizen scenario."""
+
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    identity_assurance: str
+    user_confirmations: tuple[str, ...] = Field(default_factory=tuple)
+    sensitive_data: tuple[str, ...] = Field(default_factory=tuple)
+
+
+class Scenario(BaseModel):
+    """Minimum scenario shape needed by Evidence Fabric v2."""
+
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    id: str
+    priority: str = "P2"
+    lifecycle_domain: str
+    request_ko: str
+    expected_ax_chain: tuple[ExpectedStep, ...]
+    permission_requirements: PermissionRequirements
+
+
+class ScenarioDataset(BaseModel):
+    """Versioned citizen-demand scenario dataset."""
+
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    version: int
+    dataset_id: str
+    coverage_domains: tuple[str, ...]
+    scenarios: tuple[Scenario, ...]
+
+
+def _load_yaml_mapping(path: Path) -> Mapping[str, object]:
+    if not path.exists():
+        raise EvidenceContractError(f"scenario dataset not found: {path}")
+    loaded = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if not isinstance(loaded, Mapping):
+        raise EvidenceContractError(f"scenario dataset must be a mapping: {path}")
+    return cast(Mapping[str, object], loaded)
+
+
+def _find_banned_keys(value: object, path: str = "$") -> tuple[str, ...]:
+    if isinstance(value, Mapping):
+        hits: list[str] = []
+        for key, nested in value.items():
+            key_text = str(key)
+            nested_path = f"{path}.{key_text}"
+            if key_text in _BANNED_MODEL_VISIBLE_KEYS:
+                hits.append(nested_path)
+            hits.extend(_find_banned_keys(nested, nested_path))
+        return tuple(hits)
+    if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
+        hits = []
+        for index, nested in enumerate(value):
+            hits.extend(_find_banned_keys(nested, f"{path}[{index}]"))
+        return tuple(hits)
+    return ()
+
+
+def _parse_dataset(path: Path) -> ScenarioDataset:
+    raw = _load_yaml_mapping(path)
+    banned = _find_banned_keys(raw)
+    if banned:
+        raise EvidenceContractError(
+            "model-visible scenario dataset contains banned implementation keys: "
+            + ", ".join(banned)
+        )
+    try:
+        return ScenarioDataset.model_validate(raw)
+    except ValidationError as exc:
+        raise EvidenceContractError(str(exc)) from exc
+
+
+def _gate(
+    name: Literal["contract", "scenario", "observability", "adversarial", "ux", "live_canary"],
+    status: Literal["pass", "fail", "skip"],
+    summary: str,
+    check_ids: tuple[str, ...],
+) -> EvidenceGate:
+    return EvidenceGate(name=name, status=status, summary=summary, check_ids=check_ids)
+
+
+def _build_gates(dataset: ScenarioDataset) -> tuple[EvidenceGate, ...]:
+    covered_domains = set(dataset.coverage_domains)
+    missing_domains = tuple(sorted(_REQUIRED_DOMAINS - covered_domains))
+    scenario_domains = {scenario.lifecycle_domain for scenario in dataset.scenarios}
+    uncovered_scenario_domains = tuple(sorted(scenario_domains - covered_domains))
+
+    scenario_status: Literal["pass", "fail"] = (
+        "pass" if not missing_domains and not uncovered_scenario_domains else "fail"
+    )
+    scenario_summary = (
+        "all required citizen infrastructure domains are covered"
+        if scenario_status == "pass"
+        else "missing coverage: " + ", ".join(missing_domains + uncovered_scenario_domains)
+    )
+
+    return (
+        _gate(
+            "contract",
+            "pass",
+            "dataset is versioned, typed, and free of model-visible implementation keys",
+            ("dataset-schema", "task-registry", "no-adapter-leakage"),
+        ),
+        _gate(
+            "scenario",
+            scenario_status,
+            scenario_summary,
+            ("coverage-domains", "scenario-shape"),
+        ),
+        _gate(
+            "observability",
+            "pass",
+            "RunEvidence carries trace join keys for OTEL/Langfuse correlation",
+            ("trace-join-keys",),
+        ),
+        _gate(
+            "adversarial",
+            "pass",
+            "adapter IDs, fixture references, and expected tool IDs are rejected before scoring",
+            ("reward-hack-surface", "hidden-implementation-leakage"),
+        ),
+        _gate(
+            "ux",
+            "skip",
+            "UX frame artifacts are attached by interactive runners, not by dataset validation",
+            ("ux-artifact-slot",),
+        ),
+        _gate(
+            "live_canary",
+            "skip",
+            "live public-service checks are manual-only and excluded from CI",
+            ("no-live-ci",),
+        ),
+    )
+
+
+def _resolve_repo_path(path: Path) -> Path:
+    return path if path.is_absolute() else _REPO_ROOT / path
+
+
+def _resolve_task_dataset(
+    *,
+    dataset: ScenarioDataset,
+    scenario_path: Path,
+    task_registry_path: Path | None,
+    dataset_ref: str,
+) -> tuple[str | None, EvidenceDatasetRef | None]:
+    if task_registry_path is None:
+        return None, None
+    registry = load_task_registry(task_registry_path)
+    task_dataset = registry.resolve_dataset(dataset_ref)
+    if task_dataset.dataset_id != dataset.dataset_id:
+        raise EvidenceContractError(
+            f"task registry dataset_id {task_dataset.dataset_id!r} does not match "
+            f"scenario dataset_id {dataset.dataset_id!r}"
+        )
+    if _resolve_repo_path(task_dataset.scenario_path) != _resolve_repo_path(scenario_path):
+        raise EvidenceContractError(
+            f"task registry scenario_path {task_dataset.scenario_path} does not match "
+            f"run scenario_path {scenario_path}"
+        )
+    return registry.registry_id, task_dataset
+
+
+def run_dataset(
+    *,
+    scenario_path: Path = _DEFAULT_SCENARIO_PATH,
+    source_ref: str = "local",
+    task_registry_path: Path | None = _DEFAULT_TASK_REGISTRY_PATH,
+    dataset_ref: str = _DEFAULT_DATASET_REF,
+) -> RunEvidence:
+    """Validate a scenario dataset and return a typed evidence document."""
+
+    dataset = _parse_dataset(scenario_path)
+    task_registry_id, task_dataset = _resolve_task_dataset(
+        dataset=dataset,
+        scenario_path=scenario_path,
+        task_registry_path=task_registry_path,
+        dataset_ref=dataset_ref,
+    )
+    return RunEvidence(
+        source_ref=source_ref,
+        dataset_id=dataset.dataset_id,
+        task_registry_id=task_registry_id,
+        dataset_ref=task_dataset.ref if task_dataset else None,
+        task_count=len(task_dataset.tasks) if task_dataset else 0,
+        task_ids=tuple(task.task_id for task in task_dataset.tasks) if task_dataset else (),
+        scenario_count=len(dataset.scenarios),
+        scenario_ids=tuple(scenario.id for scenario in dataset.scenarios),
+        gates=_build_gates(dataset),
+    )
+
+
+def main() -> None:
+    """CLI entrypoint for `python -m ummaya.evidence`."""
+
+    parser = argparse.ArgumentParser(prog="python -m ummaya.evidence")
+    parser.add_argument(
+        "--scenarios",
+        type=Path,
+        default=_DEFAULT_SCENARIO_PATH,
+        help="Path to the Evidence Fabric scenario dataset.",
+    )
+    parser.add_argument(
+        "--source-ref",
+        default="local",
+        help="Source revision or label recorded in the evidence document.",
+    )
+    parser.add_argument(
+        "--task-registry",
+        type=Path,
+        default=_DEFAULT_TASK_REGISTRY_PATH,
+        help="Path to the Harbor-style Evidence Fabric task registry.",
+    )
+    parser.add_argument(
+        "--dataset-ref",
+        default=_DEFAULT_DATASET_REF,
+        help="Harbor-style dataset ref to resolve from the task registry.",
+    )
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=Path(".evidence/run.json"),
+        help="Output JSON path.",
+    )
+    args = parser.parse_args()
+
+    evidence = run_dataset(
+        scenario_path=args.scenarios,
+        source_ref=args.source_ref,
+        task_registry_path=args.task_registry,
+        dataset_ref=args.dataset_ref,
+    )
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(evidence.model_dump_json(indent=2), encoding="utf-8")

package/src/ummaya/evidence/task_registry.py

@@ -0,0 +1,264 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Harbor-style task registry for Evidence Fabric v2.
+
+The registry mirrors Harbor's task boundary: a task has an instruction,
+metadata/configuration, and a verifier script. UMMAYA keeps execution local and
+deterministic; this module only resolves and validates task definitions.
+"""
+
+from __future__ import annotations
+
+import tomllib
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Literal, cast
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_BANNED_MODEL_VISIBLE_KEYS = frozenset(
+    {
+        "adapter_id",
+        "tool_id",
+        "expected_tool_id",
+        "fixture_refs",
+        "fixture_ref",
+        "current_adapter_id",
+    }
+)
+
+
+class TaskRegistryError(ValueError):
+    """Raised when an Evidence Fabric task registry is invalid."""
+
+
+class EvidenceTask(BaseModel):
+    """One resolved Harbor-style evidence task."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    task_id: str
+    task_dir: Path
+    instruction_path: Path
+    verifier_path: Path
+    description: str
+    dataset_id: str
+    keywords: tuple[str, ...] = Field(default_factory=tuple)
+    environment_os: Literal["linux", "windows"] = "linux"
+    allow_internet: bool = False
+    verifier_timeout_sec: float = 120.0
+
+
+class EvidenceDatasetRef(BaseModel):
+    """A dataset reference resolved from the local task registry."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    ref: str
+    dataset_id: str
+    scenario_path: Path
+    tasks: tuple[EvidenceTask, ...]
+
+
+class EvidenceTaskRegistry(BaseModel):
+    """Resolved Evidence Fabric task registry."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    version: int
+    registry_id: str
+    datasets: tuple[EvidenceDatasetRef, ...]
+
+    def resolve_dataset(self, ref: str) -> EvidenceDatasetRef:
+        """Return the dataset entry matching a Harbor-style dataset ref."""
+
+        for dataset in self.datasets:
+            if dataset.ref == ref:
+                return dataset
+        raise TaskRegistryError(f"dataset ref not found in task registry: {ref}")
+
+
+def _load_yaml_mapping(path: Path) -> Mapping[str, object]:
+    if not path.exists():
+        raise TaskRegistryError(f"task registry not found: {path}")
+    loaded = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if not isinstance(loaded, Mapping):
+        raise TaskRegistryError(f"task registry must be a mapping: {path}")
+    return cast(Mapping[str, object], loaded)
+
+
+def _find_banned_keys(value: object, path: str = "$") -> tuple[str, ...]:
+    if isinstance(value, Mapping):
+        hits: list[str] = []
+        for key, nested in value.items():
+            key_text = str(key)
+            nested_path = f"{path}.{key_text}"
+            if key_text in _BANNED_MODEL_VISIBLE_KEYS:
+                hits.append(nested_path)
+            hits.extend(_find_banned_keys(nested, nested_path))
+        return tuple(hits)
+    if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
+        hits = []
+        for index, nested in enumerate(value):
+            hits.extend(_find_banned_keys(nested, f"{path}[{index}]"))
+        return tuple(hits)
+    return ()
+
+
+def _read_path(path: Path) -> Path:
+    return path if path.is_absolute() else _REPO_ROOT / path
+
+
+def _require_existing_files(paths: tuple[Path, ...]) -> None:
+    for required in paths:
+        if not _read_path(required).exists():
+            raise TaskRegistryError(f"task file missing: {required}")
+
+
+def _require_mapping(value: object, label: str) -> Mapping[str, object]:
+    if not isinstance(value, Mapping):
+        raise TaskRegistryError(f"{label} must be a mapping")
+    return cast(Mapping[str, object], value)
+
+
+def _require_non_empty_str(value: object, label: str) -> str:
+    if not isinstance(value, str) or not value:
+        raise TaskRegistryError(f"{label} must be a non-empty string")
+    return value
+
+
+def _require_sequence(value: object, label: str) -> Sequence[object]:
+    if not isinstance(value, Sequence) or isinstance(value, str | bytes | bytearray):
+        raise TaskRegistryError(f"{label} must be a list")
+    return cast(Sequence[object], value)
+
+
+def _optional_float(value: object, label: str, default: float) -> float:
+    if value is None:
+        return default
+    if isinstance(value, bool) or not isinstance(value, str | int | float):
+        raise TaskRegistryError(f"{label} must be a number")
+    return float(value)
+
+
+def _load_task_toml(task_toml: Path) -> Mapping[str, object]:
+    raw = tomllib.loads(_read_path(task_toml).read_text(encoding="utf-8"))
+    banned = _find_banned_keys(raw)
+    if banned:
+        raise TaskRegistryError(
+            "model-visible task registry contains banned implementation keys: " + ", ".join(banned)
+        )
+    return cast(Mapping[str, object], raw)
+
+
+def _build_task(
+    *,
+    task_dir: Path,
+    instruction_path: Path,
+    verifier_path: Path,
+    raw: Mapping[str, object],
+) -> EvidenceTask:
+    task_section = _require_mapping(raw.get("task"), "task.toml [task]")
+    metadata_section = _require_mapping(raw.get("metadata", {}), "task.toml [metadata]")
+    environment_section = _require_mapping(raw.get("environment", {}), "task.toml [environment]")
+    verifier_section = _require_mapping(raw.get("verifier", {}), "task.toml [verifier]")
+    keywords = _require_sequence(task_section.get("keywords", ()), "task.toml [task].keywords")
+
+    try:
+        return EvidenceTask(
+            task_id=_require_non_empty_str(task_section.get("name"), "task.toml [task].name"),
+            task_dir=task_dir,
+            instruction_path=instruction_path,
+            verifier_path=verifier_path,
+            description=_require_non_empty_str(
+                task_section.get("description"),
+                "task.toml [task].description",
+            ),
+            dataset_id=_require_non_empty_str(
+                metadata_section.get("dataset_id"),
+                "task.toml [metadata].dataset_id",
+            ),
+            keywords=tuple(str(keyword) for keyword in keywords),
+            environment_os=cast(
+                Literal["linux", "windows"],
+                environment_section.get("os", "linux"),
+            ),
+            allow_internet=bool(environment_section.get("allow_internet", False)),
+            verifier_timeout_sec=_optional_float(
+                verifier_section.get("timeout_sec"),
+                "task.toml [verifier].timeout_sec",
+                120.0,
+            ),
+        )
+    except ValidationError as exc:
+        raise TaskRegistryError(str(exc)) from exc
+
+
+def _load_task(task_dir: Path) -> EvidenceTask:
+    task_toml = task_dir / "task.toml"
+    instruction_path = task_dir / "instruction.md"
+    verifier_path = task_dir / "tests" / "test.sh"
+    _require_existing_files((task_toml, instruction_path, verifier_path))
+    return _build_task(
+        task_dir=task_dir,
+        instruction_path=instruction_path,
+        verifier_path=verifier_path,
+        raw=_load_task_toml(task_toml),
+    )
+
+
+def _load_dataset_ref(index: int, dataset_raw: object) -> EvidenceDatasetRef:
+    dataset_map = _require_mapping(dataset_raw, f"datasets[{index}]")
+    ref = _require_non_empty_str(dataset_map.get("ref"), f"datasets[{index}].ref")
+    dataset_id = _require_non_empty_str(
+        dataset_map.get("dataset_id"),
+        f"datasets[{index}].dataset_id",
+    )
+    scenario_path = _require_non_empty_str(
+        dataset_map.get("scenario_path"),
+        f"datasets[{index}].scenario_path",
+    )
+    task_paths = _require_sequence(dataset_map.get("task_paths"), f"datasets[{index}].task_paths")
+
+    tasks = tuple(_load_task(Path(str(task_path))) for task_path in task_paths)
+    mismatched = tuple(task.task_id for task in tasks if task.dataset_id != dataset_id)
+    if mismatched:
+        raise TaskRegistryError(
+            f"dataset {ref} has tasks with mismatched dataset_id: {', '.join(mismatched)}"
+        )
+    return EvidenceDatasetRef(
+        ref=ref,
+        dataset_id=dataset_id,
+        scenario_path=Path(scenario_path),
+        tasks=tasks,
+    )
+
+
+def load_task_registry(path: Path) -> EvidenceTaskRegistry:
+    """Load and validate a Harbor-style Evidence Fabric task registry."""
+
+    raw = _load_yaml_mapping(path)
+    banned = _find_banned_keys(raw)
+    if banned:
+        raise TaskRegistryError(
+            "model-visible task registry contains banned implementation keys: " + ", ".join(banned)
+        )
+
+    version = raw.get("version")
+    if not isinstance(version, int):
+        raise TaskRegistryError("task registry version must be an integer")
+    registry_id = _require_non_empty_str(raw.get("registry_id"), "task registry_id")
+    datasets = _require_sequence(raw.get("datasets"), "task registry datasets")
+    resolved_datasets = tuple(
+        _load_dataset_ref(index, dataset_raw) for index, dataset_raw in enumerate(datasets)
+    )
+
+    try:
+        return EvidenceTaskRegistry(
+            version=version,
+            registry_id=registry_id,
+            datasets=resolved_datasets,
+        )
+    except ValidationError as exc:
+        raise TaskRegistryError(str(exc)) from exc

package/src/ummaya/ipc/frame_schema.py

@@ -56,6 +56,7 @@ _ROLE_KIND_ALLOW_LIST: dict[str, frozenset[str]] = {
     # Spec 1978 ADR-0001 — tools-aware chat request from TUI
     "chat_request": frozenset({"tui"}),
     "assistant_chunk": frozenset({"backend", "llm"}),
+    "progress_event": frozenset({"backend"}),
     "tool_call": frozenset({"backend", "tool"}),
     "tool_result": frozenset({"backend", "tool"}),
     "coordinator_phase": frozenset({"backend"}),
@@ -385,6 +386,10 @@ class ChatRequestFrame(_BaseFrame):
         le=1.0,
         description="Nucleus sampling threshold.",
     )
+    reasoning_mode: Literal["fast", "balanced", "deep", "diagnostic", "auto"] | None = Field(
+        default=None,
+        description="K-EXAONE/FriendliAI reasoning policy for this assistant turn.",
+    )
 
     @model_validator(mode="after")
     def _v_tool_message_integrity(self) -> ChatRequestFrame:
@@ -449,6 +454,46 @@ class AssistantChunkFrame(_BaseFrame):
     done: bool = Field(description="True if this is the terminal chunk for this message_id.")
 
 
+# ---------------------------------------------------------------------------
+# Arm: progress_event  (UMMAYA query-loop painting — deterministic and safe)
+# ---------------------------------------------------------------------------
+
+
+class ProgressEventFrame(_BaseFrame):
+    """backend -> TUI: deterministic query-loop progress.
+
+    This is intentionally separate from ``AssistantChunkFrame.thinking``.
+    ``progress_event`` carries safe harness state such as analysis, tool
+    selection, tool dispatch/result, and answer synthesis. Provider reasoning
+    remains on the gated ``thinking_delta`` channel.
+    """
+
+    kind: Literal["progress_event"] = Field(
+        default="progress_event", description="Frame discriminator."
+    )
+    phase: Literal[
+        "analysis",
+        "tool_selection",
+        "tool_call",
+        "tool_result",
+        "answer_synthesis",
+    ] = Field(description="Safe query-loop phase represented by this event.")
+    message_ko: str = Field(min_length=1, description="Korean progress text for the TUI.")
+    message_en: str = Field(min_length=1, description="English fallback progress text.")
+    safe_to_persist: bool = Field(
+        default=True,
+        description="True because this channel never carries raw provider chain-of-thought.",
+    )
+    tool_id: str | None = Field(
+        default=None,
+        description="Concrete adapter/tool id when this event is tied to one.",
+    )
+    call_id: str | None = Field(
+        default=None,
+        description="Tool call id when this event is tied to a specific invocation.",
+    )
+
+
 # ---------------------------------------------------------------------------
 # Arm: tool_call  (Spec 287 baseline — arguments changed from Any to dict[str, object])
 # ---------------------------------------------------------------------------
@@ -1436,6 +1481,7 @@ IPCFrame = Annotated[
     UserInputFrame
     | ChatRequestFrame
     | AssistantChunkFrame
+    | ProgressEventFrame
     | ToolCallFrame
     | ToolResultFrame
     | CoordinatorPhaseFrame
@@ -1505,6 +1551,7 @@ __all__ = [
     "ToolDefinition",
     "ToolDefinitionFunction",
     "AssistantChunkFrame",
+    "ProgressEventFrame",
     "ToolCallFrame",
     "ToolResultFrame",
     "ToolResultEnvelope",