world-model-optimizer 0.2.0__py3-none-any.whl

wmo/scenarios/mining/facets.py

@@ -0,0 +1,226 @@
+"""Facet extraction: one compact, embeddable summary per trace (the Clio pattern).
+
+Raw traces are dominated by boilerplate (tool schemas, retrieved content), so embedding them
+directly washes out task intent — two traces with identical scaffolding but different tasks land
+nearly on top of each other. Instead, a cheap LLM reads a compact digest of each trace and emits a
+`TraceFacet`: a short task summary (what the user was trying to get done), the outcome, and a
+failure category when the episode failed. The deterministic tool-call signature and the corpus
+domain are computed in code, not by the LLM, and join the summary in the embedded text so
+clustering groups by capability rather than phrasing. Downstream clustering/selection operates on
+facet embeddings only.
+"""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from enum import StrEnum
+
+from pydantic import BaseModel, ValidationError
+
+from wmo.core.parsing import extract_json_object
+from wmo.core.types import ActionKind, Trace
+from wmo.providers.base import Message, Provider
+
+
+class Outcome(StrEnum):
+    SUCCESS = "success"
+    FAILURE = "failure"
+    UNKNOWN = "unknown"
+
+
+class TraceFacet(BaseModel):
+    """The embeddable summary of one trace; the unit clustering and selection operate on."""
+
+    trace_id: str
+    task_summary: str  # <= ~30 words: what the user was trying to get done
+    tool_signature: str  # deterministic "tool_a>tool_b>..." with consecutive repeats collapsed
+    domain: str | None = None  # from trace metadata when the corpus records one
+    outcome: Outcome = Outcome.UNKNOWN
+    failure_category: str | None = None  # short label, only when outcome == FAILURE
+
+    def embed_text(self) -> str:
+        """The text clustering embeds: domain + task intent + capabilities exercised.
+
+        Embedding the summary alone clusters by phrasing, which splits one capability into
+        several clusters ("MMS troubleshooting" vs "International MMS troubleshooting") and lets
+        cluster-level allocation double-count it. Domain and the tool signature pull traces that
+        exercise the same capability together regardless of how the request was worded.
+        """
+        parts = []
+        if self.domain:
+            parts.append(f"[{self.domain}]")
+        parts.append(self.task_summary)
+        text = " ".join(parts)
+        if self.tool_signature:
+            text = f"{text} | tools: {self.tool_signature}"
+        return text
+
+
+def trace_domain(trace: Trace) -> str | None:
+    """The trace's domain from corpus metadata, when recorded (e.g. tau2's telecom/retail)."""
+    value = trace.metadata.get("domain")
+    if isinstance(value, str) and value.strip():
+        return value.strip()
+    return None
+
+
+def tool_signature(trace: Trace) -> str:
+    """Deterministic tool-call sequence signature with consecutive repeats collapsed.
+
+    `search>search>book` becomes `search>book`: the signature captures *which* capabilities the
+    episode exercised in what order, not how many retries each took.
+    """
+    names: list[str] = []
+    for step in trace.steps:
+        action = step.action
+        if action.kind is not ActionKind.TOOL_CALL or not action.name:
+            continue
+        if not names or names[-1] != action.name:
+            names.append(action.name)
+    return ">".join(names)
+
+
+_MAX_DIGEST_STEPS = 30
+_MAX_FIELD_CHARS = 300
+
+
+def trace_digest(trace: Trace, *, max_steps: int = _MAX_DIGEST_STEPS) -> str:
+    """A compact plain-text rendering of a trace for the facet-extraction LLM.
+
+    Includes the task, then one line per step (tool name + truncated arguments + truncated
+    observation, error-flagged). Long traces keep the first and last steps and elide the middle:
+    intent lives at the start, resolution at the end.
+    """
+    lines: list[str] = []
+    task = _trace_task(trace)
+    if task:
+        lines.append(f"TASK: {_truncate(task)}")
+    steps = trace.steps
+    if len(steps) > max_steps:
+        head = max_steps // 2
+        tail = max_steps - head
+        shown = list(enumerate(steps))[:head] + list(enumerate(steps))[-tail:]
+        elided = len(steps) - max_steps
+    else:
+        shown = list(enumerate(steps))
+        elided = 0
+    previous_index = -1
+    for index, step in shown:
+        if index > previous_index + 1:
+            lines.append(f"... ({elided} steps elided) ...")
+        previous_index = index
+        action = step.action
+        if action.kind is ActionKind.TOOL_CALL:
+            args = _truncate(str(action.arguments)) if action.arguments else ""
+            head_line = f"{index}. CALL {action.name}({args})"
+        else:
+            head_line = f"{index}. MSG {_truncate(action.content or '')}"
+        observation = step.observation
+        error_mark = " [ERROR]" if observation.is_error else ""
+        lines.append(f"{head_line} -> {_truncate(observation.content)}{error_mark}")
+    return "\n".join(lines)
+
+
+def _truncate(text: str, limit: int = _MAX_FIELD_CHARS) -> str:
+    text = " ".join(text.split())
+    return text if len(text) <= limit else text[: limit - 1] + "…"
+
+
+def _trace_task(trace: Trace) -> str | None:
+    for step in trace.steps:
+        if step.task and step.task.strip():
+            return step.task.strip()
+    return None
+
+
+FACET_SYSTEM = """You summarize one AI-agent episode (a trace of tool calls and messages) into a
+compact facet used to organize a large trace corpus.
+
+Respond with ONLY a JSON object, no prose around it:
+{"task_summary": "<what the USER was trying to get done, <=30 words, self-contained, no ids>",
+ "outcome": "success" | "failure" | "unknown",
+ "failure_category": "<short snake_case label, e.g. wrong_tool_arguments; null unless failure>"}
+
+Rules:
+- task_summary states the user's goal, not the agent's mechanics ("cancel a flight booking and get
+  a refund", NOT "called cancel_reservation").
+- outcome is "success" only if the episode visibly achieved the goal; "failure" if it visibly did
+  not (errors, refusals, wrong result); otherwise "unknown".
+- failure_category names the dominant failure mode in 1-3 words; null when outcome != "failure"."""
+
+
+class _RawFacet(BaseModel):
+    """Lenient view of the extractor's JSON before normalization."""
+
+    task_summary: str
+    outcome: str = "unknown"
+    failure_category: str | None = None
+
+
+class FacetExtractor:
+    """LLM facet extraction over a trace corpus (one completion per trace)."""
+
+    def __init__(self, provider: Provider) -> None:
+        self._provider = provider
+
+    def extract(self, trace: Trace) -> TraceFacet:
+        """Extract the facet for one trace; falls back to the raw task on an unparseable reply."""
+        completion = self._provider.complete(
+            FACET_SYSTEM,
+            [Message(role="user", content=trace_digest(trace))],
+            temperature=0.0,
+            max_tokens=512,
+        )
+        signature = tool_signature(trace)
+        domain = trace_domain(trace)
+        raw = extract_json_object(completion.text)
+        if raw is not None:
+            try:
+                parsed = _RawFacet.model_validate_json(raw)
+            except ValidationError:
+                parsed = None
+            if parsed is not None and parsed.task_summary.strip():
+                outcome = _parse_outcome(parsed.outcome)
+                category = parsed.failure_category if outcome is Outcome.FAILURE else None
+                return TraceFacet(
+                    trace_id=trace.trace_id,
+                    task_summary=parsed.task_summary.strip(),
+                    tool_signature=signature,
+                    domain=domain,
+                    outcome=outcome,
+                    failure_category=_normalize_category(category),
+                )
+        # Fallback: the recorded task prompt is still a usable intent summary; flag as UNKNOWN.
+        return TraceFacet(
+            trace_id=trace.trace_id,
+            task_summary=_truncate(_trace_task(trace) or "(no task recorded)", 200),
+            tool_signature=signature,
+            domain=domain,
+            outcome=Outcome.UNKNOWN,
+        )
+
+    def extract_all(self, traces: list[Trace], *, concurrency: int = 8) -> list[TraceFacet]:
+        """Extract facets for every trace, in order.
+
+        Each facet is one independent LLM call, so they run on a small thread pool
+        (`pool.map` preserves input order and propagates exceptions — the replay.py
+        precedent); `concurrency=1` keeps the sequential loop.
+        """
+        if concurrency > 1 and len(traces) > 1:
+            with ThreadPoolExecutor(max_workers=min(concurrency, len(traces))) as pool:
+                return list(pool.map(self.extract, traces))
+        return [self.extract(trace) for trace in traces]
+
+
+def _parse_outcome(raw: str) -> Outcome:
+    try:
+        return Outcome(raw.strip().lower())
+    except ValueError:
+        return Outcome.UNKNOWN
+
+
+def _normalize_category(category: str | None) -> str | None:
+    if category is None:
+        return None
+    normalized = "_".join(category.strip().lower().split())
+    return normalized or None

wmo/scenarios/mining/selection.py

@@ -0,0 +1,220 @@
+"""Representative selection: SemDeDup + hybrid-allocation medoid picking with failure pinning.
+
+Given clustered facet embeddings and a scenario budget K, pick which real traces become scenarios:
+
+1. SemDeDup (arXiv 2303.09540): drop near-duplicate facets within a cluster before selection, so
+   thirty rewordings of the same request can't claim thirty slots.
+2. Hybrid allocation: ~70% of the budget goes to clusters proportionally to their corpus mass (the
+   eval mirrors traffic), the rest round-robin across clusters (the long tail keeps coverage).
+3. Within a cluster the first pick is the medoid (the real trace nearest everything else); extra
+   slots go farthest-first for intra-cluster diversity.
+4. Failure pinning: every failure category present in the corpus keeps at least one exemplar,
+   regardless of frequency — rare-but-critical traces are exactly the ones proportional sampling
+   silently drops.
+
+Each selection carries `weight`: the fraction of the (deduped) corpus it stands for, so downstream
+scoring can report a traffic-weighted number. Weights sum to 1 over the selection.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from pydantic import BaseModel
+
+from wmo.scenarios.mining.clustering import normalize_rows
+from wmo.scenarios.mining.facets import Outcome, TraceFacet
+
+DEDUP_THRESHOLD = 0.95
+PROPORTIONAL_FRACTION = 0.7
+
+
+class SelectedTrace(BaseModel):
+    """One trace chosen to become a scenario, with the corpus mass it represents."""
+
+    trace_id: str
+    cluster_id: int
+    weight: float  # fraction of the deduped corpus this selection stands for
+    pinned_failure: str | None = None  # failure category this pick was retained for, if any
+
+
+def semdedup_keep(
+    embeddings: np.ndarray, labels: np.ndarray, *, threshold: float = DEDUP_THRESHOLD
+) -> list[int]:
+    """Indices that survive within-cluster near-duplicate removal (first occurrence wins).
+
+    Compares cosine similarity only within a cluster (the SemDeDup trick: k-means already grouped
+    near-duplicates, so the quadratic pass stays per-cluster).
+    """
+    matrix = normalize_rows(embeddings)
+    kept: list[int] = []
+    for cluster_id in sorted(set(labels.tolist())):
+        member_indices = np.flatnonzero(labels == cluster_id)
+        cluster_kept: list[int] = []
+        for index in member_indices.tolist():
+            duplicate = any(
+                float(matrix[index] @ matrix[other]) > threshold for other in cluster_kept
+            )
+            if not duplicate:
+                cluster_kept.append(index)
+        kept.extend(cluster_kept)
+    return sorted(kept)
+
+
+def hybrid_select(
+    facets: list[TraceFacet],
+    embeddings: np.ndarray,
+    labels: np.ndarray,
+    budget: int,
+    *,
+    proportional_fraction: float = PROPORTIONAL_FRACTION,
+    dedup_threshold: float = DEDUP_THRESHOLD,
+) -> list[SelectedTrace]:
+    """Pick `budget` representative traces from a clustered facet corpus.
+
+    See the module docstring for the algorithm. Cluster mass (and thus weights) is measured on the
+    deduped corpus. Raises when the budget is not positive; a budget larger than the deduped corpus
+    returns everything.
+    """
+    if budget < 1:
+        raise ValueError(f"budget must be >= 1, got {budget}")
+    if len(facets) != len(embeddings) or len(facets) != len(labels):
+        raise ValueError("facets, embeddings, and labels must be parallel")
+    if not facets:
+        return []
+
+    matrix = normalize_rows(embeddings)
+    kept = semdedup_keep(embeddings, labels, threshold=dedup_threshold)
+    by_cluster: dict[int, list[int]] = {}
+    for index in kept:
+        by_cluster.setdefault(int(labels[index]), []).append(index)
+    total_kept = len(kept)
+    if budget >= total_kept:
+        selections = [_selection(facets[i], int(labels[i]), 1.0 / total_kept) for i in sorted(kept)]
+        return _pin_failures(selections, facets, matrix, labels, by_cluster)
+
+    slots = _allocate_slots(by_cluster, budget, proportional_fraction)
+    selections: list[SelectedTrace] = []
+    for cluster_id, cluster_slots in slots.items():
+        member_indices = by_cluster[cluster_id]
+        chosen = _pick_representatives(matrix, member_indices, cluster_slots)
+        cluster_weight = len(member_indices) / total_kept
+        for index in chosen:
+            selections.append(_selection(facets[index], cluster_id, cluster_weight / len(chosen)))
+    selections = _pin_failures(selections, facets, matrix, labels, by_cluster)
+    # Clusters allocated zero slots contribute no selection, so their mass would silently vanish
+    # from the weights; renormalize so weights always sum to 1 over the returned selection.
+    total_weight = sum(s.weight for s in selections)
+    return [s.model_copy(update={"weight": s.weight / total_weight}) for s in selections]
+
+
+def _selection(facet: TraceFacet, cluster_id: int, weight: float) -> SelectedTrace:
+    return SelectedTrace(trace_id=facet.trace_id, cluster_id=cluster_id, weight=weight)
+
+
+def _allocate_slots(
+    by_cluster: dict[int, list[int]], budget: int, proportional_fraction: float
+) -> dict[int, int]:
+    """Split `budget` slots across clusters: proportional share + round-robin coverage share.
+
+    Proportional slots follow cluster mass (largest-remainder rounding); the remaining coverage
+    slots go one per cluster in descending-mass order, cycling. No cluster is allocated more slots
+    than it has (deduped) members; leftover slots spill to the largest clusters with capacity.
+    """
+    cluster_ids = sorted(by_cluster, key=lambda c: len(by_cluster[c]), reverse=True)
+    capacity = {c: len(by_cluster[c]) for c in cluster_ids}
+    total = sum(capacity.values())
+    proportional_budget = min(budget, round(budget * proportional_fraction))
+
+    # Largest-remainder proportional allocation, capped by capacity.
+    quotas = {c: proportional_budget * capacity[c] / total for c in cluster_ids}
+    slots = {c: min(int(quotas[c]), capacity[c]) for c in cluster_ids}
+    remainders = sorted(cluster_ids, key=lambda c: quotas[c] - int(quotas[c]), reverse=True)
+    leftover = proportional_budget - sum(slots.values())
+    for cluster_id in remainders:
+        if leftover <= 0:
+            break
+        if slots[cluster_id] < capacity[cluster_id]:
+            slots[cluster_id] += 1
+            leftover -= 1
+
+    # Coverage slots: uncovered clusters first (the long tail is the whole point of this share),
+    # then cycle clusters by descending mass, one slot each, skipping full clusters.
+    remaining = budget - sum(slots.values())
+    for cluster_id in cluster_ids:
+        if remaining <= 0:
+            break
+        if slots[cluster_id] == 0 and capacity[cluster_id] > 0:
+            slots[cluster_id] = 1
+            remaining -= 1
+    while remaining > 0:
+        progressed = False
+        for cluster_id in cluster_ids:
+            if remaining <= 0:
+                break
+            if slots[cluster_id] < capacity[cluster_id]:
+                slots[cluster_id] += 1
+                remaining -= 1
+                progressed = True
+        if not progressed:  # every cluster saturated; budget > corpus, handled by caller
+            break
+    return {c: s for c, s in slots.items() if s > 0}
+
+
+def _pick_representatives(matrix: np.ndarray, member_indices: list[int], slots: int) -> list[int]:
+    """Medoid first, then farthest-first: real, central exemplars with intra-cluster diversity."""
+    members = np.asarray(member_indices)
+    if slots >= len(members):
+        return members.tolist()
+    similarities = matrix[members] @ matrix[members].T
+    chosen: list[int] = [int(members[similarities.mean(axis=1).argmax()])]  # the medoid
+    while len(chosen) < slots:
+        chosen_rows = matrix[np.asarray(chosen)]
+        best_similarity = (matrix[members] @ chosen_rows.T).max(axis=1)
+        best_similarity[np.isin(members, chosen)] = np.inf  # never re-pick
+        chosen.append(int(members[best_similarity.argmin()]))
+    return chosen
+
+
+def _pin_failures(
+    selections: list[SelectedTrace],
+    facets: list[TraceFacet],
+    matrix: np.ndarray,
+    labels: np.ndarray,
+    by_cluster: dict[int, list[int]],
+) -> list[SelectedTrace]:
+    """Ensure every failure category in the (deduped) corpus keeps at least one exemplar.
+
+    A missing category's medoid replaces the currently lowest-weight unpinned selection, so the
+    budget holds. The replaced selection's weight transfers, keeping weights summing to 1.
+    """
+    facet_by_id = {facet.trace_id: facet for facet in facets}
+    kept_indices = [i for members in by_cluster.values() for i in members]
+    categories: dict[str, list[int]] = {}
+    for index in kept_indices:
+        facet = facets[index]
+        if facet.outcome is Outcome.FAILURE and facet.failure_category:
+            categories.setdefault(facet.failure_category, []).append(index)
+
+    covered = {
+        facet_by_id[s.trace_id].failure_category
+        for s in selections
+        if facet_by_id[s.trace_id].outcome is Outcome.FAILURE
+    }
+    for category, member_indices in sorted(categories.items()):
+        if category in covered:
+            continue
+        members = np.asarray(member_indices)
+        similarities = matrix[members] @ matrix[members].T
+        exemplar = int(members[similarities.mean(axis=1).argmax()])
+        replaceable = [s for s in selections if s.pinned_failure is None]
+        if not replaceable:
+            break
+        victim = min(replaceable, key=lambda s: s.weight)
+        selections[selections.index(victim)] = SelectedTrace(
+            trace_id=facets[exemplar].trace_id,
+            cluster_id=int(labels[exemplar]),
+            weight=victim.weight,
+            pinned_failure=category,
+        )
+        covered.add(category)
+    return selections

wmo/scenarios/synthesis/__init__.py

@@ -0,0 +1,6 @@
+"""Synthesis: write self-contained, judgeable scenarios from selected traces."""
+
+from wmo.scenarios.synthesis.scenario_set import EvalScenario, ScenarioSet
+from wmo.scenarios.synthesis.synthesizer import ScenarioSynthesizer
+
+__all__ = ["EvalScenario", "ScenarioSet", "ScenarioSynthesizer"]

wmo/scenarios/synthesis/scenario_set.py

@@ -0,0 +1,63 @@
+"""The scenario data model: `EvalScenario` records and the versioned `ScenarioSet` artifact."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from wmo.core.types import EnvState
+from wmo.env.scenarios import Scenario
+from wmo.scenarios.mining.clustering import TraceCluster
+from wmo.scenarios.mining.facets import Outcome
+
+
+class EvalScenario(BaseModel):
+    """One reusable eval scenario distilled from a real trace."""
+
+    scenario_id: str
+    task: str  # self-contained task statement handed to the agent
+    seed_state: EnvState = Field(default_factory=EnvState)  # initial env state for the world model
+    checklist: list[str] = Field(default_factory=list)  # judgeable success criteria
+    provenance: list[str] = Field(default_factory=list)  # source trace_ids
+    cluster_name: str = ""
+    weight: float = 0.0  # fraction of the corpus this scenario represents
+    source_outcome: Outcome = Outcome.UNKNOWN
+    failure_category: str | None = None
+
+    def to_scenario(self) -> Scenario:
+        """The minimal `Scenario` view consumed by existing rollout code."""
+        return Scenario(task=self.task, provenance=list(self.provenance))
+
+
+class ScenarioSet(BaseModel):
+    """The constructed scenario set plus the corpus statistics that justify it."""
+
+    scenarios: list[EvalScenario]
+    clusters: list[TraceCluster] = Field(default_factory=list)
+    corpus_traces: int = 0
+    corpus_coverage: float = 0.0  # fraction of corpus facets within tau of a selected facet
+    coverage_tau: float = 0.0
+
+    def retain(self, scenario_ids: set[str]) -> None:
+        """Keep only `scenario_ids`, renormalizing weights and invalidating coverage.
+
+        Dropping scenarios (e.g. `wmo scenarios verify --drop`) breaks two invariants the artifact
+        promises: weights sum to 1 over the set, and `corpus_coverage` describes the current
+        scenarios. Weights are renormalized over the survivors; coverage needs the facet
+        embeddings (gone by verify time), so it is zeroed rather than left stale.
+        """
+        self.scenarios = [s for s in self.scenarios if s.scenario_id in scenario_ids]
+        total_weight = sum(s.weight for s in self.scenarios)
+        if total_weight > 0:
+            for scenario in self.scenarios:
+                scenario.weight /= total_weight
+        self.corpus_coverage = 0.0
+        self.coverage_tau = 0.0
+
+    def save(self, path: str | Path) -> None:
+        Path(path).write_text(self.model_dump_json(indent=2), encoding="utf-8")
+
+    @classmethod
+    def load(cls, path: str | Path) -> ScenarioSet:
+        return cls.model_validate_json(Path(path).read_text(encoding="utf-8"))

wmo/scenarios/synthesis/synthesizer.py

@@ -0,0 +1,85 @@
+"""Scenario synthesis: turn a selected trace into a self-contained, judgeable eval scenario.
+
+The WildBench pattern: an LLM reads the source trace and writes (1) a self-contained task
+statement (the user's goal plus constraints revealed mid-episode), (2) the minimal initial
+environment state the episode needs (seeds the world model's scratchpad), and (3) a short
+checklist of success criteria a judge can grade a new trajectory against. Every scenario keeps
+provenance to its source trace so it stays auditable.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, ValidationError
+
+from wmo.core.parsing import extract_json_object
+from wmo.core.types import EnvState, Trace
+from wmo.providers.base import Message, Provider
+from wmo.scenarios.mining.facets import TraceFacet, trace_digest
+from wmo.scenarios.synthesis.scenario_set import EvalScenario
+
+SYNTHESIS_SYSTEM = """You convert one recorded AI-agent episode into a reusable evaluation
+scenario. You see a digest of the episode (task, tool calls, observations).
+
+Respond with ONLY a JSON object, no prose around it:
+{"task": "<self-contained task statement for a fresh agent: the user's goal plus any constraints
+revealed during the episode; no references to 'the trace' or 'above'>",
+ "initial_state": "<2-6 sentences of environment facts the episode started from (accounts,
+records, files, balances) that a simulator needs to answer the agent consistently>",
+ "checklist": ["<3-6 concrete, independently checkable success criteria for a NEW attempt>"]}
+
+Rules:
+- The task must be attemptable without seeing the original episode.
+- initial_state states facts about the world, not about the agent's behavior.
+- Checklist items grade the OUTCOME (what ended up true / communicated), not the exact tool
+  sequence — a different valid strategy must be able to pass."""
+
+
+class _RawSynthesis(BaseModel):
+    task: str
+    initial_state: str = ""
+    checklist: list[str] = Field(default_factory=list)
+
+
+class ScenarioSynthesizer:
+    """LLM synthesis of one `EvalScenario` per selected trace."""
+
+    def __init__(self, provider: Provider) -> None:
+        self._provider = provider
+
+    def synthesize(self, trace: Trace, facet: TraceFacet) -> EvalScenario:
+        """Synthesize the scenario for one selected trace.
+
+        On an unparseable reply, falls back to the facet's task summary with an empty checklist —
+        the scenario stays usable for rollouts, and verification will flag it (no checklist means
+        nothing to grade against).
+        """
+        completion = self._provider.complete(
+            SYNTHESIS_SYSTEM,
+            [Message(role="user", content=trace_digest(trace))],
+            temperature=0.0,
+            max_tokens=1024,
+        )
+        raw = extract_json_object(completion.text)
+        parsed: _RawSynthesis | None = None
+        if raw is not None:
+            try:
+                parsed = _RawSynthesis.model_validate_json(raw)
+            except ValidationError:
+                parsed = None
+        if parsed is not None and parsed.task.strip():
+            task = parsed.task.strip()
+            seed_state = EnvState(scratchpad=parsed.initial_state.strip())
+            checklist = [item.strip() for item in parsed.checklist if item.strip()]
+        else:
+            task = facet.task_summary
+            seed_state = EnvState()
+            checklist = []
+        return EvalScenario(
+            scenario_id=f"scenario-{trace.trace_id}",
+            task=task,
+            seed_state=seed_state,
+            checklist=checklist,
+            provenance=[trace.trace_id],
+            source_outcome=facet.outcome,
+            failure_category=facet.failure_category,
+        )

wmo/scenarios/verification/__init__.py

@@ -0,0 +1,17 @@
+"""Verification: back-agreement + solvability gates and the checklist judge that powers them."""
+
+from wmo.scenarios.verification.judge import CHECKLIST_SYSTEM, ChecklistJudge, ChecklistResult
+from wmo.scenarios.verification.verify import (
+    ScenarioVerdict,
+    VerificationReport,
+    verify_scenarios,
+)
+
+__all__ = [
+    "CHECKLIST_SYSTEM",
+    "ChecklistJudge",
+    "ChecklistResult",
+    "ScenarioVerdict",
+    "VerificationReport",
+    "verify_scenarios",
+]