world-model-optimizer 0.2.0__py3-none-any.whl

wmo/evals/harbor/tasks.py

@@ -0,0 +1,140 @@
+"""Exact task selection for the harbor scorer.
+
+Harbor's own `DatasetConfig.task_names` filter uses fnmatch semantics, so a task id containing a
+glob character would silently over-match, and an optimizer's train/heldout split firewall relies
+on exact selection. This module resolves a dataset once, post-filters by exact id, downloads any
+remote (git/package) tasks ONCE, and returns pinned `TaskConfig`s that candidate jobs run
+directly (`tasks=[...]`, `overwrite=False`): per-candidate jobs must never re-clone or clobber
+the shared task cache that concurrent jobs are reading from.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Protocol
+
+from harbor.models.job.config import DatasetConfig
+from harbor.models.trial.config import TaskConfig
+from harbor.tasks.client import BatchDownloadResult, TaskClient, TaskIdType
+
+_GIT_COMMIT_PATTERN = re.compile(r"^[0-9a-fA-F]{40}$|^[0-9a-fA-F]{64}$")
+
+
+class HarborTaskDownloader(Protocol):
+    """The task-download slice of harbor's TaskClient (fakes replace it in tests)."""
+
+    async def download_tasks(
+        self,
+        task_ids: list[TaskIdType],
+        overwrite: bool = False,
+        output_dir: Path | None = None,
+    ) -> BatchDownloadResult: ...
+
+
+async def resolve_harbor_tasks(
+    dataset: DatasetConfig | Path,
+    task_ids: Sequence[str],
+    *,
+    task_client: HarborTaskDownloader | None = None,
+) -> list[TaskConfig]:
+    """Resolve `task_ids` from a harbor dataset (or local task dir) by exact identity.
+
+    Args:
+        dataset: A harbor `DatasetConfig`, or a local directory of task dirs (shorthand for
+            `DatasetConfig(path=...)`).
+        task_ids: Exact task names to select, in the order the caller wants them evaluated.
+        task_client: Download seam; defaults to harbor's `TaskClient`.
+
+    Returns:
+        One pinned `TaskConfig` per requested id, in request order. Git and package tasks are
+        downloaded here exactly once (git with `overwrite=True`, since harbor's cache does not
+        verify that an existing checkout still matches the requested commit) and returned with
+        the resolved commit pinned and `overwrite=False`, so every candidate job reuses the
+        resolved local bytes instead of re-cloning or clobbering the shared cache.
+
+    Raises:
+        ValueError: On empty/duplicate ids, a dataset that resolves duplicate task names, ids
+            the dataset does not contain, or a git download whose commit cannot be pinned.
+    """
+    requested = list(task_ids)
+    if not requested or any(not task_id for task_id in requested):
+        raise ValueError("task_ids must be nonempty strings")
+    if len(requested) != len(set(requested)):
+        raise ValueError("task_ids must be unique")
+
+    if isinstance(dataset, Path):
+        dataset = DatasetConfig(path=dataset)
+    # Resolve the dataset WITHOUT harbor's filters, then select by exact id ourselves:
+    # task_names is an fnmatch pattern list, not an exact-selection API.
+    resolved = DatasetConfig.model_validate(dataset.model_dump(mode="python"))
+    resolved.task_names = None
+    resolved.exclude_task_names = None
+    resolved.n_tasks = None
+    configs = await resolved.get_task_configs()
+
+    by_id: dict[str, TaskConfig] = {}
+    for config in configs:
+        task_id = config.get_task_id().get_name()
+        if task_id in by_id:
+            raise ValueError(f"harbor dataset resolved duplicate task {task_id!r}")
+        by_id[task_id] = config
+    missing = sorted(set(requested) - set(by_id))
+    if missing:
+        raise ValueError(
+            f"harbor task selection was not exact: missing={missing}; "
+            "check the ids against the dataset's task names"
+        )
+
+    selected = [by_id[task_id].model_copy(deep=True) for task_id in requested]
+    return await _pin_remote_tasks(
+        selected,
+        dataset=resolved,
+        task_client=task_client or TaskClient(),
+    )
+
+
+async def _pin_remote_tasks(
+    selected: list[TaskConfig],
+    *,
+    dataset: DatasetConfig,
+    task_client: HarborTaskDownloader,
+) -> list[TaskConfig]:
+    """Download remote tasks once and pin their provenance with `overwrite=False`."""
+    remote_indexes = [
+        index
+        for index, config in enumerate(selected)
+        if config.is_git_task() or config.is_package_task()
+    ]
+    if not remote_indexes:
+        return selected
+    downloads = await task_client.download_tasks(
+        [selected[index].get_task_id() for index in remote_indexes],
+        # Refresh git checkouts once, here: harbor's cache does not verify that an existing
+        # checkout still came from the requested commit.
+        overwrite=dataset.overwrite
+        or any(selected[index].is_git_task() for index in remote_indexes),
+        output_dir=dataset.download_dir,
+    )
+    if len(downloads.results) != len(remote_indexes):
+        raise ValueError("harbor returned an incomplete task download result")
+    for index, download in zip(remote_indexes, downloads.results, strict=True):
+        config = selected[index]
+        updates: dict[str, object] = {"overwrite": False}
+        if config.is_git_task():
+            commit = download.resolved_git_commit_id
+            if commit is None or _GIT_COMMIT_PATTERN.fullmatch(commit) is None:
+                raise ValueError(
+                    "harbor did not resolve a git commit for task "
+                    f"{config.get_task_id().get_name()!r}"
+                )
+            requested_commit = config.git_commit_id
+            if requested_commit is not None and requested_commit.lower() != commit.lower():
+                raise ValueError(
+                    "harbor resolved a different git commit for task "
+                    f"{config.get_task_id().get_name()!r}"
+                )
+            updates["git_commit_id"] = commit.lower()
+        selected[index] = config.model_copy(update=updates)
+    return selected

wmo/evals/open_loop.py

@@ -0,0 +1,194 @@
+"""Open-loop evaluation: reconstruction fidelity over trace files (the default `wmo eval` mode).
+
+`replay` (in `wmo.engine.replay`) scores one corpus of held-out steps teacher-forced. This
+orchestration layer is what `wmo eval` calls: it loads one or more OTel trace files, splits each
+into train/holdout, replays the holdout through a world-model prompt with leak-free RAG, and
+aggregates a per-file + overall scorecard. Its closed-loop counterpart
+(`wmo eval --mode closed-loop`, `wmo.evals.closed_loop`) runs a live agent instead of replaying;
+both implement the `Evaluation` interface in `wmo.evals.base`.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from statistics import fmean, pstdev
+
+from pydantic import BaseModel, Field
+
+from wmo.engine.build import split_traces, split_traces_3way
+from wmo.engine.knowledge import seeded_knowledge_text
+from wmo.engine.replay import ReplayReport, replay, valid_scores
+from wmo.ingest import get_adapter
+from wmo.optimize.judge import Judge
+from wmo.providers.base import Embedder, Provider
+from wmo.retrieval import EmbeddingRetriever
+
+
+class EvalReport(BaseModel):
+    """Per-file fidelity reports plus the step-weighted overall mean ± std.
+
+    `per_file` maps a trace file's clean name to its `ReplayReport` (per-step `StepResult`s), and
+    `overall_fidelity`/`overall_std` are the step-weighted aggregates across files.
+    """
+
+    per_file: dict[str, ReplayReport] = Field(default_factory=dict)
+    overall_fidelity: float = 0.0  # step-weighted mean of valid per-step scores across all files
+    overall_std: float = 0.0  # std of valid per-step scores across all files
+    total_steps: int = 0  # all steps attempted, including judge-invalid ones
+    total_invalid: int = 0  # judge failures across files; excluded from fidelity/std
+
+    @property
+    def headline(self) -> float:
+        """The `EvalResult` headline: per-step reconstruction fidelity."""
+        return self.overall_fidelity
+
+    @property
+    def total_valid(self) -> int:
+        """Steps that actually back the fidelity mean (judge-invalid ones excluded)."""
+        return self.total_steps - self.total_invalid
+
+    def summary(self) -> str:
+        invalid = f", {self.total_invalid} judge-invalid excluded" if self.total_invalid else ""
+        return (
+            f"fidelity={self.overall_fidelity:.3f}±{self.overall_std:.3f} "
+            f"({self.total_steps} steps, {len(self.per_file)} file(s){invalid})"
+        )
+
+
+def evaluate_files(
+    files: list[Path],
+    prompt: str,
+    provider: Provider,
+    judge: Judge,
+    *,
+    embedder: Embedder | None = None,
+    train_split: float = 0.7,
+    val_frac: float | None = None,
+    top_k: int = 5,
+    sample_turns: str = "all",
+    seed: int = 0,
+    adapter_name: str = "otel-genai",
+    max_holdout_traces: int | None = None,
+    knowledge: bool = False,
+    reasoning: bool = False,
+) -> EvalReport:
+    """Replay-score each trace file's held-out split. `embedder=None` -> zero-shot (no retrieval).
+
+    Each file is split deterministically; tiny corpora with no held-out trace fall back to scoring
+    every trace. RAG, when enabled, retrieves from that file's own train split only (leak-free).
+    `sample_turns`/`seed` are forwarded to `replay` (see its docstring). `max_holdout_traces` caps
+    how many held-out traces are scored per file (a deterministic prefix by trace_id) - for cheap
+    dry-runs; the train side stays full so retrieval is unaffected.
+
+    `val_frac` makes the split leak-free against a GEPA-evolved prompt: when it is a positive
+    fraction, the traces are cut 3-way (`train`/`val`/`test`) on the same hash line GEPA used, and
+    only the reserved `test` band is scored, so a prompt selected on `val` is never graded on those
+    same traces. Retrieval still draws from `train` only. `val_frac` of `None` or `0` keeps the
+    plain 2-way `train`/held-out split (`split_traces_3way` requires a strictly positive band).
+
+    `knowledge` seeds an ephemeral knowledge base from each file's TRAIN split (never the holdout -
+    the same leak-free discipline as RAG) and renders it into every prediction; `reasoning`
+    switches predictions to the deliberate-then-answer contract. Both mirror the serving engine's
+    agentic mode. Closed-loop evals get agentic mode from the ARTIFACT instead (the served
+    WorldModel's config / --max-fidelity winner), not from these flags.
+    """
+    adapter = get_adapter(adapter_name)
+    per_file: dict[str, ReplayReport] = {}
+    for path in files:
+        traces = adapter.from_file(str(path))
+        if not traces:
+            continue
+        if val_frac:  # truthy (a positive val band); None or 0.0 keeps the plain 2-way split
+            train, _val, holdout = split_traces_3way(traces, train_split, val_frac)
+        else:
+            train, holdout = split_traces(traces, train_split)
+        if not holdout:  # tiny corpus: evaluate on everything
+            train, holdout = traces, traces
+        if max_holdout_traces is not None:
+            holdout = sorted(holdout, key=lambda t: t.trace_id)[:max_holdout_traces]
+        retriever = EmbeddingRetriever(embedder) if embedder is not None else None
+        # Ephemeral, per-file, train-only KB: rendered text only — nothing under models/ is read
+        # or written, so eval can never leak a serve-time learned.md into scoring.
+        knowledge_text = seeded_knowledge_text(train, provider) if knowledge else None
+        name = _display_name(path)
+        per_file[name] = replay(
+            prompt,
+            holdout,
+            provider,
+            judge,
+            retriever=retriever,
+            train=train if embedder is not None else None,
+            top_k=top_k,
+            sample_turns=sample_turns,
+            seed=seed,
+            knowledge=knowledge_text,
+            reasoning=reasoning,
+        )
+
+    # Step-weighted aggregate over every validly-judged step across files (judge failures are
+    # counted in total_invalid, never as spurious zeros — see replay.valid_scores).
+    step_scores = valid_scores(r for rep in per_file.values() for r in rep.results)
+    overall = fmean(step_scores) if step_scores else 0.0
+    overall_std = pstdev(step_scores) if len(step_scores) > 1 else 0.0
+    return EvalReport(
+        per_file=per_file,
+        overall_fidelity=overall,
+        overall_std=overall_std,
+        total_steps=sum(rep.n_steps for rep in per_file.values()),
+        total_invalid=sum(rep.n_invalid for rep in per_file.values()),
+    )
+
+
+def _display_name(path: Path) -> str:
+    """Human label for a corpus, using the example folder name for `traces.otel.jsonl`."""
+    name = path.name.removesuffix(".jsonl").removesuffix(".otel")
+    return path.parent.name if name == "traces" else name
+
+
+class OpenLoopEval:
+    """The open-loop `Evaluation`: teacher-forced replay of held-out trace steps."""
+
+    def __init__(
+        self,
+        files: list[Path],
+        prompt: str,
+        provider: Provider,
+        judge: Judge,
+        *,
+        embedder: Embedder | None = None,
+        train_split: float = 0.7,
+        top_k: int = 5,
+        sample_turns: str = "all",
+        seed: int = 0,
+        adapter_name: str = "otel-genai",
+        knowledge: bool = False,
+        reasoning: bool = False,
+    ) -> None:
+        self._files = files
+        self._prompt = prompt
+        self._provider = provider
+        self._judge = judge
+        self._embedder = embedder
+        self._train_split = train_split
+        self._top_k = top_k
+        self._sample_turns = sample_turns
+        self._seed = seed
+        self._adapter_name = adapter_name
+        self._knowledge = knowledge
+        self._reasoning = reasoning
+
+    def run(self) -> EvalReport:
+        return evaluate_files(
+            self._files,
+            self._prompt,
+            self._provider,
+            self._judge,
+            embedder=self._embedder,
+            train_split=self._train_split,
+            top_k=self._top_k,
+            sample_turns=self._sample_turns,
+            seed=self._seed,
+            adapter_name=self._adapter_name,
+            knowledge=self._knowledge,
+            reasoning=self._reasoning,
+        )

wmo/evals/tasks.py

@@ -0,0 +1,53 @@
+"""Task specs for closed-loop evaluation: an instruction plus gold assertions that define success.
+
+Gold assertions are semantic post-conditions the `GoldJudge` checks against the run transcript —
+conditions on the final state, made robust to wording by an LLM judge instead of brittle exact
+matching. Tasks are typically derived from the same benchmark the world model's traces came from —
+`Trace.metadata` already carries gold assertions for traces captured with them.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel, Field, field_validator
+
+from wmo.core.text import normalize_durable_text
+
+
+class TaskSpec(BaseModel):
+    """One task: what the agent must do, and the assertions that must hold afterwards."""
+
+    task_id: str
+    instruction: str
+    gold: list[str] = Field(default_factory=list)  # assertions that define success
+
+    @field_validator("task_id", "instruction")
+    @classmethod
+    def _normalize_scalar_text(cls, value: str) -> str:
+        return normalize_durable_text(value)
+
+    @field_validator("gold")
+    @classmethod
+    def _normalize_gold(cls, value: list[str]) -> list[str]:
+        return [normalize_durable_text(assertion) for assertion in value]
+
+
+def load_tasks(path: str | Path) -> list[TaskSpec]:
+    """Read a JSONL task file (one TaskSpec per line; blank lines ignored).
+
+    Duplicate `task_id`s are an error: reports key outcomes by task_id, so a duplicate would run
+    (and cost) k passes twice while silently keeping only the last outcome.
+    """
+    tasks: list[TaskSpec] = []
+    for line in Path(path).read_text(encoding="utf-8").splitlines():
+        stripped = line.strip()
+        if stripped:
+            tasks.append(TaskSpec.model_validate_json(stripped))
+    if not tasks:
+        raise ValueError(f"no tasks in {path}")
+    ids = [t.task_id for t in tasks]
+    duplicates = sorted({i for i in ids if ids.count(i) > 1})
+    if duplicates:
+        raise ValueError(f"duplicate task_id(s) in {path}: {duplicates}")
+    return tasks

wmo/harness/__init__.py

@@ -0,0 +1,51 @@
+"""The agent harness: the scaffold a live agent runs with, and the machinery to improve it.
+
+A minimal, fixed agent loop (`AgentRuntime`) drives one action at a time against an
+`AgentEnvironment` — an interface, not a backend: closed-loop eval (`wmo.evals.closed_loop`) binds
+it to the world model, and a real execution backend can bind the same loop to reality. What the
+loop runs with is a `HarnessDoc` — a typed document of identity-keyed surfaces (prompt sections,
+tool policy, loop params, skills) stored as immutable versions with movable aliases
+(`wmo.harness.store`) and updated through audited `HarnessDelta`s (`wmo.harness.delta`,
+docs/reference/harness_delta.md) proposed by a meta-agent and gated on non-regression
+(`wmo.harness.create`, the `wmo optimize` search).
+
+`create` and `mutate` are imported directly (not re-exported here): they depend on
+`wmo.evals.closed_loop`, which itself binds to this package's runtime — re-exporting them would
+make `import wmo.evals` observe a partially initialized module.
+"""
+
+from wmo.harness.delta import (
+    FailureSignature,
+    GateRecord,
+    HarnessDelta,
+    SurfaceOp,
+    apply_delta,
+)
+from wmo.harness.doc import HarnessDoc, Surface, SurfaceKind
+from wmo.harness.environment import AgentEnvironment, is_env_action
+from wmo.harness.runtime import AgentRuntime, RunResult, StopReason
+from wmo.harness.skills import Skill, SkillLibrary
+from wmo.harness.store import HarnessStore
+from wmo.harness.tools import TOOL_REGISTRY, ToolCall, parse_tool_call
+
+__all__ = [
+    "TOOL_REGISTRY",
+    "AgentEnvironment",
+    "AgentRuntime",
+    "FailureSignature",
+    "GateRecord",
+    "HarnessDelta",
+    "HarnessDoc",
+    "HarnessStore",
+    "RunResult",
+    "Skill",
+    "SkillLibrary",
+    "StopReason",
+    "Surface",
+    "SurfaceKind",
+    "SurfaceOp",
+    "ToolCall",
+    "apply_delta",
+    "is_env_action",
+    "parse_tool_call",
+]