agent-exam 0.1.0__py3-none-any.whl

agent_exam/_models.py

@@ -0,0 +1,120 @@
+"""Pydantic foundations for assertion config models.
+
+Each assertion declares a `<Foo>Config` pydantic model; `_parse_assertion`
+constructs the model from raw YAML and surfaces a clean UsageError if
+the config is malformed. `check` then receives the typed config directly
+— no preamble, no re-extraction, types flow.
+
+Two shared base classes cover what most assertions need:
+
+- `_StrictModel`: rejects unknown keys (`extra="forbid"`) — the anti-typo
+  default for every config in the codebase.
+- `_ScalarShorthandModel`: enables `<assertion>: <scalar>` shorthand
+  alongside the full `<assertion>: {<key>: <scalar>}` form. Subclasses
+  set `_shorthand_key: ClassVar[str]` to declare which field the scalar
+  populates.
+
+`render_validation_error` turns a pydantic `ValidationError` into a flat
+CLI message — one line per error, no `pydantic.dev` URL, with the
+"Value error," wrapping that pydantic adds to validator-raised
+`ValueError`s stripped. Used by the registry to wrap pydantic errors as
+`UsageError` so they read like every other config error in the system.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Annotated, Any, ClassVar
+
+from pydantic import (
+    BaseModel,
+    BeforeValidator,
+    ConfigDict,
+    Field,
+    ValidationError,
+    model_validator,
+)
+
+# Strips pydantic's `... or instance of <ModelClass>` suffix from
+# type-mismatch messages — internal class names mean nothing to a YAML
+# author and just make errors look noisy.
+_MODEL_CLASS_SUFFIX = re.compile(r" or instance of _?[A-Z]\w+")
+
+
+class _StrictModel(BaseModel):
+    """Reject unknown keys — every config model in the codebase wants this."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class _ScalarShorthandModel(_StrictModel):
+    """Enable the `<assertion>: <scalar>` shorthand form.
+
+    Subclasses set `_shorthand_key: ClassVar[str]` to declare which
+    field the scalar value populates. Any non-dict input is treated as
+    the shorthand value — `model_validate(x)` becomes
+    `model_validate({<key>: x})`. The full mapping form is unchanged.
+
+    The wrap is type-agnostic: if `x` is the wrong type for the field,
+    pydantic surfaces a per-field message (e.g. `path: Input should be a
+    valid string`) instead of the misleading top-level
+    `Input should be a valid dictionary`.
+    """
+
+    _shorthand_key: ClassVar[str]
+
+    @model_validator(mode="before")
+    @classmethod
+    def _apply_shorthand(cls, v: Any) -> Any:
+        if isinstance(v, dict):
+            return v
+        return {cls._shorthand_key: v}
+
+
+def _reject_bool_and_str(v: Any) -> Any:
+    if isinstance(v, bool):
+        raise ValueError("must be a number, not a bool")
+    if isinstance(v, str):
+        raise ValueError("must be a number, not a string")
+    return v
+
+
+# A numeric field that accepts int OR float and rejects bool (otherwise
+# an int subclass) and strings (otherwise lax-coerced). Single-error
+# message — avoids pydantic's two-branch verbose `StrictInt | StrictFloat`
+# union error.
+Number = Annotated[int | float, BeforeValidator(_reject_bool_and_str)]
+
+# Number plus a positivity constraint (`> 0`). Type-mismatch errors come
+# from the BeforeValidator; sign errors are pydantic's
+# "Input should be greater than 0".
+PositiveNumber = Annotated[
+    int | float, BeforeValidator(_reject_bool_and_str), Field(gt=0)
+]
+
+
+def render_validation_error(label: str, exc: ValidationError) -> str:
+    """Convert a pydantic `ValidationError` into a flat CLI message.
+
+    Single error: `<label>: [<loc>: ]<msg>` on one line.
+    Multiple errors: `<label>: invalid config (N errors)` header, then
+    `<loc>: <msg>` per error.
+
+    `include_url=False` drops pydantic's "see docs at ..." appendage,
+    and the "Value error, " prefix pydantic adds around validator-raised
+    `ValueError` messages is stripped — both are noise for CLI use.
+    """
+    errors = exc.errors(include_url=False)
+    if len(errors) == 1:
+        err = errors[0]
+        loc = ".".join(str(x) for x in err["loc"])
+        msg = _MODEL_CLASS_SUFFIX.sub("", err["msg"].removeprefix("Value error, "))
+        if loc:
+            return f"{label}: {loc}: {msg}"
+        return f"{label}: {msg}"
+    lines = [f"{label}: invalid config ({len(errors)} errors)"]
+    for err in errors:
+        loc = ".".join(str(x) for x in err["loc"]) or "<root>"
+        msg = _MODEL_CLASS_SUFFIX.sub("", err["msg"].removeprefix("Value error, "))
+        lines.append(f"  {loc}: {msg}")
+    return "\n".join(lines)

agent_exam/_validate.py

@@ -0,0 +1,117 @@
+"""Small shared validation primitives.
+
+These collapse the repetitive shape/type checks that recur across the
+task loader (`tasks.py`) and the assertion validators (`assertions/*`).
+Each raises `UsageError` with a uniform message so callers don't
+hand-roll — and drift — their own.
+
+Deliberately not a schema DSL: the genuinely per-field logic (cross-field
+rules like `tool_count`'s min/max, shorthand forms, the str-or-dict judge
+config) stays as plain Python in the caller. Only the boilerplate lives
+here.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from .errors import UsageError
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+
+def reject_unknown_keys(
+    items: Iterable[Any],
+    allowed: Iterable[str],
+    *,
+    label: str,
+    noun: str = "key",
+) -> None:
+    """Raise UsageError if `items` (a mapping — iterated as its keys — or
+    any other iterable) contains an entry outside `allowed`. Catches
+    typos that would otherwise be silently ignored.
+    """
+    allowed_set = set(allowed)
+    unknown = sorted(str(k) for k in items if k not in allowed_set)
+    if unknown:
+        raise UsageError(
+            f"{label}: unknown {noun}(s) {unknown}; "
+            f"valid: {', '.join(sorted(allowed_set))}"
+        )
+
+
+def require_str(
+    value: Any,
+    label: str,
+    *,
+    allow_none: bool = False,
+    non_empty: bool = False,
+) -> None:
+    """Raise UsageError unless `value` is a string — optionally non-empty
+    (after strip), optionally None."""
+    if value is None and allow_none:
+        return
+    if not isinstance(value, str) or (non_empty and not value.strip()):
+        kind = "a non-empty string" if non_empty else "a string"
+        raise UsageError(f"{label} must be {kind} (got {value!r})")
+
+
+def require_str_list(
+    value: Any,
+    label: str,
+    *,
+    allow_none: bool = False,
+    non_empty: bool = False,
+) -> None:
+    """Raise UsageError unless `value` is a list of non-empty strings —
+    optionally itself non-empty, optionally None."""
+    if value is None and allow_none:
+        return
+    if (
+        not isinstance(value, list)
+        or (non_empty and len(value) == 0)
+        or not all(isinstance(x, str) and x for x in value)
+    ):
+        kind = "a non-empty list of strings" if non_empty else "a list of strings"
+        raise UsageError(f"{label} must be {kind} (got {value!r})")
+
+
+def require_int(
+    value: Any,
+    label: str,
+    *,
+    allow_none: bool = False,
+    minimum: int | None = None,
+) -> None:
+    """Raise UsageError unless `value` is an integer — optionally >=
+    `minimum`, optionally None. `bool` is rejected: it's an `int`
+    subclass, but `timeout_seconds: true` is a typo, not a 1."""
+    if value is None and allow_none:
+        return
+    if (
+        not isinstance(value, int)
+        or isinstance(value, bool)
+        or (minimum is not None and value < minimum)
+    ):
+        bound = f" >= {minimum}" if minimum is not None else ""
+        raise UsageError(f"{label} must be an integer{bound} (got {value!r})")
+
+
+def require_number(
+    value: Any,
+    label: str,
+    *,
+    allow_none: bool = False,
+    positive: bool = False,
+) -> None:
+    """Raise UsageError unless `value` is a number (int or float —
+    `bool` rejected), optionally strictly positive, optionally None.
+    Use this for fields where sub-integer precision is meaningful
+    (e.g. `timeout_seconds: 0.5`); use `require_int` for counts."""
+    if value is None and allow_none:
+        return
+    if not isinstance(value, (int, float)) or isinstance(value, bool):
+        raise UsageError(f"{label} must be a number (got {value!r})")
+    if positive and value <= 0:
+        raise UsageError(f"{label} must be a positive number (got {value!r})")

agent_exam/artifacts.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class RunPaths:
+    """All on-disk paths for a single run, derived from evals_dir + run_id."""
+
+    evals_dir: Path
+    run_id: str
+
+    @property
+    def run_dir(self) -> Path:
+        return self.evals_dir / "runs" / self.run_id
+
+    @property
+    def run_json(self) -> Path:
+        return self.run_dir / "run.json"
+
+    @property
+    def artifacts_dir(self) -> Path:
+        return self.run_dir / "artifacts"
+
+    @property
+    def reports_dir(self) -> Path:
+        return self.run_dir / "reports"
+
+    @property
+    def judge_cache(self) -> Path:
+        return self.run_dir / "judge-cache.json"
+
+    def attempt_dir(self, suite: str, task: str, attempt_n: int) -> Path:
+        return self.artifacts_dir / suite / task / f"attempt-{attempt_n}"
+
+    def attempt_json(self, suite: str, task: str, attempt_n: int) -> Path:
+        return self.attempt_dir(suite, task, attempt_n) / "attempt.json"
+
+    def trajectory_json(self, suite: str, task: str, attempt_n: int) -> Path:
+        return self.attempt_dir(suite, task, attempt_n) / "trajectory.json"
+
+    def attempt_cwd(self, suite: str, task: str, attempt_n: int) -> Path:
+        return self.attempt_dir(suite, task, attempt_n) / "cwd"
+
+    def report_file(self, report_ts: str) -> Path:
+        return self.reports_dir / f"{report_ts}.json"

agent_exam/assertions/_shared.py

@@ -0,0 +1,34 @@
+"""Config models shared across multiple assertions."""
+
+from __future__ import annotations
+
+from typing import ClassVar
+
+from pydantic import Field, StrictBool, field_validator
+
+from .._models import _ScalarShorthandModel
+
+
+class JudgeConfigBase(_ScalarShorthandModel):
+    """Shared config for `judge` and `judge_agent`: a criterion plus
+    optional `include_trajectory` / `pass_on`. Accepts the scalar
+    shorthand `judge: <criterion>` and the full mapping form."""
+
+    _shorthand_key: ClassVar[str] = "criterion"
+    criterion: str
+    include_trajectory: StrictBool = True
+    pass_on: list[str] | None = Field(default=None, min_length=1)
+
+    @field_validator("criterion")
+    @classmethod
+    def _non_whitespace(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("must not be empty or whitespace")
+        return v
+
+    @field_validator("pass_on")
+    @classmethod
+    def _no_empty_entries(cls, v: list[str] | None) -> list[str] | None:
+        if v is not None and not all(s for s in v):
+            raise ValueError("entries must be non-empty strings")
+        return v

agent_exam/assertions/file_contains.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from pydantic import StrictBool, model_validator
+
+from .._models import _StrictModel
+from ..schemas import AssertionResult, RunResult
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class FileContainsConfig(_StrictModel):
+    """`file_contains: {path: ..., pattern: ..., regex?: false}`."""
+
+    path: str
+    pattern: str
+    # StrictBool — lax `bool` would coerce YAML-style "yes"/"no" strings.
+    regex: StrictBool = False
+
+    @model_validator(mode="after")
+    def _validate_regex(self) -> FileContainsConfig:
+        if self.regex:
+            try:
+                re.compile(self.pattern)
+            except re.error as exc:
+                raise ValueError(
+                    f"invalid regex pattern {self.pattern!r}: {exc}"
+                ) from exc
+        return self
+
+
+def check(config: FileContainsConfig, result: RunResult, cwd: Path) -> AssertionResult:
+    full = cwd / config.path
+    if not full.is_file():
+        return AssertionResult(
+            pass_=False,
+            reason=f"missing {config.path}",
+            details={"path": str(full)},
+        )
+    text = full.read_text(errors="replace")
+    if config.regex:
+        found = re.search(config.pattern, text) is not None
+    else:
+        found = config.pattern in text
+    return AssertionResult(
+        pass_=found,
+        reason=(
+            f"{'matched' if found else 'no match for'} "
+            f"{config.pattern!r} in {config.path}"
+        ),
+        details={
+            "path": str(full),
+            "pattern": config.pattern,
+            "regex": config.regex,
+        },
+    )

agent_exam/assertions/file_exists.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from .._models import _ScalarShorthandModel
+from ..schemas import AssertionResult, RunResult
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class FileExistsConfig(_ScalarShorthandModel):
+    """`file_exists: out/x.py` or `file_exists: {path: out/x.py}`."""
+
+    _shorthand_key: ClassVar[str] = "path"
+    path: str
+
+
+def check(config: FileExistsConfig, result: RunResult, cwd: Path) -> AssertionResult:
+    full = cwd / config.path
+    exists = full.is_file()
+    return AssertionResult(
+        pass_=exists,
+        reason=f"{'found' if exists else 'missing'} {config.path}",
+        details={"path": str(full)},
+    )

agent_exam/assertions/first_skill.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from .._models import _ScalarShorthandModel
+from ..schemas import AssertionResult, RunResult
+from ..trajectory_walk import first_skill_invocation
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class FirstSkillConfig(_ScalarShorthandModel):
+    """`first_skill: scrape-codegen` or `first_skill: {skill: scrape-codegen}`."""
+
+    _shorthand_key: ClassVar[str] = "skill"
+    skill: str
+
+
+def check(
+    config: FirstSkillConfig, result: RunResult, cwd: Path, context: Any
+) -> AssertionResult:
+    expected = config.skill
+    first = first_skill_invocation(result.trajectory)
+    if first is None:
+        return AssertionResult(
+            pass_=False,
+            reason=f"no skill invocation in trajectory (expected {expected!r})",
+            details={"expected": expected},
+        )
+    ok = context.provider.is_same_skill(first.skill_name, expected)
+    return AssertionResult(
+        pass_=ok,
+        reason=(
+            f"first skill was {first.skill_name!r}"
+            if ok
+            else f"first skill was {first.skill_name!r}, expected {expected!r}"
+        ),
+        details={
+            "expected": expected,
+            "actual": first.skill_name,
+            "trigger_kind": first.trigger_kind,
+        },
+    )

agent_exam/assertions/judge.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from ..judge import (
+    agent_output_hash,
+    build_prompt,
+    call_judge,
+    format_trajectory,
+    key_for,
+    parse_verdict,
+)
+from ..judge.format_trajectory import final_output_text
+from ..schemas import AssertionResult, RunResult
+from ._shared import JudgeConfigBase
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class JudgeConfig(JudgeConfigBase):
+    """`judge: <criterion>` or
+    `judge: {criterion: ..., include_trajectory?: ..., pass_on?: [...]}`."""
+
+
+def check(
+    config: JudgeConfig,
+    result: RunResult,
+    cwd: Path,
+    context=None,
+) -> AssertionResult:
+    if context is None or context.judge_call is None:
+        return AssertionResult(
+            pass_=False,
+            reason="judge: no judge_call in scoring context "
+            "(runner did not configure a judge)",
+        )
+
+    pass_on = config.pass_on or context.judge_pass_on or ["YES"]
+    output_hash = agent_output_hash(result.trajectory)
+    cache_key = key_for(config.criterion, output_hash, context.judge_call.judge_model)
+
+    cached = context.judge_cache.get(cache_key) if context.judge_cache else None
+    if cached is not None:
+        verdict = cached["verdict"]
+        reasoning = cached.get("reasoning", "")
+        cache_hit = True
+    else:
+        final_output = final_output_text(result.trajectory)
+        traj_text = (
+            format_trajectory(result.trajectory) if config.include_trajectory else ""
+        )
+        prompt = build_prompt(
+            config.criterion, final_output, traj_text, config.include_trajectory
+        )
+        raw_response = call_judge(context.judge_call, prompt)
+        verdict, reasoning = parse_verdict(raw_response)
+        if context.judge_cache is not None:
+            context.judge_cache.put(cache_key, config.criterion, verdict, reasoning)
+        cache_hit = False
+
+    passed = verdict in pass_on
+    return AssertionResult(
+        pass_=passed,
+        reason=f"judge said {verdict}",
+        details={
+            "verdict": verdict,
+            "reasoning": reasoning,
+            "criterion": config.criterion,
+            "cache": "hit" if cache_hit else "miss",
+        },
+    )

agent_exam/assertions/judge_agent.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from ..judge import (
+    agent_output_hash,
+    build_prompt,
+    call_judge_agent,
+    format_trajectory,
+    key_for_judge_agent,
+    parse_verdict,
+    tools_signature,
+)
+from ..judge.format_trajectory import final_output_text
+from ..schemas import AssertionResult, RunResult
+from ._shared import JudgeConfigBase
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class JudgeAgentConfig(JudgeConfigBase):
+    """`judge_agent: <criterion>` or
+    `judge_agent: {criterion: ..., include_trajectory?: ..., pass_on?: [...]}`.
+
+    Same config shape as `judge`; the difference is the runtime — the
+    judge runs against the attempt's archived cwd with read-only file
+    tools available."""
+
+
+def check(
+    config: JudgeAgentConfig,
+    result: RunResult,
+    cwd: Path,
+    context=None,
+) -> AssertionResult:
+    """Agentic judge — same config shape as the plain ``judge`` assertion
+    but the judge runs against the attempt's archived cwd with read-only
+    file tools available, so it can verify criteria that depend on the
+    artifacts the skill produced.
+
+    Cache key extends ``judge``'s key with the cwd content hash and the
+    safe-tools signature: any change to the criterion, the agent's
+    output, the cwd, the toolset, or the judge model invalidates the
+    cached verdict.
+    """
+    if context is None or context.judge_call is None:
+        return AssertionResult(
+            pass_=False,
+            reason="judge_agent: no judge_call in scoring context "
+            "(runner did not configure a judge)",
+        )
+
+    provider = context.judge_call.provider
+    if not provider.safe_judge_tools:
+        return AssertionResult(
+            pass_=False,
+            reason=(
+                f"judge_agent: provider {provider.name!r} has no "
+                "safe_judge_tools configured"
+            ),
+        )
+
+    pass_on = config.pass_on or context.judge_pass_on or ["YES"]
+    output_hash = agent_output_hash(result.trajectory)
+    cwd_h = context.cwd_hash_for(cwd)
+    tools_sig = tools_signature(provider.safe_judge_tools)
+    cache_key = key_for_judge_agent(
+        config.criterion,
+        output_hash,
+        cwd_h,
+        tools_sig,
+        context.judge_call.judge_model,
+    )
+
+    cached = context.judge_cache.get(cache_key) if context.judge_cache else None
+    if cached is not None:
+        verdict = cached["verdict"]
+        reasoning = cached.get("reasoning", "")
+        cache_hit = True
+    else:
+        final_output = final_output_text(result.trajectory)
+        traj_text = (
+            format_trajectory(result.trajectory) if config.include_trajectory else ""
+        )
+        prompt = build_prompt(
+            config.criterion,
+            final_output,
+            traj_text,
+            config.include_trajectory,
+            inspect_cwd=True,
+        )
+        raw_response = call_judge_agent(context.judge_call, prompt, cwd)
+        verdict, reasoning = parse_verdict(raw_response)
+        if context.judge_cache is not None:
+            context.judge_cache.put(cache_key, config.criterion, verdict, reasoning)
+        cache_hit = False
+
+    passed = verdict in pass_on
+    return AssertionResult(
+        pass_=passed,
+        reason=f"judge_agent said {verdict}",
+        details={
+            "verdict": verdict,
+            "reasoning": reasoning,
+            "criterion": config.criterion,
+            "cache": "hit" if cache_hit else "miss",
+        },
+    )