agent-eval-contract 0.2.0__py3-none-any.whl

agent_eval_contract/__init__.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+from .clean_room import run_clean_room_contract_check
+from .external import (
+    normalize_external_result,
+    normalize_swe_bench_result,
+    normalize_terminal_bench_result,
+    to_swe_bench_format,
+    to_terminal_bench_format,
+)
+from .models import (
+    CONTEXT_PROFILES,
+    EVAL_RUN_MODES,
+    EVAL_TASK_SOURCES,
+    EXTERNAL_HARNESSES,
+    FAILURE_PRIORITIES,
+    FINAL_STATUSES,
+    ContextProfile,
+    EvalFailure,
+    EvalRun,
+    EvalRunMode,
+    EvalScore,
+    EvalTask,
+    EvalTaskSource,
+    ExternalHarness,
+    ExternalResult,
+    FailurePriority,
+    FinalStatus,
+    FixtureBundleManifest,
+    JsonValue,
+    NormalizedRun,
+)
+from .release import load_release_metadata, validate_release_metadata
+from .samples import load_sample, validate_all_samples, validate_sample
+from .schema_export import export_json_schemas
+from .templates import (
+    render_eval_template,
+    supported_template_ids,
+    validate_eval_template,
+    validate_eval_template_file,
+    validate_template_directory,
+)
+from .validators import (
+    HARNESS_DIMENSION_NAMES,
+    validate_context_profile,
+    validate_eval_failure,
+    validate_eval_run,
+    validate_eval_score,
+    validate_eval_task,
+    validate_external_result,
+    validate_final_status,
+    validate_harness_fixture_components,
+    validate_priority,
+)
+
+__all__ = [
+    "CONTEXT_PROFILES",
+    "EVAL_RUN_MODES",
+    "EVAL_TASK_SOURCES",
+    "EXTERNAL_HARNESSES",
+    "FAILURE_PRIORITIES",
+    "FINAL_STATUSES",
+    "HARNESS_DIMENSION_NAMES",
+    "ContextProfile",
+    "EvalFailure",
+    "EvalRun",
+    "EvalRunMode",
+    "EvalScore",
+    "EvalTask",
+    "EvalTaskSource",
+    "ExternalHarness",
+    "ExternalResult",
+    "FailurePriority",
+    "FinalStatus",
+    "FixtureBundleManifest",
+    "JsonValue",
+    "NormalizedRun",
+    "export_json_schemas",
+    "load_release_metadata",
+    "load_sample",
+    "normalize_external_result",
+    "normalize_swe_bench_result",
+    "normalize_terminal_bench_result",
+    "render_eval_template",
+    "run_clean_room_contract_check",
+    "supported_template_ids",
+    "to_swe_bench_format",
+    "to_terminal_bench_format",
+    "validate_all_samples",
+    "validate_context_profile",
+    "validate_eval_failure",
+    "validate_eval_run",
+    "validate_eval_score",
+    "validate_eval_task",
+    "validate_eval_template",
+    "validate_eval_template_file",
+    "validate_external_result",
+    "validate_final_status",
+    "validate_harness_fixture_components",
+    "validate_priority",
+    "validate_release_metadata",
+    "validate_sample",
+    "validate_template_directory",
+]

agent_eval_contract/clean_room.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from .samples import validate_all_samples
+from .templates import validate_template_directory
+
+
+def run_clean_room_contract_check(
+    *,
+    template_root: Path,
+    sample_root: Path | None = None,
+) -> dict[str, Any]:
+    templates = validate_template_directory(template_root)
+    samples = (
+        validate_all_samples()
+        if sample_root is None
+        else validate_all_samples(sample_root=sample_root)
+    )
+    return {
+        "ok": True,
+        "template_count": len(templates),
+        "sample_count": len(samples),
+        "templates": templates,
+        "samples": samples,
+    }

agent_eval_contract/cli.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+from .external import normalize_external_result
+from .fixture_runner import write_contract_fixture_bundle
+from .schema_export import export_json_schemas
+from .validators import (
+    validate_eval_failure,
+    validate_eval_run,
+    validate_eval_score,
+    validate_eval_task,
+)
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    loaded = json.loads(path.expanduser().resolve().read_text(encoding="utf-8"))
+    if not isinstance(loaded, dict):
+        raise ValueError(f"{path} must contain a JSON object")
+    return loaded
+
+
+def _print_json(value: BaseModel | dict[str, Any] | list[str]) -> None:
+    payload = value.model_dump(mode="json") if isinstance(value, BaseModel) else value
+    print(json.dumps(payload, indent=2, sort_keys=True))
+
+
+def _run_fixtures(args: argparse.Namespace) -> int:
+    _print_json(write_contract_fixture_bundle(Path(args.output_dir)))
+    return 0
+
+
+def _run_schemas(args: argparse.Namespace) -> int:
+    _print_json({"schemas": export_json_schemas(Path(args.output_dir))})
+    return 0
+
+
+def _run_validate(args: argparse.Namespace) -> int:
+    data = _load_json(Path(args.file))
+    validators = {
+        "task": validate_eval_task,
+        "run": validate_eval_run,
+        "score": validate_eval_score,
+        "failure": validate_eval_failure,
+    }
+    _print_json(validators[args.kind](data))
+    return 0
+
+
+def _run_normalize(args: argparse.Namespace) -> int:
+    data = _load_json(Path(args.file))
+    normalized = normalize_external_result(
+        data,
+        eval_task_id=args.task_id,
+        harness=args.harness,
+        model=args.model,
+    )
+    _print_json(normalized)
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="agent-eval-contract",
+        description="Validate, normalize, and export portable agent evaluation contracts.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    fixtures = subparsers.add_parser(
+        "fixtures", help="Write sample records, templates, and schemas."
+    )
+    fixtures.add_argument(
+        "--output-dir", required=True, help="Directory to write fixture artifacts into."
+    )
+    fixtures.set_defaults(func=_run_fixtures)
+
+    schemas = subparsers.add_parser(
+        "schemas", help="Export JSON Schemas for public contract models."
+    )
+    schemas.add_argument(
+        "--output-dir", required=True, help="Directory to write schema files into."
+    )
+    schemas.set_defaults(func=_run_schemas)
+
+    validate = subparsers.add_parser(
+        "validate", help="Validate a JSON record against a contract model."
+    )
+    validate.add_argument("--kind", choices=("task", "run", "score", "failure"), required=True)
+    validate.add_argument("--file", required=True, help="JSON file to validate.")
+    validate.set_defaults(func=_run_validate)
+
+    normalize = subparsers.add_parser("normalize", help="Normalize external harness output.")
+    normalize.add_argument("--harness", choices=("terminal-bench", "swe-bench"), required=True)
+    normalize.add_argument("--file", required=True, help="External result JSON file to normalize.")
+    normalize.add_argument("--task-id", help="Task id to use when the external result omits one.")
+    normalize.add_argument("--model", help="Model name to use when the external result omits one.")
+    normalize.set_defaults(func=_run_normalize)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return int(args.func(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agent_eval_contract/external.py

@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from .models import ExternalHarness, FinalStatus, JsonValue, NormalizedRun
+
+
+def _as_mapping(value: object) -> Mapping[str, Any]:
+    if isinstance(value, Mapping):
+        return value
+    raise TypeError("external result must be a mapping")
+
+
+def _string_list(value: object) -> list[str]:
+    if isinstance(value, list):
+        return [str(item) for item in value]
+    if isinstance(value, tuple):
+        return [str(item) for item in value]
+    return []
+
+
+def _json_metadata(data: Mapping[str, Any]) -> dict[str, JsonValue]:
+    return {
+        str(key): value
+        for key, value in data.items()
+        if isinstance(key, str) and _is_json_value(value)
+    }
+
+
+def _is_json_value(value: object) -> bool:
+    if value is None or isinstance(value, str | int | float | bool):
+        return True
+    if isinstance(value, list):
+        return all(_is_json_value(item) for item in value)
+    if isinstance(value, dict):
+        return all(isinstance(key, str) and _is_json_value(item) for key, item in value.items())
+    return False
+
+
+def _passed_status(data: Mapping[str, Any]) -> FinalStatus:
+    passed_value = data.get("passed", data.get("success", data.get("resolved")))
+    if passed_value is None:
+        status = data.get("status")
+        if isinstance(status, str):
+            lowered = status.lower()
+            if lowered in {"passed", "pass", "success", "resolved"}:
+                return "success"
+            if lowered in {"failed", "fail", "failure", "unresolved"}:
+                return "failed"
+            if lowered in {"error", "errored"}:
+                return "error"
+        return "partial"
+    return "success" if bool(passed_value) else "failed"
+
+
+def _score(data: Mapping[str, Any]) -> float | None:
+    value = data.get("score")
+    if value is None:
+        return None
+    try:
+        numeric = float(value)
+    except (TypeError, ValueError):
+        return None
+    if 0.0 <= numeric <= 1.0:
+        return numeric
+    if 0.0 <= numeric <= 100.0:
+        return numeric / 100.0
+    return None
+
+
+def _duration_ms(data: Mapping[str, Any]) -> int | None:
+    value = data.get("duration_ms")
+    if value is None:
+        seconds = data.get("duration_seconds", data.get("elapsed_seconds"))
+        if seconds is None:
+            return None
+        try:
+            return int(float(seconds) * 1000)
+        except (TypeError, ValueError):
+            return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _terminal_checks(data: Mapping[str, Any]) -> list[str]:
+    checks = data.get("tests_run", data.get("checks"))
+    if checks is not None:
+        return _string_list(checks)
+    command = data.get("command")
+    return [str(command)] if command else []
+
+
+def _swe_bench_checks(data: Mapping[str, Any]) -> list[str]:
+    checks = _string_list(data.get("tests_run", data.get("checks", [])))
+    if checks:
+        return checks
+    fail_to_pass = _string_list(data.get("FAIL_TO_PASS", data.get("fail_to_pass", [])))
+    pass_to_pass = _string_list(data.get("PASS_TO_PASS", data.get("pass_to_pass", [])))
+    return [*fail_to_pass, *pass_to_pass]
+
+
+def to_swe_bench_format(eval_task: Mapping[str, Any]) -> dict[str, Any]:
+    task = _as_mapping(eval_task)
+    return {
+        "repo": task.get("repo"),
+        "instance_id": task.get("task_id"),
+        "problem_statement": task.get("description"),
+        "base_commit": task.get("start_revision"),
+        "FAIL_TO_PASS": task.get("fail_to_pass", []),
+        "PASS_TO_PASS": task.get("pass_to_pass", []),
+    }
+
+
+def to_terminal_bench_format(eval_task: Mapping[str, Any]) -> dict[str, Any]:
+    task = _as_mapping(eval_task)
+    return {
+        "task_id": task.get("task_id"),
+        "command": task.get("command", "pytest"),
+        "expected_exit_code": int(task.get("expected_exit_code", 0)),
+        "setup_commands": task.get("setup_commands", []),
+        "timeout_seconds": int(task.get("timeout_seconds", 600)),
+    }
+
+
+def normalize_external_result(
+    external_result: Mapping[str, Any],
+    *,
+    eval_task_id: str | None = None,
+    harness: ExternalHarness | str,
+    model: str | None = None,
+) -> NormalizedRun:
+    if harness == "terminal-bench":
+        return normalize_terminal_bench_result(
+            external_result,
+            eval_task_id=eval_task_id,
+            model=model,
+        )
+    if harness == "swe-bench":
+        return normalize_swe_bench_result(
+            external_result,
+            eval_task_id=eval_task_id,
+            model=model,
+        )
+    data = _as_mapping(external_result)
+    task_id = eval_task_id or data.get("task_id") or data.get("instance_id")
+    if not task_id:
+        raise ValueError("eval_task_id is required when the external result has no task_id")
+    resolved_model = model or data.get("model") or "unknown"
+    checks = data.get("tests_run", data.get("checks", []))
+    return NormalizedRun(
+        task_id=str(task_id),
+        harness=str(harness),
+        model=str(resolved_model),
+        final_status=_passed_status(data),
+        checks=_string_list(checks),
+        duration_ms=_duration_ms(data),
+        score=_score(data),
+        metadata={
+            "source": "external_result",
+            "raw": _json_metadata(data),
+        },
+    )
+
+
+def normalize_terminal_bench_result(
+    external_result: Mapping[str, Any],
+    *,
+    eval_task_id: str | None = None,
+    model: str | None = None,
+) -> NormalizedRun:
+    data = _as_mapping(external_result)
+    task_id = eval_task_id or data.get("task_id")
+    if not task_id:
+        raise ValueError("eval_task_id is required when the Terminal-Bench result has no task_id")
+    return NormalizedRun(
+        task_id=str(task_id),
+        harness="terminal-bench",
+        model=str(model or data.get("model") or "unknown"),
+        final_status=_passed_status(data),
+        checks=_terminal_checks(data),
+        duration_ms=_duration_ms(data),
+        score=_score(data),
+        metadata={
+            "source": "terminal-bench",
+            "raw": _json_metadata(data),
+        },
+    )
+
+
+def normalize_swe_bench_result(
+    external_result: Mapping[str, Any],
+    *,
+    eval_task_id: str | None = None,
+    model: str | None = None,
+) -> NormalizedRun:
+    data = _as_mapping(external_result)
+    task_id = eval_task_id or data.get("instance_id") or data.get("task_id")
+    if not task_id:
+        raise ValueError("eval_task_id is required when the SWE-bench result has no instance_id")
+    return NormalizedRun(
+        task_id=str(task_id),
+        harness="swe-bench",
+        model=str(model or data.get("model_name_or_path") or data.get("model") or "unknown"),
+        final_status=_passed_status(data),
+        checks=_swe_bench_checks(data),
+        duration_ms=_duration_ms(data),
+        score=_score(data),
+        metadata={
+            "source": "swe-bench",
+            "raw": _json_metadata(data),
+        },
+    )

agent_eval_contract/fixture_runner.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+from .clean_room import run_clean_room_contract_check
+from .models import FixtureBundleManifest
+from .release import load_release_metadata
+from .samples import SAMPLE_FILES, SAMPLE_ROOT, load_sample
+from .schema_export import export_json_schemas
+from .templates import render_eval_template, supported_template_ids
+
+
+def write_contract_fixture_bundle(
+    output_dir: Path,
+    *,
+    sample_root: Path = SAMPLE_ROOT,
+) -> dict[str, Any]:
+    resolved = output_dir.expanduser().resolve()
+    samples_dir = resolved / "samples"
+    templates_dir = resolved / "templates"
+    schemas_dir = resolved / "schemas"
+    samples_dir.mkdir(parents=True, exist_ok=True)
+    templates_dir.mkdir(parents=True, exist_ok=True)
+    schemas_dir.mkdir(parents=True, exist_ok=True)
+
+    sample_ids: list[str] = []
+    for sample_id, filename in sorted(SAMPLE_FILES.items()):
+        sample = load_sample(sample_id, sample_root=sample_root)
+        (samples_dir / filename).write_text(
+            json.dumps(sample, indent=2, sort_keys=True) + "\n",
+            encoding="utf-8",
+        )
+        sample_ids.append(sample_id)
+
+    template_ids: list[str] = []
+    for template_id in supported_template_ids():
+        (templates_dir / f"{template_id}.md").write_text(
+            render_eval_template(template_id),
+            encoding="utf-8",
+        )
+        template_ids.append(template_id)
+
+    check = run_clean_room_contract_check(template_root=templates_dir, sample_root=samples_dir)
+    metadata = load_release_metadata()
+    schema_files = export_json_schemas(schemas_dir)
+    manifest = FixtureBundleManifest(
+        version=str(metadata["version"]),
+        contract_version=str(metadata["contract_version"]),
+        samples=sample_ids,
+        templates=template_ids,
+        schemas=schema_files,
+        metadata={
+            "clean_room_check": check,
+            "public_surfaces": metadata["public_surfaces"],
+            "out_of_scope": metadata["out_of_scope"],
+        },
+    ).model_dump(mode="json")
+    (resolved / "manifest.json").write_text(
+        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    return manifest
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Produce an agent-eval-contract fixture bundle with samples, templates, and JSON Schemas."
+    )
+    parser.add_argument(
+        "--output-dir", required=True, help="Directory to write fixture artifacts into."
+    )
+    args = parser.parse_args(argv)
+    manifest = write_contract_fixture_bundle(Path(args.output_dir))
+    print(json.dumps(manifest, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agent_eval_contract/models.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator
+
+ContextProfile = Literal[
+    "repo_only",
+    "provided_context",
+    "clean_room",
+    "tool_augmented",
+    "full_workspace",
+]
+EvalTaskSource = Literal[
+    "manual",
+    "ci",
+    "benchmark",
+    "production_trace",
+    "synthetic",
+]
+EvalRunMode = Literal[
+    "interactive",
+    "autonomous",
+    "shadow",
+    "replay",
+    "benchmark",
+]
+FinalStatus = Literal["success", "partial", "failed", "abandoned", "error"]
+FailurePriority = Literal["low", "medium", "high", "critical"]
+ExternalHarness = Literal["terminal-bench", "swe-bench"]
+
+CONTEXT_PROFILES: frozenset[str] = frozenset(
+    {"repo_only", "provided_context", "clean_room", "tool_augmented", "full_workspace"}
+)
+EVAL_TASK_SOURCES: frozenset[str] = frozenset(
+    {"manual", "ci", "benchmark", "production_trace", "synthetic"}
+)
+EVAL_RUN_MODES: frozenset[str] = frozenset(
+    {"interactive", "autonomous", "shadow", "replay", "benchmark"}
+)
+FINAL_STATUSES: frozenset[str] = frozenset({"success", "partial", "failed", "abandoned", "error"})
+FAILURE_PRIORITIES: frozenset[str] = frozenset({"low", "medium", "high", "critical"})
+EXTERNAL_HARNESSES: frozenset[str] = frozenset({"terminal-bench", "swe-bench"})
+
+Score = Annotated[float, Field(ge=0.0, le=1.0)]
+NonNegativeInt = Annotated[int, Field(ge=0)]
+NonNegativeFloat = Annotated[float, Field(ge=0.0)]
+
+
+def utc_now() -> datetime:
+    return datetime.now(UTC)
+
+
+class ContractModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+
+class EvalTask(ContractModel):
+    task_id: str = Field(min_length=1)
+    title: str = Field(min_length=1)
+    description: str = Field(min_length=1)
+    source: EvalTaskSource = "manual"
+    context_profile: ContextProfile = "repo_only"
+    acceptance_criteria: list[str] = Field(default_factory=list)
+    repo: str | None = None
+    start_revision: str | None = None
+    tags: list[str] = Field(default_factory=list)
+    created_at: datetime = Field(default_factory=utc_now)
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class EvalRun(ContractModel):
+    run_id: str = Field(min_length=1)
+    task_id: str = Field(min_length=1)
+    harness: str = Field(min_length=1)
+    model: str = Field(min_length=1)
+    mode: EvalRunMode = "interactive"
+    context_profile: ContextProfile = "repo_only"
+    final_status: FinalStatus
+    started_at: datetime | None = None
+    completed_at: datetime | None = None
+    duration_ms: NonNegativeInt | None = None
+    total_tokens: NonNegativeInt | None = None
+    estimated_cost_usd: NonNegativeFloat | None = None
+    tool_calls: NonNegativeInt = 0
+    failed_steps: NonNegativeInt = 0
+    files_changed: NonNegativeInt = 0
+    checks: list[str] = Field(default_factory=list)
+    output_summary: str | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class EvalScore(ContractModel):
+    run_id: str = Field(min_length=1)
+    overall_score: Score
+    metrics: dict[str, Score] = Field(default_factory=dict)
+    passed: bool | None = None
+    reviewer_notes: str | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class EvalFailure(ContractModel):
+    failure_id: str = Field(min_length=1)
+    run_id: str = Field(min_length=1)
+    failure_types: list[str] = Field(min_length=1)
+    summary: str = Field(min_length=1)
+    suspected_cause: str | None = None
+    affected_components: list[str] = Field(default_factory=list)
+    recommended_fixes: list[str] = Field(default_factory=list)
+    priority: FailurePriority = "medium"
+    regression_task_id: str | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class ExternalResult(ContractModel):
+    harness: str = Field(min_length=1)
+    task_id: str | None = None
+    model: str | None = None
+    passed: bool | None = None
+    success: bool | None = None
+    score: Score | None = None
+    tests_run: list[str] = Field(default_factory=list)
+    duration_ms: NonNegativeInt | None = None
+    raw: dict[str, JsonValue] = Field(default_factory=dict)
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class NormalizedRun(ContractModel):
+    task_id: str = Field(min_length=1)
+    harness: str = Field(min_length=1)
+    model: str = Field(min_length=1)
+    mode: EvalRunMode = "benchmark"
+    context_profile: ContextProfile = "clean_room"
+    final_status: FinalStatus
+    checks: list[str] = Field(default_factory=list)
+    duration_ms: NonNegativeInt | None = None
+    score: Score | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class FixtureBundleManifest(ContractModel):
+    package: str = "agent-eval-contract"
+    version: str = Field(min_length=1)
+    contract_version: str = Field(min_length=1)
+    generated_at: datetime = Field(default_factory=utc_now)
+    samples: list[str] = Field(default_factory=list)
+    templates: list[str] = Field(default_factory=list)
+    schemas: list[str] = Field(default_factory=list)
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("package")
+    @classmethod
+    def package_name_matches(_cls, value: str) -> str:
+        if value != "agent-eval-contract":
+            raise ValueError("package must be agent-eval-contract")
+        return value

agent_eval_contract/py.typed

@@ -0,0 +1 @@
+