agent-eval-contract 0.2.0__py3-none-any.whl

agent_eval_contract/release.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+RELEASE_METADATA_PATH = Path(__file__).resolve().parent / "release_metadata.json"
+REQUIRED_RELEASE_METADATA_KEYS = (
+    "package_name",
+    "version",
+    "contract_version",
+    "status",
+    "python_requires",
+    "public_promise",
+    "public_surfaces",
+    "out_of_scope",
+    "public_modules",
+    "release_blockers",
+)
+
+
+def load_release_metadata(path: Path = RELEASE_METADATA_PATH) -> dict[str, Any]:
+    loaded = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(loaded, dict):
+        raise ValueError("Agent eval release metadata must be a JSON object.")
+    validate_release_metadata(loaded)
+    return loaded
+
+
+def validate_release_metadata(metadata: dict[str, Any]) -> None:
+    missing = [key for key in REQUIRED_RELEASE_METADATA_KEYS if key not in metadata]
+    if missing:
+        raise ValueError(f"Agent eval release metadata is missing keys: {', '.join(missing)}")
+    if metadata["package_name"] != "agent-eval-contract":
+        raise ValueError("Agent eval release metadata package_name must be agent-eval-contract.")
+    if metadata["status"] not in {"public_package_candidate", "published"}:
+        raise ValueError("Agent eval release metadata status is invalid.")
+    if not isinstance(metadata["public_promise"], str) or not metadata["public_promise"].strip():
+        raise ValueError("Agent eval release metadata public_promise must be a non-empty string.")
+    for key in ("public_surfaces", "out_of_scope", "public_modules"):
+        value = metadata[key]
+        if (
+            not isinstance(value, list)
+            or not value
+            or not all(isinstance(item, str) for item in value)
+        ):
+            raise ValueError(f"Agent eval release metadata field '{key}' must be a string list.")
+    blockers = metadata["release_blockers"]
+    if not isinstance(blockers, list) or not all(isinstance(item, str) for item in blockers):
+        raise ValueError(
+            "Agent eval release metadata field 'release_blockers' must be a string list."
+        )

agent_eval_contract/release_metadata.json

@@ -0,0 +1,39 @@
+{
+  "package_name": "agent-eval-contract",
+  "version": "0.2.0",
+  "contract_version": "0.1",
+  "status": "published",
+  "python_requires": ">=3.12",
+  "public_promise": "Define, validate, serialize, export JSON Schema for, and normalize portable agent evaluation records.",
+  "public_surfaces": [
+    "Pydantic evaluation record models",
+    "runtime validation helpers",
+    "JSON Schema export",
+    "external harness result normalization",
+    "sample records",
+    "template rendering and validation",
+    "fixture bundle generation",
+    "agent-eval-contract CLI"
+  ],
+  "out_of_scope": [
+    "evaluation execution",
+    "model provider calls",
+    "leaderboard hosting",
+    "dashboard storage",
+    "private workflow vocabulary",
+    "agent orchestration"
+  ],
+  "public_modules": [
+    "agent_eval_contract.models",
+    "agent_eval_contract.validators",
+    "agent_eval_contract.external",
+    "agent_eval_contract.schema_export",
+    "agent_eval_contract.templates",
+    "agent_eval_contract.samples",
+    "agent_eval_contract.clean_room",
+    "agent_eval_contract.fixture_runner",
+    "agent_eval_contract.cli",
+    "agent_eval_contract.release"
+  ],
+  "release_blockers": []
+}

agent_eval_contract/samples/eval_failure.json

@@ -0,0 +1,22 @@
+{
+  "failure_id": "failure-login-flow-001",
+  "run_id": "run-login-flow-001",
+  "failure_types": [
+    "verification_gap"
+  ],
+  "summary": "The first run checked the happy path but skipped the failed-auth redirect case.",
+  "suspected_cause": "The harness did not include a regression test for unauthenticated navigation.",
+  "affected_components": [
+    "app/auth.py",
+    "tests/test_auth_redirect.py"
+  ],
+  "recommended_fixes": [
+    "Add a regression case that starts from a protected nested route.",
+    "Record failed command output in the eval run metadata."
+  ],
+  "priority": "medium",
+  "regression_task_id": "task-login-flow-001",
+  "metadata": {
+    "found_by": "review"
+  }
+}

agent_eval_contract/samples/eval_run.json

@@ -0,0 +1,25 @@
+{
+  "run_id": "run-login-flow-001",
+  "task_id": "task-login-flow-001",
+  "harness": "pytest",
+  "model": "gpt-5",
+  "mode": "autonomous",
+  "context_profile": "repo_only",
+  "final_status": "success",
+  "started_at": "2026-07-04T00:01:00Z",
+  "completed_at": "2026-07-04T00:03:05Z",
+  "duration_ms": 125000,
+  "total_tokens": 8420,
+  "estimated_cost_usd": 0.54,
+  "tool_calls": 18,
+  "failed_steps": 1,
+  "files_changed": 4,
+  "checks": [
+    "pytest tests/test_auth_redirect.py -q",
+    "ruff check app tests"
+  ],
+  "output_summary": "Implemented redirect preservation and added regression coverage.",
+  "metadata": {
+    "branch": "agent/login-redirect"
+  }
+}

agent_eval_contract/samples/eval_score.json

@@ -0,0 +1,16 @@
+{
+  "run_id": "run-login-flow-001",
+  "overall_score": 0.91,
+  "metrics": {
+    "task_success": 1.0,
+    "quality_adherence": 0.9,
+    "verification_strength": 0.85,
+    "cost_efficiency": 0.78,
+    "human_trust": 1.0
+  },
+  "passed": true,
+  "reviewer_notes": "The agent fixed the bug and added a focused regression test.",
+  "metadata": {
+    "reviewer": "human"
+  }
+}

agent_eval_contract/samples/eval_task.json

@@ -0,0 +1,23 @@
+{
+  "task_id": "task-login-flow-001",
+  "title": "Repair login redirect handling",
+  "description": "Fix an agent task where a web app should preserve the requested path after sign-in.",
+  "source": "benchmark",
+  "context_profile": "repo_only",
+  "acceptance_criteria": [
+    "Unauthenticated users are redirected back to the original path after sign-in.",
+    "Existing authenticated navigation keeps working.",
+    "The regression test fails before the fix and passes after it."
+  ],
+  "repo": "example/web-app",
+  "start_revision": "abc1234",
+  "tags": [
+    "web",
+    "auth",
+    "regression"
+  ],
+  "created_at": "2026-07-04T00:00:00Z",
+  "metadata": {
+    "difficulty": "medium"
+  }
+}

agent_eval_contract/samples/external_result_normalization.json

@@ -0,0 +1,39 @@
+{
+  "external_result": {
+    "passed": true,
+    "tests_run": [
+      "pytest tests/test_auth_redirect.py -q"
+    ],
+    "duration_ms": 125000,
+    "score": 0.91
+  },
+  "request": {
+    "eval_task_id": "task-login-flow-001",
+    "harness": "terminal-bench",
+    "model": "gpt-5"
+  },
+  "expected_normalized": {
+    "task_id": "task-login-flow-001",
+    "harness": "terminal-bench",
+    "model": "gpt-5",
+    "mode": "benchmark",
+    "context_profile": "clean_room",
+    "final_status": "success",
+    "checks": [
+      "pytest tests/test_auth_redirect.py -q"
+    ],
+    "duration_ms": 125000,
+    "score": 0.91,
+    "metadata": {
+      "source": "terminal-bench",
+      "raw": {
+        "passed": true,
+        "tests_run": [
+          "pytest tests/test_auth_redirect.py -q"
+        ],
+        "duration_ms": 125000,
+        "score": 0.91
+      }
+    }
+  }
+}

agent_eval_contract/samples.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from .external import normalize_external_result
+from .validators import (
+    validate_eval_failure,
+    validate_eval_run,
+    validate_eval_score,
+    validate_eval_task,
+)
+
+SAMPLE_ROOT = Path(__file__).resolve().parent / "samples"
+SAMPLE_FILES = {
+    "eval_task": "eval_task.json",
+    "eval_run": "eval_run.json",
+    "eval_score": "eval_score.json",
+    "eval_failure": "eval_failure.json",
+    "external_result_normalization": "external_result_normalization.json",
+}
+
+
+def load_sample(sample_id: str, *, sample_root: Path = SAMPLE_ROOT) -> dict[str, Any]:
+    filename = SAMPLE_FILES.get(sample_id)
+    if filename is None:
+        allowed = ", ".join(sorted(SAMPLE_FILES))
+        raise ValueError(f"Unknown agent eval sample '{sample_id}'. Use one of: {allowed}.")
+    loaded = json.loads((sample_root / filename).read_text(encoding="utf-8"))
+    if not isinstance(loaded, dict):
+        raise ValueError(f"Agent eval sample '{sample_id}' must be a JSON object.")
+    return loaded
+
+
+def validate_sample(sample_id: str, sample: dict[str, Any]) -> None:
+    if sample_id == "eval_task":
+        validate_eval_task(sample)
+        return
+    if sample_id == "eval_run":
+        validate_eval_run(sample)
+        return
+    if sample_id == "eval_score":
+        validate_eval_score(sample)
+        return
+    if sample_id == "eval_failure":
+        validate_eval_failure(sample)
+        return
+    if sample_id == "external_result_normalization":
+        _validate_external_result_normalization(sample)
+        return
+    allowed = ", ".join(sorted(SAMPLE_FILES))
+    raise ValueError(f"Unknown agent eval sample '{sample_id}'. Use one of: {allowed}.")
+
+
+def validate_all_samples(*, sample_root: Path = SAMPLE_ROOT) -> list[str]:
+    validated: list[str] = []
+    for sample_id in sorted(SAMPLE_FILES):
+        validate_sample(sample_id, load_sample(sample_id, sample_root=sample_root))
+        validated.append(sample_id)
+    return validated
+
+
+def _validate_external_result_normalization(sample: dict[str, Any]) -> None:
+    external_result = sample.get("external_result")
+    request = sample.get("request")
+    expected = sample.get("expected_normalized")
+    if (
+        not isinstance(external_result, dict)
+        or not isinstance(request, dict)
+        or not isinstance(expected, dict)
+    ):
+        raise ValueError("Agent eval external normalization sample fields must be objects.")
+    normalized = normalize_external_result(
+        external_result,
+        eval_task_id=str(request["eval_task_id"]),
+        harness=str(request["harness"]),
+        model=str(request["model"]),
+    )
+    if normalized.model_dump(mode="json") != expected:
+        raise ValueError("Agent eval external normalization sample expected output does not match.")

agent_eval_contract/schema_export.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from .models import (
+    EvalFailure,
+    EvalRun,
+    EvalScore,
+    EvalTask,
+    ExternalResult,
+    FixtureBundleManifest,
+    NormalizedRun,
+)
+
+type ModelType = type[BaseModel]
+
+SCHEMA_MODELS: dict[str, ModelType] = {
+    "eval_failure": EvalFailure,
+    "eval_run": EvalRun,
+    "eval_score": EvalScore,
+    "eval_task": EvalTask,
+    "external_result": ExternalResult,
+    "fixture_bundle_manifest": FixtureBundleManifest,
+    "normalized_run": NormalizedRun,
+}
+
+
+def export_json_schemas(output_dir: Path) -> list[str]:
+    resolved = output_dir.expanduser().resolve()
+    resolved.mkdir(parents=True, exist_ok=True)
+    exported: list[str] = []
+    for schema_id, model in sorted(SCHEMA_MODELS.items()):
+        path = resolved / f"{schema_id}.schema.json"
+        path.write_text(
+            json.dumps(model.model_json_schema(), indent=2, sort_keys=True) + "\n",
+            encoding="utf-8",
+        )
+        exported.append(path.name)
+    return exported

agent_eval_contract/schemas.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from .models import (
+    CONTEXT_PROFILES,
+    EVAL_RUN_MODES,
+    EVAL_TASK_SOURCES,
+    EXTERNAL_HARNESSES,
+    FAILURE_PRIORITIES,
+    FINAL_STATUSES,
+    ContextProfile,
+    EvalFailure,
+    EvalRun,
+    EvalRunMode,
+    EvalScore,
+    EvalTask,
+    EvalTaskSource,
+    ExternalHarness,
+    ExternalResult,
+    FailurePriority,
+    FinalStatus,
+    FixtureBundleManifest,
+    JsonValue,
+    NormalizedRun,
+)
+
+__all__ = [
+    "CONTEXT_PROFILES",
+    "EVAL_RUN_MODES",
+    "EVAL_TASK_SOURCES",
+    "EXTERNAL_HARNESSES",
+    "FAILURE_PRIORITIES",
+    "FINAL_STATUSES",
+    "ContextProfile",
+    "EvalFailure",
+    "EvalRun",
+    "EvalRunMode",
+    "EvalScore",
+    "EvalTask",
+    "EvalTaskSource",
+    "ExternalHarness",
+    "ExternalResult",
+    "FailurePriority",
+    "FinalStatus",
+    "FixtureBundleManifest",
+    "JsonValue",
+    "NormalizedRun",
+]

agent_eval_contract/templates.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+TEMPLATE_SECTIONS: dict[str, tuple[str, ...]] = {
+    "major-task-eval": (
+        "Task Summary",
+        "Original Acceptance Criteria",
+        "Context Profile",
+        "Harness Condition",
+        "Checks Run",
+        "Check Results",
+        "Failure Taxonomy Labels",
+        "Final Confidence Level",
+    ),
+    "failure-record": (
+        "Identifiers",
+        "Condition",
+        "Context Profile",
+        "Failure Types",
+        "Summary",
+        "Priority",
+        "Standards or Project Truth Update Needed",
+    ),
+    "shadow-branch-comparison": (
+        "Branches",
+        "Task Description",
+        "Acceptance Criteria",
+        "Context Profile",
+        "Harness Condition",
+        "Test Results",
+        "Shadow Branch Delta",
+        "Recommendation",
+    ),
+    "backfill-hotspot": (
+        "Hotspot",
+        "Evidence",
+        "Risk",
+        "Suggested Fix",
+        "Priority",
+        "Blocking Status",
+        "Related Failure Taxonomy Label",
+    ),
+    "portable-context-packet": (
+        "Packet ID",
+        "Scope",
+        "Task Summary",
+        "Included Repo Files/Docs",
+        "Excluded Content",
+        "Privacy/Secret Review",
+        "Expiration/Staleness Notes",
+    ),
+}
+
+
+def supported_template_ids() -> tuple[str, ...]:
+    return tuple(sorted(TEMPLATE_SECTIONS))
+
+
+def render_eval_template(template_id: str) -> str:
+    required_sections = TEMPLATE_SECTIONS.get(template_id)
+    if required_sections is None:
+        allowed = ", ".join(supported_template_ids())
+        raise ValueError(f"Unknown eval template id '{template_id}'. Use one of: {allowed}.")
+    title = template_id.replace("-", " ").title()
+    sections = "\n\n".join(f"## {section}\n\nTODO" for section in required_sections)
+    return f"# {title}\n\n{sections}\n"
+
+
+def validate_eval_template(template_id: str, markdown: str) -> None:
+    required_sections = TEMPLATE_SECTIONS.get(template_id)
+    if required_sections is None:
+        allowed = ", ".join(supported_template_ids())
+        raise ValueError(f"Unknown eval template id '{template_id}'. Use one of: {allowed}.")
+    if not markdown.strip():
+        raise ValueError(f"Eval template '{template_id}' must not be empty.")
+    missing = [
+        section
+        for section in required_sections
+        if f"## {section}" not in markdown and f"# {section}" not in markdown
+    ]
+    if missing:
+        raise ValueError(
+            f"Eval template '{template_id}' is missing required sections: {', '.join(missing)}"
+        )
+
+
+def validate_eval_template_file(path: Path, *, template_id: str | None = None) -> str:
+    resolved = path.expanduser().resolve()
+    actual_id = template_id or resolved.stem
+    validate_eval_template(actual_id, resolved.read_text(encoding="utf-8"))
+    return actual_id
+
+
+def validate_template_directory(template_root: Path) -> list[str]:
+    resolved = template_root.expanduser().resolve()
+    validated: list[str] = []
+    for template_id in supported_template_ids():
+        validate_eval_template_file(resolved / f"{template_id}.md", template_id=template_id)
+        validated.append(template_id)
+    return validated

agent_eval_contract/validators.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from .models import (
+    CONTEXT_PROFILES,
+    FAILURE_PRIORITIES,
+    FINAL_STATUSES,
+    EvalFailure,
+    EvalRun,
+    EvalScore,
+    EvalTask,
+    ExternalResult,
+)
+
+HARNESS_DIMENSION_NAMES = (
+    "task_success",
+    "quality_adherence",
+    "runtime_reliability",
+    "context_usefulness",
+    "verification_strength",
+    "cost_efficiency",
+    "latency",
+    "human_trust",
+)
+
+
+def _allowed_message(label: str, allowed: frozenset[str]) -> str:
+    return f"Invalid {label}. Use one of: {', '.join(sorted(allowed))}."
+
+
+def validate_context_profile(context_profile: str) -> None:
+    if context_profile not in CONTEXT_PROFILES:
+        raise ValueError(_allowed_message("context_profile", CONTEXT_PROFILES))
+
+
+def validate_final_status(final_status: str) -> None:
+    if final_status not in FINAL_STATUSES:
+        raise ValueError(_allowed_message("final_status", FINAL_STATUSES))
+
+
+def validate_priority(priority: str) -> None:
+    if priority not in FAILURE_PRIORITIES:
+        raise ValueError(_allowed_message("priority", FAILURE_PRIORITIES))
+
+
+def validate_eval_task(data: Mapping[str, Any]) -> EvalTask:
+    return EvalTask.model_validate(data)
+
+
+def validate_eval_run(data: Mapping[str, Any]) -> EvalRun:
+    return EvalRun.model_validate(data)
+
+
+def validate_eval_score(data: Mapping[str, Any]) -> EvalScore:
+    return EvalScore.model_validate(data)
+
+
+def validate_eval_failure(data: Mapping[str, Any]) -> EvalFailure:
+    return EvalFailure.model_validate(data)
+
+
+def validate_external_result(data: Mapping[str, Any]) -> ExternalResult:
+    return ExternalResult.model_validate(data)
+
+
+def _is_string_list(value: Any) -> bool:
+    return isinstance(value, list) and all(isinstance(item, str) for item in value)
+
+
+def _validate_expected_gates(expected_gates: Any) -> None:
+    if not isinstance(expected_gates, list):
+        raise ValueError("harness fixture expected_gates must be a list")
+    for item in expected_gates:
+        if not isinstance(item, Mapping):
+            raise ValueError("harness fixture expected_gates entries must be objects")
+        if not item.get("id") or not item.get("expected_decision"):
+            raise ValueError("harness fixture expected_gates entries need id and expected_decision")
+
+
+def _validate_runs(runs: Any) -> None:
+    if not isinstance(runs, Mapping) or not runs:
+        raise ValueError("harness fixture runs must be a non-empty object")
+    for run_id, run in runs.items():
+        if not isinstance(run_id, str) or not run_id:
+            raise ValueError("harness fixture run ids must be non-empty strings")
+        if not isinstance(run, Mapping):
+            raise ValueError("harness fixture run artifacts must be objects")
+        harness = run.get("harness")
+        if not isinstance(harness, Mapping) or not harness.get("name"):
+            raise ValueError("harness fixture run artifacts need harness.name")
+
+
+def validate_harness_fixture_components(
+    *,
+    task_markdown: str,
+    expected_context_packets: list[str],
+    expected_gates: list[dict[str, str]],
+    expected_success_criteria: list[str],
+    golden_outcome_markdown: str,
+    scoring: Mapping[str, Any],
+    runs: Mapping[str, Mapping[str, Any]],
+) -> None:
+    if not task_markdown.strip():
+        raise ValueError("harness fixture task_markdown must not be empty")
+    if not golden_outcome_markdown.strip():
+        raise ValueError("harness fixture golden_outcome_markdown must not be empty")
+    if not _is_string_list(expected_context_packets):
+        raise ValueError("harness fixture expected_context_packets must be a string list")
+    if not _is_string_list(expected_success_criteria):
+        raise ValueError("harness fixture expected_success_criteria must be a string list")
+    if not isinstance(scoring, Mapping):
+        raise ValueError("harness fixture scoring must be an object")
+    _validate_expected_gates(expected_gates)
+    _validate_runs(runs)