ageval-cli 0.7.1__py3-none-any.whl

ageval/__init__.py

@@ -0,0 +1,26 @@
+"""ageval — agent evaluation runtime.
+
+Lock a dataset, run one Attempt through the five phases, and bind PASS only
+from an independent evaluator. CLI: ``ageval``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def _load_version() -> str:
+    try:
+        from importlib.metadata import version
+
+        return version("ageval-cli")
+    except Exception:
+        pass
+    # Editable / source tree: repo-root VERSION (src/ageval/__init__.py → parents[2]).
+    version_file = Path(__file__).resolve().parents[2] / "VERSION"
+    if version_file.is_file():
+        return version_file.read_text(encoding="utf-8").strip()
+    return "0.0.0"
+
+
+__version__ = _load_version()

ageval/agents/__init__.py

@@ -0,0 +1,6 @@
+"""Local Agent (`ageval.agent/1`) cache and manifest (design/14).
+
+An Agent is one versioned job binding plus display metadata. Install only
+writes the local cache; run-time use goes through the --agent projection
+into the existing profiles lane.
+"""

ageval/agents/builtin/anthropic-http/agent.yaml

@@ -0,0 +1,15 @@
+format: ageval.agent/1
+agent_id: anthropic-http
+version: "0.1.0"
+label: "Anthropic HTTP"
+description: >-
+  Anthropic Messages HTTP. No default model; pass --model on the run.
+  Locators stay names until invoke.
+tags: [anthropic-http]
+binding:
+  executor: anthropic-http
+  api_key: ${ANTHROPIC_API_KEY}
+  base_url: https://api.anthropic.com/v1
+  extensions:
+    - plugin: anthropic-http
+    - plugin: local

ageval/agents/builtin/catalog.json

@@ -0,0 +1,51 @@
+[
+  {
+    "harness_id": "pi",
+    "kind": "acp",
+    "product": "pi",
+    "label": "Pi",
+    "description": "Pi coding agent via ACP (`executor: acp` + `options.entry: pi`). No default model; pass --model on the run."
+  },
+  {
+    "harness_id": "opencode",
+    "kind": "acp",
+    "product": "opencode",
+    "label": "OpenCode",
+    "description": "OpenCode via ACP. Ships a LiteLLM locator overlay into $HOME/.config/opencode. No default model; pass --model on the run."
+  },
+  {
+    "harness_id": "codex",
+    "kind": "acp",
+    "product": "codex",
+    "label": "Codex",
+    "description": "OpenAI Codex via ACP (`options.entry: codex`). No default model; pass --model on the run."
+  },
+  {
+    "harness_id": "claude-code",
+    "kind": "acp",
+    "product": "claude-code",
+    "label": "Claude Code",
+    "description": "Anthropic Claude Code via ACP (`options.entry: claude-code`). No default model; pass --model on the run."
+  },
+  {
+    "harness_id": "grok-build",
+    "kind": "acp",
+    "product": "grok-build",
+    "label": "Grok Build",
+    "description": "xAI Grok via ACP (`options.entry: grok-build`). No default model; pass --model on the run."
+  },
+  {
+    "harness_id": "openai-http",
+    "kind": "executor",
+    "product": "openai-http",
+    "label": "OpenAI HTTP",
+    "description": "OpenAI-compatible HTTP Chat Completions (`executor: openai-http`). Locators for base_url and api_key. No default model; pass --model on the run."
+  },
+  {
+    "harness_id": "anthropic-http",
+    "kind": "executor",
+    "product": "anthropic-http",
+    "label": "Anthropic HTTP",
+    "description": "Anthropic Messages HTTP (`executor: anthropic-http`). Locators for base_url and api_key. No default model; pass --model on the run."
+  }
+]

ageval/agents/builtin/claude-code/agent.yaml

@@ -0,0 +1,11 @@
+format: ageval.agent/1
+agent_id: claude-code
+version: "0.1.0"
+label: "Claude Code"
+description: "Anthropic Claude Code via ACP. No default model; pass --model on the run."
+tags: [coding, acp]
+binding:
+  executor: acp
+  extensions:
+    - plugin: acp
+      options: { entry: claude-code }

ageval/agents/builtin/codex/agent.yaml

@@ -0,0 +1,11 @@
+format: ageval.agent/1
+agent_id: codex
+version: "0.1.0"
+label: "Codex"
+description: "OpenAI Codex via ACP. No default model; pass --model on the run."
+tags: [coding, acp]
+binding:
+  executor: acp
+  extensions:
+    - plugin: acp
+      options: { entry: codex }

ageval/agents/builtin/grok-build/agent.yaml

@@ -0,0 +1,11 @@
+format: ageval.agent/1
+agent_id: grok-build
+version: "0.1.0"
+label: "Grok Build"
+description: "xAI Grok via ACP. No default model; pass --model on the run."
+tags: [coding, acp]
+binding:
+  executor: acp
+  extensions:
+    - plugin: acp
+      options: { entry: grok-build }

ageval/agents/builtin/openai-http/agent.yaml

@@ -0,0 +1,15 @@
+format: ageval.agent/1
+agent_id: openai-http
+version: "0.1.0"
+label: "OpenAI HTTP"
+description: >-
+  OpenAI-compatible HTTP Chat Completions. No default model; pass --model
+  on the run. Locators stay names until invoke.
+tags: [openai-http]
+binding:
+  executor: openai-http
+  api_key: ${litellm_api_key}
+  base_url: ${litellm_base_url}
+  extensions:
+    - plugin: openai-http
+    - plugin: local

ageval/agents/builtin/opencode/agent.yaml

@@ -0,0 +1,23 @@
+format: ageval.agent/1
+agent_id: opencode
+version: "0.1.0"
+label: "OpenCode"
+description: >-
+  OpenCode via ACP. Bundles overlays/opencode.litellm.json into
+  $HOME/.config/opencode. Locators stay names until invoke. No default
+  model; pass --model on the run.
+tags: [coding, acp, opencode]
+binding:
+  executor: acp
+  overlays:
+    - overlays/opencode.litellm.json
+  extensions:
+    - plugin: acp
+      options:
+        entry: opencode
+    - plugin: home-files
+      options:
+        files:
+          - src: overlays/opencode.litellm.json
+            dest: .config/opencode/opencode.json
+            dest_root: home

ageval/agents/builtin/opencode/overlays/opencode.litellm.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "https://opencode.ai/config.json",
+  "provider": {
+    "litellm": {
+      "npm": "@ai-sdk/openai-compatible",
+      "name": "LiteLLM",
+      "options": {
+        "baseURL": "{env:litellm_base_url}",
+        "apiKey": "{env:litellm_api_key}"
+      },
+      "models": {
+        "dashscope/qwen3.8-max": { "name": "qwen3.8-max" },
+        "dashscope/qwen3-coder-plus": { "name": "qwen3-coder-plus" },
+        "dashscope/kimi-k2.7-code": { "name": "kimi-k2.7-code" },
+        "MiniMax-M2.7": { "name": "MiniMax-M2.7" },
+        "glm-5.2": { "name": "glm-5.2" },
+        "anthropic/claude-sonnet-4-6": { "name": "claude-sonnet-4-6" },
+        "openai/gpt-5.4": { "name": "gpt-5.4" },
+        "dashscope/deepseek-v4-pro": { "name": "deepseek-v4-pro" }
+      }
+    }
+  }
+}

ageval/agents/builtin/pi/agent.yaml

@@ -0,0 +1,11 @@
+format: ageval.agent/1
+agent_id: pi
+version: "0.1.0"
+label: "Pi"
+description: "Pi coding agent via ACP. No default model; pass --model on the run."
+tags: [coding, acp]
+binding:
+  executor: acp
+  extensions:
+    - plugin: acp
+      options: { entry: pi }

ageval/agents/manifest.py

@@ -0,0 +1,234 @@
+"""Parse and validate ``ageval.agent/1`` (``agent.yaml``) — design/14.
+
+The ``binding`` block is exactly one ``ageval.profiles/1`` binding and is
+validated by the same parser (key allowlist, options denylist, overlay path
+shape). ``binding.agent_ref`` is reserved for the --agent projection and
+rejected here. Every file in the package is secret-scanned fail-closed.
+"""
+
+from __future__ import annotations
+
+import copy
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from ageval.config.errors import (
+    ERROR_INVALID_FORMAT,
+    ERROR_INVALID_PACKAGE,
+    ERROR_INVALID_SCHEMA,
+    ConfigError,
+)
+from ageval.config.overlay_files import (
+    assert_overlays_at_lock,
+    parse_overlay_paths,
+    scan_overlay_files,
+)
+from ageval.config.profiles import PROFILES_FORMAT, parse_job_mapping
+
+AGENT_FILENAME = "agent.yaml"
+AGENT_FORMAT = "ageval.agent/1"
+
+# Short id, or org/name Hub address (one slash), matching plugin id rules.
+_AGENT_ID_RE = re.compile(r"^(?:[a-z0-9][a-z0-9_-]*/)?[a-z0-9][a-z0-9_-]*$")
+
+_TOP_LEVEL_KEYS = frozenset(
+    {"format", "agent_id", "version", "label", "description", "tags", "binding"}
+)
+
+# Placeholder role used only to run the profiles binding validator.
+_VALIDATION_ROLE = "agent"
+
+
+@dataclass(frozen=True)
+class AgentManifest:
+    agent_id: str
+    version: str
+    binding: dict[str, Any]
+    label: str | None = None
+    description: str | None = None
+    tags: tuple[str, ...] = ()
+    root: Path | None = None  # package dir (None when loaded from a bare yaml)
+
+    def as_dict(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "format": AGENT_FORMAT,
+            "agent_id": self.agent_id,
+            "version": self.version,
+            "binding": copy.deepcopy(self.binding),
+        }
+        if self.label:
+            out["label"] = self.label
+        if self.description:
+            out["description"] = self.description
+        if self.tags:
+            out["tags"] = list(self.tags)
+        return out
+
+
+def _optional_str(raw: Any, *, key: str, location: str) -> str | None:
+    if raw is None:
+        return None
+    if not isinstance(raw, str) or not raw.strip():
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            f"{key} must be a non-empty string when set",
+            location=f"{location}:/{key}",
+        )
+    return raw.strip()
+
+
+def parse_agent_document(raw: Any, *, location: str = AGENT_FILENAME) -> AgentManifest:
+    """Validate a loaded ``agent.yaml`` mapping into an :class:`AgentManifest`."""
+    if not isinstance(raw, dict):
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "agent document root must be a mapping",
+            location=location,
+        )
+    fmt = raw.get("format")
+    if fmt != AGENT_FORMAT:
+        raise ConfigError(
+            ERROR_INVALID_FORMAT,
+            f"agent format must be {AGENT_FORMAT}; got {fmt!r}",
+            location=f"{location}:/format",
+        )
+    unknown = set(raw) - _TOP_LEVEL_KEYS
+    if unknown:
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            f"unknown agent keys: {sorted(unknown)}",
+            location=location,
+        )
+
+    agent_id_raw = raw.get("agent_id")
+    if not isinstance(agent_id_raw, str) or not _AGENT_ID_RE.fullmatch(agent_id_raw.strip()):
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "agent_id must match ^[a-z0-9][a-z0-9_-]*$ (optionally org/name)",
+            location=f"{location}:/agent_id",
+        )
+    agent_id = agent_id_raw.strip()
+
+    version_raw = raw.get("version")
+    if not isinstance(version_raw, str) or not version_raw.strip():
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "version must be a non-empty string",
+            location=f"{location}:/version",
+        )
+    version = version_raw.strip()
+
+    label = _optional_str(raw.get("label"), key="label", location=location)
+    description = _optional_str(raw.get("description"), key="description", location=location)
+
+    tags_raw = raw.get("tags")
+    tags: tuple[str, ...] = ()
+    if tags_raw is not None:
+        if not isinstance(tags_raw, list) or not all(
+            isinstance(t, str) and t.strip() for t in tags_raw
+        ):
+            raise ConfigError(
+                ERROR_INVALID_SCHEMA,
+                "tags must be a list of non-empty strings",
+                location=f"{location}:/tags",
+            )
+        tags = tuple(t.strip() for t in tags_raw)
+
+    binding_raw = raw.get("binding")
+    if not isinstance(binding_raw, dict) or not binding_raw:
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "binding must be a non-empty mapping (one ageval.profiles/1 binding)",
+            location=f"{location}:/binding",
+        )
+    if "agent_ref" in binding_raw:
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "binding.agent_ref is reserved (injected by --agent projection)",
+            location=f"{location}:/binding/agent_ref",
+        )
+    parsed = parse_job_mapping(
+        {"format": PROFILES_FORMAT, "agent_profiles": {_VALIDATION_ROLE: binding_raw}},
+        location=f"{location}:/binding",
+    )
+    binding = parsed.profiles[_VALIDATION_ROLE]
+    if label and "label" not in binding:
+        binding["label"] = label
+
+    return AgentManifest(
+        agent_id=agent_id,
+        version=version,
+        binding=binding,
+        label=label,
+        description=description,
+        tags=tags,
+    )
+
+
+def _package_files(root: Path) -> list[Path]:
+    files: list[Path] = []
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in {"__pycache__", ".git", ".ageval"}]
+        for name in filenames:
+            files.append(Path(dirpath) / name)
+    return sorted(files)
+
+
+def load_agent_manifest(path: Path) -> AgentManifest:
+    """Load ``agent.yaml`` from a package dir or a direct yaml path.
+
+    Fail closed on schema errors, on secret material anywhere in the
+    package (locator names are fine; values are not), and when
+    ``binding.overlays`` lists a path that is missing from the package.
+    """
+    path = path.expanduser().resolve(strict=False)
+    if path.is_dir():
+        root: Path | None = path
+        yaml_path = path / AGENT_FILENAME
+    else:
+        root = None
+        yaml_path = path
+    if not yaml_path.is_file():
+        raise ConfigError(
+            ERROR_INVALID_PACKAGE,
+            f"agent manifest not found: {yaml_path}",
+            location=str(yaml_path),
+        )
+    try:
+        data = yaml.safe_load(yaml_path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            f"invalid YAML: {exc}",
+            location=str(yaml_path),
+        ) from exc
+    manifest = parse_agent_document(data, location=str(yaml_path))
+
+    scan_files = _package_files(root) if root is not None else [yaml_path]
+    scan_overlay_files(scan_files, location=str(yaml_path))
+    listed = parse_overlay_paths(
+        manifest.binding.get("overlays"),
+        location=f"{yaml_path}:/binding/overlays",
+    )
+    if listed:
+        # Direct agent.yaml: listed overlays resolve from the sibling package dir.
+        assert_overlays_at_lock(
+            root if root is not None else yaml_path.parent,
+            manifest.binding,
+            location=f"{yaml_path}:/binding/overlays",
+        )
+
+    return AgentManifest(
+        agent_id=manifest.agent_id,
+        version=manifest.version,
+        binding=manifest.binding,
+        label=manifest.label,
+        description=manifest.description,
+        tags=manifest.tags,
+        root=root,
+    )

ageval/agents/paths.py

@@ -0,0 +1,25 @@
+"""Local agent cache paths (design/14).
+
+Mirrors the plugins cache: ``$AGEVAL_HOME/agents`` + ``index.json``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ageval.plugins.paths import ageval_home
+
+AGENTS_DIRNAME = "agents"
+INDEX_FILENAME = "index.json"
+
+
+def agents_root() -> Path:
+    return ageval_home() / AGENTS_DIRNAME
+
+
+def index_path() -> Path:
+    return agents_root() / INDEX_FILENAME
+
+
+def package_dir(agent_id: str, version: str) -> Path:
+    return agents_root() / agent_id / version

ageval/agents/refs.py

@@ -0,0 +1,134 @@
+"""Parse ``agent_ref`` and resolve the Agent package root (design/14).
+
+``agent_ref`` is provenance injected by ``--agent``. Overlay bytes listed on
+that binding resolve only against this package, never the Dataset tree.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ageval.agents.store import resolve_installed_ref
+from ageval.config.errors import (
+    ERROR_INVALID_PACKAGE,
+    ERROR_INVALID_SCHEMA,
+    ERROR_MISSING_REFERENCE,
+    ConfigError,
+)
+
+_FILE_PREFIX = "file:"
+_DEV_PLUS = "@dev+"
+_DEV_SUFFIX = "@dev"
+_SHA_PREFIX = "sha256:"
+_LOCAL_PREFIX = "local/"
+
+
+def published_agent_ref_parts(ref: object) -> tuple[str, str] | None:
+    """Hub-publishable ``(org/name, version)`` from an ``agent_ref``.
+
+    ``file:`` and ``local/`` refs return ``None`` — they must not create
+    Hub Performance (design/12, design/14).
+    """
+    if not isinstance(ref, str):
+        return None
+    text = ref.strip()
+    if not text or text.startswith(_FILE_PREFIX):
+        return None
+    at = text.find("@")
+    if at <= 0:
+        return None
+    package_id = text[:at]
+    if "/" not in package_id or package_id.startswith(_LOCAL_PREFIX):
+        return None
+    rest = text[at + 1 :]
+    plus = rest.find("+")
+    version = (rest[:plus] if plus >= 0 else rest).strip()
+    if not version:
+        return None
+    return package_id, version
+
+
+def package_root_from_agent_ref(ref: str) -> Path:
+    """Return the installed (or ``file:``) Agent package root. Fail closed."""
+    text = ref.strip()
+    if not text:
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "agent_ref must not be empty",
+            location="/agent_ref",
+        )
+    if text.startswith(_FILE_PREFIX):
+        return _file_package_root(text)
+    return _cache_package_root(text)
+
+
+def _file_package_root(ref: str) -> Path:
+    rest = ref[len(_FILE_PREFIX) :]
+    if _DEV_PLUS in rest:
+        path_s = rest.rsplit(_DEV_PLUS, 1)[0]
+    elif rest.endswith(_DEV_SUFFIX):
+        path_s = rest[: -len(_DEV_SUFFIX)]
+    else:
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "file: agent_ref must be file:<path>@dev[+sha256:…]",
+            location=ref,
+        )
+    if not path_s:
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "file: agent_ref is missing a path",
+            location=ref,
+        )
+    path = Path(path_s).expanduser()
+    if path.is_file():
+        path = path.parent
+    if not path.is_dir():
+        raise ConfigError(
+            ERROR_MISSING_REFERENCE,
+            f"agent package not found for agent_ref: {path_s}",
+            location=ref,
+        )
+    return path.resolve(strict=False)
+
+
+def _cache_package_root(ref: str) -> Path:
+    id_ver, plus, digest = ref.partition("+")
+    agent_id, at, version = id_ver.rpartition("@")
+    if not at or not agent_id.strip() or not version.strip():
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "agent_ref must be <id>@<version>+sha256:…",
+            location=ref,
+        )
+    from ageval.agents.reserved import builtin_harness_root, canonical_harness_id
+    from ageval.plugins.store import compute_tree_digest
+
+    hit = canonical_harness_id(agent_id.strip())
+    if hit is not None:
+        root = builtin_harness_root(hit)
+        have = compute_tree_digest(root)
+        if plus and digest.strip():
+            want = digest.strip()
+            hex_want = want[len(_SHA_PREFIX) :] if want.startswith(_SHA_PREFIX) else want
+            hex_have = have[len(_SHA_PREFIX) :] if have.startswith(_SHA_PREFIX) else have
+            if not hex_have.startswith(hex_want):
+                raise ConfigError(
+                    ERROR_INVALID_PACKAGE,
+                    "agent_ref digest does not match the installed package",
+                    location=ref,
+                )
+        return root
+    entry, root = resolve_installed_ref(agent_id.strip(), version.strip())
+    if plus and digest.strip():
+        want = digest.strip()
+        hex_want = want[len(_SHA_PREFIX) :] if want.startswith(_SHA_PREFIX) else want
+        have = entry.digest
+        hex_have = have[len(_SHA_PREFIX) :] if have.startswith(_SHA_PREFIX) else have
+        if not hex_have.startswith(hex_want):
+            raise ConfigError(
+                ERROR_INVALID_PACKAGE,
+                "agent_ref digest does not match the installed package",
+                location=ref,
+            )
+    return root