ageval-cli 0.7.1__py3-none-any.whl

ageval/application/agent_ops/install_remote.py

@@ -0,0 +1,156 @@
+"""Install an agent from a local path or the Registry into the agents cache."""
+
+from __future__ import annotations
+
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from ageval.agents.manifest import load_agent_manifest
+from ageval.agents.paths import package_dir
+from ageval.agents.reserved import reject_reserved_harness_id
+from ageval.agents.store import AgentIndexEntry, install_from_path
+from ageval.application.agent_ops.install_plugins import install_declared_plugins
+from ageval.config.errors import ConfigError
+from ageval.plugins.install import InstalledItem
+from ageval.registry.archive import extract_archive
+from ageval.registry.client import RegistryError
+from ageval.registry.media_types import AGENT_MEDIA_TYPE
+
+HubFetch = Callable[[str], Path]
+
+
+class AgentInstallCommand:
+    def __init__(
+        self,
+        client_factory: Any,
+        *,
+        hub_fetch: HubFetch | None = None,
+        cleanup_plugin_tmp: Callable[[], None] | None = None,
+    ) -> None:
+        self._client_factory = client_factory
+        self._hub_fetch = hub_fetch
+        self._cleanup_plugin_tmp = cleanup_plugin_tmp
+        self._tmp_dirs: list[tempfile.TemporaryDirectory[str]] = []
+
+    def install_agent_from_path(self, source: Path) -> dict[str, Any]:
+        """Copy a local agent package, then install declared plugins."""
+        entry = install_from_path(source)
+        try:
+            plugins = self._install_plugins(entry)
+        finally:
+            self.cleanup_tmp()
+        return _summary(entry, plugins=plugins)
+
+    def install_agent_from_registry(
+        self,
+        locator: str,
+        *,
+        registry_url: str | None = None,
+        token: str | None = None,
+    ) -> dict[str, Any]:
+        """Locator forms: ``org/agent_id@version`` or ``org/agent_id@sha256:…``."""
+        if "@" not in locator:
+            raise ConfigError(
+                "invalid_ref",
+                "locator must be package_id@version or package_id@sha256:…",
+                location=locator,
+            )
+        package_id, ver_or_digest = locator.rsplit("@", 1)
+        package_id = package_id.strip()
+        ver_or_digest = ver_or_digest.strip()
+        if not package_id or not ver_or_digest:
+            raise ConfigError("invalid_ref", "empty package_id or version", location=locator)
+        reject_reserved_harness_id(package_id)
+
+        client = self._client_factory(
+            registry_url=registry_url,
+            token=token,
+            require_token=True,
+        )
+        try:
+            extract_dir = self._download_agent(
+                client,
+                package_id=package_id,
+                version=None if ver_or_digest.startswith("sha256:") else ver_or_digest,
+                package_digest=ver_or_digest if ver_or_digest.startswith("sha256:") else None,
+                location=locator,
+            )
+            entry = install_from_path(extract_dir, agent_id=package_id)
+            plugins = self._install_plugins(entry)
+        except RegistryError as exc:
+            raise ConfigError(exc.code, exc.message, location="registry") from exc
+        finally:
+            self.cleanup_tmp()
+        summary = _summary(entry, plugins=plugins)
+        summary["from"] = locator
+        return summary
+
+    def _download_agent(
+        self,
+        client: Any,
+        *,
+        package_id: str,
+        version: str | None,
+        package_digest: str | None,
+        location: str,
+    ) -> Path:
+        try:
+            if package_digest:
+                meta = client.get_metadata(dataset_id=package_id, package_digest=package_digest)
+            else:
+                meta = client.get_metadata(dataset_id=package_id, version=version)
+            if meta.media_type != AGENT_MEDIA_TYPE:
+                raise ConfigError(
+                    "invalid_package_kind",
+                    f"release is not an agent (media_type={meta.media_type})",
+                    location=location,
+                )
+            tmp = tempfile.TemporaryDirectory(prefix="ageval-agent-dl-")
+            self._tmp_dirs.append(tmp)
+            archive_path = Path(tmp.name) / "agent.tar.gz"
+            client.fetch_content(
+                dataset_id=package_id,
+                package_digest=meta.package_digest,
+                dest=archive_path,
+            )
+        except RegistryError as exc:
+            raise ConfigError(exc.code, exc.message, location="registry") from exc
+        extract_dir = Path(tmp.name) / "pkg"
+        extract_dir.mkdir()
+        extract_archive(archive_path, extract_dir)
+        archive_path.unlink(missing_ok=True)
+        return extract_dir
+
+    def _install_plugins(self, entry: AgentIndexEntry) -> list[InstalledItem]:
+        root = package_dir(entry.agent_id, entry.version)
+        manifest = load_agent_manifest(root)
+        return install_declared_plugins(
+            manifest.binding,
+            agent_root=root,
+            hub_fetch=self._hub_fetch,
+        )
+
+    def cleanup_tmp(self) -> None:
+        while self._tmp_dirs:
+            tmp = self._tmp_dirs.pop()
+            tmp.cleanup()
+        if self._cleanup_plugin_tmp is not None:
+            self._cleanup_plugin_tmp()
+
+
+def _summary(entry: AgentIndexEntry, *, plugins: list[InstalledItem]) -> dict[str, Any]:
+    summary = entry.as_dict()
+    summary["ok"] = True
+    summary["ref"] = f"{entry.agent_id}@{entry.version}"
+    summary["plugins"] = [
+        {
+            "plugin_id": item.plugin_id,
+            "version": item.version,
+            "digest": item.digest,
+            "status": item.status,
+        }
+        for item in plugins
+    ]
+    return summary

ageval/application/agent_ops/publish.py

@@ -0,0 +1,97 @@
+"""Publish a ageval.agent/1 package to the Registry (design/14)."""
+
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from ageval.agents.manifest import load_agent_manifest
+from ageval.agents.reserved import reject_reserved_harness_id
+from ageval.config.errors import ConfigError
+from ageval.registry.agent_package import (
+    AGENT_MEDIA_TYPE,
+    build_agent_archive,
+    compute_agent_digest,
+)
+from ageval.registry.client import RegistryError
+
+
+def hub_agent_package_id(agent_id: str, *, org: str) -> str:
+    """Hub address for publish: short id concatenates; namespaced must match *org*."""
+    org_id = org.strip()
+    if not org_id:
+        raise ConfigError("org_required", "agent publish requires --org", location="registry")
+    prefix, _, name = agent_id.rpartition("/")
+    if not prefix:
+        return f"{org_id}/{agent_id}"
+    if prefix != org_id:
+        raise ConfigError(
+            "agent_org_mismatch",
+            f"agent_id prefix {prefix!r} does not match upload org {org_id!r}",
+            location="agent.yaml:/agent_id",
+        )
+    return agent_id
+
+
+class AgentPublishCommand:
+    def __init__(self, client_factory: Any) -> None:
+        self._client_factory = client_factory
+
+    def publish_agent(
+        self,
+        agent_root: Path,
+        *,
+        public: bool = False,
+        org: str | None = None,
+        registry_url: str | None = None,
+        token: str | None = None,
+    ) -> dict[str, Any]:
+        root = agent_root.expanduser().resolve(strict=False)
+        manifest = load_agent_manifest(root)
+        reject_reserved_harness_id(manifest.agent_id)
+        package_id = hub_agent_package_id(manifest.agent_id, org=org or "")
+        reject_reserved_harness_id(package_id)
+        package_digest = compute_agent_digest(root)
+        archive, blob_digest, size = build_agent_archive(root)
+
+        visibility = "public" if public else "private"
+        client = self._client_factory(
+            registry_url=registry_url,
+            token=token,
+            require_token=True,
+        )
+        with tempfile.TemporaryDirectory(prefix="ageval-agent-") as tmp:
+            archive_path = Path(tmp) / "agent.tar.gz"
+            archive_path.write_bytes(archive)
+            try:
+                info = client.publish(
+                    dataset_id=package_id,
+                    version=manifest.version,
+                    package_digest=package_digest,
+                    blob_digest=blob_digest,
+                    size=size,
+                    media_type=AGENT_MEDIA_TYPE,
+                    visibility=visibility,
+                    archive=archive_path,
+                    org_id=(org or "").strip(),
+                    package_kind="agent",
+                )
+            except RegistryError as exc:
+                raise ConfigError(exc.code, exc.message, location="registry") from exc
+
+        return {
+            "ok": True,
+            "package_kind": "agent",
+            "agent_id": manifest.agent_id,
+            "package_id": info.dataset_id,
+            "version": info.version,
+            "visibility": info.visibility,
+            "package_digest": info.package_digest,
+            "blob_digest": info.blob_digest,
+            "size": info.size,
+            "media_type": info.media_type,
+            "ref": f"{info.dataset_id}@{info.version}",
+            "org_id": info.org_id or (org or "").strip(),
+            "label": manifest.label,
+        }

ageval/application/agent_ops/resolve.py

@@ -0,0 +1,164 @@
+"""Project ``--agent`` specs into a ``ageval.profiles/1`` document (design/14).
+
+Spec forms (repeatable):
+
+* ``<ref>``            — bind ALL role slots (wildcard ``"*"`` binding)
+* ``<role>=<ref>``     — bind one role; exact role wins over wildcard
+* ``<path>``           — local dir or ``agent.yaml`` (dev loop; ref = ``file:…@dev``)
+
+``<ref>`` is a short builtin id (``pi``), a pinned local-cache id
+``local/<id>@<version>``, or ``org/name@<version>`` (short ``<id>@<version>``
+falls back to ``local/<id>``).
+The synthesized document enters the existing ``profiles_path`` lane, so lock /
+job_overlay / fingerprint behave exactly as with a hand-written profiles file.
+``agent_ref`` (``<id>@<version>+sha256:<digest12>``) is injected per binding as
+provenance; it never enters suite fingerprint identity. Optional ``model``
+patches ``binding.model`` on every role this ``--agent`` bound (package default
+otherwise).
+"""
+
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from ageval.agents.manifest import AgentManifest, load_agent_manifest
+from ageval.agents.paths import agents_root
+from ageval.agents.store import LOCAL_NAMESPACE, resolve_installed_ref
+from ageval.config.errors import ERROR_INVALID_SCHEMA, ConfigError
+from ageval.config.profiles import (
+    PROFILES_FORMAT,
+    WILDCARD_ROLE,
+    parse_job_mapping,
+    write_profiles_yaml,
+)
+from ageval.plugins.store import compute_tree_digest
+
+_ROLE_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
+_DIGEST_SHORT_HEX = 12
+
+PROJECTIONS_DIRNAME = ".projections"
+
+
+def _short_digest(digest: str) -> str:
+    if digest.startswith("sha256:"):
+        return "sha256:" + digest[len("sha256:") :][:_DIGEST_SHORT_HEX]
+    return digest[:_DIGEST_SHORT_HEX]
+
+
+def _looks_like_path(ref: str) -> bool:
+    if ref.startswith(("./", "../", "/", "~")):
+        return True
+    return Path(ref).exists()
+
+
+def _load_from_path(ref: str) -> tuple[AgentManifest, str]:
+    path = Path(ref).expanduser().resolve(strict=False)
+    manifest = load_agent_manifest(path)
+    if manifest.root is not None:
+        digest = compute_tree_digest(manifest.root)
+    else:
+        digest = "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()
+    return manifest, f"file:{path}@dev+{_short_digest(digest)}"
+
+
+def _load_from_builtin(ref: str) -> tuple[AgentManifest, str] | None:
+    from ageval.agents.reserved import builtin_harness_root, canonical_harness_id
+
+    hit = canonical_harness_id(ref)
+    if hit is None:
+        return None
+    root = builtin_harness_root(hit)
+    manifest = load_agent_manifest(root)
+    digest = compute_tree_digest(root)
+    return manifest, f"{hit}@{manifest.version}+{_short_digest(digest)}"
+
+
+def _load_ref(ref: str) -> tuple[AgentManifest, str]:
+    if _looks_like_path(ref):
+        return _load_from_path(ref)
+    if "@" not in ref and "/" not in ref:
+        loaded = _load_from_builtin(ref)
+        if loaded is not None:
+            return loaded
+    return _load_from_cache(ref)
+
+
+def _load_from_cache(ref: str) -> tuple[AgentManifest, str]:
+    agent_id, _, version = ref.rpartition("@")
+    if not agent_id or not version:
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "--agent ref must be <id>@<version> or a local path "
+            "(install first: ageval agent install …)",
+            location=ref,
+        )
+    if version.startswith("sha256:"):
+        raise ConfigError(
+            ERROR_INVALID_SCHEMA,
+            "--agent by digest requires registry install; pin @<version> locally",
+            location=ref,
+        )
+    try:
+        entry, root = resolve_installed_ref(agent_id, version)
+    except ConfigError:
+        if "/" in agent_id:
+            raise
+        entry, root = resolve_installed_ref(f"{LOCAL_NAMESPACE}/{agent_id}", version)
+    manifest = load_agent_manifest(root)
+    return manifest, f"{entry.agent_id}@{entry.version}+{_short_digest(entry.digest)}"
+
+
+def parse_agent_spec(raw: str) -> tuple[str, str]:
+    """Split one --agent value into ``(role_or_wildcard, ref)``."""
+    spec = raw.strip()
+    if not spec:
+        raise ConfigError(ERROR_INVALID_SCHEMA, "--agent must not be empty", location=raw)
+    role, sep, rest = spec.partition("=")
+    if sep and _ROLE_RE.fullmatch(role.strip()) and rest.strip():
+        return role.strip(), rest.strip()
+    return WILDCARD_ROLE, spec
+
+
+def bindings_from_agent_specs(specs: list[str]) -> dict[str, dict[str, Any]]:
+    """Resolve specs into a profiles bindings map with ``agent_ref`` injected."""
+    bindings: dict[str, dict[str, Any]] = {}
+    for raw in specs:
+        role, ref = parse_agent_spec(raw)
+        if role in bindings:
+            raise ConfigError(
+                ERROR_INVALID_SCHEMA,
+                f"duplicate --agent for role {role!r}",
+                location=raw,
+            )
+        manifest, agent_ref = _load_ref(ref)
+        binding = copy.deepcopy(manifest.binding)
+        binding["agent_ref"] = agent_ref
+        bindings[role] = binding
+    return bindings
+
+
+def resolve_agent_specs(specs: list[str], model: str | None = None) -> Path:
+    """Synthesize a ``ageval.profiles/1`` file for the profiles lane; return its path.
+
+    Written content-addressed under ``$AGEVAL_HOME/agents/.projections`` so the
+    exact document a run used stays inspectable and re-runnable (also works
+    when the Dataset itself is a registry ref with no local root).
+    ``model`` (from ``--model``) overrides ``binding.model`` on bound roles.
+    """
+    bindings = bindings_from_agent_specs(specs)
+    if model:
+        for binding in bindings.values():
+            binding["model"] = model
+    document = {"format": PROFILES_FORMAT, "agent_profiles": bindings}
+    # Re-parse for shape safety before anything reads the file.
+    parse_job_mapping(document, location="--agent")
+    canon = json.dumps(document, sort_keys=True, separators=(",", ":"))
+    name = hashlib.sha256(canon.encode("utf-8")).hexdigest()[:16]
+    out = agents_root() / PROJECTIONS_DIRNAME / f"{name}.yaml"
+    write_profiles_yaml(out, document)
+    return out

ageval/application/campaign.py

@@ -0,0 +1,149 @@
+"""Campaign / matrix expansion (v0.11) — foreground serial Trials."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from ageval.application.composition import build_run_attempt
+from ageval.config.errors import ConfigError
+from ageval.evidence.locators import campaign_locator
+
+
+@dataclass(frozen=True, slots=True)
+class MatrixAxis:
+    pointer: str
+    values: tuple[Any, ...]
+
+
+def expand_matrix(axes: list[MatrixAxis]) -> list[dict[str, Any]]:
+    """Deterministic cartesian expansion; empty axes → single empty variant."""
+    variants: list[dict[str, Any]] = [{}]
+    for axis in axes:
+        if not axis.values:
+            raise ConfigError("campaign_matrix_empty_axis", f"empty axis {axis.pointer}")
+        next_variants: list[dict[str, Any]] = []
+        for base in variants:
+            for val in axis.values:
+                item = dict(base)
+                item[axis.pointer] = val
+                next_variants.append(item)
+        variants = next_variants
+    # Stable order by JSON of pointer/value pairs
+    return sorted(variants, key=lambda d: json.dumps(d, sort_keys=True, default=str))
+
+
+def parse_matrix_arg(raw: str) -> MatrixAxis:
+    """Parse ``/parameters/x=[1,2]`` or ``/bindings/<role>/model=[…]`` matrix axis."""
+    if "=" not in raw:
+        raise ConfigError("campaign_matrix_invalid", "matrix must be pointer=json-array")
+    pointer, _, arr = raw.partition("=")
+    if not pointer.startswith("/"):
+        raise ConfigError("campaign_matrix_invalid", "pointer must start with /")
+    # #59: parameters/* and binding axes (/bindings/<role>/…)
+    from ageval.config.overrides import is_allowlisted_override_pointer
+
+    if not is_allowlisted_override_pointer(pointer):
+        raise ConfigError(
+            "campaign_matrix_pointer_unsupported",
+            f"unsupported matrix pointer: {pointer}",
+        )
+    try:
+        values = json.loads(arr)
+    except json.JSONDecodeError as exc:
+        raise ConfigError("campaign_matrix_invalid", "matrix values must be JSON") from exc
+    if not isinstance(values, list):
+        raise ConfigError("campaign_matrix_invalid", "matrix values must be a JSON array")
+    return MatrixAxis(pointer=pointer, values=tuple(values))
+
+
+async def run_campaign(
+    package_root: Path,
+    task_ids: Sequence[str] | str,
+    *,
+    matrix_args: list[str],
+    profiles_path: Path | str | None = None,
+    keep_vendor_raw: bool = False,
+) -> dict[str, Any]:
+    """Run every (task, variant) cell serially through the production Attempt.
+
+    One cell that cannot even lock is one failed cell: the others still run and
+    still keep their results. There is no second lifecycle here — each cell is
+    an ordinary ``ageval run``.
+    """
+    from ageval.registry.resolve import resolve_dataset_root
+
+    dataset_root = resolve_dataset_root(package_root)
+    tasks = [task_ids] if isinstance(task_ids, str) else list(task_ids)
+    variants = expand_matrix([parse_matrix_arg(arg) for arg in matrix_args])
+    cells = [(task, variant) for task in tasks for variant in (variants or [{}])]
+
+    trials: list[dict[str, Any]] = []
+    dataset_id = ""
+    for index, (task_id, variant) in enumerate(cells):
+        row: dict[str, Any] = {
+            "trial_index": index,
+            "task_id": task_id,
+            "variant": variant,
+        }
+        try:
+            code, result = await build_run_attempt()(
+                dataset_root,
+                task_id,
+                overrides=variant or None,
+                profiles_path=profiles_path,
+                keep_vendor_raw=keep_vendor_raw,
+            )
+        except ConfigError as exc:
+            row.update({"exit_code": 2, "status": "ERROR", "error": str(exc)})
+            trials.append(row)
+            continue
+        dataset_id = dataset_id or result.as_dict().get("dataset_id") or dataset_id
+        row.update(
+            {
+                "exit_code": code,
+                "status": result.status,
+                "score": result.score,
+                "evidence_path": _portable(result, dataset_root),
+                "logs": _portable(result, dataset_root),
+            }
+        )
+        trials.append(row)
+
+    summary: dict[str, Any] = {
+        "campaign": True,
+        "dataset_id": dataset_id,
+        "task_ids": tasks,
+        "trial_count": len(trials),
+        "trials": trials,
+        "all_pass": bool(trials) and all(t.get("status") == "PASS" for t in trials),
+    }
+    summary["summary_path"] = _write_summary(dataset_root, tasks, summary)
+    return summary
+
+
+def _portable(result: Any, dataset_root: Path) -> str | None:
+    """Never let an absolute host path into a campaign row."""
+    from ageval.evidence.locators import portable_run_locator
+
+    for raw in (result.evidence_path, result.logs):
+        if raw:
+            return portable_run_locator(raw, dataset_root=dataset_root)
+    return None
+
+
+def _write_summary(dataset_root: Path, tasks: Sequence[str], summary: dict[str, Any]) -> str:
+    campaigns = dataset_root / ".ageval" / "campaigns"
+    campaigns.mkdir(parents=True, exist_ok=True)
+    name = "-".join(tasks) if len(tasks) <= 3 else f"{len(tasks)}-tasks"
+    out = campaigns / f"summary_{name}_{summary['trial_count']}.json"
+    tmp = out.with_suffix(".tmp")
+    tmp.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    tmp.replace(out)
+    try:
+        return out.relative_to(dataset_root.resolve()).as_posix()
+    except ValueError:
+        return campaign_locator(out.name)