agentinputlab 0.1.0a1__py3-none-any.whl

agentinputlab/experiments/auxiliary_matrix.py

@@ -0,0 +1,203 @@
+"""Capability-locked 27-method by seven-static-dataset execution matrix.
+
+The matrix distinguishes structural eligibility from execution evidence.  A
+static benchmark is eligible only when it declares the method's semantic
+family.  Ineligible pairs remain explicit rows; they are never converted into
+zero scores or silently omitted from denominators.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from importlib import resources
+from types import MappingProxyType
+from typing import Any
+
+from agentinputlab.benchmarks import BenchmarkRegistry, production_benchmark_specs
+from agentinputlab.contracts.enums import ProfileKind
+from agentinputlab.contracts.errors import ArtifactError, ValidationError
+from agentinputlab.contracts.serialization import content_hash, freeze_mapping
+from agentinputlab.registry import AlgorithmRegistry
+
+AUXILIARY_MATRIX_PROTOCOL = "agentinputlab.auxiliary-method-cell-matrix.v1"
+AUXILIARY_PROFILE_PROTOCOL = "agentinputlab.auxiliary-method-profiles.v1"
+
+
+@dataclass(frozen=True, slots=True)
+class AuxiliaryMethodProfile:
+    method_id: str
+    release_id: str
+    profile_kind: ProfileKind
+
+    def __post_init__(self) -> None:
+        for label, value in (
+            ("method_id", self.method_id),
+            ("release_id", self.release_id),
+        ):
+            if not isinstance(value, str) or not value.strip():
+                raise ValidationError(f"auxiliary profile {label} must be non-empty")
+        if not isinstance(self.profile_kind, ProfileKind):
+            raise TypeError("auxiliary profile_kind must be ProfileKind")
+
+
+def auxiliary_static_cells() -> Mapping[str, str]:
+    """Return the exact seven production static cells and frozen splits."""
+
+    cells = {
+        spec.cell_id: spec.split
+        for spec in production_benchmark_specs()
+        if spec.adapter_kind in {"retrieval", "qa"}
+    }
+    if len(cells) != 7:
+        raise ValidationError(f"auxiliary matrix requires seven static cells, got {len(cells)}")
+    return MappingProxyType(dict(sorted(cells.items())))
+
+
+def load_auxiliary_method_profiles() -> Mapping[str, AuxiliaryMethodProfile]:
+    """Load and validate the packaged method-to-executable-profile selection."""
+
+    try:
+        source = resources.files("agentinputlab.registry").joinpath(
+            "data/auxiliary_method_profiles.json"
+        )
+        raw_bytes = source.read_bytes()
+        raw = json.loads(raw_bytes.decode("utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise ArtifactError("auxiliary method profile catalog is unavailable") from exc
+    if (
+        not isinstance(raw, dict)
+        or set(raw) != {"schema_version", "protocol", "method_count", "methods"}
+        or raw.get("schema_version") != 1
+        or raw.get("protocol") != AUXILIARY_PROFILE_PROTOCOL
+        or not isinstance(raw.get("methods"), list)
+        or raw.get("method_count") != len(raw["methods"])
+    ):
+        raise ValidationError("auxiliary method profile catalog schema is invalid")
+
+    registry = AlgorithmRegistry.default()
+    profiles: dict[str, AuxiliaryMethodProfile] = {}
+    for index, item in enumerate(raw["methods"]):
+        if not isinstance(item, dict) or set(item) != {
+            "method_id",
+            "release_id",
+            "profile_kind",
+        }:
+            raise ValidationError(f"auxiliary method profile row {index} is invalid")
+        try:
+            profile = AuxiliaryMethodProfile(
+                method_id=item["method_id"],
+                release_id=item["release_id"],
+                profile_kind=ProfileKind(item["profile_kind"]),
+            )
+            method, release = registry.get_release(profile.release_id)
+        except (KeyError, TypeError, ValueError) as exc:
+            raise ValidationError(
+                f"auxiliary method profile row {index} cannot be resolved"
+            ) from exc
+        if profile.method_id in profiles:
+            raise ValidationError("auxiliary method profile catalog contains duplicates")
+        if method.method_id != profile.method_id:
+            raise ValidationError("auxiliary profile release belongs to another method")
+        if profile.profile_kind not in release.profile_modes:
+            raise ValidationError("auxiliary profile kind is unsupported by its release")
+        if release.entrypoint is None:
+            raise ValidationError("auxiliary profile release has no executable entrypoint")
+        profiles[profile.method_id] = profile
+
+    registered = {method.method_id for method in registry.list_methods()}
+    if set(profiles) != registered or len(profiles) != 27:
+        raise ValidationError("auxiliary profiles must cover the exact 27-method registry")
+    return MappingProxyType(dict(sorted(profiles.items())))
+
+
+def build_auxiliary_method_cell_matrix() -> Mapping[str, Any]:
+    """Build the deterministic 27x7 eligibility and launch matrix."""
+
+    registry = AlgorithmRegistry.default()
+    benchmark_registry = BenchmarkRegistry.default()
+    profiles = load_auxiliary_method_profiles()
+    cells = auxiliary_static_cells()
+    rows: list[dict[str, Any]] = []
+    family_method_counts: dict[str, int] = {}
+    for method in sorted(registry.list_methods(), key=lambda value: value.method_id):
+        family = method.semantic_family.value
+        family_method_counts[family] = family_method_counts.get(family, 0) + 1
+        profile = profiles[method.method_id]
+        for cell_id, split in cells.items():
+            cell = benchmark_registry.get(cell_id)
+            eligible = method.semantic_family in cell.families
+            reason = (
+                None
+                if eligible
+                else (
+                    f"{cell_id} does not declare {method.semantic_family.value} support; "
+                    "the pair is structurally ineligible"
+                )
+            )
+            rows.append(
+                {
+                    "method_id": method.method_id,
+                    "display_name": method.display_name,
+                    "release_id": profile.release_id,
+                    "profile_kind": profile.profile_kind.value,
+                    "family": family,
+                    "benchmark_cell_id": cell_id,
+                    "split": split,
+                    "benchmark_family_support": tuple(
+                        item.value for item in sorted(cell.families, key=lambda value: value.value)
+                    ),
+                    "eligible": eligible,
+                    "state": "launchable" if eligible else "structurally_ineligible",
+                    "reason": reason,
+                    "one_click_command": (
+                        "python scripts/experiments/run_auxiliary_method_cell.py "
+                        "<frozen-campaign.json> "
+                        f"--method {method.method_id} --cell {cell_id}"
+                        if eligible
+                        else None
+                    ),
+                }
+            )
+    eligible_count = sum(bool(row["eligible"]) for row in rows)
+    if len(rows) != 189 or eligible_count != 117:
+        raise ValidationError(
+            f"auxiliary matrix invariant failed: rows={len(rows)}, eligible={eligible_count}"
+        )
+    payload: dict[str, Any] = {
+        "schema_version": 1,
+        "protocol": AUXILIARY_MATRIX_PROTOCOL,
+        "record_type": "EXECUTION_DESIGN",
+        "observed_results": False,
+        "method_count": 27,
+        "dataset_count": 7,
+        "row_count": len(rows),
+        "eligible_pair_count": eligible_count,
+        "structurally_ineligible_pair_count": len(rows) - eligible_count,
+        "family_method_counts": dict(sorted(family_method_counts.items())),
+        "cells": dict(cells),
+        "rows": rows,
+    }
+    payload["matrix_hash"] = content_hash(payload)
+    return freeze_mapping(payload)
+
+
+def auxiliary_pair(method_id: str, cell_id: str) -> Mapping[str, Any]:
+    """Resolve one row and reject unknown identities without guessing."""
+
+    for row in build_auxiliary_method_cell_matrix()["rows"]:
+        if row["method_id"] == method_id and row["benchmark_cell_id"] == cell_id:
+            return row
+    raise KeyError(f"unknown auxiliary method-cell pair: {method_id} x {cell_id}")
+
+
+__all__ = [
+    "AUXILIARY_MATRIX_PROTOCOL",
+    "AUXILIARY_PROFILE_PROTOCOL",
+    "AuxiliaryMethodProfile",
+    "auxiliary_pair",
+    "auxiliary_static_cells",
+    "build_auxiliary_method_cell_matrix",
+    "load_auxiliary_method_profiles",
+]

agentinputlab/experiments/bindings.py

@@ -0,0 +1,430 @@
+"""Content-bound receipts for assets admitted by experiment preflight.
+
+The manifest deliberately stores environment-variable *names*, not machine-local
+paths or secret values.  This module resolves those names at preflight time and
+turns every admitted prerequisite into a canonical, secret-safe identity.  The
+resulting receipt is embedded in the effective experiment-driver fingerprint.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import stat
+import subprocess
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from agentinputlab.contracts.errors import ArtifactError, ValidationError
+from agentinputlab.contracts.serialization import (
+    content_hash,
+    freeze_mapping,
+    to_jsonable,
+)
+
+from .schema import ExperimentManifest, PrerequisiteKind, require_sha256
+
+_GIT_OBJECT_ID_LENGTHS = frozenset({40, 64})
+_CHECKOUT_SUFFIX = "_CHECKOUT"
+
+
+@dataclass(frozen=True, slots=True)
+class FrozenBindingEntry:
+    """One prerequisite identity without its raw environment value."""
+
+    name: str
+    kind: str
+    identity: Mapping[str, Any]
+    schema_version: int = 1
+
+    def __post_init__(self) -> None:
+        if self.schema_version != 1:
+            raise ValidationError("unsupported frozen binding entry schema")
+        if not isinstance(self.name, str) or not self.name.strip():
+            raise ValidationError("frozen binding entry requires a name")
+        if not isinstance(self.kind, str) or not self.kind.strip():
+            raise ValidationError("frozen binding entry requires a kind")
+        try:
+            identity = freeze_mapping(self.identity)
+        except (TypeError, ValueError) as exc:
+            raise ValidationError("frozen binding identity must be canonical JSON") from exc
+        if not identity:
+            raise ValidationError("frozen binding identity must not be empty")
+        object.__setattr__(self, "identity", identity)
+
+    @property
+    def binding_hash(self) -> str:
+        return content_hash(
+            {
+                "schema_version": self.schema_version,
+                "name": self.name,
+                "kind": self.kind,
+                "identity": self.identity,
+            }
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "name": self.name,
+            "kind": self.kind,
+            "identity": to_jsonable(self.identity),
+            "binding_hash": self.binding_hash,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class FrozenBindingReceipt:
+    """Canonical receipt for all manifest prerequisites accepted by preflight."""
+
+    manifest_hash: str
+    bindings: tuple[FrozenBindingEntry, ...]
+    schema_version: int = 1
+
+    def __post_init__(self) -> None:
+        if self.schema_version != 1:
+            raise ValidationError("unsupported frozen binding receipt schema")
+        require_sha256(self.manifest_hash, label="binding receipt manifest_hash")
+        bindings = tuple(self.bindings)
+        if any(not isinstance(item, FrozenBindingEntry) for item in bindings):
+            raise ValidationError("frozen binding receipt entries must be typed")
+        identities = [(item.kind, item.name) for item in bindings]
+        if len(identities) != len(set(identities)):
+            raise ValidationError("frozen binding receipt entries must be unique")
+        canonical = tuple(sorted(bindings, key=lambda item: (item.kind, item.name)))
+        object.__setattr__(self, "bindings", canonical)
+
+    @property
+    def receipt_hash(self) -> str:
+        return content_hash(
+            {
+                "schema_version": self.schema_version,
+                "manifest_hash": self.manifest_hash,
+                "bindings": [
+                    {
+                        "schema_version": item.schema_version,
+                        "name": item.name,
+                        "kind": item.kind,
+                        "identity": item.identity,
+                    }
+                    for item in self.bindings
+                ],
+            }
+        )
+
+    def binding(self, name: str, *, kind: str | None = None) -> FrozenBindingEntry:
+        matches = tuple(
+            item
+            for item in self.bindings
+            if item.name == name and (kind is None or item.kind == kind)
+        )
+        if len(matches) != 1:
+            raise ArtifactError(f"frozen binding receipt has no unique entry for {name!r}")
+        return matches[0]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "manifest_hash": self.manifest_hash,
+            "bindings": [item.to_dict() for item in self.bindings],
+            "receipt_hash": self.receipt_hash,
+        }
+
+
+def resolve_frozen_binding_receipt(
+    manifest: ExperimentManifest,
+    *,
+    environment: Mapping[str, str] | None = None,
+) -> FrozenBindingReceipt:
+    """Resolve and independently attest every declared prerequisite.
+
+    Checkout directories are required to be clean Git roots.  Other directories
+    receive a deterministic content-tree hash.  Raw paths and environment values
+    never enter the receipt; only their SHA-256 identities do.
+    """
+
+    if not isinstance(manifest, ExperimentManifest):
+        raise TypeError("binding resolution requires ExperimentManifest")
+    env = dict(os.environ if environment is None else environment)
+    expectations = _checkout_expectations(manifest, environment=env)
+    declared_directories = {
+        item.name for item in manifest.prerequisites if item.kind is PrerequisiteKind.DIRECTORY_ENV
+    }
+    unknown_expectations = sorted(set(expectations) - declared_directories)
+    if unknown_expectations:
+        raise ValidationError(
+            "checkout_expected_revisions references undeclared directories: "
+            + ", ".join(unknown_expectations)
+        )
+
+    bindings: list[FrozenBindingEntry] = []
+    for prerequisite in manifest.prerequisites:
+        if prerequisite.kind is PrerequisiteKind.ENV:
+            raw = env.get(prerequisite.name)
+            if not raw:
+                raise ArtifactError(f"missing environment prerequisite {prerequisite.name}")
+            encoded = raw.encode("utf-8")
+            identity = {
+                "identity_mode": "value_sha256",
+                "value_sha256": hashlib.sha256(encoded).hexdigest(),
+                "value_bytes": len(encoded),
+            }
+        elif prerequisite.kind is PrerequisiteKind.EXECUTABLE:
+            executable = shutil.which(prerequisite.name, path=env.get("PATH"))
+            if executable is None:
+                raise ArtifactError(f"missing executable prerequisite {prerequisite.name}")
+            candidate = Path(executable).expanduser().resolve()
+            if not candidate.is_file():
+                raise ArtifactError(f"executable prerequisite {prerequisite.name} is not a file")
+            identity = {
+                "identity_mode": "executable_sha256",
+                "resolved_path_sha256": _sha256_text(str(candidate)),
+                "content_sha256": _sha256_file(candidate),
+                "size_bytes": candidate.stat().st_size,
+            }
+        else:
+            raw = env.get(prerequisite.name)
+            if not raw:
+                raise ArtifactError(f"missing environment prerequisite {prerequisite.name}")
+            candidate = Path(raw).expanduser().resolve()
+            if prerequisite.kind is PrerequisiteKind.FILE_ENV:
+                if not candidate.is_file():
+                    raise ArtifactError(f"{prerequisite.name} does not identify an existing file")
+                actual = _sha256_file(candidate)
+                if prerequisite.sha256_env is not None:
+                    expected = env.get(prerequisite.sha256_env)
+                    if not expected:
+                        raise ArtifactError(
+                            f"missing checksum prerequisite {prerequisite.sha256_env}"
+                        )
+                    require_sha256(
+                        expected,
+                        label=f"environment {prerequisite.sha256_env}",
+                    )
+                    if actual != expected:
+                        raise ArtifactError(
+                            f"{prerequisite.name} checksum differs from {prerequisite.sha256_env}"
+                        )
+                identity = {
+                    "identity_mode": "file_sha256",
+                    "resolved_path_sha256": _sha256_text(str(candidate)),
+                    "content_sha256": actual,
+                    "size_bytes": candidate.stat().st_size,
+                    "checksum_environment": prerequisite.sha256_env,
+                }
+            else:
+                if not candidate.is_dir():
+                    raise ArtifactError(
+                        f"{prerequisite.name} does not identify an existing directory"
+                    )
+                expected_revision = expectations.get(prerequisite.name)
+                if expected_revision is not None or prerequisite.name.endswith(_CHECKOUT_SUFFIX):
+                    identity = _git_checkout_identity(
+                        candidate,
+                        expected_revision=expected_revision,
+                    )
+                else:
+                    identity = {
+                        "identity_mode": "directory_tree_sha256",
+                        "resolved_path_sha256": _sha256_text(str(candidate)),
+                        "tree_sha256": _sha256_directory_tree(candidate),
+                    }
+        bindings.append(
+            FrozenBindingEntry(
+                name=prerequisite.name,
+                kind=prerequisite.kind.value,
+                identity=identity,
+            )
+        )
+    return FrozenBindingReceipt(
+        manifest_hash=manifest.manifest_hash,
+        bindings=tuple(bindings),
+    )
+
+
+def build_guarded_driver_fingerprint(
+    operator_fingerprint: Mapping[str, Any],
+    binding_receipt: FrozenBindingReceipt,
+    *,
+    parity_contract: Mapping[str, Any] | None = None,
+) -> Mapping[str, Any]:
+    """Build the exact fingerprint persisted by the guarded runtime driver."""
+
+    if not isinstance(binding_receipt, FrozenBindingReceipt):
+        raise TypeError("guarded fingerprint requires FrozenBindingReceipt")
+    try:
+        operator = freeze_mapping(operator_fingerprint)
+        contract = freeze_mapping(parity_contract or {})
+    except (TypeError, ValueError) as exc:
+        raise ValidationError("guarded driver identity must be canonical JSON") from exc
+    if not operator:
+        raise ValidationError("operator driver fingerprint must not be empty")
+    return freeze_mapping(
+        {
+            "schema_version": 1,
+            "guard": "agentinputlab.frozen_binding_driver_guard",
+            "operator_driver_fingerprint": operator,
+            "frozen_binding_receipt": binding_receipt.to_dict(),
+            "frozen_binding_receipt_hash": binding_receipt.receipt_hash,
+            "parity_contract": contract,
+        }
+    )
+
+
+def _checkout_expectations(
+    manifest: ExperimentManifest,
+    *,
+    environment: Mapping[str, str],
+) -> dict[str, str]:
+    raw = manifest.metadata.get("checkout_expected_revisions", {})
+    if not isinstance(raw, Mapping):
+        raise ValidationError("checkout_expected_revisions metadata must be an object")
+    result: dict[str, str] = {}
+    for name, revision in raw.items():
+        if not isinstance(name, str) or not name:
+            raise ValidationError("checkout expectation names must be non-empty strings")
+        if (
+            not isinstance(revision, str)
+            or len(revision) not in _GIT_OBJECT_ID_LENGTHS
+            or any(character not in "0123456789abcdef" for character in revision)
+        ):
+            raise ValidationError(
+                f"checkout expectation {name} must be a full lowercase Git object ID"
+            )
+        result[name] = revision
+
+    dynamic = manifest.metadata.get("checkout_expected_revision_from_env", {})
+    if not isinstance(dynamic, Mapping):
+        raise ValidationError("checkout_expected_revision_from_env metadata must be an object")
+    declared_value_environments = {
+        item.name for item in manifest.prerequisites if item.kind is PrerequisiteKind.ENV
+    }
+    for checkout_environment, revision_environment in dynamic.items():
+        if not isinstance(checkout_environment, str) or not checkout_environment:
+            raise ValidationError("dynamic checkout expectation names must be non-empty strings")
+        if not isinstance(revision_environment, str) or not revision_environment:
+            raise ValidationError(
+                "dynamic checkout revision environments must be non-empty strings"
+            )
+        if revision_environment not in declared_value_environments:
+            raise ValidationError(
+                "checkout_expected_revision_from_env references an undeclared "
+                f"value prerequisite: {revision_environment}"
+            )
+        revision = environment.get(revision_environment)
+        if not revision:
+            raise ArtifactError(f"missing checkout revision prerequisite {revision_environment}")
+        if (
+            not isinstance(revision, str)
+            or len(revision) not in _GIT_OBJECT_ID_LENGTHS
+            or any(character not in "0123456789abcdef" for character in revision)
+        ):
+            raise ValidationError(
+                f"environment {revision_environment} must be a full lowercase Git object ID"
+            )
+        existing = result.get(checkout_environment)
+        if existing is not None and existing != revision:
+            raise ValidationError(
+                f"dynamic checkout expectation for {checkout_environment} "
+                "conflicts with the manifest-pinned revision"
+            )
+        result[checkout_environment] = revision
+    return result
+
+
+def _git_checkout_identity(
+    path: Path,
+    *,
+    expected_revision: str | None,
+) -> Mapping[str, Any]:
+    if shutil.which("git") is None:
+        raise ArtifactError("git is required to attest checkout prerequisites")
+    top_level = Path(_git(path, "rev-parse", "--show-toplevel")).resolve()
+    if top_level != path:
+        raise ArtifactError("checkout prerequisite must identify the Git repository root")
+    head = _git(path, "rev-parse", "HEAD")
+    tree = _git(path, "rev-parse", "HEAD^{tree}")
+    for label, value in (("HEAD", head), ("tree", tree)):
+        if len(value) not in _GIT_OBJECT_ID_LENGTHS or any(
+            character not in "0123456789abcdef" for character in value
+        ):
+            raise ArtifactError(f"checkout {label} is not a full Git object ID")
+    if expected_revision is not None and head != expected_revision:
+        raise ArtifactError("checkout HEAD differs from the manifest-pinned revision")
+    status = _git(path, "status", "--porcelain=v1", "--untracked-files=all")
+    if status:
+        raise ArtifactError("checkout prerequisite must have a clean working tree")
+    return freeze_mapping(
+        {
+            "identity_mode": "clean_git_checkout",
+            "resolved_path_sha256": _sha256_text(str(path)),
+            "git_head": head,
+            "git_tree": tree,
+            "git_clean": True,
+            "expected_revision": expected_revision,
+        }
+    )
+
+
+def _git(path: Path, *arguments: str) -> str:
+    try:
+        completed = subprocess.run(
+            ("git", "-C", str(path), *arguments),
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+    except (OSError, subprocess.SubprocessError) as exc:
+        raise ArtifactError(f"cannot attest Git checkout with {' '.join(arguments)}") from exc
+    return completed.stdout.strip()
+
+
+def _sha256_directory_tree(root: Path) -> str:
+    digest = hashlib.sha256()
+    for candidate in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()):
+        relative = candidate.relative_to(root).as_posix()
+        mode = candidate.lstat().st_mode
+        if stat.S_ISDIR(mode):
+            digest.update(f"D\0{relative}\0".encode())
+        elif stat.S_ISREG(mode):
+            digest.update(f"F\0{relative}\0".encode())
+            digest.update(_sha256_file(candidate).encode())
+            digest.update(b"\0")
+        elif stat.S_ISLNK(mode):
+            target = candidate.resolve(strict=True)
+            if not target.is_file():
+                raise ArtifactError(f"directory binding symlink must resolve to a file: {relative}")
+            digest.update(f"L\0{relative}\0".encode())
+            digest.update(_sha256_text(os.readlink(candidate)).encode())
+            digest.update(b"\0")
+            digest.update(_sha256_file(target).encode())
+            digest.update(b"\0")
+        else:
+            raise ArtifactError(
+                f"directory binding contains unsupported filesystem entry: {relative}"
+            )
+    return digest.hexdigest()
+
+
+def _sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for block in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def _sha256_text(value: str) -> str:
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+__all__ = [
+    "FrozenBindingEntry",
+    "FrozenBindingReceipt",
+    "build_guarded_driver_fingerprint",
+    "resolve_frozen_binding_receipt",
+]