agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/amend.py

@@ -0,0 +1,312 @@
+"""Changing the contract after the fact, and being honest about having done it.
+
+The contract is what the agent verifiably is, read from its own source. That makes it the thing
+everything downstream is confined to, and it is why the harness cannot invent a tool or a value.
+
+But it is not permanent. Two situations genuinely require changing it, and they are different:
+
+- **It was read wrong.** Stage one missed a value the agent really accepts. Correcting that is
+  restoring the truth, and the correction should come from the source.
+- **The agent is being changed.** Somebody adds an item to the world because the real menu is
+  gaining one. The world and the action space have to move together: an item the world holds but
+  the agent cannot name is dead data, and a scenario about it can only fail.
+
+Either way the amendment is recorded on the contract itself rather than blended into what was
+read, so that a month later it is still possible to tell what came from the agent and what came
+from us. That distinction is the whole value of the contract; quietly widening it would make it
+the same kind of guess it exists to prevent.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from .contract import MODALITIES, AgentContract, validate_contract
+
+CONTRACT = "contract.json"
+
+
+def widen(
+    contract: AgentContract,
+    destination: Path,
+    *,
+    tool_name: str,
+    argument: str,
+    values: list[str],
+    why: str,
+) -> tuple[bool, str]:
+    """Let a tool's argument accept values it did not before.
+
+    Amends the contract the stage is holding, then persists it. Loading a second copy from disk
+    and writing that back would leave the stage still working from the old one, so the world it
+    goes on to check would be checked against an action space that no longer matches.
+
+    Returns whether it was amended, and what happened.
+    """
+    spec = next((tool for tool in contract.tools if tool.name == tool_name), None)
+    if spec is None:
+        return False, (
+            f"{tool_name!r} is not a tool this agent has. It has: "
+            f"{', '.join(sorted(contract.tool_names()))}"
+        )
+    if argument not in spec.args:
+        return False, (
+            f"{tool_name} takes no argument called {argument!r}. It takes: "
+            f"{', '.join(spec.args) or 'nothing'}"
+        )
+    if not why.strip():
+        return (
+            False,
+            "say why: an unexplained amendment is indistinguishable from a guess",
+        )
+
+    existing = spec.arg_values.get(argument)
+    current = list(existing) if isinstance(existing, (list, tuple)) else []
+    fresh = [value for value in values if value and value not in current]
+    if not fresh:
+        return False, f"{argument} already accepts {', '.join(values) or 'nothing new'}"
+
+    spec.arg_values[argument] = [*current, *fresh]
+    contract.amendments.append(
+        f"{tool_name}.{argument} widened by {', '.join(fresh)}: {why.strip()}"
+    )
+
+    problems = validate_contract(contract)
+    if problems:
+        spec.arg_values[argument] = current
+        contract.amendments.pop()
+        return False, "the amended contract would not be valid: " + "; ".join(problems)
+
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    (destination / CONTRACT).write_text(
+        json.dumps(contract.model_dump(), indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    return True, (
+        f"{tool_name}.{argument} now accepts {', '.join(fresh)}. "
+        f"{len(contract.amendments)} amendment(s) recorded on the contract."
+    )
+
+
+def add_rule(
+    contract: AgentContract, destination: Path, *, rule: str, why: str
+) -> tuple[bool, str]:
+    """Give the agent a rule its source did not state.
+
+    A hard constraint is not decoration: the agent under test is told it, and the judge grades
+    against it. So this is a real change to what is being tested, and like a widened argument it
+    is recorded rather than blended into what was read from the source.
+    """
+    rule = rule.strip()
+    if not rule:
+        return False, "no rule given"
+    if not why.strip():
+        return False, "say why: an unexplained rule is indistinguishable from a guess"
+    if any(rule.lower() == existing.lower() for existing in contract.hard_constraints):
+        return False, f"the agent already has that rule: {rule}"
+
+    contract.hard_constraints.append(rule)
+    contract.amendments.append(f"rule added — {rule}: {why.strip()}")
+    problems = validate_contract(contract)
+    if problems:
+        contract.hard_constraints.pop()
+        contract.amendments.pop()
+        return False, "the amended contract would not be valid: " + "; ".join(problems)
+
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    (destination / CONTRACT).write_text(
+        json.dumps(contract.model_dump(), indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    return True, (
+        f"added. The agent now has {len(contract.hard_constraints)} rules, and this one is "
+        "graded from here on."
+    )
+
+
+def set_modality(
+    contract: AgentContract, destination: Path, *, modality: str, why: str
+) -> tuple[bool, str]:
+    """Correct how a person actually reaches this agent.
+
+    Worth its own amendment because modality is the one field that reroutes everything: it picks
+    the world, the simulator and the transport, so a wrong value does not degrade a run, it runs
+    a different test. And it is the field the source is least able to settle. An agent's code
+    reads the same whether it is answering a chat window or a phone call, so a reader with no
+    other evidence concludes whatever the repository looks like, which for a text benchmark is
+    text, even when the person asking said they had deployed it to a phone number.
+
+    Where the agent is deployed is a fact about somebody's setup rather than about the source, so
+    when the two disagree the person is right and the source is describing a different runtime of
+    the same agent. Recorded like every other amendment, because it is still a change to what was
+    read.
+    """
+    named = (modality or "").strip().lower()
+    if named not in MODALITIES:
+        return (
+            False,
+            f"{named!r} is not a modality. It is one of: {', '.join(MODALITIES)}",
+        )
+    if not why.strip():
+        return False, "say why: modality decides how every scenario is run"
+    if named == contract.modality:
+        return False, f"the contract already says {named}"
+
+    was = contract.modality
+    contract.modality = named
+    contract.amendments.append(f"modality {was} -> {named}: {why.strip()}")
+    problems = validate_contract(contract)
+    if problems:
+        contract.modality = was
+        contract.amendments.pop()
+        return False, "the amended contract would not be valid: " + "; ".join(problems)
+
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    (destination / CONTRACT).write_text(
+        json.dumps(contract.model_dump(), indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    reached = {
+        "voice": "a call placed to the agent where it is hosted, its own tools answered over a "
+        "webhook",
+        "chat": "a typed conversation, the agent reconstructed here from its contract",
+        "browser": "a browser the agent drives",
+    }[named]
+    return True, f"modality is now {named}. Every scenario will be run as {reached}."
+
+
+def drop_rule(
+    contract: AgentContract, destination: Path, *, rule: str, why: str
+) -> tuple[bool, str]:
+    """Take away a rule the agent does not really have.
+
+    Stage one can misread a comment as a constraint, and a rule nobody has is worse than a
+    missing one: the agent under test is told to obey it and the judge fails it for not doing
+    something it was never supposed to do.
+    """
+    if not why.strip():
+        return False, "say why: removing a rule changes what is being graded"
+    match = next(
+        (
+            existing
+            for existing in contract.hard_constraints
+            if existing.lower() == rule.strip().lower()
+        ),
+        None,
+    ) or next(
+        (
+            existing
+            for existing in contract.hard_constraints
+            if rule.strip().lower() in existing.lower()
+        ),
+        None,
+    )
+    if match is None:
+        return False, (
+            "no rule like that. It has:\n  - "
+            + "\n  - ".join(contract.hard_constraints)
+        )
+    contract.hard_constraints.remove(match)
+    contract.amendments.append(f"rule removed — {match}: {why.strip()}")
+    _persist(contract, destination)
+    return True, f"removed. {len(contract.hard_constraints)} rules left"
+
+
+def fix_tool(
+    contract: AgentContract,
+    destination: Path,
+    *,
+    tool_name: str,
+    why: str,
+    args: list[str] | None = None,
+    arg_types: dict[str, str] | None = None,
+    description: str = "",
+    remove: bool = False,
+) -> tuple[bool, str]:
+    """Correct a tool that was read wrong, or take away one the agent does not have.
+
+    The most damaging thing stage one can get wrong. Every argument name flows into the world's
+    handlers, the probes and the scenarios, so a tool recorded with the wrong argument produces
+    a world that refuses everything and a suite that blames the agent for it.
+    """
+    if not why.strip():
+        return False, "say why: this changes what everything downstream is built from"
+    spec = next((tool for tool in contract.tools if tool.name == tool_name), None)
+    if spec is None:
+        return False, (
+            f"{tool_name!r} is not a tool this agent has. It has: "
+            f"{', '.join(sorted(contract.tool_names()))}"
+        )
+
+    if remove:
+        contract.tools.remove(spec)
+        contract.amendments.append(f"tool removed — {tool_name}: {why.strip()}")
+        problems = validate_contract(contract)
+        if problems:
+            contract.tools.append(spec)
+            contract.amendments.pop()
+            return False, "cannot remove it: " + "; ".join(problems)
+        _persist(contract, destination)
+        return True, f"{tool_name} removed. {len(contract.tools)} tools left"
+
+    changed = []
+    if args is not None:
+        # Values recorded against an argument that no longer exists would silently be lost, so
+        # they are carried across by name and anything orphaned is said out loud.
+        orphaned = sorted(set(spec.arg_values) - set(args))
+        spec.args = list(args)
+        spec.arg_types = {k: v for k, v in spec.arg_types.items() if k in spec.args}
+        spec.arg_values = {k: v for k, v in spec.arg_values.items() if k in spec.args}
+        changed.append(f"arguments are now {', '.join(args)}")
+        if orphaned:
+            changed.append(f"dropped values recorded for {', '.join(orphaned)}")
+    if arg_types:
+        unknown = sorted(set(arg_types) - set(spec.args))
+        if unknown:
+            return False, f"{tool_name} takes no argument called {', '.join(unknown)}"
+        spec.arg_types.update(arg_types)
+        changed.append("types updated")
+    if description:
+        spec.description = description
+        changed.append("description updated")
+    if not changed:
+        return False, "nothing to change: give args, arg_types, description, or remove"
+
+    contract.amendments.append(f"tool corrected — {tool_name}: {why.strip()}")
+    problems = validate_contract(contract)
+    if problems:
+        return False, "the amended contract would not be valid: " + "; ".join(problems)
+    _persist(contract, destination)
+    return True, f"{tool_name}: {', '.join(changed)}"
+
+
+def _persist(contract: AgentContract, destination: Path) -> None:
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    (destination / CONTRACT).write_text(
+        json.dumps(contract.model_dump(), indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+
+
+def not_offered(contract: AgentContract, candidates: dict[str, set[str]]) -> list[str]:
+    """For each argument, the candidate values the contract does not let the agent send.
+
+    ``candidates`` maps an argument name to identifiers found in the world that plausibly belong
+    to it. Kept as an argument rather than inferred here, because which column feeds which
+    argument is knowledge about one agent, not something a schema states.
+    """
+    missing: list[str] = []
+    for tool in contract.tools:
+        for argument, values in (tool.arg_values or {}).items():
+            if argument not in candidates or not isinstance(values, (list, tuple)):
+                continue
+            permitted = {str(value) for value in values}
+            absent = sorted(candidates[argument] - permitted)
+            if absent:
+                missing.append(f"{tool.name}.{argument}: {', '.join(absent)}")
+    return missing

fi/alk/harness/artifacts.py

@@ -0,0 +1,319 @@
+"""Content-addressed artifact manifests and call-evidence integrity checks."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import mimetypes
+import os
+import re
+from pathlib import Path, PurePosixPath
+from pydantic import BaseModel, Field, model_validator
+
+ARTIFACT_MANIFEST_VERSION = "futureagi.harness-artifacts.v1"
+ARTIFACT_MANIFEST_NAME = "artifact-manifest.json"
+
+
+class ArtifactIntegrityError(RuntimeError):
+    pass
+
+
+class ArtifactRecord(BaseModel):
+    path: str
+    sha256: str
+    size: int = Field(ge=0)
+    kind: str
+    media_type: str
+    scenario: str | None = None
+
+    @model_validator(mode="after")
+    def _safe_path(self) -> "ArtifactRecord":
+        path = PurePosixPath(self.path)
+        if path.is_absolute() or ".." in path.parts or not self.path:
+            raise ValueError(f"artifact_path_unsafe: {self.path}")
+        if not re.fullmatch(r"[0-9a-f]{64}", self.sha256):
+            raise ValueError(f"artifact_digest_invalid: {self.path}")
+        return self
+
+
+class ScenarioEvidence(BaseModel):
+    scenario: str
+    result_path: str
+    canonical_status: str
+    transcript_path: str | None = None
+    recording_paths: list[str] = Field(default_factory=list)
+    tool_trace_path: str | None = None
+
+
+class ArtifactManifest(BaseModel):
+    schema_version: str = ARTIFACT_MANIFEST_VERSION
+    run_id: str
+    digest: str
+    files: list[ArtifactRecord] = Field(default_factory=list)
+    scenarios: list[ScenarioEvidence] = Field(default_factory=list)
+    total_bytes: int = Field(ge=0)
+    complete: bool = True
+
+    @model_validator(mode="after")
+    def _valid(self) -> "ArtifactManifest":
+        if self.schema_version != ARTIFACT_MANIFEST_VERSION:
+            raise ValueError("artifact_manifest_version_unsupported")
+        if not re.fullmatch(r"sha256:[0-9a-f]{64}", self.digest):
+            raise ValueError("artifact_manifest_digest_invalid")
+        paths = [item.path for item in self.files]
+        if len(paths) != len(set(paths)):
+            raise ValueError("artifact_manifest_duplicate_path")
+        return self
+
+
+_SECRET_CONTENT = (
+    re.compile(rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
+    re.compile(rb"\b(?:ghp|github_pat)_[A-Za-z0-9_]{20,}\b"),
+    re.compile(rb"\bAKIA[0-9A-Z]{16}\b"),
+    re.compile(rb"\bsk-[A-Za-z0-9_-]{20,}\b"),
+)
+_SECRET_FILES = {".env", ".env.local", ".npmrc", ".pypirc", "id_rsa"}
+_TERMINAL_STATUSES = {
+    "completed",
+    "failed",
+    "timed_out",
+    "cancelled",
+    "canceled",
+    # Chat conversations use domain-specific endings. All three mean execution
+    # stopped and produced durable evidence; only ``finished`` implies that the
+    # simulated user achieved its conversational goal. Artifact sealing must
+    # preserve the other two as findings rather than turn them into an
+    # infrastructure failure.
+    "finished",
+    "gave-up",
+    "ran-out-of-turns",
+}
+
+
+def seal_artifacts(
+    root: str | Path,
+    *,
+    run_id: str,
+    max_bytes: int = 1_073_741_824,
+    expected_scenarios: int | None = None,
+) -> ArtifactManifest:
+    """Validate evidence, hash every retained file, and atomically seal the directory."""
+    root = Path(root).expanduser().resolve()
+    if not root.is_dir():
+        raise ArtifactIntegrityError(f"artifact_root_missing: {root}")
+    records: list[ArtifactRecord] = []
+    total = 0
+    for path in sorted(root.rglob("*")):
+        # The outbox remains append-only until the platform acknowledges the terminal event.
+        # It is transport state, not immutable run evidence, and is therefore not sealed.
+        if (
+            path.name in {ARTIFACT_MANIFEST_NAME, "harness-events.jsonl"}
+            or path.is_dir()
+        ):
+            continue
+        if path.is_symlink():
+            raise ArtifactIntegrityError(
+                f"artifact_symlink_forbidden: {path.relative_to(root)}"
+            )
+        relative = path.relative_to(root).as_posix()
+        if path.name in _SECRET_FILES or path.suffix.lower() in {
+            ".pem",
+            ".key",
+            ".p12",
+            ".pfx",
+        }:
+            raise ArtifactIntegrityError(f"artifact_secret_file_forbidden: {relative}")
+        digest, size = _hash_file(path, relative)
+        total += size
+        if total > max_bytes:
+            raise ArtifactIntegrityError(
+                f"artifact_size_limit_exceeded: {total} > {max_bytes}"
+            )
+        records.append(
+            ArtifactRecord(
+                path=relative,
+                sha256=digest,
+                size=size,
+                kind=_kind(path),
+                media_type=mimetypes.guess_type(path.name)[0]
+                or "application/octet-stream",
+                scenario=_scenario(relative),
+            )
+        )
+
+    scenarios = _scenario_evidence(root)
+    if expected_scenarios is not None and len(scenarios) != expected_scenarios:
+        raise ArtifactIntegrityError(
+            "artifact_scenario_count_mismatch: "
+            f"expected {expected_scenarios}, found {len(scenarios)}"
+        )
+    core = {
+        "schema_version": ARTIFACT_MANIFEST_VERSION,
+        "run_id": run_id,
+        "files": [record.model_dump(mode="json") for record in records],
+        "scenarios": [item.model_dump(mode="json") for item in scenarios],
+        "total_bytes": total,
+        "complete": True,
+    }
+    digest = (
+        "sha256:"
+        + hashlib.sha256(
+            json.dumps(core, sort_keys=True, separators=(",", ":")).encode()
+        ).hexdigest()
+    )
+    manifest = ArtifactManifest(**core, digest=digest)
+    target = root / ARTIFACT_MANIFEST_NAME
+    temporary = root / f".{ARTIFACT_MANIFEST_NAME}.tmp-{os.getpid()}"
+    temporary.write_text(manifest.model_dump_json(indent=2) + "\n", encoding="utf-8")
+    temporary.replace(target)
+    return manifest
+
+
+def load_artifact_manifest(
+    root: str | Path, *, verify: bool = True
+) -> ArtifactManifest:
+    root = Path(root).expanduser().resolve()
+    path = root / ARTIFACT_MANIFEST_NAME
+    try:
+        manifest = ArtifactManifest.model_validate_json(
+            path.read_text(encoding="utf-8")
+        )
+    except (OSError, ValueError) as exc:
+        raise ArtifactIntegrityError(f"artifact_manifest_invalid: {exc}") from exc
+    if verify:
+        for record in manifest.files:
+            candidate = (root / record.path).resolve()
+            if root not in candidate.parents or not candidate.is_file():
+                raise ArtifactIntegrityError(f"artifact_missing: {record.path}")
+            digest, size = _hash_file(candidate, record.path)
+            if digest != record.sha256 or size != record.size:
+                raise ArtifactIntegrityError(f"artifact_changed: {record.path}")
+        expected = (
+            "sha256:"
+            + hashlib.sha256(
+                json.dumps(
+                    manifest.model_dump(mode="json", exclude={"digest"}),
+                    sort_keys=True,
+                    separators=(",", ":"),
+                ).encode()
+            ).hexdigest()
+        )
+        if expected != manifest.digest:
+            raise ArtifactIntegrityError("artifact_manifest_digest_mismatch")
+    return manifest
+
+
+def _scenario_evidence(root: Path) -> list[ScenarioEvidence]:
+    evidence: list[ScenarioEvidence] = []
+    for result_path in sorted((root / "runs").glob("*/*/result.json")):
+        try:
+            result = json.loads(result_path.read_text(encoding="utf-8"))
+        except (OSError, ValueError) as exc:
+            raise ArtifactIntegrityError(
+                f"result_invalid: {result_path.relative_to(root)}"
+            ) from exc
+        relative = result_path.relative_to(root).as_posix()
+        scenario = result_path.parent.name
+        status = str(
+            result.get("ended") or (result.get("measured") or {}).get("status") or ""
+        ).lower()
+        if status not in _TERMINAL_STATUSES:
+            raise ArtifactIntegrityError(
+                f"result_not_terminal: {scenario}: {status or 'missing'}"
+            )
+        transcript_path = result_path.parent / "transcript.txt"
+        transcript = str(result.get("transcript") or "").strip()
+        if not transcript and (
+            not transcript_path.is_file()
+            or not transcript_path.read_text(encoding="utf-8").strip()
+        ):
+            raise ArtifactIntegrityError(f"transcript_missing: {scenario}")
+
+        recordings: list[str] = []
+        for track in result.get("tracks") or []:
+            raw = str(track.get("path") or "") if isinstance(track, dict) else ""
+            if not raw:
+                continue
+            candidate = Path(raw)
+            if not candidate.is_absolute():
+                candidate = result_path.parent / candidate
+            candidate = candidate.resolve()
+            if (
+                root not in candidate.parents
+                or not candidate.is_file()
+                or candidate.stat().st_size == 0
+            ):
+                raise ArtifactIntegrityError(f"recording_missing: {scenario}: {raw}")
+            recordings.append(candidate.relative_to(root).as_posix())
+        if (
+            result.get("recording") or (result.get("measured") or {}).get("room")
+        ) and not recordings:
+            raise ArtifactIntegrityError(f"recording_evidence_missing: {scenario}")
+
+        tool_trace = result_path.parent / "agent-tool-calls.jsonl"
+        evidence.append(
+            ScenarioEvidence(
+                scenario=scenario,
+                result_path=relative,
+                canonical_status=status,
+                transcript_path=(
+                    transcript_path.relative_to(root).as_posix()
+                    if transcript_path.is_file()
+                    else None
+                ),
+                recording_paths=sorted(recordings),
+                tool_trace_path=(
+                    tool_trace.relative_to(root).as_posix()
+                    if tool_trace.is_file()
+                    else None
+                ),
+            )
+        )
+    return evidence
+
+
+def _hash_file(path: Path, relative: str) -> tuple[str, int]:
+    digest = hashlib.sha256()
+    size = 0
+    with path.open("rb") as stream:
+        while chunk := stream.read(1024 * 1024):
+            size += len(chunk)
+            digest.update(chunk)
+            if any(pattern.search(chunk) for pattern in _SECRET_CONTENT):
+                raise ArtifactIntegrityError(
+                    f"artifact_secret_material_detected: {relative}"
+                )
+    return digest.hexdigest(), size
+
+
+def _kind(path: Path) -> str:
+    if path.name == "result.json":
+        return "result"
+    if path.name == "transcript.txt":
+        return "transcript"
+    if path.name == "agent-tool-calls.jsonl":
+        return "tool_trace"
+    if path.suffix.lower() in {".wav", ".mp3", ".ogg", ".webm", ".m4a"}:
+        return "recording"
+    if path.name == "harness-events.jsonl":
+        return "event_stream"
+    return "artifact"
+
+
+def _scenario(relative: str) -> str | None:
+    parts = PurePosixPath(relative).parts
+    if len(parts) >= 4 and parts[0] == "runs":
+        return parts[2]
+    return None
+
+
+__all__ = [
+    "ARTIFACT_MANIFEST_NAME",
+    "ARTIFACT_MANIFEST_VERSION",
+    "ArtifactIntegrityError",
+    "ArtifactManifest",
+    "ArtifactRecord",
+    "ScenarioEvidence",
+    "load_artifact_manifest",
+    "seal_artifacts",
+]