agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/executor.py

@@ -0,0 +1,304 @@
+"""One ALK harness executor used by the local CLI and hosted sandbox workers."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+from typing import Callable, Protocol
+
+from .job import (
+    FailureDomain,
+    HarnessFailure,
+    HarnessJob,
+    HarnessJobStatus,
+    HarnessStage,
+    SourceKind,
+    SourceVisibility,
+)
+
+
+class SourceAcquirer(Protocol):
+    """Materialize job source inside an executor-owned ephemeral workspace."""
+
+    async def acquire(self, job: HarnessJob, workspace: Path) -> Path: ...
+
+
+class SourceAcquisitionError(RuntimeError):
+    pass
+
+
+class GitHubSourceAcquirer:
+    """Clone one platform-authorized repository without exposing its token in argv or logs."""
+
+    _REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
+
+    def __init__(self, installation_token: Callable[[str], str]) -> None:
+        self._installation_token = installation_token
+
+    async def acquire(self, job: HarnessJob, workspace: Path) -> Path:
+        source = job.source
+        repository = str(source.repository or "")
+        installation_id = str(source.installation_id or "")
+        if not self._REPOSITORY.fullmatch(repository):
+            raise SourceAcquisitionError("github_repository_invalid")
+        is_public = source.visibility is SourceVisibility.PUBLIC
+        if not is_public and not installation_id:
+            raise SourceAcquisitionError("github_installation_missing")
+        token = "" if is_public else self._installation_token(installation_id)
+        if not is_public and not token:
+            raise SourceAcquisitionError("github_installation_token_missing")
+        if source.ref and (
+            ".." in source.ref or not re.fullmatch(r"[A-Za-z0-9._/-]+", source.ref)
+        ):
+            raise SourceAcquisitionError("github_ref_invalid")
+        workspace = workspace.expanduser().resolve()
+        workspace.mkdir(parents=True, exist_ok=True)
+        destination = workspace / "repository"
+        if destination.exists():
+            raise SourceAcquisitionError(f"source_destination_not_empty: {destination}")
+
+        command = ["git", "clone", "--depth", "1"]
+        if source.ref:
+            command.extend(["--branch", source.ref])
+        command.extend([f"https://github.com/{repository}.git", str(destination)])
+        # Git reads the authorization header from its child environment. It never appears in
+        # the process command, exception, persisted job, or event stream.
+        environment = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
+        if token:
+            environment.update(
+                {
+                    "GIT_CONFIG_COUNT": "1",
+                    "GIT_CONFIG_KEY_0": "http.extraHeader",
+                    "GIT_CONFIG_VALUE_0": f"Authorization: Bearer {token}",
+                }
+            )
+
+        def clone() -> None:
+            try:
+                completed = subprocess.run(
+                    command,
+                    env=environment,
+                    cwd=workspace,
+                    capture_output=True,
+                    text=True,
+                    timeout=300,
+                    check=False,
+                )
+            except (OSError, subprocess.TimeoutExpired) as exc:
+                raise SourceAcquisitionError(
+                    f"github_clone_unavailable: {type(exc).__name__}"
+                ) from exc
+            if completed.returncode:
+                detail = (
+                    completed.stderr or completed.stdout or "clone failed"
+                ).strip()
+                # Git errors should not contain an env-only header, but redact defensively.
+                detail = detail.replace(token, "[REDACTED]")[:1000]
+                raise SourceAcquisitionError(f"github_clone_failed: {detail}")
+
+            if source.commit_sha:
+                verified = subprocess.run(
+                    ["git", "rev-parse", "HEAD"],
+                    cwd=destination,
+                    capture_output=True,
+                    text=True,
+                    timeout=30,
+                    check=False,
+                    env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
+                )
+                actual = verified.stdout.strip().lower()
+                if verified.returncode or actual != source.commit_sha.lower():
+                    raise SourceAcquisitionError("github_commit_mismatch")
+
+        await asyncio.to_thread(clone)
+        if not destination.is_dir():
+            raise SourceAcquisitionError("github_clone_missing_checkout")
+        return destination
+
+
+class HarnessExecutor:
+    """Execute the autonomous pipeline without platform or scheduler dependencies.
+
+    A hosted worker first resolves a GitHub installation/archive/image through its
+    ``SourceAcquirer``. A local invocation already has a path. From that point onward both call
+    exactly the same ``auto`` pipeline and produce the same job, bundle, event, scenario, trace,
+    and result artifacts.
+    """
+
+    async def run(
+        self,
+        job: HarnessJob,
+        *,
+        source: Path,
+        output: Path,
+        model: str | None = None,
+        run_model: str | None = None,
+        adjustments_path: Path | None = None,
+    ) -> HarnessJobStatus:
+        from .cli import _auto
+
+        output = output.expanduser().resolve()
+        source = source.expanduser().resolve()
+        # A branch name is acquisition input, not immutable provenance. Resolve the checkout to
+        # its exact commit before job.json and the environment plan are written. The source
+        # content digest remains authoritative for uploads and non-Git sources.
+        if job.source.kind is SourceKind.GITHUB and not job.source.commit_sha:
+            completed = await asyncio.to_thread(
+                subprocess.run,
+                ["git", "rev-parse", "HEAD"],
+                cwd=source,
+                capture_output=True,
+                text=True,
+                timeout=30,
+                check=False,
+                env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
+            )
+            commit_sha = completed.stdout.strip().lower()
+            if completed.returncode or not re.fullmatch(r"[0-9a-f]{40}", commit_sha):
+                return HarnessJobStatus(
+                    job_id=job.job_id,
+                    run_id=job.run_id,
+                    stage=HarnessStage.FAILED,
+                    updated_at=datetime.now(timezone.utc),
+                    detail="could not resolve the acquired GitHub revision",
+                    failure=HarnessFailure(
+                        domain=FailureDomain.INFRASTRUCTURE,
+                        stage=HarnessStage.ACQUIRING_SOURCE,
+                        code="github_commit_resolution_failed",
+                        message="The runner could not resolve the acquired GitHub revision",
+                    ),
+                    total_scenarios=job.scenario_count,
+                )
+            job = job.model_copy(
+                update={
+                    "source": job.source.model_copy(update={"commit_sha": commit_sha})
+                }
+            )
+        # Source acquisition has already materialized GitHub/archive/local inputs as a local
+        # checkout.  The understanding registry describes how to inspect that materialized
+        # content (``repo``), while the immutable job retains its original source provenance.
+        # Passing transport kinds such as ``archive`` into source resolution makes a valid
+        # uploaded repository fail before its code is inspected.
+        understanding_kind = str(job.metadata.get("source_kind") or "repo")
+        if understanding_kind not in {"repo", "spec"}:
+            understanding_kind = "repo"
+        args = argparse.Namespace(
+            path=str(source),
+            name=str(job.metadata.get("agent_name") or source.name),
+            kind=understanding_kind,
+            out=str(output),
+            count=job.scenario_count,
+            model=model,
+            run_model=run_model,
+            job=job,
+            adjustments_path=str(adjustments_path) if adjustments_path else None,
+        )
+        status = await _auto(args)
+        failure = _failure_from_events(output) if status not in (0, 2) else None
+        return HarnessJobStatus(
+            job_id=job.job_id,
+            run_id=job.run_id,
+            stage=HarnessStage.COMPLETED if status in (0, 2) else HarnessStage.FAILED,
+            updated_at=datetime.now(timezone.utc),
+            detail=(
+                "agent checks failed"
+                if status == 2
+                else None
+                if status == 0
+                else f"exit {status}"
+            ),
+            failure=failure,
+            completed_scenarios=_scenario_count(output) if status in (0, 2) else 0,
+            total_scenarios=_scenario_count(output) or job.scenario_count,
+        )
+
+    async def acquire_and_run(
+        self,
+        job: HarnessJob,
+        *,
+        acquirer: SourceAcquirer,
+        workspace: Path,
+        output: Path,
+        model: str | None = None,
+        run_model: str | None = None,
+    ) -> HarnessJobStatus:
+        source = await acquirer.acquire(job, workspace)
+        return await self.run(
+            job,
+            source=source,
+            output=output,
+            model=model,
+            run_model=run_model,
+        )
+
+
+def run_sync(job: HarnessJob, *, source: Path, output: Path) -> HarnessJobStatus:
+    """Small synchronous adapter for job consumers that do not own an event loop."""
+    return asyncio.run(HarnessExecutor().run(job, source=source, output=output))
+
+
+def _scenario_count(output: Path) -> int:
+    try:
+        value = json.loads((output / "scenarios.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return 0
+    return len(value) if isinstance(value, list) else 0
+
+
+def _failure_from_events(output: Path) -> HarnessFailure:
+    failed: dict = {}
+    path = output / "harness-events.jsonl"
+    if path.is_file():
+        for raw in path.read_text(encoding="utf-8", errors="replace").splitlines():
+            try:
+                event = json.loads(raw)
+            except ValueError:
+                continue
+            if event.get("type") == "harness.stage.failed":
+                failed = event.get("payload") or {}
+    label = str(failed.get("stage") or "running")
+    stage, domain = {
+        "understand": (HarnessStage.UNDERSTANDING_AGENT, FailureDomain.AGENT),
+        "environment": (
+            HarnessStage.VALIDATING_ENVIRONMENT,
+            FailureDomain.ENVIRONMENT,
+        ),
+        "scenarios": (
+            HarnessStage.VALIDATING_SCENARIOS,
+            FailureDomain.SIMULATOR,
+        ),
+        "calls": (HarnessStage.RUNNING, FailureDomain.CONNECTIVITY),
+        "cleaning_up": (
+            HarnessStage.CLEANING_UP,
+            FailureDomain.INFRASTRUCTURE,
+        ),
+        "uploading_artifacts": (
+            HarnessStage.UPLOADING_ARTIFACTS,
+            FailureDomain.ARTIFACT,
+        ),
+    }.get(label, (HarnessStage.RUNNING, FailureDomain.INFRASTRUCTURE))
+    return HarnessFailure(
+        domain=domain,
+        stage=stage,
+        code=str(failed.get("code") or f"{label}_failed"),
+        message=str(failed.get("detail") or f"Harness stage {label} failed"),
+        # A job replay can repeat real calls. Only a failing stage that explicitly proves
+        # it is safe may opt in to retry; deterministic agent/grading failures never do.
+        retryable=bool(failed.get("retryable", False)),
+        details={"status": failed.get("status", 1)},
+    )
+
+
+__all__ = [
+    "GitHubSourceAcquirer",
+    "HarnessExecutor",
+    "SourceAcquirer",
+    "SourceAcquisitionError",
+    "run_sync",
+]

fi/alk/harness/folder.py

@@ -0,0 +1,234 @@
+"""A scenario as a folder of files, and running the code inside it.
+
+A scenario used to be a row in one big JSON file, and its setup was a list of rows to insert.
+That was enough while every world was a database. It stopped being enough the moment a world
+could hold a service as well as a table: "the weather service starts returning errors" is not
+expressible as rows, and neither is "the file is missing" or "the queue is backed up".
+
+So a scenario owns a folder, and the parts that are logic are files:
+
+    scenarios/<name>/
+        scenario.json     what it is: instruction, solution, which sub-goals
+        setup.py          def setup(world)  — the changes this scenario makes
+        ready.py          def ready(world)  — is the world ready for this scenario
+        checks/<goal>.py  def check(world, calls) — one per deterministic sub-goal
+
+The files are the artifact, not a rendering of one. Each is executable on its own, so a check
+can be run by hand against what a run left behind and answer exactly what it answers inside the
+harness. That is the whole point of them being files: something you can open, read and run is
+something you can argue with.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .catalogue import Catalogue
+from .scenario import Scenario
+from .world.runtime import GeneratedWorld
+
+SCENARIOS = "scenarios"
+INDEX = "scenarios.json"
+
+# Appended to every check file the harness writes. The model writes only ``check(world, calls)``;
+# this is what makes that same file runnable by a person, so nobody has to keep two versions of
+# one truth in step.
+_RUNNABLE = """
+
+if __name__ == "__main__":
+    # Run this check by hand against what a run left behind. The first argument is anything
+    # inside the saved world's folder, because not every world has a database to name:
+    #     python <this file> <world folder>/manifest.json [calls.json]
+    import json as _json
+    import sys as _sys
+    from pathlib import Path as _Path
+
+    from fi.alk.harness.world.runtime import Call as _Call
+    from fi.alk.harness.world.snapshot import restore as _restore
+
+    _world = _restore(_Path(_sys.argv[1]).parent) if len(_sys.argv) > 1 else None
+    _calls = []
+    if len(_sys.argv) > 2:
+        _calls = [_Call(**_one) for _one in _json.loads(_Path(_sys.argv[2]).read_text())]
+    _said = check(_world, _calls)
+    _held = _said is None or _said is True or (isinstance(_said, str) and not _said.strip())
+    print("held" if _held else f"FAILED: {_said}")
+    raise SystemExit(0 if _held else 1)
+"""
+
+
+@dataclass
+class Outcome:
+    """What one piece of a scenario's own code did."""
+
+    ok: bool
+    said: str = ""
+    broken: bool = False
+
+
+def _run(source: str, name: str, entry: str, *args: Any) -> Outcome:
+    """Execute one function out of a scenario's own code.
+
+    A file that will not compile, or that raises, is **broken** rather than failing: it is our
+    mistake, and scoring it as though the world were wrong would send somebody looking in the
+    wrong place.
+    """
+    if not source.strip():
+        return Outcome(True)
+    namespace: dict[str, Any] = {}
+    try:
+        exec(compile(source, f"<{name}>", "exec"), namespace)
+    except Exception as failed:
+        return Outcome(False, f"{name} would not compile: {failed}", broken=True)
+
+    function = namespace.get(entry)
+    if not callable(function):
+        return Outcome(False, f"{name} defines no {entry}()", broken=True)
+    try:
+        said = function(*args)
+    except Exception as failed:
+        return Outcome(
+            False, f"{name} raised {type(failed).__name__}: {failed}", broken=True
+        )
+    # The convention is that a complaint is a sentence, and anything else means it held. An empty
+    # string is the case worth naming: it reads as "no complaint" to whoever wrote it, and taking
+    # it as a failure produces a rejection with no reason attached, which cannot be acted on and
+    # sends the author hunting for a problem that is not there.
+    if said is None or said is True or (isinstance(said, str) and not said.strip()):
+        return Outcome(True)
+    if said is False:
+        # Bare False from ready() names no precondition, so a scenario that hits it cannot be
+        # told apart from one whose ready.py is simply wrong — that is our mistake, not a
+        # generation-time precondition failure, so ready() alone reports it broken.
+        return Outcome(
+            False,
+            f"{name} returned False without saying what is wrong. Return the sentence instead, "
+            "or None if it holds.",
+            broken=(entry == "ready"),
+        )
+    if entry == "ready" and not isinstance(said, str):
+        # Same reasoning as bare False, widened: ready() has no way to turn a non-string value
+        # into a precondition sentence, so any of them is our mistake rather than the world's.
+        return Outcome(
+            False,
+            f"{name} returned {type(said).__name__} {repr(said)[:200]}. Return the sentence "
+            "naming what is missing, or None if it holds.",
+            broken=True,
+        )
+    return Outcome(False, str(said))
+
+
+def apply_setup(scenario: Scenario, world: GeneratedWorld) -> Outcome:
+    """Make this scenario's changes to the world."""
+    return _run(scenario.setup_code, f"{scenario.name}/setup.py", "setup", world)
+
+
+def check_ready(scenario: Scenario, world: GeneratedWorld) -> Outcome:
+    """Whether the world now holds what this scenario presumes."""
+    return _run(scenario.ready_code, f"{scenario.name}/ready.py", "ready", world)
+
+
+def folder_for(destination: Path, name: str) -> Path:
+    return Path(destination) / SCENARIOS / name
+
+
+def write_folder(scenario: Scenario, catalogue: Catalogue, destination: Path) -> Path:
+    """Write one scenario out as its own folder of files."""
+    root = folder_for(destination, scenario.name)
+    (root / "checks").mkdir(parents=True, exist_ok=True)
+
+    body = scenario.model_dump()
+    # The code lives in its own files; keeping a second copy in the JSON would let the two drift
+    # and leave nobody able to say which one ran.
+    body.pop("setup_code", None)
+    body.pop("ready_code", None)
+    (root / "scenario.json").write_text(
+        json.dumps(body, indent=2, ensure_ascii=False), encoding="utf-8"
+    )
+
+    (root / "setup.py").write_text(
+        scenario.setup_code
+        or 'def setup(world):\n    """This scenario runs on the base world unchanged."""\n',
+        encoding="utf-8",
+    )
+    (root / "ready.py").write_text(
+        scenario.ready_code
+        or 'def ready(world):\n    """Nothing beyond the base world is presumed."""\n',
+        encoding="utf-8",
+    )
+
+    for name in scenario.sub_goals:
+        sub_goal = catalogue.named(name)
+        if sub_goal is None or not sub_goal.deterministic():
+            continue
+        (root / "checks" / f"{name}.py").write_text(
+            sub_goal.check.rstrip() + "\n" + _RUNNABLE, encoding="utf-8"
+        )
+    return root
+
+
+def read_folder(destination: Path, name: str) -> Scenario | None:
+    """One scenario, reassembled from its folder."""
+    root = folder_for(destination, name)
+    body = root / "scenario.json"
+    if not body.exists():
+        return None
+    payload = json.loads(body.read_text(encoding="utf-8"))
+    for field, filename in (("setup_code", "setup.py"), ("ready_code", "ready.py")):
+        path = root / filename
+        payload[field] = path.read_text(encoding="utf-8") if path.exists() else ""
+    return Scenario.model_validate(payload)
+
+
+def write_index(scenarios: list[Scenario], destination: Path) -> Path:
+    """The whole suite at a glance, over the folders.
+
+    Regenerated from the folders rather than maintained alongside them, so it can never disagree
+    with what is actually on disk.
+    """
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    path = destination / INDEX
+    path.write_text(
+        json.dumps(
+            [
+                {
+                    "name": one.name,
+                    "use_case": one.use_case,
+                    "tests": one.tests,
+                    "instruction": one.instruction,
+                    "sub_goals": one.sub_goals,
+                    "steps": len(one.solution),
+                    "folder": f"{SCENARIOS}/{one.name}",
+                }
+                for one in scenarios
+            ],
+            indent=2,
+            ensure_ascii=False,
+        ),
+        encoding="utf-8",
+    )
+    return path
+
+
+def read_all(destination: Path) -> list[Scenario]:
+    """Every scenario on disk, read from the folders."""
+    root = Path(destination) / SCENARIOS
+    if not root.exists():
+        return []
+    found: list[Scenario] = []
+    for folder in sorted(root.iterdir()):
+        if not folder.is_dir():
+            continue
+        try:
+            scenario = read_folder(destination, folder.name)
+        except Exception:
+            # A folder we cannot read is skipped rather than crashing the stage: the rest of the
+            # suite is still usable, and the gap shows up as a missing scenario.
+            continue
+        if scenario is not None:
+            found.append(scenario)
+    return found