puremethod 0.1.0__py3-none-any.whl

executor/__init__.py

@@ -0,0 +1,47 @@
+"""executor: runs experiment code on a backend the user chooses
+(docs/knowledge_base/05-executor.md). v1 backends: local-docker, local-process, ssh."""
+
+from .base import CredentialLookup, Executor
+from .local_docker import LocalDockerExecutor
+from .local_process import LocalProcessExecutor
+from .ssh import SSHExecutor
+from .types import (
+    DEFAULT_IMAGE,
+    METRICS_FILE,
+    DataMount,
+    EnvRef,
+    EnvSpec,
+    ExecutorError,
+    ExecutorInfo,
+    JobHandle,
+    JobResult,
+    JobSpec,
+    JobState,
+    NotDone,
+    PreflightReport,
+    Resources,
+    UnknownJob,
+)
+
+__all__ = [
+    "DEFAULT_IMAGE",
+    "METRICS_FILE",
+    "CredentialLookup",
+    "DataMount",
+    "EnvRef",
+    "EnvSpec",
+    "Executor",
+    "ExecutorError",
+    "ExecutorInfo",
+    "JobHandle",
+    "JobResult",
+    "JobSpec",
+    "JobState",
+    "LocalDockerExecutor",
+    "LocalProcessExecutor",
+    "NotDone",
+    "PreflightReport",
+    "Resources",
+    "SSHExecutor",
+    "UnknownJob",
+]

executor/base.py

@@ -0,0 +1,161 @@
+"""Shared executor behaviour: handles, queueing beyond the concurrency limit (§5 step 8),
+and the `poll` / `result` / `kill` contract. Backends implement the `_` hooks."""
+
+from __future__ import annotations
+
+import uuid
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from .common import check_tag, redact, sanitized_tar
+from .types import (
+    EnvRef,
+    EnvSpec,
+    ExecutorInfo,
+    JobHandle,
+    JobResult,
+    JobSpec,
+    JobState,
+    NotDone,
+    PreflightReport,
+    Status,
+    UnknownJob,
+)
+
+CredentialLookup = Callable[[str], str]
+
+
+@dataclass
+class _Job:
+    spec: JobSpec
+    state: JobState = "queued"
+    backend: Any = None
+    killed: bool = False
+    result: JobResult | None = None
+    secrets: dict[str, str] = field(default_factory=dict)
+
+
+class Executor(ABC):
+    """One per configured backend, built from its configuration and a credential lookup
+    provided by the caller (05-executor.md §3, KB-015)."""
+
+    def __init__(self, credentials: CredentialLookup) -> None:
+        self._credentials = credentials
+        self._jobs: dict[str, _Job] = {}
+        self._charged_envs: set[str] = set()
+
+    # ---- interface ---------------------------------------------------------------------
+
+    @abstractmethod
+    def describe(self) -> ExecutorInfo: ...
+
+    @abstractmethod
+    def preflight(self) -> PreflightReport: ...
+
+    @abstractmethod
+    def build_env(self, spec: EnvSpec) -> EnvRef: ...
+
+    @abstractmethod
+    def reap(self, tags: list[str]) -> list[str]:
+        """Stops and removes every job, container and workspace carrying one of these
+        tags, whether or not this instance started it; returns the tags found (INV-9)."""
+
+    def submit(self, job: JobSpec) -> JobHandle:
+        if job.env.status != "ok":
+            raise ValueError("the job's environment failed to build")
+        if job.deadline_s <= 0:
+            raise ValueError("deadline_s must be positive")
+        sanitized_tar(job.code)  # refuse an unsafe bundle now, not when it is dequeued
+        if job.tag is not None:
+            check_tag(job.tag)
+        secrets = {name: self._credentials(name) for name in job.credentials}
+        job_id = uuid.uuid4().hex[:16]
+        self._jobs[job_id] = _Job(spec=job, secrets=secrets)
+        self._start_queued()
+        return JobHandle(job_id=job_id, executor_id=self.describe().id)
+
+    def poll(self, handle: JobHandle) -> JobState:
+        job = self._job(handle)
+        if job.state == "running" and self._done(job.backend):
+            job.result = self._finish(handle.job_id, job)
+            job.state = "done"
+        self._start_queued()
+        return job.state
+
+    def result(self, handle: JobHandle) -> JobResult:
+        job = self._job(handle)
+        if job.state != "done" or job.result is None:
+            raise NotDone(handle.job_id)
+        return job.result
+
+    def kill(self, handle: JobHandle) -> None:
+        job = self._job(handle)
+        if job.state == "done":
+            return
+        job.killed = True
+        if job.state == "queued":
+            job.result = self._never_started(job)
+            job.state = "done"
+            return
+        self._kill(job.backend)
+
+    # ---- internals ---------------------------------------------------------------------
+
+    def _job(self, handle: JobHandle) -> _Job:
+        job = self._jobs.get(handle.job_id)
+        if job is None:
+            raise UnknownJob(handle.job_id)
+        return job
+
+    def _start_queued(self) -> None:
+        running = sum(1 for j in self._jobs.values() if j.state == "running")
+        for job_id, job in self._jobs.items():
+            if running >= self.describe().concurrency:
+                break
+            if job.state == "queued" and not job.killed:
+                try:
+                    job.backend = self._start(job_id, job.spec, job.secrets)
+                except Exception as exc:  # a backend fault ends this job, not the queue
+                    job.result = self._never_started(job, status="crashed",
+                                                     reason=f"start failed: "
+                                                            f"{type(exc).__name__}: {exc}")
+                    job.state = "done"
+                    continue
+                job.state = "running"
+                running += 1
+
+    def _finish(self, job_id: str, job: _Job) -> JobResult:
+        result = self._collect(job.backend, job.spec, job.secrets, killed=job.killed)
+        env = job.spec.env
+        if env.built and env.digest not in self._charged_envs:
+            # §5 step 6: a fresh environment build is charged to the first job using it.
+            self._charged_envs.add(env.digest)
+            rate = self.describe().rate_usd_per_hour
+            if result.cost_usd is not None and rate is not None:
+                result.cost_usd += rate * env.build_seconds / 3600.0
+        return result
+
+    def _never_started(self, job: _Job, status: Status = "killed",
+                       reason: str = "") -> JobResult:
+        stderr, _ = redact(reason.encode(), job.secrets)
+        return JobResult(status=status, exit_code=None, stdout=b"", stderr=stderr, outputs={},
+                         metrics=None, started_at=None, ended_at=None, duration_s=0.0,
+                         cost_usd=0.0 if self.describe().metered else None,
+                         metered=self.describe().metered, env_digest=job.spec.env.digest,
+                         code_digest=job.spec.code_digest, redactions=[],
+                         output_cap_bytes=0)
+
+    @abstractmethod
+    def _start(self, job_id: str, spec: JobSpec, secrets: dict[str, str]) -> Any: ...
+
+    @abstractmethod
+    def _done(self, backend: Any) -> bool: ...
+
+    @abstractmethod
+    def _collect(self, backend: Any, spec: JobSpec, secrets: dict[str, str], *,
+                 killed: bool) -> JobResult: ...
+
+    @abstractmethod
+    def _kill(self, backend: Any) -> None: ...

executor/common.py

@@ -0,0 +1,139 @@
+"""Helpers shared by the backends: workspaces, redaction, outputs, status, cost."""
+
+from __future__ import annotations
+
+import io
+import os
+import tarfile
+from datetime import UTC, datetime
+from pathlib import Path
+
+from .types import Status
+
+TAG_FILE = "pm_tag"  # in the job directory, outside the workspace the job can write
+_TAG_OK = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.:-")
+
+
+def check_tag(tag: str) -> str:
+    """Tags travel into container labels and shell commands: a closed character set."""
+    if not tag or len(tag) > 200 or not set(tag) <= _TAG_OK:
+        raise ValueError(f"invalid job tag {tag!r}")
+    return tag
+
+OUTPUT_CAP = 2 * 1024 * 1024  # bytes kept per captured stream or text output
+EXIT_TIMEOUT = 124            # coreutils/busybox `timeout` when the deadline passes
+
+
+def now_iso() -> str:
+    return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def unpack(code: bytes, dest: Path) -> None:
+    """Unpack a code bundle with the `data` filter: no absolute paths, no `..`, no links
+    out of the workspace, no device files (§5 step 1)."""
+    dest.mkdir(parents=True, exist_ok=True)
+    try:
+        with tarfile.open(fileobj=io.BytesIO(code), mode="r:*") as tar:
+            tar.extractall(dest, filter="data")
+    except (tarfile.FilterError, tarfile.ReadError) as exc:
+        raise ValueError(f"unsafe or unreadable code bundle: {type(exc).__name__}") from None
+    (dest / "outputs").mkdir(exist_ok=True)
+
+
+def sanitized_tar(code: bytes) -> bytes:
+    """Re-pack a bundle after the same checks, for hosts whose `tar` may be less strict."""
+    out = io.BytesIO()
+    try:
+        with tarfile.open(fileobj=io.BytesIO(code), mode="r:*") as src, \
+                tarfile.open(fileobj=out, mode="w") as dst:
+            for member in src.getmembers():
+                safe = tarfile.data_filter(member, "/nonexistent-root")
+                # The filter clears ownership ("don't apply"); a new archive needs values.
+                safe = safe.replace(uid=0, gid=0, uname="", gname="", deep=False)
+                if safe.isfile():
+                    dst.addfile(safe, src.extractfile(member))
+                elif safe.isdir():
+                    dst.addfile(safe)
+    except (tarfile.FilterError, tarfile.ReadError) as exc:
+        raise ValueError(f"unsafe or unreadable code bundle: {type(exc).__name__}") from None
+    return out.getvalue()
+
+
+def redact(data: bytes, secrets: dict[str, str]) -> tuple[bytes, list[str]]:
+    """Replace every value of the job's credentials (§5 step 5). Returns names, never values."""
+    hit: list[str] = []
+    for name, value in secrets.items():
+        if value and value.encode() in data:
+            data = data.replace(value.encode(), f"[REDACTED:{name}]".encode())
+            hit.append(name)
+    return data, hit
+
+
+def cap(data: bytes, limit: int = OUTPUT_CAP) -> tuple[bytes, bool]:
+    return (data[:limit], True) if len(data) > limit else (data, False)
+
+
+def read_outputs(outputs_dir: Path, secrets: dict[str, str]
+                 ) -> tuple[dict[str, bytes], list[str], list[str]]:
+    """Files under `outputs/`, with text files redacted and capped."""
+    files: dict[str, bytes] = {}
+    redacted: list[str] = []
+    truncated: list[str] = []
+    if not outputs_dir.is_dir():
+        return files, redacted, truncated
+    for path in sorted(outputs_dir.rglob("*")):
+        if not path.is_file() or path.is_symlink():
+            continue
+        rel = "outputs/" + path.relative_to(outputs_dir).as_posix()
+        data = path.read_bytes()
+        files[rel], names, cut = process_text(data, secrets)
+        redacted += names
+        if cut:
+            truncated.append(rel)
+    return files, sorted(set(redacted)), truncated
+
+
+def process_text(data: bytes, secrets: dict[str, str]) -> tuple[bytes, list[str], bool]:
+    """Redact and cap a file if it is text; binary files pass through unchanged."""
+    try:
+        data.decode("utf-8")
+    except UnicodeDecodeError:
+        return data, [], False
+    data, names = redact(data, secrets)
+    data, cut = cap(data)
+    return data, names, cut
+
+
+def classify(exit_code: int | None, *, killed: bool, timed_out: bool,
+             crashed: bool) -> Status:
+    """§4 statuses. A kill by the user wins; then the deadline; then crashes."""
+    if killed:
+        return "killed"
+    if timed_out or exit_code == EXIT_TIMEOUT:
+        return "timeout"
+    if crashed:
+        return "crashed"
+    if exit_code is None:
+        return "lost"
+    if exit_code == 0:
+        return "ok"
+    if exit_code < 0 or exit_code in (137, 139, 143):  # ended by a signal
+        return "crashed"
+    return "failed"
+
+
+def cost(rate_usd_per_hour: float | None, seconds: float | None) -> tuple[float | None, bool]:
+    """§5 step 6: rate times wall-clock, or unknown and unmetered. Never a default 0."""
+    if rate_usd_per_hour is None or seconds is None:
+        return None, False
+    return rate_usd_per_hour * seconds / 3600.0, True
+
+
+def minimal_env(extra: dict[str, str]) -> dict[str, str]:
+    """A small environment for agent-written code. The engine's own environment (which may
+    hold provider keys) is never inherited."""
+    keep = ("PATH", "SYSTEMROOT", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT", "TEMP", "TMP",
+            "LANG", "LC_ALL", "TZ")
+    env = {k: os.environ[k] for k in keep if k in os.environ}
+    env.update(extra)
+    return env

executor/local_docker.py

@@ -0,0 +1,239 @@
+"""`local-docker` backend: isolation `container` (05-executor.md §5, §6).
+
+The container sees only its workspace (read-write) and the configured data mounts
+(read-only); no host home directory, Docker socket or host credentials (§5 step 2). The
+deadline is enforced inside the container by `timeout`, so it holds without the engine
+(INV-2). Credentials are passed as `-e NAME` with the value in the Docker CLI's own
+environment, never on its command line or on disk (INV-7).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+from .base import CredentialLookup, Executor
+from .common import TAG_FILE, cap, check_tag, cost, minimal_env, read_outputs, redact, unpack
+from .common import classify as classify_status
+from .local_process import _tar
+from .types import (
+    DEFAULT_IMAGE,
+    METRICS_FILE,
+    EnvRef,
+    EnvSpec,
+    ExecutorInfo,
+    JobResult,
+    JobSpec,
+    Network,
+    PreflightReport,
+    Resources,
+)
+
+LABEL = "puremethod.job"
+TAG_LABEL = "puremethod.tag"
+
+
+@dataclass
+class _Container:
+    name: str
+    ws: Path
+    deadline_s: float
+    started: float
+    killed: bool = False
+    info: dict[str, Any] = field(default_factory=dict)
+
+
+class LocalDockerExecutor(Executor):
+    def __init__(self, credentials: CredentialLookup, *, executor_id: str = "local-docker",
+                 root: str | Path, concurrency: int = 1,
+                 rate_usd_per_hour: float | None = None, network: Network = "on",
+                 grace_s: float = 30.0, resources: Resources | None = None,
+                 docker: str = "docker") -> None:
+        super().__init__(credentials)
+        self._info = ExecutorInfo(id=executor_id, kind="local-docker", isolation="container",
+                                  network=network, concurrency=concurrency,
+                                  resources=resources or Resources(),
+                                  rate_usd_per_hour=rate_usd_per_hour)
+        self._root = Path(root)
+        self._grace = grace_s
+        self._docker = docker
+
+    def describe(self) -> ExecutorInfo:
+        return self._info
+
+    def _cli(self, *args: str, env: dict[str, str] | None = None, check: bool = True,
+             timeout: float = 120) -> subprocess.CompletedProcess[str]:
+        return subprocess.run([self._docker, *args], capture_output=True, text=True,  # noqa: S603
+                              check=check, env=env, timeout=timeout)
+
+    def preflight(self) -> PreflightReport:
+        try:
+            self._cli("version", "--format", "{{.Server.Version}}", timeout=30)
+        except (OSError, subprocess.SubprocessError) as exc:
+            return PreflightReport(False, [f"docker unavailable: {type(exc).__name__}"])
+        env = self.build_env(EnvSpec())
+        if env.status != "ok":
+            return PreflightReport(False, [f"environment: {env.log[-500:]}"])
+        handle = self.submit(JobSpec(code=_tar({"main.py": b"print('ok')\n"}),
+                                     entrypoint="python main.py", env=env, seed=0,
+                                     deadline_s=60))
+        deadline = time.monotonic() + 120
+        while self.poll(handle) != "done" and time.monotonic() < deadline:
+            time.sleep(0.5)
+        if self.poll(handle) != "done" or self.result(handle).status != "ok":
+            return PreflightReport(False, ["a trivial job did not run"])
+        return PreflightReport(True, [])
+
+    def build_env(self, spec: EnvSpec) -> EnvRef:
+        digest = spec.digest()
+        tag = "pm-env:" + digest.removeprefix("sha256:")[:24]
+        if self._cli("image", "inspect", tag, check=False).returncode == 0:
+            return EnvRef(digest=digest, status="ok", ref=tag)
+        lines = [f"FROM {spec.base_image or DEFAULT_IMAGE}"]
+        if spec.system_packages:
+            pkgs = " ".join(spec.system_packages)
+            lines.append("RUN apt-get update && apt-get install -y --no-install-recommends "
+                         f"{pkgs} && rm -rf /var/lib/apt/lists/*")
+        if spec.requirements_lock.strip():
+            lines += ["COPY requirements.lock /tmp/requirements.lock",
+                      "RUN pip install --no-cache-dir -r /tmp/requirements.lock"]
+        lines.append(f'LABEL puremethod.env="{digest}"')
+        t0 = time.monotonic()
+        with tempfile.TemporaryDirectory(prefix="pm-env-") as ctx:
+            Path(ctx, "Dockerfile").write_text("\n".join(lines) + "\n", encoding="utf-8")
+            Path(ctx, "requirements.lock").write_text(spec.requirements_lock, encoding="utf-8")
+            out = self._cli("build", "-t", tag, ctx, check=False, timeout=3600)
+        log = (out.stdout + out.stderr)[-20000:]
+        status: Literal["ok", "failed"] = "ok" if out.returncode == 0 else "failed"
+        return EnvRef(digest=digest, status=status, ref=tag if status == "ok" else "",
+                      log=log, build_seconds=time.monotonic() - t0, built=status == "ok")
+
+    # ---- backend hooks -----------------------------------------------------------------
+
+    def _start(self, job_id: str, spec: JobSpec, secrets: dict[str, str]) -> _Container:
+        ws = (self._root / "jobs" / job_id / "ws").resolve()
+        unpack(spec.code, ws)
+        name = f"pm-job-{job_id}"
+        args = ["run", "-d", "--name", name, "--label", f"{LABEL}={job_id}",
+                "-v", f"{ws}:/work", "-w", "/work", "-e", f"PM_SEED={spec.seed}"]
+        if spec.tag is not None:
+            (ws.parent / TAG_FILE).write_text(spec.tag, encoding="utf-8")
+            args += ["--label", f"{TAG_LABEL}={spec.tag}"]
+        if self._info.network == "off":
+            args += ["--network", "none"]
+        r = spec.resources
+        if r.cpus:
+            args += ["--cpus", str(r.cpus)]
+        if r.memory_mb:
+            args += ["--memory", f"{r.memory_mb}m"]
+        if r.gpus:
+            args += ["--gpus", str(r.gpus)]
+        for m in spec.data_mounts:
+            args += ["-v", f"{Path(m.source).resolve()}:/data/{m.name}:ro"]
+        for cred in secrets:
+            args += ["-e", cred]  # value comes from the CLI's environment below
+        grace = str(int(max(self._grace, 1)))
+        deadline = str(max(int(spec.deadline_s), 1))
+        args += [spec.env.ref, "timeout", "-s", "TERM", "-k", grace, deadline,
+                 "sh", "-c", spec.entrypoint]
+        self._cli(*args, env=minimal_env(secrets) | _docker_env())
+        return _Container(name=name, ws=ws, deadline_s=spec.deadline_s,
+                          started=time.monotonic())
+
+    def _inspect(self, c: _Container) -> dict[str, Any] | None:
+        out = self._cli("inspect", c.name, check=False)
+        if out.returncode != 0:
+            return None
+        data = json.loads(out.stdout)
+        return data[0] if data else None
+
+    def _done(self, backend: Any) -> bool:
+        c: _Container = backend
+        info = self._inspect(c)
+        if info is None:
+            return True  # container gone: lost
+        c.info = info
+        return info["State"]["Status"] in ("exited", "dead")
+
+    def _collect(self, backend: Any, spec: JobSpec, secrets: dict[str, str], *,
+                 killed: bool) -> JobResult:
+        c: _Container = backend
+        state = c.info.get("State", {}) if c.info else {}
+        logs = self._cli("logs", c.name, check=False, timeout=300)
+        stdout, n1 = redact(logs.stdout.encode(), secrets)
+        stderr, n2 = redact(logs.stderr.encode(), secrets)
+        stdout, t1 = cap(stdout)
+        stderr, t2 = cap(stderr)
+        outputs, n3, truncated = read_outputs(c.ws / "outputs", secrets)
+        truncated += [n for n, t in (("stdout", t1), ("stderr", t2)) if t]
+        started, ended = state.get("StartedAt"), state.get("FinishedAt")
+        duration = _duration(started, ended)
+        exit_code = state.get("ExitCode") if state else None
+        timed_out = duration is not None and duration >= c.deadline_s and exit_code != 0
+        status = classify_status(exit_code, killed=killed or c.killed, timed_out=timed_out,
+                                 crashed=bool(state.get("OOMKilled")))
+        if not state:
+            status = "lost"
+        self._cli("rm", "-f", c.name, check=False)
+        # Outputs are captured and redacted above; the workspace may hold whatever the job
+        # wrote, credentials included, so it does not outlive the job.
+        shutil.rmtree(c.ws.parent, ignore_errors=True)
+        usd, metered = cost(self._info.rate_usd_per_hour, duration)
+        return JobResult(status=status, exit_code=exit_code, stdout=stdout, stderr=stderr,
+                         outputs=outputs, metrics=outputs.get(METRICS_FILE),
+                         started_at=started, ended_at=ended, duration_s=duration,
+                         cost_usd=usd, metered=metered, env_digest=spec.env.digest,
+                         code_digest=spec.code_digest, redactions=sorted(set(n1 + n2 + n3)),
+                         output_cap_bytes=2 * 1024 * 1024, truncated=truncated)
+
+    def _kill(self, backend: Any) -> None:
+        c: _Container = backend
+        c.killed = True
+        self._cli("kill", c.name, check=False)
+
+    def reap(self, tags: list[str]) -> list[str]:
+        found: set[str] = set()
+        for tag in map(check_tag, tags):
+            out = self._cli("ps", "-aq", "--filter", f"label={TAG_LABEL}={tag}", check=False)
+            ids = out.stdout.split()
+            if ids:
+                self._cli("rm", "-f", *ids, check=False)
+                found.add(tag)
+        jobs = self._root / "jobs"
+        for marker in jobs.glob(f"*/{TAG_FILE}") if jobs.is_dir() else []:
+            tag = marker.read_text(encoding="utf-8").strip()
+            if tag in tags:
+                shutil.rmtree(marker.parent, ignore_errors=True)
+                found.add(tag)
+        return sorted(found)
+
+
+def _docker_env() -> dict[str, str]:
+    """Variables the Docker CLI itself needs to reach the daemon."""
+    keep = ("DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG", "DOCKER_CERT_PATH",
+            "DOCKER_TLS_VERIFY", "USERPROFILE", "HOME", "APPDATA", "LOCALAPPDATA",
+            "PROGRAMDATA", "ProgramFiles")
+    return {k: os.environ[k] for k in keep if k in os.environ}
+
+
+def _duration(started: str | None, ended: str | None) -> float | None:
+    from datetime import datetime
+
+    if not started or not ended or ended.startswith("0001"):
+        return None
+
+    def parse(s: str) -> datetime:
+        s = s.rstrip("Z")
+        if "." in s:
+            head, frac = s.split(".", 1)
+            s = f"{head}.{frac[:6]}"
+        return datetime.fromisoformat(s)
+
+    return max((parse(ended) - parse(started)).total_seconds(), 0.0)