puremethod 0.1.0__py3-none-any.whl

executor/watchdog.py

@@ -0,0 +1,108 @@
+"""Deadline watchdog for `local-process` jobs (05-executor.md §5 step 3, INV-2, INV-3).
+
+Run as its own process (`python -m executor.watchdog ...`), detached from the engine, so
+the deadline holds even if the engine dies. It starts the job in a new process group, and
+at the deadline or on a kill request it sends a termination signal to the whole group,
+waits the grace period, then force-kills the tree. It writes the outcome to a result file.
+
+Standard library only.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import subprocess
+import sys
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+POLL_S = 0.1
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def _terminate(proc: subprocess.Popen[bytes]) -> None:
+    if sys.platform == "win32":
+        try:
+            proc.send_signal(signal.CTRL_BREAK_EVENT)
+        except (OSError, ValueError):
+            pass
+    else:
+        try:
+            os.killpg(proc.pid, signal.SIGTERM)
+        except ProcessLookupError:
+            pass
+
+
+def _force_kill(proc: subprocess.Popen[bytes]) -> None:
+    if sys.platform == "win32":
+        subprocess.run(["taskkill", "/F", "/T", "/PID", str(proc.pid)],  # noqa: S603, S607
+                       capture_output=True, check=False)
+    else:
+        try:
+            os.killpg(proc.pid, signal.SIGKILL)
+        except ProcessLookupError:
+            pass
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--deadline", type=float, required=True)
+    p.add_argument("--grace", type=float, required=True)
+    p.add_argument("--cwd", required=True)
+    p.add_argument("--stdout", required=True)
+    p.add_argument("--stderr", required=True)
+    p.add_argument("--result", required=True)
+    p.add_argument("--killfile", required=True)
+    p.add_argument("command")
+    a = p.parse_args(argv)
+
+    started_at, t0 = _now(), time.monotonic()
+    kwargs: dict[str, object] = {}
+    if sys.platform == "win32":
+        kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
+        args: list[str] | str = a.command
+    else:
+        kwargs["start_new_session"] = True
+        args = ["sh", "-c", a.command]
+    env = dict(os.environ)
+    env.pop("PYTHONPATH", None)  # the watchdog's own import path is not the job's
+    with open(a.stdout, "wb") as out, open(a.stderr, "wb") as err:
+        proc = subprocess.Popen(args, cwd=a.cwd, stdout=out, stderr=err,  # noqa: S603
+                                stdin=subprocess.DEVNULL, shell=os.name == "nt", env=env,
+                                **kwargs)  # type: ignore[call-overload]
+        # For `reap` after an engine restart, if this watchdog dies (KB-020).
+        Path(a.result).with_name("job.pid").write_text(str(proc.pid), encoding="utf-8")
+        reason = "exited"
+        while proc.poll() is None:
+            if Path(a.killfile).exists():
+                reason = "killed"
+            elif time.monotonic() - t0 >= a.deadline:
+                reason = "timeout"
+            if reason != "exited":
+                _terminate(proc)
+                end = time.monotonic() + a.grace
+                while proc.poll() is None and time.monotonic() < end:
+                    time.sleep(POLL_S)
+                _force_kill(proc)
+                proc.wait()
+                break
+            time.sleep(POLL_S)
+        if reason == "exited":
+            _force_kill(proc)  # leftovers of the job's group, if any
+    result = {"exit_code": proc.returncode, "reason": reason, "started_at": started_at,
+              "ended_at": _now(), "duration_s": time.monotonic() - t0, "pid": proc.pid}
+    tmp = Path(a.result + ".tmp")
+    tmp.write_text(json.dumps(result), encoding="utf-8")
+    os.replace(tmp, a.result)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ledger/__init__.py

@@ -0,0 +1,46 @@
+"""ledger: the append-only record of a run (docs/knowledge_base/04-ledger.md)."""
+
+from .canonical import canonical_json, entry_hash
+from .core import Ledger
+from .errors import (
+    ArtifactCorrupt,
+    ArtifactMissing,
+    ConcurrentWrite,
+    ExportError,
+    LedgerError,
+    LedgerRejection,
+    UnknownRun,
+)
+from .model import (
+    ARTIFACT_KINDS,
+    CONFIRM_STAGE,
+    ENTRY_TYPES,
+    SMOKE_STAGE,
+    ChainReport,
+    Entry,
+    Rejection,
+    RunState,
+    SpendSummary,
+)
+
+__all__ = [
+    "ARTIFACT_KINDS",
+    "CONFIRM_STAGE",
+    "ENTRY_TYPES",
+    "SMOKE_STAGE",
+    "ArtifactCorrupt",
+    "ArtifactMissing",
+    "ChainReport",
+    "ConcurrentWrite",
+    "Entry",
+    "ExportError",
+    "Ledger",
+    "LedgerError",
+    "LedgerRejection",
+    "Rejection",
+    "RunState",
+    "SpendSummary",
+    "UnknownRun",
+    "canonical_json",
+    "entry_hash",
+]

ledger/artifacts.py

@@ -0,0 +1,62 @@
+"""Content-addressed artifact files (04-ledger.md §4, "Artifact").
+
+v0.1 stores artifacts on the local filesystem under `<root>/<run_id>/<hex>`. Files are
+written once and never replaced; reads re-check the digest.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import tempfile
+from pathlib import Path
+
+from .canonical import sha256_hex
+from .errors import ArtifactCorrupt, ArtifactMissing
+
+DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
+_RUN_ID_RE = re.compile(r"^[A-Za-z0-9_-]+$")
+
+
+def digest_of(data: bytes) -> str:
+    return "sha256:" + sha256_hex(data)
+
+
+class FileArtifactStore:
+    def __init__(self, root: str | Path) -> None:
+        self._root = Path(root)
+        self._root.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, run_id: str, digest: str) -> Path:
+        if not _RUN_ID_RE.match(run_id) or not DIGEST_RE.match(digest):
+            raise ArtifactMissing("malformed run id or digest")
+        return self._root / run_id / digest.removeprefix("sha256:")
+
+    def put(self, run_id: str, digest: str, data: bytes) -> None:
+        path = self._path(run_id, digest)
+        if path.exists():
+            return  # idempotent: same digest, same content
+        path.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".tmp-")
+        try:
+            with os.fdopen(fd, "wb") as f:
+                f.write(data)
+            try:
+                os.link(tmp, path)  # fails if another writer won the race; never overwrites
+            except FileExistsError:
+                pass
+        finally:
+            os.unlink(tmp)
+
+    def get(self, run_id: str, digest: str) -> bytes:
+        path = self._path(run_id, digest)
+        try:
+            data = path.read_bytes()
+        except FileNotFoundError:
+            raise ArtifactMissing(digest) from None
+        if digest_of(data) != digest:
+            raise ArtifactCorrupt(digest)
+        return data
+
+    def exists(self, run_id: str, digest: str) -> bool:
+        return self._path(run_id, digest).exists()

ledger/canonical.py

@@ -0,0 +1,48 @@
+"""Canonical JSON and hashing for ledger entries (04-ledger.md §4)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from typing import Any
+
+GENESIS_HASH = "0" * 64
+
+
+class NonCanonicalValue(ValueError):
+    """A value that has no stable JSON form (NaN, infinity, or a non-JSON type)."""
+
+
+def _check(value: Any, path: str) -> None:
+    if isinstance(value, float):
+        if math.isnan(value) or math.isinf(value):
+            raise NonCanonicalValue(f"{path}: NaN and infinity are not allowed")
+    elif isinstance(value, dict):
+        for key, item in value.items():
+            if not isinstance(key, str):
+                raise NonCanonicalValue(f"{path}: object keys must be strings")
+            _check(item, f"{path}.{key}")
+    elif isinstance(value, list | tuple):
+        for i, item in enumerate(value):
+            _check(item, f"{path}[{i}]")
+    elif value is not None and not isinstance(value, str | int | bool):
+        raise NonCanonicalValue(f"{path}: unsupported type {type(value).__name__}")
+
+
+def canonical_json(value: Any) -> str:
+    """Sorted keys, no whitespace, UTF-8 text. Same output on every platform."""
+    _check(value, "$")
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
+                      allow_nan=False)
+
+
+def sha256_hex(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def entry_hash(fields: dict[str, Any]) -> str:
+    """SHA-256 over the canonical JSON of every entry field except `hash`."""
+    if "hash" in fields:
+        raise ValueError("entry_hash takes the fields without `hash`")
+    return sha256_hex(canonical_json(fields).encode("utf-8"))

ledger/core.py

@@ -0,0 +1,337 @@
+"""The Ledger (04-ledger.md §3): the append-only record of a run."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import tempfile
+import uuid
+from collections.abc import Callable, Iterable, Iterator
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from . import payloads
+from .artifacts import DIGEST_RE, FileArtifactStore, digest_of
+from .canonical import GENESIS_HASH, NonCanonicalValue, canonical_json, entry_hash
+from .errors import ConcurrentWrite, ExportError, LedgerRejection, UnknownRun
+from .model import (
+    ARTIFACT_KINDS,
+    Actor,
+    ChainReport,
+    Entry,
+    Rejection,
+    RunState,
+    SpendSummary,
+)
+from .rules import RunIndex, check
+from .storage import Storage, open_storage
+
+
+def _utc_now() -> str:
+    return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+
+def _strings(value: Any) -> Iterator[str]:
+    if isinstance(value, str):
+        yield value
+    elif isinstance(value, dict):
+        for k, v in value.items():
+            yield k
+            yield from _strings(v)
+    elif isinstance(value, list | tuple):
+        for item in value:
+            yield from _strings(item)
+
+
+def artifact_refs(e: Entry) -> list[str]:
+    """Digests of stored artifacts an entry points to (04-ledger.md §4).
+
+    Only these fields name artifacts. Other digests, such as the content digest of an
+    evaluation data set, describe data that lives outside the ledger.
+    """
+    pl = e.payload
+    found: list[Any] = []
+    if e.type == "call.recorded":
+        found = [pl.get("request_artifact"), pl.get("response_artifact")]
+    elif e.type == "prereg.registered":
+        found = [pl.get("code_digest")]
+    elif e.type in ("experiment.finished", "paper.built"):
+        found = list(pl.get("artifacts", {}).values())
+    elif e.type == "selfcheck.result":
+        found = [pl.get("paper")]
+    return [d for d in found if isinstance(d, str) and DIGEST_RE.match(d)]
+
+
+class Ledger:
+    """One writer per run (`loop`); any number of readers.
+
+    Secret values live only in this object's memory (INV-7, KB-013). A new `Ledger` is a
+    restart: writes to a run are refused until `register_secrets` is called for it.
+    """
+
+    def __init__(self, storage_url: str, artifact_root: str | Path,
+                 clock: Callable[[], str] = _utc_now) -> None:
+        self._storage: Storage = open_storage(storage_url)
+        self._artifacts = FileArtifactStore(artifact_root)
+        self._clock = clock
+        self._secrets: dict[str, tuple[str, ...]] = {}
+        self._index: dict[str, RunIndex] = {}
+
+    def close(self) -> None:
+        self._storage.close()
+
+    # ---- secrets -------------------------------------------------------------------------
+
+    def register_secrets(self, run_id: str, secrets: list[str]) -> None:
+        if not self._storage.run_exists(run_id):
+            raise UnknownRun(run_id)
+        self._secrets[run_id] = self._clean_secrets(secrets)
+
+    @staticmethod
+    def _clean_secrets(secrets: Iterable[str]) -> tuple[str, ...]:
+        values = tuple(secrets)
+        if any(not isinstance(s, str) or s == "" for s in values):
+            raise ValueError("secret values must be non-empty strings")
+        return values
+
+    def _secret_hit(self, run_id: str, texts: Iterable[str]) -> int | None:
+        secrets = self._secrets.get(run_id, ())
+        for text in texts:
+            for i, secret in enumerate(secrets):
+                if secret in text:
+                    return i
+        return None
+
+    # ---- rejections ----------------------------------------------------------------------
+
+    def _reject(self, run_id: str, target: str, rule: str, message: str) -> LedgerRejection:
+        # The message is built by the ledger and never contains payload content (INV-7).
+        self._storage.add_rejection(Rejection(run_id, self._clock(), target, rule, message))
+        return LedgerRejection(rule, message)
+
+    def rejections(self, run_id: str) -> list[Rejection]:
+        return self._storage.rejections(run_id)
+
+    # ---- writing -------------------------------------------------------------------------
+
+    def open_run(self, config: dict[str, Any], secrets: list[str]) -> str:
+        run_id = "run_" + uuid.uuid4().hex
+        values = self._clean_secrets(secrets)
+        payload = {"idea": config.get("idea"), "config": config}
+        self._secrets[run_id] = values
+        try:
+            self._append(run_id, "run.created", payload, [], "loop", new_run=True)
+        except LedgerRejection:
+            del self._secrets[run_id]
+            raise
+        return run_id
+
+    def append(self, run_id: str, type: str, payload: dict[str, Any],
+               refs: list[str] | None = None, *, actor: Actor = "loop") -> Entry:
+        return self._append(run_id, type, payload, list(refs or []), actor, new_run=False)
+
+    def _append(self, run_id: str, entry_type: str, payload: dict[str, Any], refs: list[str],
+                actor: Actor, *, new_run: bool) -> Entry:
+        if actor not in ("loop", "user", "system"):
+            raise ValueError("actor must be loop, user or system")
+        if not new_run and not self._storage.run_exists(run_id):
+            raise UnknownRun(run_id)
+        if run_id not in self._secrets:
+            raise self._reject(run_id, entry_type, "INV-7",
+                               "secrets not registered for this run since the ledger started")
+        # 1. schema
+        try:
+            canonical_json({"payload": payload, "refs": refs})
+            model = payloads.validate_payload(entry_type, payload)
+        except NonCanonicalValue as exc:
+            raise self._reject(run_id, entry_type, "schema", str(exc)) from None
+        except LedgerRejection as exc:
+            raise self._reject(run_id, entry_type, exc.rule, exc.message) from None
+        # 2-3. refs and the type's rules
+        idx = self._refresh(run_id)
+        try:
+            check(idx, entry_type, model, refs)
+        except LedgerRejection as exc:
+            raise self._reject(run_id, entry_type, exc.rule, exc.message) from None
+        # 4. secret scan
+        hit = self._secret_hit(run_id, _strings([payload, refs]))
+        if hit is not None:
+            raise self._reject(run_id, entry_type, "INV-7",
+                               f"content contains the value of registered secret #{hit}")
+
+        at = self._clock()
+        expected_seq = idx.last_seq + 1
+
+        def build(prev_hash: str | None, seq: int) -> Entry:
+            if seq != expected_seq:
+                raise ConcurrentWrite(f"expected seq {expected_seq}, storage is at {seq - 1}")
+            fields: dict[str, Any] = {
+                "run_id": run_id, "seq": seq, "entry_id": f"{run_id}:{seq}",
+                "type": entry_type, "at": at, "actor": actor, "payload": payload,
+                "refs": refs, "prev_hash": prev_hash or GENESIS_HASH,
+            }
+            return Entry(**fields, hash=entry_hash(fields))
+
+        try:
+            entry = self._storage.append(run_id, build)
+        except ConcurrentWrite:
+            self._index.pop(run_id, None)
+            raise
+        idx.apply(entry)
+        return entry
+
+    def close_dangling(self, run_id: str) -> list[Entry]:
+        """INV-6: on resume, close every unmatched experiment.started as `lost`."""
+        idx = self._refresh(run_id)
+        closed = []
+        for started in idx.open_experiments():
+            closed.append(self.append(
+                run_id, "experiment.finished",
+                {"experiment": started, "status": "lost", "metrics": [], "artifacts": {},
+                 "duration_s": None, "cost_usd": None, "metered": False},
+                [started], actor="system"))
+        return closed
+
+    def put_artifact(self, run_id: str, data: bytes | Path, kind: str) -> str:
+        if kind not in ARTIFACT_KINDS:
+            raise ValueError(f"unknown artifact kind {kind!r}")
+        if not self._storage.run_exists(run_id):
+            raise UnknownRun(run_id)
+        if run_id not in self._secrets:
+            raise self._reject(run_id, "artifact", "INV-7",
+                               "secrets not registered for this run since the ledger started")
+        content = data.read_bytes() if isinstance(data, Path) else bytes(data)
+        for i, secret in enumerate(self._secrets[run_id]):
+            if secret.encode("utf-8") in content:
+                raise self._reject(run_id, "artifact", "INV-7",
+                                   f"content contains the value of registered secret #{i}")
+        digest = digest_of(content)
+        self._artifacts.put(run_id, digest, content)
+        self._storage.add_artifact(run_id, digest, kind, len(content), self._clock())
+        return digest
+
+    # ---- reading -------------------------------------------------------------------------
+
+    def _refresh(self, run_id: str) -> RunIndex:
+        idx = self._index.get(run_id)
+        if idx is None:
+            idx = self._index[run_id] = RunIndex(run_id)
+        for e in self._storage.entries(run_id, since_seq=idx.last_seq):
+            idx.apply(e)
+        return idx
+
+    def get_artifact(self, run_id: str, digest: str) -> bytes:
+        return self._artifacts.get(run_id, digest)
+
+    def entries(self, run_id: str, types: Iterable[str] | None = None,
+                refs: Iterable[str] | None = None,
+                since_seq: int | None = None) -> Iterator[Entry]:
+        type_set = set(types) if types is not None else None
+        ref_set = set(refs) if refs is not None else None
+        for e in self._storage.entries(run_id, since_seq=since_seq or 0):
+            if type_set is not None and e.type not in type_set:
+                continue
+            if ref_set is not None and not ref_set.intersection(e.refs):
+                continue
+            yield e
+
+    def state(self, run_id: str) -> RunState:
+        if not self._storage.run_exists(run_id):
+            raise UnknownRun(run_id)
+        idx = RunIndex(run_id)
+        for e in self._storage.entries(run_id):
+            idx.apply(e)
+        return idx.to_state()
+
+    def spend(self, run_id: str) -> SpendSummary:
+        if not self._storage.run_exists(run_id):
+            raise UnknownRun(run_id)
+        total = 0.0
+        by_source: dict[str, float] = {}
+        unmetered = 0
+        limit: float | None = None
+        for e in self._storage.entries(run_id):
+            pl = e.payload
+            if e.type == "run.created":
+                money = pl.get("config", {}).get("limits", {}).get("money")
+                limit = float(money) if isinstance(money, int | float) else None
+                continue
+            if e.type == "call.recorded":
+                source = pl["source"]
+            elif e.type == "experiment.finished":
+                source = "executor"
+            else:
+                continue
+            if not pl.get("metered") or pl.get("cost_usd") is None:
+                unmetered += 1
+                continue
+            cost = float(pl["cost_usd"])
+            total += cost
+            by_source[source] = by_source.get(source, 0.0) + cost
+        return SpendSummary(total_usd=total, by_source=by_source, unmetered_items=unmetered,
+                            money_limit_usd=limit)
+
+    def verify(self, run_id: str) -> ChainReport:
+        entries = self._storage.entries(run_id)
+        prev = GENESIS_HASH
+        for i, e in enumerate(entries, start=1):
+            reason = None
+            if e.seq != i:
+                reason = f"expected seq {i}, found {e.seq}"
+            elif e.run_id != run_id or e.entry_id != f"{run_id}:{e.seq}":
+                reason = "entry does not belong to this run"
+            elif e.prev_hash != prev:
+                reason = "prev_hash does not match the previous entry"
+            elif entry_hash(e.hashed_fields()) != e.hash:
+                reason = "hash does not match the entry content"
+            if reason:
+                return ChainReport(run_id, len(entries), ok=False, first_break_seq=i,
+                                   reason=reason)
+            prev = e.hash
+        return ChainReport(run_id, len(entries), ok=True,
+                           head_hash=entries[-1].hash if entries else None)
+
+    def export(self, run_id: str, dest: Path) -> Path:
+        """INV-8: every entry and every referenced artifact, verified. All or nothing."""
+        dest = Path(dest)
+        if dest.exists() and any(dest.iterdir()):
+            raise ExportError("destination exists and is not empty")
+        if not self._storage.run_exists(run_id):
+            raise UnknownRun(run_id)
+        report = self.verify(run_id)
+        if not report.ok:
+            raise ExportError(f"chain broken at seq {report.first_break_seq}: {report.reason}")
+        entries = self._storage.entries(run_id)
+        digests = sorted({d for e in entries for d in artifact_refs(e)})
+
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        tmp = Path(tempfile.mkdtemp(dir=dest.parent, prefix=".export-"))
+        try:
+            (tmp / "artifacts").mkdir()
+            kinds: dict[str, str | None] = {}
+            for d in digests:
+                try:
+                    data = self._artifacts.get(run_id, d)
+                except Exception as exc:
+                    raise ExportError(f"referenced artifact {d} unavailable: "
+                                      f"{type(exc).__name__}") from None
+                (tmp / "artifacts" / d.removeprefix("sha256:")).write_bytes(data)
+                kinds[d] = self._storage.artifact_kind(run_id, d)
+            with open(tmp / "ledger.jsonl", "w", encoding="utf-8", newline="\n") as f:
+                for e in entries:
+                    f.write(canonical_json(e.to_dict()) + "\n")
+            with open(tmp / "rejections.jsonl", "w", encoding="utf-8", newline="\n") as f:
+                for r in self._storage.rejections(run_id):
+                    f.write(canonical_json(r.__dict__) + "\n")
+            (tmp / "artifacts" / "index.json").write_text(
+                json.dumps(kinds, indent=2, sort_keys=True), encoding="utf-8")
+            (tmp / "chain_report.json").write_text(
+                json.dumps(report.__dict__, indent=2, sort_keys=True), encoding="utf-8")
+            if dest.exists():
+                dest.rmdir()
+            tmp.rename(dest)
+        except BaseException:
+            shutil.rmtree(tmp, ignore_errors=True)
+            raise
+        return dest

ledger/errors.py

@@ -0,0 +1,36 @@
+"""Ledger errors."""
+
+from __future__ import annotations
+
+
+class LedgerError(Exception):
+    """Base class for ledger errors."""
+
+
+class LedgerRejection(LedgerError):
+    """An append or artifact write refused by a rule. Nothing was stored (04-ledger.md §5)."""
+
+    def __init__(self, rule: str, message: str) -> None:
+        super().__init__(f"{rule}: {message}")
+        self.rule = rule
+        self.message = message
+
+
+class UnknownRun(LedgerError):
+    """The run id does not exist."""
+
+
+class ConcurrentWrite(LedgerError):
+    """Another writer appended to the run between validation and storage."""
+
+
+class ArtifactMissing(LedgerError):
+    """No artifact with this digest exists for the run."""
+
+
+class ArtifactCorrupt(LedgerError):
+    """A stored artifact no longer matches its digest."""
+
+
+class ExportError(LedgerError):
+    """An export could not be completed (INV-8)."""