weir-run 0.1.0__py3-none-any.whl

weir_run/__init__.py

@@ -0,0 +1,6 @@
+"""The canary runner: plants fake canaries, runs a program's flows against a
+loopback mock, and judges the captured telemetry with `weir canary judge`.
+
+Its contract is weir's docs/canary.md. The runner is a test-time tool for a
+developer's machine or a CI runner, never imported by production code.
+"""

weir_run/capture.py

@@ -0,0 +1,52 @@
+"""The capture directory the judge reads (docs/canary.md, "The capture").
+
+    capture/otlp/<signal>.jsonl   telemetry files (--otlp): one decompressed
+                                  OTLP/JSON export request per line
+    capture/swept/...             swept files: every other telemetry body and
+                                  each flow's stdout and stderr, one file each
+
+The roles are fixed by directory, never guessed from content.
+"""
+
+from __future__ import annotations
+
+import re
+import threading
+from pathlib import Path
+
+SIGNALS = ("traces", "logs", "metrics")
+
+
+def safe_name(text: str) -> str:
+    """A file-name segment: id characters other than [A-Za-z0-9._-] become `_`."""
+    return re.sub(r"[^A-Za-z0-9._-]", "_", text)
+
+
+class Capture:
+    def __init__(self, root: Path) -> None:
+        self.root = root
+        self.otlp = root / "otlp"
+        self.swept = root / "swept"
+        self.otlp.mkdir(parents=True)
+        self.swept.mkdir(parents=True)
+        self._lock = threading.Lock()
+        self._counts: dict[str, int] = {}
+
+    def otlp_line(self, signal: str, line: bytes) -> None:
+        if signal not in SIGNALS or b"\n" in line:
+            raise ValueError("an OTLP capture line is one export request of a known signal")
+        with self._lock, (self.otlp / f"{signal}.jsonl").open("ab") as f:
+            f.write(line + b"\n")
+
+    def swept_file(self, stem: str, data: bytes, suffix: str = ".body") -> str:
+        """Write one swept file and return its name under capture/swept."""
+        with self._lock:
+            n = self._counts.get(stem, 0) + 1
+            self._counts[stem] = n
+            name = f"{safe_name(stem)}-{n}{suffix}"
+            (self.swept / name).write_bytes(data)
+        return name
+
+    def flow_streams(self, index: int, flow_id: str) -> tuple[Path, Path]:
+        stem = f"flow-{index:02d}-{safe_name(flow_id)}"
+        return self.swept / f"{stem}.stdout", self.swept / f"{stem}.stderr"

weir_run/check.py

@@ -0,0 +1,181 @@
+"""One check run, start to finish (docs/canary.md, "What the runner owes
+the judge"):
+
+1. plant one fresh unique value per placement and write the ledger before
+   any flow starts;
+2. start the mock on 127.0.0.1 and run each flow against it, recording one
+   `flow` record per flow (the mock records minted values, credential-field
+   evidence and unexpected credentials as they happen);
+3. stop the mock, relay each flow's output with every needle replaced;
+4. call the judge, then delete the ledger and the capture.
+
+The CLI runs a whole config through `run_spec`; the pytest plugin drives a
+`CheckRun` call by call.
+"""
+
+from __future__ import annotations
+
+import shutil
+import sys
+import tempfile
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TextIO
+
+from weir_run import flows
+from weir_run import judge as judge_cli
+from weir_run.capture import Capture
+from weir_run.ledger import Ledger
+from weir_run.mock import Mock
+from weir_run.shapes import ByteSource, shaped
+from weir_run.spec import FlowSpec, RunSpec, SpecError, validate
+
+
+@dataclass
+class FlowOutcome:
+    id: str
+    status: str
+    stdout: Path = field(repr=False)
+    stderr: Path = field(repr=False)
+
+
+class CheckRun:
+    def __init__(
+        self,
+        spec: RunSpec,
+        judge: Sequence[str],
+        *,
+        seed: int | None = None,
+        python: str = sys.executable,
+        validate_spec: bool = True,
+    ) -> None:
+        if validate_spec:
+            validate(spec)
+        self.spec = spec
+        self.judge_command = list(judge)
+        self.python = python
+        self.source = ByteSource(seed)
+        self.outcomes: list[FlowOutcome] = []
+        self.verdict: judge_cli.Verdict | None = None
+        self.json_verdict: judge_cli.Verdict | None = None
+        # 0700, private to this user: it holds the ledger and the capture.
+        self.run_dir = Path(tempfile.mkdtemp(prefix="canary-run-"))
+        self._flow_dirs = Path(tempfile.mkdtemp(prefix="canary-flows-"))
+        try:
+            self.ledger = Ledger(self.run_dir / judge_cli.LEDGER, self.source, spec.reserved_ids())
+            self.capture = Capture(self.run_dir / "capture")
+            self.planted: dict[str, str] = {}
+            for p in spec.placements:
+                made = shaped(self.source, p.shape)
+                self.planted[p.variable] = self.ledger.plant(p.id, p.kind, made, p.required).value
+            self.mock = Mock(spec.mock, self.ledger, self.capture, spec.placements, self.source)
+            self.mock.start()
+        except BaseException:
+            for directory in (self.run_dir, self._flow_dirs):
+                shutil.rmtree(directory, ignore_errors=True)
+            raise
+        self._stopped = False
+
+    @property
+    def mock_url(self) -> str:
+        return self.mock.url
+
+    def run_flow(self, flow: FlowSpec) -> FlowOutcome:
+        if self._stopped:
+            raise RuntimeError("the check run has already been judged")
+        if any(o.id == flow.id for o in self.outcomes):
+            raise SpecError(f"flow id {flow.id} is defined twice")
+        index = len(self.outcomes) + 1
+        home = self._flow_dirs / f"{index:02d}" / "home"
+        tmp = self._flow_dirs / f"{index:02d}" / "tmp"
+        home.mkdir(parents=True)
+        tmp.mkdir()
+        env = flows.flow_environment(self.mock_url, self.planted, flow.env, home, tmp)
+        stdout, stderr = self.capture.flow_streams(index, flow.id)
+        status = flows.run(flow, env, stdout, stderr, self.python)
+        self.ledger.flow(flow.id, status)
+        outcome = FlowOutcome(flow.id, status, stdout, stderr)
+        self.outcomes.append(outcome)
+        return outcome
+
+    def stop(self) -> None:
+        if not self._stopped:
+            self._stopped = True
+            self.mock.stop()
+            self.ledger.close()
+
+    def safe(self, text: str) -> str:
+        """Program-derived text, printed only if it holds no trace of a value."""
+        return "[withheld]" if self.ledger.withheld(text) else text
+
+    def relay(self, out: TextIO) -> None:
+        """Each flow's own output, every needle replaced by `<canary:ID>`."""
+        for o in self.outcomes:
+            out.write(f"flow {o.id}: {o.status}\n")
+            for name, path in (("stdout", o.stdout), ("stderr", o.stderr)):
+                data = path.read_bytes() if path.exists() else b""
+                if not data:
+                    continue
+                out.write(f"--- flow {o.id} {name} (canaries redacted) ---\n")
+                text = self.ledger.redact(data).decode("utf-8", "replace")
+                out.write(text if text.endswith("\n") else text + "\n")
+                out.write(f"--- end of flow {o.id} {name} ---\n")
+        mock = self.mock
+        for host, count in sorted(mock.egress.items()):
+            out.write(f"mock: refused egress to {self.safe(host)} ({count}x)\n")
+        for request, count in sorted(mock.unmocked.items()):
+            out.write(f"mock: no mock for {self.safe(request)} ({count}x), answered 404\n")
+        if mock.errors:
+            out.write(f"mock: {mock.errors} request(s) failed inside the mock, answered 500\n")
+
+    def judge(self, *, json_report: bool = False) -> judge_cli.Verdict:
+        self.stop()
+        self.verdict = judge_cli.judge(self.judge_command, self.run_dir)
+        if json_report:
+            self.json_verdict = judge_cli.judge(self.judge_command, self.run_dir, json_report=True)
+        return self.verdict
+
+    def redacted(self, data: bytes) -> str:
+        """Judge output is value-free by contract; redact it all the same."""
+        return self.ledger.redact(data).decode("utf-8", "replace")
+
+    def cleanup(self) -> list[str]:
+        """Delete the ledger, the capture and the flows' homes. Returns the
+        directories that could not be removed."""
+        self.stop()
+        failed: list[str] = []
+        for directory in (self.run_dir, self._flow_dirs):
+            shutil.rmtree(directory, ignore_errors=True)
+            if directory.exists():
+                failed.append(str(directory))
+        return failed
+
+
+def run_spec(
+    spec: RunSpec,
+    judge: Sequence[str],
+    *,
+    out: TextIO,
+    err: TextIO,
+    seed: int | None = None,
+    json_report: Path | None = None,
+    python: str = sys.executable,
+) -> int:
+    """Run every flow of spec, judge, clean up. Returns the exit code."""
+    run = CheckRun(spec, judge, seed=seed, python=python)
+    try:
+        for flow in spec.flows:
+            run.run_flow(flow)
+        run.stop()
+        run.relay(err)
+        verdict = run.judge(json_report=json_report is not None)
+        out.write(run.redacted(verdict.stdout))
+        if verdict.stderr:
+            err.write(run.redacted(verdict.stderr))
+        if json_report is not None and run.json_verdict is not None:
+            json_report.write_text(run.redacted(run.json_verdict.stdout), encoding="utf-8")
+        return verdict.code
+    finally:
+        for directory in run.cleanup():
+            err.write(f"weir-run: could not delete {directory}; delete it by hand\n")

weir_run/cli.py

@@ -0,0 +1,95 @@
+"""`weir-run run`: one check run from a maintainer's config.
+
+Exit codes are the judge's (0 clean, 1 leak, 3 incomplete), and 2 for a
+usage, configuration or input error, the runner's own or the judge's, and
+for any other failure of the run (docs/canary.md, "Exit codes"). A runner
+fault never exits 1, which would read as a leak.
+"""
+
+from __future__ import annotations
+
+import signal
+import subprocess
+import sys
+from pathlib import Path
+from types import FrameType
+
+import click
+
+from weir_run import config
+from weir_run import judge as judge_cli
+from weir_run.check import run_spec
+from weir_run.ledger import LedgerError
+from weir_run.spec import SpecError
+
+USAGE_ERROR = 2
+
+
+@click.group()
+def main() -> None:
+    # Flow output and judge reports can hold any text the program wrote.
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)
+        if reconfigure is not None:
+            reconfigure(encoding="utf-8", errors="replace")
+
+
+@main.command("run")
+@click.option(
+    "--config",
+    "config_path",
+    required=True,
+    type=click.Path(path_type=Path, dir_okay=False),
+    help="The check's TOML config: placements, flows and mocked endpoints.",
+)
+@click.option(
+    "--report",
+    type=click.Path(path_type=Path, dir_okay=False),
+    help="Also write the judge's JSON report (canary-report/1) here.",
+)
+@click.option(
+    "--seed",
+    type=int,
+    help="Draw canary values from a seeded stream, for local reproduction only.",
+)
+@click.option(
+    "--weir",
+    "weir_path",
+    help="The weir executable to judge with. Default: the one beside this Python, else PATH.",
+)
+def run_command(
+    config_path: Path, report: Path | None, seed: int | None, weir_path: str | None
+) -> None:
+    """Plant canaries, run every flow against the loopback mock, judge the capture."""
+    # A CI cancel or timeout sends SIGTERM: stop the same way as Ctrl-C, so
+    # the flow's processes are stopped and the ledger and capture deleted.
+    previous = signal.signal(signal.SIGTERM, _terminate)
+    try:
+        if report is not None and not report.parent.is_dir():
+            raise SpecError(f"--report {report}: no such directory {report.parent}")
+        spec = config.load(config_path)
+        command = judge_cli.resolve(weir_path)
+        judge_cli.check_available(command)
+        code = run_spec(
+            spec, command, out=sys.stdout, err=sys.stderr, seed=seed, json_report=report
+        )
+    except (SpecError, LedgerError, OSError, subprocess.SubprocessError, RuntimeError) as exc:
+        click.echo(f"weir-run: {exc}", err=True)
+        sys.exit(USAGE_ERROR)
+    except KeyboardInterrupt:
+        click.echo("weir-run: interrupted", err=True)
+        sys.exit(USAGE_ERROR)
+    except Exception as exc:
+        click.echo(f"weir-run: the run failed ({type(exc).__name__})", err=True)
+        sys.exit(USAGE_ERROR)
+    finally:
+        signal.signal(signal.SIGTERM, previous)
+    sys.exit(code)
+
+
+def _terminate(signum: int, frame: FrameType | None) -> None:
+    raise KeyboardInterrupt
+
+
+if __name__ == "__main__":
+    main()

weir_run/config.py

@@ -0,0 +1,169 @@
+"""The maintainer's config file: TOML in, a validated `RunSpec` out.
+
+Strict on purpose: an unknown table or key is a usage error, never ignored,
+so a typo cannot quietly drop a placement or a flow from the check. Paths
+are relative to the config file's directory.
+"""
+
+from __future__ import annotations
+
+import tomllib
+from collections.abc import Mapping
+from pathlib import Path
+from typing import cast
+
+from weir_run.spec import (
+    CredentialField,
+    EndpointSpec,
+    FlowSpec,
+    MockSpec,
+    PlacementSpec,
+    RunSpec,
+    SpecError,
+    default_variable,
+    validate,
+)
+
+_TOP = {"mock", "placement", "flow"}
+_MOCK = {"oauth", "model_reply", "tool_calls", "credential_fields", "endpoint"}
+_ENDPOINT = {"id", "credential_fields", "status", "reply", "content_type", "capture"}
+_PLACEMENT = {"id", "kind", "variable", "shape", "required", "provider_host"}
+_FLOW = {"id", "command", "cwd", "timeout_seconds", "env", "skip"}
+
+
+def _table(value: object, where: str) -> dict[str, object]:
+    if not isinstance(value, dict):
+        raise SpecError(f"{where}: expected a table")
+    return cast(dict[str, object], value)
+
+
+def _tables(value: object, where: str) -> list[dict[str, object]]:
+    if not isinstance(value, list):
+        raise SpecError(f"{where}: expected an array of tables ([[{where}]])")
+    return [_table(v, where) for v in cast(list[object], value)]
+
+
+def _keys(table: Mapping[str, object], allowed: set[str], where: str) -> None:
+    unknown = sorted(set(table) - allowed)
+    if unknown:
+        raise SpecError(f"{where}: unknown key {unknown[0]!r}")
+
+
+def _str(table: Mapping[str, object], key: str, where: str, default: str | None = None) -> str:
+    value = table.get(key, default)
+    if not isinstance(value, str):
+        raise SpecError(f"{where}: {key} must be a string")
+    return value
+
+
+def _opt_str(table: Mapping[str, object], key: str, where: str) -> str | None:
+    return None if key not in table else _str(table, key, where)
+
+
+def _bool(table: Mapping[str, object], key: str, where: str, default: bool) -> bool:
+    value = table.get(key, default)
+    if not isinstance(value, bool):
+        raise SpecError(f"{where}: {key} must be true or false")
+    return value
+
+
+def _str_list(table: Mapping[str, object], key: str, where: str) -> list[str]:
+    value = table.get(key, [])
+    if not isinstance(value, list) or not all(
+        isinstance(v, str) for v in cast(list[object], value)
+    ):
+        raise SpecError(f"{where}: {key} must be an array of strings")
+    return cast(list[str], value)
+
+
+def _fields(table: Mapping[str, object], key: str, where: str) -> tuple[CredentialField, ...]:
+    return tuple(CredentialField.parse(f) for f in _str_list(table, key, where))
+
+
+def _endpoint(t: dict[str, object]) -> EndpointSpec:
+    where = f"mock.endpoint {t.get('id', '?')}"
+    _keys(t, _ENDPOINT, where)
+    status = t.get("status", 200)
+    if not isinstance(status, int) or isinstance(status, bool):
+        raise SpecError(f"{where}: status must be an integer")
+    return EndpointSpec(
+        id=_str(t, "id", where),
+        credential_fields=_fields(t, "credential_fields", where),
+        status=status,
+        reply=_str(t, "reply", where, "{}"),
+        content_type=_str(t, "content_type", where, "application/json"),
+        capture=_bool(t, "capture", where, False),
+    )
+
+
+def _mock(t: dict[str, object]) -> MockSpec:
+    _keys(t, _MOCK, "mock")
+    fields = _table(t.get("credential_fields", {}), "mock.credential_fields")
+    return MockSpec(
+        oauth=_bool(t, "oauth", "mock", False),
+        model_reply=_str(t, "model_reply", "mock", "Done."),
+        tool_calls=_str(t, "tool_calls", "mock", "each-offered-tool-once"),
+        credential_fields={
+            name: _fields(fields, name, "mock.credential_fields") for name in fields
+        },
+        endpoints=tuple(_endpoint(e) for e in _tables(t.get("endpoint", []), "mock.endpoint")),
+    )
+
+
+def _placement(t: dict[str, object]) -> PlacementSpec:
+    where = f"placement {t.get('id', '?')}"
+    _keys(t, _PLACEMENT, where)
+    pid = _str(t, "id", where)
+    kind = _str(t, "kind", where)
+    variable = _opt_str(t, "variable", where)
+    if variable is None:
+        if kind == "env" and not pid.startswith("env."):
+            raise SpecError(
+                f"{where}: an env placement whose id does not start with env. names its variable"
+            )
+        variable = default_variable(pid, kind)
+    return PlacementSpec(
+        id=pid,
+        kind=kind,
+        variable=variable,
+        shape=_str(t, "shape", where, "generic"),
+        required=_bool(t, "required", where, False),
+        provider_host=_opt_str(t, "provider_host", where),
+    )
+
+
+def _flow(t: dict[str, object], base: Path) -> FlowSpec:
+    where = f"flow {t.get('id', '?')}"
+    _keys(t, _FLOW, where)
+    timeout = t.get("timeout_seconds", 300)
+    if not isinstance(timeout, int | float) or isinstance(timeout, bool):
+        raise SpecError(f"{where}: timeout_seconds must be a number")
+    env = _table(t.get("env", {}), f"{where} env")
+    if not all(isinstance(v, str) for v in env.values()):
+        raise SpecError(f"{where}: env values must be strings")
+    return FlowSpec(
+        id=_str(t, "id", where),
+        command=tuple(_str_list(t, "command", where)),
+        cwd=(base / _str(t, "cwd", where, ".")).resolve(),
+        timeout_seconds=float(timeout),
+        env=cast(dict[str, str], env),
+        skip=_opt_str(t, "skip", where),
+    )
+
+
+def load(path: Path) -> RunSpec:
+    try:
+        raw = tomllib.loads(path.read_text(encoding="utf-8"))
+    except OSError as exc:
+        raise SpecError(f"cannot read config {path}: {exc.strerror}") from None
+    except (tomllib.TOMLDecodeError, UnicodeDecodeError) as exc:
+        raise SpecError(f"config {path} is not valid TOML: {exc}") from None
+    _keys(raw, _TOP, str(path))
+    base = path.resolve().parent
+    spec = RunSpec(
+        placements=tuple(_placement(p) for p in _tables(raw.get("placement", []), "placement")),
+        flows=tuple(_flow(f, base) for f in _tables(raw.get("flow", []), "flow")),
+        mock=_mock(_table(raw.get("mock", {}), "mock")),
+    )
+    validate(spec)
+    return spec

weir_run/flows.py

@@ -0,0 +1,104 @@
+"""Running one flow: an allowlisted environment and a bounded subprocess.
+
+A flow starts from an empty environment, the `env -i` of the contract, and
+gets only what is listed here: PATH, a private HOME and temporary
+directory, a fixed locale, its planted canaries, the mock's URLs and its
+own non-secret settings from the config. Nothing else from the job's
+environment reaches it, so a real provider key in CI secrets never does.
+"""
+
+from __future__ import annotations
+
+import os
+import signal
+import subprocess
+import sys
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from weir_run.spec import FlowSpec
+
+PYTHON_TOKEN = "{python}"
+MOCK_TOKEN = "{mock}"
+
+
+def flow_environment(
+    mock_url: str,
+    planted: Mapping[str, str],
+    extra: Mapping[str, str],
+    home: Path,
+    tmp: Path,
+) -> dict[str, str]:
+    """The whole environment of a flow: variable name -> value."""
+    env = {
+        "PATH": os.environ.get("PATH", os.defpath),
+        "HOME": str(home),
+        "TMPDIR": str(tmp),
+        "TEMP": str(tmp),
+        "TMP": str(tmp),
+        "LANG": "C.UTF-8",
+        "PYTHONUTF8": "1",
+        "CANARY_MOCK_URL": mock_url,
+        "OPENAI_BASE_URL": f"{mock_url}/v1",
+        "OPENAI_API_BASE": f"{mock_url}/v1",
+        "OTEL_EXPORTER_OTLP_ENDPOINT": mock_url,
+        "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
+    }
+    for signal_name in ("TRACES", "LOGS", "METRICS"):
+        env[f"OTEL_EXPORTER_OTLP_{signal_name}_ENDPOINT"] = f"{mock_url}/v1/{signal_name.lower()}"
+        env[f"OTEL_EXPORTER_OTLP_{signal_name}_PROTOCOL"] = "http/protobuf"
+    # Every client that honours the proxy variables reaches other hosts only
+    # through the mock, which refuses them. The mock itself is reached direct.
+    for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"):
+        env[name] = env[name.lower()] = mock_url
+    env["NO_PROXY"] = env["no_proxy"] = "127.0.0.1,localhost"
+    if sys.platform == "win32":
+        env["USERPROFILE"] = str(home)
+        # Windows processes cannot start without these; neither holds a secret.
+        for name in ("SYSTEMROOT", "PATHEXT"):
+            if name in os.environ:
+                env[name] = os.environ[name]
+    env.update(planted)
+    env.update({k: v.replace(MOCK_TOKEN, mock_url) for k, v in extra.items()})
+    return env
+
+
+def command_line(flow: FlowSpec, python: str) -> list[str]:
+    return [python if part == PYTHON_TOKEN else part for part in flow.command]
+
+
+def run(flow: FlowSpec, env: Mapping[str, str], stdout: Path, stderr: Path, python: str) -> str:
+    """Run a flow to completion; return its ledger status."""
+    if flow.skip is not None:
+        return "skipped"
+    with stdout.open("wb") as out, stderr.open("wb") as err:
+        # Its own process group, so a timeout or the end of the flow can
+        # stop everything it started.
+        options: dict[str, Any] = {"cwd": flow.cwd, "env": dict(env), "stdout": out, "stderr": err}
+        if sys.platform == "win32":
+            options["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
+        else:
+            options["start_new_session"] = True
+        try:
+            proc = subprocess.Popen(command_line(flow, python), stdin=subprocess.DEVNULL, **options)
+        except OSError as exc:
+            err.write(f"weir-run: could not start the flow: {exc.strerror}\n".encode())
+            return "failed"
+        try:
+            code = proc.wait(timeout=flow.timeout_seconds)
+            status = "ok" if code == 0 else "failed"
+        except subprocess.TimeoutExpired:
+            status = "timeout"
+        finally:
+            # A flow's leftover children do not outlive it: the mock and the
+            # telemetry receiver exist only for the length of the run.
+            if sys.platform == "win32":
+                proc.kill()
+            else:
+                try:
+                    os.killpg(proc.pid, signal.SIGKILL)
+                except (ProcessLookupError, PermissionError):
+                    pass
+            proc.wait()
+    return status