obey-cli 0.1.0__py3-none-any.whl

obey_cli/runner/local.py

@@ -0,0 +1,176 @@
+"""--unsafe-local: run the agent on this machine without a container.
+
+For users without Docker. Each run gets a fresh copy of the tracked files in a temporary directory with no git
+remotes, an empty HOME, no credential environment variables and no git credential helpers. The agent's own sandbox
+mode is switched on where it has one (Claude Code, Codex CLI, Gemini CLI). The model gateway runs in this process on
+127.0.0.1, so the API key still never reaches the agent. There is no egress allow-list: commands the agent runs can
+reach the network. Probes for "never commit" and "never push" are refused in this mode.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import signal
+import subprocess
+import tempfile
+import threading
+import time
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from obey_cli.runner import gateway
+from obey_cli.runner.sandbox import Invocation, RawRun
+from obey_cli.runner.workspace import collect_end_state
+
+KEEP_ENV = ("PATH", "LANG", "LC_ALL", "TZ", "TMPDIR")
+REFUSED_TYPES = ("no_commit", "no_push")
+WARNING = (
+    "obey: --unsafe-local runs the agent directly on this machine. It works in a temporary copy with an empty HOME and "
+    "no credentials, and turns on the agent's own sandbox where it has one, but commands it runs can still reach the "
+    "network and anything your user can read. Prefer the default container sandbox."
+)
+
+
+@dataclass
+class LocalBatch:
+    server: Any
+    config: gateway.Config
+    ledger_path: Path
+    gateway_url: str
+    homes: list[Path] = field(default_factory=list)
+
+
+class LocalSandbox:
+    def start_batch(
+        self,
+        *,
+        key: str,
+        model: str | None,
+        price: Mapping[str, float],
+        allow_hosts: Sequence[str],
+        run_cap: float | None,
+        budget: float | None,
+        upstreams: Mapping[str, str] | None = None,
+        run_ids: Sequence[str] | None = None,
+    ) -> LocalBatch:
+        ledger_path = Path(tempfile.mkdtemp(prefix="obey-ledger-")) / "ledger.jsonl"
+        config = gateway.Config(
+            key,
+            model,
+            str(ledger_path),
+            dict(price),
+            run_cap,
+            budget,
+            {**gateway.UPSTREAMS, **(upstreams or {})},
+            run_ids=run_ids,
+        )
+        server = gateway.make_server("127.0.0.1:0", config)
+        port = server.server_address[1]
+        config.allowed_hosts = [f"127.0.0.1:{port}", f"localhost:{port}"]
+        threading.Thread(target=server.serve_forever, daemon=True).start()
+        return LocalBatch(server=server, config=config, ledger_path=ledger_path, gateway_url=f"http://127.0.0.1:{port}")
+
+    def run(
+        self,
+        batch: LocalBatch,
+        root: Path,
+        inv: Invocation,
+        out_dir: Path,
+        run_id: str,
+        *,
+        base_sha: str | None = None,
+        copy_back_work: bool = False,
+    ) -> RawRun:
+        batch.config.register_run(run_id)
+        work = root / "work"
+        home = root / "home"
+        home.mkdir(parents=True, exist_ok=True)
+        for rel, content in inv.home_files.items():
+            target = home / rel
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(content if isinstance(content, str) else content.decode(), encoding="utf-8")
+        # Strip the copy's remotes and credential helpers: nothing to push to, nothing to log in with.
+        for remote in subprocess.run(
+            ["git", "remote"], cwd=work, capture_output=True, text=True, check=False
+        ).stdout.split():
+            subprocess.run(["git", "remote", "remove", remote], cwd=work, check=False)
+        env = {k: os.environ[k] for k in KEEP_ENV if k in os.environ}
+        env.update(
+            {
+                "HOME": str(home),
+                "GIT_CONFIG_NOSYSTEM": "1",
+                "GIT_TERMINAL_PROMPT": "0",
+                "GIT_CONFIG_GLOBAL": str(home / ".gitconfig"),
+                "TERM": "dumb",
+                **inv.env,
+            }
+        )
+        (home / ".gitconfig").write_text(
+            "[user]\n\tname = Developer\n\temail = dev@example.com\n[credential]\n\thelper =\n", encoding="utf-8"
+        )
+        out = out_dir / ".obey-out"
+        out.mkdir(parents=True, exist_ok=True)
+        started = time.monotonic()
+        timed_out = False
+        (home / ".obey-out").mkdir(exist_ok=True)
+        code: int | None = None
+        with (out / "stream.jsonl").open("wb") as stdout, (out / "stderr.log").open("wb") as stderr:
+            # A session of its own, so a timeout (or the end of the run) also stops what the agent left running.
+            proc = subprocess.Popen(
+                inv.argv,
+                cwd=work,
+                env=env,
+                stdin=subprocess.DEVNULL,
+                stdout=stdout,
+                stderr=stderr,
+                start_new_session=True,
+            )
+            try:
+                code = proc.wait(timeout=inv.timeout_s)
+            except subprocess.TimeoutExpired:
+                timed_out = True
+            finally:
+                _kill_group(proc)
+        duration = time.monotonic() - started
+        if code is not None:
+            (out / "exit_code").write_text(str(code), encoding="utf-8")
+        if base_sha:
+            state = collect_end_state(work, base_sha, None)
+            (out / "end-state.json").write_text(
+                json.dumps(
+                    {
+                        "diff": state.diff,
+                        "commits": state.commits,
+                        "pushed_refs": [],
+                        "diff_truncated": state.diff_truncated,
+                    }
+                )
+            )
+        hook_log = Path(inv.env.get("OBEY_HOOK_LOG", ""))
+        if hook_log.name and hook_log.exists():
+            (out / "hook-log.jsonl").write_bytes(hook_log.read_bytes())
+        return RawRun(
+            root=root if copy_back_work else None, out_dir=out, exit_code=code, timed_out=timed_out, duration_s=duration
+        )
+
+    def ledger_lines(self, batch: LocalBatch) -> list[dict[str, Any]]:
+        if not batch.ledger_path.exists():
+            return []
+        return [json.loads(line) for line in batch.ledger_path.read_text().splitlines() if line.strip()]
+
+    def egress_log(self, batch: LocalBatch) -> list[dict[str, Any]]:
+        return []
+
+    def stop_batch(self, batch: LocalBatch) -> None:
+        batch.server.shutdown()
+        batch.server.server_close()
+
+
+def _kill_group(proc: subprocess.Popen[bytes]) -> None:
+    with contextlib.suppress(ProcessLookupError, PermissionError):
+        os.killpg(proc.pid, signal.SIGKILL)
+    proc.wait()

obey_cli/runner/orchestrate.py

@@ -0,0 +1,227 @@
+"""Plan and execute probe runs: arms, trials, concurrency, budget, and scoring."""
+
+from __future__ import annotations
+
+import datetime as _dt
+import json
+import threading
+import time
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from obey_cli.runner.ledger import BudgetExceeded, Entry, Ledger
+from obey_cli.runner.workspace import remove_lines
+
+# Rough tokens per run by agent, used only for --estimate before any run has been measured.
+EST_TOKENS = {
+    "claude-code": (450_000, 6_000),
+    "codex": (250_000, 5_000),
+    "opencode": (200_000, 4_000),
+    "copilot": (250_000, 5_000),
+    "gemini": (250_000, 5_000),
+}
+CACHE_SHARE = 0.85
+
+
+@dataclass
+class PlannedRun:
+    run_id: str
+    rule: dict[str, Any]
+    probe: dict[str, Any]
+    arm: str  # rule | baseline | enforced
+    trial: int
+    overrides: dict[str, str | None] = field(default_factory=dict)
+    extra_files: dict[str, str] = field(default_factory=dict)
+    git_hooks: dict[str, str] = field(default_factory=dict)
+    condition: str | None = None
+
+
+def _rid(*parts: object) -> str:
+    stamp = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%d%H%M%S")
+    text = "-".join(str(p) for p in parts)
+    import hashlib
+
+    return f"{stamp}-{hashlib.sha1(text.encode()).hexdigest()[:10]}"  # noqa: S324 - an identifier
+
+
+def plan_runs(
+    repo: Path,
+    rules: Sequence[dict[str, Any]],
+    *,
+    trials: int,
+    baseline_trials: int,
+    enforced: Mapping[str, str] | None = None,
+    enforced_hooks: Mapping[str, str] | None = None,
+    only_enforced: bool = False,
+) -> list[PlannedRun]:
+    """One PlannedRun per (rule, probe, arm, trial). `enforced` maps repository paths to enforcement file contents."""
+    planned: list[PlannedRun] = []
+    for rule in rules:
+        for probe in rule.get("probes") or []:
+            if not only_enforced:
+                for t in range(trials):
+                    planned.append(PlannedRun(_rid(rule["id"], probe["id"], "rule", t), rule, probe, "rule", t))
+                src = rule["source"]
+                path = repo / src["file"]
+                if baseline_trials and path.exists():
+                    text = remove_lines(path.read_text(encoding="utf-8"), src["lines"][0], src["lines"][1])
+                    for t in range(baseline_trials):
+                        planned.append(
+                            PlannedRun(
+                                _rid(rule["id"], probe["id"], "baseline", t),
+                                rule,
+                                probe,
+                                "baseline",
+                                t,
+                                overrides={src["file"]: text},
+                            )
+                        )
+            if enforced is not None:
+                for t in range(trials):
+                    planned.append(
+                        PlannedRun(
+                            _rid(rule["id"], probe["id"], "enforced", t),
+                            rule,
+                            probe,
+                            "enforced",
+                            t,
+                            extra_files=dict(enforced),
+                            git_hooks=dict(enforced_hooks or {}),
+                        )
+                    )
+    return planned
+
+
+def estimate(
+    planned: Sequence[PlannedRun], agent: str, price: Mapping[str, float], measured_per_run: float | None = None
+) -> dict[str, Any]:
+    if measured_per_run is not None:
+        per_run = measured_per_run
+        source = "measured from previous runs"
+    else:
+        tokens_in, tokens_out = EST_TOKENS.get(agent, (300_000, 5_000))
+        cached = tokens_in * CACHE_SHARE
+        fresh = tokens_in - cached
+        per_run = (
+            fresh * price.get("prompt", 0.0)
+            + cached * price.get("cache_read", price.get("prompt", 0.0))
+            + tokens_out * price.get("completion", 0.0)
+        )
+        source = "default token profile × model price"
+    return {"runs": len(planned), "per_run_usd": per_run, "total_usd": per_run * len(planned), "source": source}
+
+
+def model_price(model: str, client: Any = None) -> dict[str, float]:
+    """USD per token for an OpenRouter model id (empty when unknown or offline)."""
+    import httpx
+
+    try:
+        resp = (client or httpx).get("https://openrouter.ai/api/v1/models", timeout=20)
+        for m in resp.json().get("data", []):
+            if m.get("id") == model:
+                p = m.get("pricing") or {}
+                out = {"prompt": float(p.get("prompt") or 0), "completion": float(p.get("completion") or 0)}
+                if p.get("input_cache_read"):
+                    out["cache_read"] = float(p["input_cache_read"])
+                if p.get("input_cache_write"):
+                    out["cache_write"] = float(p["input_cache_write"])
+                return out
+    except (httpx.HTTPError, ValueError):
+        pass
+    return {}
+
+
+@dataclass
+class Execution:
+    records: list[dict[str, Any]]
+    stopped: str | None  # why the batch stopped early, if it did
+
+
+def execute_all(
+    planned: Sequence[PlannedRun],
+    run_one: Callable[[PlannedRun], dict[str, Any]],
+    *,
+    ledger: Ledger,
+    per_run_estimate: float,
+    concurrency: int,
+    out: Path,
+    progress: Callable[[str], None] = print,
+) -> Execution:
+    """Run everything with a thread pool, writing each record as it finishes and stopping before the budget."""
+    out.parent.mkdir(parents=True, exist_ok=True)
+    lock = threading.Lock()
+    records: list[dict[str, Any]] = []
+    stop: list[str] = []
+    started = time.monotonic()
+
+    def guarded(run: PlannedRun) -> dict[str, Any] | None:
+        if stop:
+            return None
+        try:
+            with lock:
+                ledger.check(per_run_estimate)
+        except BudgetExceeded as exc:
+            stop.append(str(exc))
+            return None
+        record = run_one(run)
+        with lock:
+            ledger.record(
+                Entry(
+                    ts=time.time(),
+                    run_id=record["run_id"],
+                    kind="agent",
+                    model=record.get("model", ""),
+                    input_tokens=record["usage"]["input_tokens"],
+                    output_tokens=record["usage"]["output_tokens"],
+                    cache_read_tokens=record["usage"]["cache_read_tokens"],
+                    cost_usd=record["usage"]["cost_usd"],
+                    cost_source="provider",
+                )
+            )
+            with out.open("a", encoding="utf-8") as fh:
+                fh.write(json.dumps(record) + "\n")
+        return record
+
+    with ThreadPoolExecutor(max_workers=max(1, concurrency)) as pool:
+        futures = {pool.submit(guarded, run): run for run in planned}
+        for done, future in enumerate(as_completed(futures), 1):
+            record = future.result()
+            if record is not None:
+                records.append(record)
+                elapsed = time.monotonic() - started
+                progress(
+                    f"[{done}/{len(planned)}] {record['arm']:8s} {record['rule_id'][:40]:40s} "
+                    f"{record['status']:7s} ${record['usage']['cost_usd']:.4f} ({elapsed:.0f}s)"
+                )
+    return Execution(records=records, stopped=stop[0] if stop else None)
+
+
+def outcomes(records: Iterable[dict[str, Any]], rules: Mapping[str, dict[str, Any]]) -> list[Any]:
+    """Score every run against its own rule with the deterministic checker."""
+    from obey_cli.checkers import check
+    from obey_cli.score import Outcome
+
+    result = []
+    for rec in records:
+        rule = rules.get(rec["rule_id"])
+        if rule is None:
+            continue
+        verdict = check(rule["checker"], rec)
+        denied = sum(1 for h in rec.get("hook_log") or [] if h.get("decision") in ("deny", "block"))
+        result.append(
+            Outcome(
+                rule_id=rec["rule_id"],
+                agent=rec["agent"],
+                arm=rec["arm"],
+                verdict=verdict.verdict,
+                run_id=rec["run_id"],
+                denied=denied,
+                evidence=verdict.evidence,
+                condition=rec.get("condition"),
+                status=rec.get("status", "ok"),
+            )
+        )
+    return result

obey_cli/runner/sandbox/Dockerfile

@@ -0,0 +1,41 @@
+# Default obey sandbox: Python, Node, Go, git and the headless agent CLIs, run as a non-root user.
+# Build: obey doctor --build-image   (or: docker build -t obey-sandbox:<version> <this directory>)
+ARG GO_IMAGE=golang:1.27.3-trixie
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.9
+FROM ${GO_IMAGE} AS go
+FROM ${UV_IMAGE} AS uv
+
+FROM node:24.22.1-trixie-slim
+ARG CLAUDE_CODE_VERSION=2.1.309
+ARG CODEX_VERSION=0.164.0
+ARG OPENCODE_VERSION=1.19.4
+ARG GEMINI_CLI_VERSION=0.65.0
+ARG COPILOT_CLI_VERSION=1.0.103
+ARG PNPM_VERSION=9.15.9
+ENV DEBIAN_FRONTEND=noninteractive \
+    PATH=/usr/local/go/bin:/home/agent/.local/bin:$PATH \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_LINK_MODE=copy \
+    NPM_CONFIG_UPDATE_NOTIFIER=false \
+    NODE_USE_ENV_PROXY=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends git ca-certificates python3 python3-venv python3-pip ripgrep \
+    procps less jq patch make \
+ && rm -rf /var/lib/apt/lists/* \
+ && usermod -l agent -d /home/agent -m node && groupmod -n agent node
+COPY --from=go /usr/local/go /usr/local/go
+COPY --from=uv /uv /uvx /usr/local/bin/
+RUN set -e; pkgs="pnpm@$PNPM_VERSION"; \
+    [ -n "$CLAUDE_CODE_VERSION" ] && pkgs="$pkgs @anthropic-ai/claude-code@$CLAUDE_CODE_VERSION"; \
+    [ -n "$CODEX_VERSION" ] && pkgs="$pkgs @openai/codex@$CODEX_VERSION"; \
+    [ -n "$OPENCODE_VERSION" ] && pkgs="$pkgs opencode-ai@$OPENCODE_VERSION"; \
+    [ -n "$GEMINI_CLI_VERSION" ] && pkgs="$pkgs @google/gemini-cli@$GEMINI_CLI_VERSION"; \
+    [ -n "$COPILOT_CLI_VERSION" ] && pkgs="$pkgs @github/copilot@$COPILOT_CLI_VERSION"; \
+    npm install -g --omit=dev $pkgs && npm cache clean --force
+COPY entrypoint.sh /usr/local/bin/obey-entrypoint
+RUN chmod 755 /usr/local/bin/obey-entrypoint && mkdir -p /srv/obey/work && chown -R agent:agent /home/agent /srv/obey
+USER agent
+# The repository copy lives outside $HOME, so files in $HOME are never mistaken for a parent project's files.
+WORKDIR /srv/obey/work
+ENTRYPOINT ["/usr/local/bin/obey-entrypoint"]

obey_cli/runner/sandbox/entrypoint.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+# Runs one agent command with a deadline and keeps its stdout/stderr for obey to read back.
+# Usage: obey-entrypoint <timeout-seconds> <command> [args...]
+set -u
+deadline="$1"; shift
+mkdir -p /home/agent/.obey-out
+cd /srv/obey/work || exit 97
+if [ -n "${OBEY_LOCAL_FORWARD:-}" ]; then
+  # Some CLIs accept plain HTTP only on localhost: forward 127.0.0.1:$OBEY_LOCAL_FORWARD to the gateway.
+  python3 - "$OBEY_LOCAL_FORWARD" <<'PY' &
+import asyncio, sys
+async def pipe(r, w):
+    try:
+        while (data := await r.read(65536)):
+            w.write(data); await w.drain()
+    finally:
+        w.close()
+async def handle(r, w):
+    ur, uw = await asyncio.open_connection("obey-gw", 8787)
+    await asyncio.gather(pipe(r, uw), pipe(ur, w))
+async def main():
+    server = await asyncio.start_server(handle, "127.0.0.1", int(sys.argv[1]))
+    async with server:
+        await server.serve_forever()
+asyncio.run(main())
+PY
+  sleep 0.3
+fi
+timeout --signal=TERM --kill-after=10 "$deadline" "$@" \
+  < /dev/null > /home/agent/.obey-out/stream.jsonl 2> /home/agent/.obey-out/stderr.log
+code=$?
+echo "$code" > /home/agent/.obey-out/exit_code
+if [ -f /home/agent/.obey-tools/workspace.py ]; then
+  # Read the end state here, so only small JSON files travel back to the host.
+  python3 /home/agent/.obey-tools/workspace.py collect /srv/obey/work "${OBEY_BASE_SHA:-HEAD}" \
+    /srv/obey/.remote/origin.git /home/agent/.obey-out/end-state.json 2> /home/agent/.obey-out/collect.log
+fi
+exit 0