doesitfire 0.1.2__py3-none-any.whl

doesitfire/runner.py

@@ -0,0 +1,283 @@
+"""Run the activation matrix: skills x prompts x agents x repeats."""
+
+from __future__ import annotations
+
+import datetime as dt
+import platform
+import tempfile
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from . import __version__
+from .agents import CLI_VERSIONS, EXPERIMENTAL, Agent, Observation, make_agent
+from .config import Config
+from .errors import BudgetExceeded, DoesItFireError
+from .providers import Provider, get_provider, resolve_key
+from .sandbox import IMAGE, Sandbox, build_image, check_docker, image_exists, stage_skills
+
+# Used for budgeting before the first real usage figure is known (doesitfire's plan
+# assumed ~20k input tokens per CLI run).
+EST_INPUT_TOKENS = 20_000
+EST_OUTPUT_TOKENS = 1_000
+
+
+def run_cost(provider: Provider, agent_cfg, obs: Observation) -> float:
+    if provider.local:
+        return 0.0
+    pin, pout = agent_cfg.price_input or 0.0, agent_cfg.price_output or 0.0
+    computed = obs.input_tokens * pin / 1e6 + obs.output_tokens * pout / 1e6
+    if not obs.usage_seen:  # killed before reporting usage: charge the estimate, never zero
+        computed = max(computed, EST_INPUT_TOKENS * pin / 1e6 + EST_OUTPUT_TOKENS * pout / 1e6)
+    return round(max(computed, obs.reported_cost or 0.0), 6)
+
+
+def preflight(cfg: Config) -> dict[str, Provider]:
+    """Validate keys, prices and model availability before any container starts."""
+    providers: dict[str, Provider] = {}
+    for a in cfg.agents:
+        p = get_provider(a.provider)
+        resolve_key(a.name, p)  # raises a clear error naming the missing variable
+        if p.paid and (a.price_input is None or a.price_output is None):
+            raise DoesItFireError(
+                f"Agent '{a.name}' uses paid provider '{p.name}' but has no price. Add "
+                "'price: {input: <USD per 1M tokens>, output: <USD per 1M tokens>}' to it in "
+                "doesitfire.yaml so --max-cost can be enforced."
+            )
+        if p.name == "ollama":
+            check_ollama_model(p, a.model)
+        providers[a.name] = p
+    return providers
+
+
+def check_ollama_model(p: Provider, model: str) -> None:
+    import openai
+
+    client = openai.OpenAI(base_url=p.openai_base, api_key="ollama", timeout=10, max_retries=0)
+    try:
+        ids = {m.id for m in client.models.list()}
+    except (openai.APIConnectionError, openai.APITimeoutError):
+        raise DoesItFireError(
+            f"Ollama is not reachable at {p.openai_base.removesuffix('/v1')}. Start it with 'ollama serve' "
+            "or set OLLAMA_HOST. On Linux, make it listen on all interfaces (OLLAMA_HOST=0.0.0.0) so "
+            "containers can reach it."
+        ) from None
+    except openai.APIError as exc:
+        raise DoesItFireError(f"Ollama answered with an error while listing models: {exc}") from None
+    if model not in ids and f"{model}:latest" not in ids:
+        raise DoesItFireError(
+            f"Model '{model}' not found in Ollama. Available: {', '.join(sorted(ids)) or 'none'}. "
+            "Agent prompts need a 16k context; create the model with: "
+            "ollama pull qwen3:4b && ollama create qwen3:4b-16k -f ollama/Modelfile"
+        )
+
+
+def hardware() -> str | None:
+    """A short CPU and memory label, e.g. 'Apple M4, 16 GB'. Best effort, no network."""
+    import os
+    import subprocess
+
+    cpu, mem = None, None
+    try:
+        if platform.system() == "Darwin":
+            cpu = subprocess.run(
+                ["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True, timeout=5
+            ).stdout.strip()
+            mem = int(
+                subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True, text=True, timeout=5).stdout.strip()
+                or 0
+            )
+        elif platform.system() == "Linux":
+            with open("/proc/cpuinfo", encoding="utf-8") as fh:
+                for line in fh:
+                    if line.lower().startswith("model name"):
+                        cpu = line.split(":", 1)[1].strip()
+                        break
+            mem = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+    except (OSError, ValueError, subprocess.SubprocessError):
+        pass
+    parts = [p for p in (cpu, f"{round(mem / 2**30)} GB" if mem else None) if p]
+    return ", ".join(parts) or None
+
+
+def host_info(docker_version: str | None) -> dict:
+    return {
+        "os": platform.system(),
+        "arch": platform.machine(),
+        "hardware": hardware(),
+        "python": platform.python_version(),
+        "docker": docker_version,
+    }
+
+
+def run_matrix(
+    cfg: Config,
+    max_cost: float = 1.0,
+    log: Callable[[str], None] = print,
+    progress: Callable[[dict], None] | None = None,
+    checkpoint: Callable[[dict], None] | None = None,
+    logs_dir: Path | None = None,
+    resume: dict | None = None,
+    image: str = IMAGE,
+    rebuild: bool = False,
+) -> dict:
+    providers = preflight(cfg)
+    docker_version = check_docker()
+    if rebuild or not image_exists(image):
+        build_image(image, log)
+
+    agents: list[Agent] = [make_agent(a, providers[a.name], cfg.max_turns) for a in cfg.agents]
+    skill_names = {s.name for s in cfg.skills}
+    started = dt.datetime.now(dt.UTC)
+    results: dict = {
+        "schema": 1,
+        "tool": "doesitfire",
+        "version": __version__,
+        "started_at": started.isoformat(timespec="seconds"),
+        "finished_at": None,
+        "duration_s": None,
+        "repeats": cfg.repeats,
+        "max_turns": cfg.max_turns,
+        "host": host_info(docker_version),
+        "agents": [
+            {
+                "name": a.cfg.name,
+                "cli_version": CLI_VERSIONS[a.cfg.name],
+                "provider": a.provider.name,
+                "model": a.cfg.model,
+                "experimental": a.cfg.name in EXPERIMENTAL,
+                "price": (
+                    {"input": a.cfg.price_input, "output": a.cfg.price_output}
+                    if a.cfg.price_input is not None
+                    else None
+                ),
+            }
+            for a in agents
+        ],
+        "skills": [{"name": s.name, "description": s.description} for s in cfg.skills],
+        "trials": [],
+        "cost_usd": 0.0,
+        "partial": False,
+        "stop_reason": None,
+        "sessions": 1,
+        "recorded": False,
+    }
+
+    plan = [
+        (rep, agent, skill, prompt, expect)
+        for rep in range(cfg.repeats)
+        for agent in agents
+        for skill in cfg.skills
+        for expect, prompts in (("fire", skill.should_fire), ("no_fire", skill.should_not_fire))
+        for prompt in prompts
+    ]
+    previous_duration = 0.0
+    if resume:
+        # Keep finished trials of an interrupted run with the same plan, run only the rest.
+        keys = {(a.cfg.name, s.name, p, e, r) for r, a, s, p, e in plan}
+        kept = [
+            t
+            for t in resume.get("trials", [])
+            if (t["agent"], t["skill"], t["prompt"], t["expect"], t["repeat"]) in keys
+        ]
+        done = {(t["agent"], t["skill"], t["prompt"], t["expect"], t["repeat"]) for t in kept}
+        plan = [x for x in plan if (x[1].cfg.name, x[2].name, x[3], x[4], x[0]) not in done]
+        results["trials"] = kept
+        results["cost_usd"] = round(sum(t.get("cost_usd") or 0.0 for t in kept), 6)
+        results["started_at"] = resume.get("started_at") or results["started_at"]
+        results["sessions"] = int(resume.get("sessions") or 1) + 1
+        previous_duration = float(resume.get("duration_s") or 0.0)
+        log(f"Resuming: {len(kept)} runs already done, {len(plan)} to go.")
+    max_seen: dict[str, float] = {}
+    t0 = time.monotonic()
+    with tempfile.TemporaryDirectory(prefix="doesitfire-") as tmp:
+        staged = stage_skills([(s.name, s.path) for s in cfg.skills], Path(tmp) / "skills")
+        allow = [u for a in agents for u in a.build("x").allow]
+        developer_as_system = any(a.cfg.name == "codex" and a.provider.name == "ollama" for a in agents)
+        with Sandbox(allow, image=image, developer_as_system=developer_as_system) as box:
+            for i, (rep, agent, skill, prompt, expect) in enumerate(plan, 1):
+                spent = results["cost_usd"]
+                if agent.provider.paid:
+                    pin, pout = agent.cfg.price_input or 0, agent.cfg.price_output or 0
+                    estimate = max_seen.get(
+                        agent.cfg.name, EST_INPUT_TOKENS * pin / 1e6 + EST_OUTPUT_TOKENS * pout / 1e6
+                    )
+                    if spent + estimate > max_cost:
+                        results["partial"] = True
+                        results["stop_reason"] = (
+                            f"budget: spent {spent:.4f} USD, next {agent.cfg.name} run would cost about "
+                            f"{estimate:.4f} USD, --max-cost is {max_cost:.2f} USD"
+                        )
+                        break
+                trial = run_trial(box, agent, skill.name, prompt, expect, rep, staged, cfg, skill_names)
+                raw = trial.pop("_raw")
+                if logs_dir is not None:
+                    logs_dir.mkdir(parents=True, exist_ok=True)
+                    n = len(results["trials"]) + 1
+                    name = f"{n:04d}-{agent.cfg.name}-{skill.name}-{expect}-r{rep}.jsonl"
+                    (logs_dir / name).write_text(raw, encoding="utf-8")
+                    trial["log"] = f"logs/{name}"
+                cost = run_cost(agent.provider, agent.cfg, trial.pop("_obs"))
+                trial["cost_usd"] = cost
+                max_seen[agent.cfg.name] = max(max_seen.get(agent.cfg.name, 0.0), cost)
+                results["cost_usd"] = round(results["cost_usd"] + cost, 6)
+                results["trials"].append(trial)
+                if progress:
+                    progress({"done": i, "total": len(plan), "trial": trial})
+                if checkpoint:
+                    results["duration_s"] = round(previous_duration + time.monotonic() - t0, 1)
+                    checkpoint(results)
+            denials = box.proxy_denials()
+    results["blocked_egress"] = sorted(set(denials))
+    finished = dt.datetime.now(dt.UTC)
+    results["finished_at"] = finished.isoformat(timespec="seconds")
+    results["duration_s"] = round(previous_duration + time.monotonic() - t0, 1)
+    if results["partial"]:
+        raise BudgetExceeded(results["stop_reason"], results)
+    return results
+
+
+def run_trial(
+    box: Sandbox,
+    agent: Agent,
+    skill: str,
+    prompt: str,
+    expect: str,
+    rep: int,
+    staged: Path,
+    cfg: Config,
+    skill_names: set[str],
+) -> dict:
+    inv = agent.build(prompt)
+    obs = Observation()
+
+    def on_line(line: str) -> bool:
+        agent.feed(line, obs, skill_names)
+        return agent.should_stop(obs, skill_names)
+
+    out = box.run(inv, agent.skills_dir, staged, cfg.timeout, cfg.cpus, cfg.memory, on_line)
+    fired = skill in obs.activated
+    error = None
+    if not fired:
+        if out.timed_out:
+            error = f"timeout after {cfg.timeout}s"
+        elif obs.error:
+            error = obs.error
+        elif out.returncode not in (0,) and not out.stopped_early and not out.stdout.strip():
+            tail = out.stderr.strip().splitlines()[-3:]
+            error = f"exit {out.returncode}: {' | '.join(tail)[:300]}" if tail else f"exit {out.returncode}"
+    return {
+        "agent": agent.cfg.name,
+        "skill": skill,
+        "prompt": prompt,
+        "expect": expect,
+        "repeat": rep,
+        "fired": fired,
+        "activated": obs.activated,
+        "error": error,
+        "duration_s": out.duration_s,
+        "input_tokens": obs.input_tokens,
+        "output_tokens": obs.output_tokens,
+        "_obs": obs,
+        "_raw": out.stdout + (f"\n# stderr\n{out.stderr[-4000:]}" if out.stderr.strip() else ""),
+    }

doesitfire/sandbox.py

@@ -0,0 +1,276 @@
+"""Docker isolation: every agent run happens in a throwaway container.
+
+Layout per `doesitfire run`:
+
+    [agent container] --internal network--> [egress proxy] --bridge--> model API only
+
+- The internal network has no route out. The only peer is the egress proxy,
+  which allows exactly the model API host:port and returns 403 for anything else.
+- Agent containers: non-root, read-only root filesystem, tmpfs home, all Linux
+  capabilities dropped, no-new-privileges, CPU/memory/pids limits, hard timeout.
+- Skills are mounted read-only and copied into the agent's discovery directory.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import threading
+import time
+import uuid
+from collections.abc import Callable
+from contextlib import ExitStack
+from dataclasses import dataclass
+from importlib import resources
+from pathlib import Path
+
+from . import __version__
+from .agents import Invocation
+from .errors import DoesItFireError
+from .providers import host_port
+
+IMAGE = f"doesitfire-agents:{__version__}"
+LABEL = "io.github.doesitfire.run"
+
+
+def _docker(*args: str, timeout: float = 120, check: bool = True) -> subprocess.CompletedProcess:
+    try:
+        proc = subprocess.run(["docker", *args], capture_output=True, text=True, timeout=timeout)
+    except FileNotFoundError:
+        raise DoesItFireError(
+            "Docker is not installed. doesitfire runs every agent inside a container and refuses to run "
+            "without one. Install Docker Desktop (macOS/Windows) or Docker Engine (Linux)."
+        ) from None
+    except subprocess.TimeoutExpired:
+        raise DoesItFireError(
+            f"'docker {args[0]}' timed out after {timeout:.0f}s. Is the Docker daemon healthy?"
+        ) from None
+    if check and proc.returncode != 0:
+        raise DoesItFireError(f"'docker {' '.join(args[:2])}' failed: {proc.stderr.strip()[:400]}")
+    return proc
+
+
+def check_docker() -> str:
+    """Return the Docker server version, or raise a clear error if Docker is not usable."""
+    proc = _docker("info", "--format", "{{.ServerVersion}}", timeout=30, check=False)
+    if proc.returncode != 0 or not proc.stdout.strip():
+        raise DoesItFireError(
+            "Docker is not running. Start Docker Desktop (or the Docker daemon) and try again. "
+            "doesitfire runs every agent inside a container and will not run without it."
+        )
+    return proc.stdout.strip()
+
+
+def image_exists(image: str = IMAGE) -> bool:
+    return _docker("image", "inspect", image, check=False).returncode == 0
+
+
+def build_image(image: str = IMAGE, log: Callable[[str], None] = print) -> None:
+    log(f"Building the agent image {image} (first run only, a few minutes)...")
+    with ExitStack() as stack:
+        ctx = stack.enter_context(resources.as_file(resources.files("doesitfire") / "docker"))
+        proc = subprocess.run(["docker", "build", "-t", image, str(ctx)], capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise DoesItFireError(f"Building {image} failed:\n{proc.stderr.strip()[-1500:]}")
+
+
+def stage_skills(skill_dirs: list[tuple[str, Path]], dest: Path) -> Path:
+    """Copy each skill directory to dest/<name> (what gets mounted read-only at /skills)."""
+    dest.mkdir(parents=True, exist_ok=True)
+    for name, src in skill_dirs:
+        shutil.copytree(
+            src,
+            dest / name,
+            dirs_exist_ok=True,
+            ignore=shutil.ignore_patterns(".git", "node_modules", "__pycache__", ".venv"),
+        )
+    return dest
+
+
+@dataclass
+class RunOutput:
+    stdout: str
+    stderr: str
+    returncode: int
+    timed_out: bool
+    stopped_early: bool
+    duration_s: float
+
+
+class Sandbox:
+    """An internal Docker network plus an egress proxy, for one `doesitfire run`."""
+
+    def __init__(self, allow_urls: list[str], image: str = IMAGE, developer_as_system: bool = False):
+        self.run_id = uuid.uuid4().hex[:10]
+        self.image = image
+        self.network = f"doesitfire-{self.run_id}"
+        self.proxy = f"doesitfire-{self.run_id}-egress"
+        self.allow = sorted({host_port(u) for u in allow_urls})
+        self.developer_as_system = developer_as_system
+
+    def __enter__(self) -> Sandbox:
+        _docker("network", "create", "--internal", "--label", f"{LABEL}={self.run_id}", self.network)
+        try:
+            _docker(
+                "run",
+                "-d",
+                "--name",
+                self.proxy,
+                "--label",
+                f"{LABEL}={self.run_id}",
+                "--network",
+                "bridge",
+                "--add-host",
+                "host.docker.internal:host-gateway",
+                "--read-only",
+                "--cap-drop",
+                "ALL",
+                "--security-opt",
+                "no-new-privileges",
+                "--memory",
+                "128m",
+                "--cpus",
+                "0.5",
+                "--pids-limit",
+                "64",
+                "-e",
+                f"DOESITFIRE_ALLOW={','.join(self.allow)}",
+                "-e",
+                f"DOESITFIRE_DEVELOPER_AS_SYSTEM={'1' if self.developer_as_system else '0'}",
+                "--entrypoint",
+                "node",
+                self.image,
+                "/opt/doesitfire/egress-proxy.mjs",
+            )
+            _docker("network", "connect", "--alias", "egress", self.network, self.proxy)
+            deadline = time.monotonic() + 20
+            while time.monotonic() < deadline:
+                logs = _docker("logs", self.proxy, check=False)
+                if "egress proxy on" in logs.stderr + logs.stdout:
+                    break
+                time.sleep(0.3)
+            else:
+                raise DoesItFireError("The egress proxy container did not start.")
+        except BaseException:
+            self.close()
+            raise
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+    def close(self) -> None:
+        ids = _docker("ps", "-aq", "--filter", f"label={LABEL}={self.run_id}", check=False).stdout.split()
+        if ids:
+            _docker("rm", "-f", *ids, check=False)
+        _docker("network", "rm", self.network, check=False)
+
+    def proxy_denials(self) -> list[str]:
+        logs = _docker("logs", self.proxy, check=False)
+        return [ln for ln in (logs.stderr + logs.stdout).splitlines() if ln.startswith("deny ")]
+
+    def run(
+        self,
+        inv: Invocation,
+        skills_dir: str,
+        staged: Path,
+        timeout: int,
+        cpus: float = 1.0,
+        memory: str = "1g",
+        on_line: Callable[[str], bool] | None = None,
+    ) -> RunOutput:
+        name = f"doesitfire-{self.run_id}-{uuid.uuid4().hex[:8]}"
+        proxy = "http://egress:8888"
+        args = [
+            "docker",
+            "run",
+            "--rm",
+            "-i",
+            "--name",
+            name,
+            "--label",
+            f"{LABEL}={self.run_id}",
+            "--network",
+            self.network,
+            "--read-only",
+            "--tmpfs",
+            "/home/node:rw,exec,uid=1000,gid=1000,size=512m",
+            "--tmpfs",
+            "/tmp:rw,exec,size=256m",
+            "--cap-drop",
+            "ALL",
+            "--security-opt",
+            "no-new-privileges",
+            "--cpus",
+            str(cpus),
+            "--memory",
+            memory,
+            "--pids-limit",
+            "512",
+            "-v",
+            f"{staged}:/skills:ro",
+            "-e",
+            f"DOESITFIRE_SKILLS_DIR={skills_dir}",
+        ]
+        for var in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"):
+            args += ["-e", f"{var}={proxy}"]
+        args += ["-e", "NO_PROXY=", "-e", "no_proxy="]
+        for key, value in inv.env.items():
+            args += ["-e", f"{key}={value}"]
+        child_env = dict(os.environ)
+        for key, value in inv.secret_env.items():
+            args += ["-e", key]  # value travels through the environment, never argv
+            child_env[key] = value
+        args += [self.image, "timeout", "-s", "KILL", str(timeout + 5), *inv.argv]
+
+        start = time.monotonic()
+        proc = subprocess.Popen(
+            args,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            env=child_env,
+        )
+        out_lines: list[str] = []
+        err_chunks: list[str] = []
+        stopped = threading.Event()
+
+        def read_stderr() -> None:
+            assert proc.stderr is not None
+            err_chunks.append(proc.stderr.read())
+
+        def kill() -> None:
+            _docker("kill", name, check=False, timeout=30)
+
+        t_err = threading.Thread(target=read_stderr, daemon=True)
+        t_err.start()
+        timer = threading.Timer(timeout, kill)
+        timer.start()
+        try:
+            assert proc.stdout is not None
+            for line in proc.stdout:
+                out_lines.append(line)
+                if on_line and not stopped.is_set() and on_line(line):
+                    stopped.set()
+                    kill()
+            proc.wait()
+        except KeyboardInterrupt:
+            kill()
+            raise
+        finally:
+            timed_out = not timer.is_alive() and not stopped.is_set() and proc.returncode not in (0,)
+            timer.cancel()
+            t_err.join(timeout=5)
+        duration = time.monotonic() - start
+        return RunOutput(
+            stdout="".join(out_lines),
+            stderr="".join(err_chunks),
+            returncode=proc.returncode,
+            timed_out=timed_out and duration >= timeout,
+            stopped_early=stopped.is_set(),
+            duration_s=round(duration, 2),
+        )