dispatchbox 0.1.0__py3-none-any.whl

dispatchbox/slack.py

@@ -0,0 +1,251 @@
+"""Slack adapter over plain HTTP (Events API + Interactivity), standard library only.
+
+Formats follow docs.slack.dev: v0 request signing, chat.postMessage / chat.update, Block Kit
+buttons, block_actions payloads, app_mention and message.im events.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import logging
+import re
+import time
+from typing import Any
+from urllib.parse import parse_qs
+
+from .chat import (ChatAdapter, ChatError, Response, answer_text, decided_line, decision_text, empty_response,
+                   http_json, in_background, json_response, status_line)
+
+log = logging.getLogger("dispatchbox.slack")
+
+REPLAY_WINDOW = 300
+_MENTION = re.compile(r"<@[A-Z0-9]+>")
+
+
+def esc(text: str) -> str:
+    """Escape the three characters Slack mrkdwn treats as control characters."""
+    return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+
+
+def link_md(link: str, label: str) -> str:
+    return f"<{link}|{label}>" if link.startswith(("http://", "https://")) else ""
+
+
+def sign(secret: str, timestamp: str, body: bytes) -> str:
+    base = b"v0:" + timestamp.encode() + b":" + body
+    return "v0=" + hmac.new(secret.encode(), base, hashlib.sha256).hexdigest()
+
+
+class SlackAdapter(ChatAdapter):
+    name = "slack"
+
+    def __init__(self, hub, bot_token: str, signing_secret: str, api_base: str = "https://slack.com/api",
+                 clock=time.time):
+        super().__init__(hub)
+        self.bot_token = bot_token
+        self.signing_secret = signing_secret
+        self.api_base = api_base.rstrip("/")
+        self.clock = clock
+
+    # ------------------------------------------------------------------ Slack Web API
+
+    def api(self, method: str, body: dict[str, Any]) -> dict[str, Any]:
+        resp = http_json("POST", f"{self.api_base}/{method}", body,
+                         headers={"Authorization": f"Bearer {self.bot_token}"})
+        if not resp.get("ok"):
+            raise ChatError(f"slack {method} failed: {resp.get('error', resp)}")
+        return resp
+
+    def post(self, channel: str, text: str, blocks: list[dict[str, Any]] | None = None,
+             thread_ts: str | None = None) -> dict[str, Any]:
+        body: dict[str, Any] = {"channel": channel, "text": text}
+        if blocks:
+            body["blocks"] = blocks
+        if thread_ts:
+            body["thread_ts"] = thread_ts
+        return self.api("chat.postMessage", body)
+
+    # ------------------------------------------------------------------ incoming requests
+
+    def verify(self, headers: dict[str, str], body: bytes) -> bool:
+        ts = headers.get("x-slack-request-timestamp", "")
+        sig = headers.get("x-slack-signature", "")
+        if not ts.isdigit() or abs(self.clock() - int(ts)) > REPLAY_WINDOW:
+            return False
+        return hmac.compare_digest(sig, sign(self.signing_secret, ts, body))
+
+    def handle(self, headers: dict[str, str], body: bytes) -> Response:
+        if not self.verify(headers, body):
+            return json_response({"error": "bad slack signature"}, 401)
+        if headers.get("content-type", "").startswith("application/x-www-form-urlencoded"):
+            form = parse_qs(body.decode())
+            try:
+                payload = json.loads(form.get("payload", ["{}"])[-1])
+            except json.JSONDecodeError:
+                return json_response({"error": "bad payload"}, 400)
+            return self._interaction(payload)
+        try:
+            data = json.loads(body or b"{}")
+        except json.JSONDecodeError:
+            return json_response({"error": "bad json"}, 400)
+        if data.get("type") == "url_verification":
+            return json_response({"challenge": data.get("challenge", "")})
+        if data.get("type") == "event_callback":
+            self._event(data)
+        return empty_response()
+
+    def _event(self, data: dict[str, Any]) -> None:
+        ev = data.get("event") or {}
+        kind = ev.get("type")
+        if ev.get("bot_id") or ev.get("subtype") or not ev.get("user"):
+            return  # bots (including ourselves), edits, deletes, joins
+        if kind == "app_mention":
+            etype = "message.mention"
+        elif kind == "message" and ev.get("channel_type") == "im":
+            etype = "message.dm"
+        else:
+            return
+        user, channel = ev["user"], ev.get("channel", "")
+        thread = ev.get("thread_ts") or ev.get("ts")
+        if not self.user_allowed(user):
+            log.warning("slack user %s is not in chat.slack.allowed_users", user)
+            in_background(self._reply_safe, channel, thread,
+                          f"You are not allowed to start dispatchbox runs. Ask the owner to add `{user}` to "
+                          "chat.slack.allowed_users.")
+            return
+        text = _MENTION.sub("", ev.get("text", "")).strip()
+        result = self.hub.ingest({
+            "source": "slack", "type": etype, "subject": channel, "summary": text[:200],
+            "payload": {"chat": channel, "thread": thread, "ts": ev.get("ts"), "user": user, "text": text},
+            "dedupe_key": f"slack:{data.get('event_id') or ev.get('ts')}",
+            "dedupe_window": 86400,
+        })
+        if self.settings.get("ack", True):
+            msg = ack_text(result)
+            if msg:
+                in_background(self._reply_safe, channel, thread, msg)
+
+    def _interaction(self, p: dict[str, Any]) -> Response:
+        if p.get("type") != "block_actions":
+            return empty_response()
+        user = (p.get("user") or {}).get("id", "")
+        channel = (p.get("container") or {}).get("channel_id") or (p.get("channel") or {}).get("id")
+        ts = (p.get("container") or {}).get("message_ts") or (p.get("message") or {}).get("ts")
+        for action in p.get("actions") or []:
+            if not str(action.get("action_id", "")).startswith("dispatchbox_choice_"):
+                continue
+            try:
+                value = json.loads(action.get("value") or "{}")
+                run_id, choice = int(value["run"]), str(value["choice"])
+            except (ValueError, KeyError, TypeError):
+                continue
+            ok, message, run = self.decide_from_chat(run_id, choice, user)
+            if run and channel and ts:
+                in_background(self._refresh_card, channel, ts, run)
+            if not ok and p.get("response_url"):
+                in_background(http_json, "POST", p["response_url"],
+                              {"response_type": "ephemeral", "replace_original": False, "text": message})
+        return empty_response()
+
+    # ------------------------------------------------------------------ hub hooks
+
+    def notify(self, run: dict[str, Any], link: str) -> None:
+        if self.from_chat(run):
+            return  # the originating conversation hears about it in run_finished
+        channel = self.settings.get("notify_channel")
+        if not channel:
+            log.warning("slack: chat.slack.notify_channel is not set, skipping notification for run %s", run["id"])
+            return
+        if run["status"] == "awaiting_decision":
+            self.post(channel, decision_fallback(run), decision_blocks(run, link))
+        else:
+            self.post(channel, result_text(run, link))
+
+    def run_finished(self, run: dict[str, Any], link: str) -> None:
+        origin = self.origin(run)
+        if not origin or not self.settings.get("reply_on_finish", True):
+            return
+        if run["status"] == "awaiting_decision":
+            self.post(origin["chat"], decision_fallback(run), decision_blocks(run, link), thread_ts=origin.get("thread"))
+        elif run["status"] in {"succeeded", "failed", "lost"}:
+            self.post(origin["chat"], result_text(run, link), thread_ts=origin.get("thread"))
+
+    # ------------------------------------------------------------------ helpers
+
+    def _reply_safe(self, channel: str, thread: str | None, text: str) -> None:
+        try:
+            self.post(channel, text, thread_ts=thread)
+        except ChatError as e:
+            log.error("slack reply failed: %s", e)
+
+    def _refresh_card(self, channel: str, ts: str, run: dict[str, Any]) -> None:
+        link = self.hub.run_link(run["id"])
+        if run["status"] == "awaiting_decision":
+            blocks, text = decision_blocks(run, link), decision_fallback(run)
+        else:
+            blocks, text = decided_blocks(run, link), decided_line(run)
+        self.api("chat.update", {"channel": channel, "ts": ts, "text": text, "blocks": blocks})
+
+
+def ack_text(result: dict[str, Any]) -> str:
+    status = result.get("status")
+    if status == "routed" and result.get("runs"):
+        return "Got it. Queued as run " + ", ".join(f"#{r}" for r in result["runs"]) + "."
+    if status == "coalesced":
+        return "Added to queued run " + ", ".join(f"#{r}" for r in result.get("coalesced_into", [])) + "."
+    if status == "unmatched":
+        return "No dispatchbox rule matches this message, so nothing was started."
+    return ""
+
+
+def decision_fallback(run: dict[str, Any]) -> str:
+    question, _, _ = decision_text(run)
+    return f"Run #{run['id']} ({run['rule']}) needs your decision: {question}"
+
+
+def decision_blocks(run: dict[str, Any], link: str) -> list[dict[str, Any]]:
+    question, context, options = decision_text(run)
+    blocks: list[dict[str, Any]] = [
+        {"type": "section", "text": {"type": "mrkdwn", "text": f"*{esc(question)}*"}},
+        {"type": "context", "elements": [{"type": "mrkdwn", "text": esc(status_line(run))}]},
+    ]
+    if context:
+        blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": esc(context)[:2900]}})
+    if options:
+        blocks.append({"type": "actions", "block_id": f"dispatchbox_run_{run['id']}", "elements": [
+            {"type": "button", "action_id": f"dispatchbox_choice_{i}", "text": {"type": "plain_text", "text": opt[:75]},
+             "value": json.dumps({"run": run["id"], "choice": opt}, ensure_ascii=False)[:2000],
+             **({"style": "primary"} if i == 0 else {})}
+            for i, opt in enumerate(options)
+        ]})
+    else:
+        blocks.append({"type": "context", "elements": [{"type": "mrkdwn", "text": "No options were given. Answer on the dashboard."}]})
+    md = link_md(link, "Open in dispatchbox")
+    if md:
+        blocks.append({"type": "context", "elements": [{"type": "mrkdwn", "text": md}]})
+    return blocks
+
+
+def decided_blocks(run: dict[str, Any], link: str) -> list[dict[str, Any]]:
+    question, _, _ = decision_text(run)
+    blocks: list[dict[str, Any]] = [
+        {"type": "section", "text": {"type": "mrkdwn", "text": f"~{esc(question)}~\n*{esc(decided_line(run))}*"}},
+        {"type": "context", "elements": [{"type": "mrkdwn", "text": esc(status_line(run))}]},
+    ]
+    md = link_md(link, "Open in dispatchbox")
+    if md:
+        blocks.append({"type": "context", "elements": [{"type": "mrkdwn", "text": md}]})
+    return blocks
+
+
+def result_text(run: dict[str, Any], link: str) -> str:
+    lines = [esc(status_line(run))]
+    answer = answer_text(run, 3000)
+    if answer:
+        lines.append(esc(answer))
+    md = link_md(link, "details")
+    if md:
+        lines.append(md)
+    return "\n".join(lines)

dispatchbox/store.py

@@ -0,0 +1,99 @@
+"""SQLite persistence. All access goes through one connection guarded by the hub's lock."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from typing import Any
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS events (
+    id INTEGER PRIMARY KEY,
+    received_at REAL NOT NULL,
+    source TEXT NOT NULL,
+    type TEXT NOT NULL,
+    subject TEXT NOT NULL,
+    summary TEXT NOT NULL,
+    link TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    dedupe_key TEXT NOT NULL,
+    status TEXT NOT NULL,          -- routed | coalesced | duplicate | unmatched
+    note TEXT NOT NULL DEFAULT ''
+);
+CREATE INDEX IF NOT EXISTS events_dedupe ON events(dedupe_key, received_at);
+
+CREATE TABLE IF NOT EXISTS runs (
+    id INTEGER PRIMARY KEY,
+    rule TEXT NOT NULL,
+    worker_pattern TEXT NOT NULL,
+    group_key TEXT NOT NULL,
+    concurrency INTEGER NOT NULL,
+    spec TEXT NOT NULL,
+    status TEXT NOT NULL,          -- queued | running | succeeded | failed | lost | awaiting_decision | decided
+    created_at REAL NOT NULL,
+    claimed_by TEXT,
+    claimed_at REAL,
+    heartbeat_at REAL,
+    finished_at REAL,
+    exit_code INTEGER,
+    summary TEXT,
+    session_id TEXT,
+    prompt TEXT,
+    output_tail TEXT,
+    decision_request TEXT,
+    decision TEXT,
+    parent_run INTEGER,
+    cost_usd REAL,
+    input_tokens INTEGER,
+    output_tokens INTEGER,
+    attempt INTEGER NOT NULL DEFAULT 1,
+    retry_of INTEGER
+);
+CREATE INDEX IF NOT EXISTS runs_status ON runs(status, id);
+
+CREATE TABLE IF NOT EXISTS run_events (
+    run_id INTEGER NOT NULL,
+    event_id INTEGER NOT NULL,
+    PRIMARY KEY (run_id, event_id)
+);
+"""
+
+
+def connect(path: str) -> sqlite3.Connection:
+    conn = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA foreign_keys=ON")
+    conn.executescript(SCHEMA)
+    _migrate(conn)
+    return conn
+
+
+# Columns added after the first release. Older databases get them on startup.
+_ADDED_RUN_COLUMNS = {
+    "cost_usd": "REAL",
+    "input_tokens": "INTEGER",
+    "output_tokens": "INTEGER",
+    "attempt": "INTEGER NOT NULL DEFAULT 1",
+    "retry_of": "INTEGER",
+}
+
+
+def _migrate(conn: sqlite3.Connection) -> None:
+    have = {row["name"] for row in conn.execute("PRAGMA table_info(runs)")}
+    for name, decl in _ADDED_RUN_COLUMNS.items():
+        if name not in have:
+            conn.execute(f"ALTER TABLE runs ADD COLUMN {name} {decl}")
+
+
+def row_to_dict(row: sqlite3.Row | None) -> dict[str, Any] | None:
+    if row is None:
+        return None
+    d = dict(row)
+    for key in ("payload", "spec", "decision_request", "decision"):
+        if key in d and isinstance(d[key], str) and d[key]:
+            try:
+                d[key] = json.loads(d[key])
+            except json.JSONDecodeError:
+                pass
+    return d

dispatchbox/worker.py

@@ -0,0 +1,314 @@
+"""Worker: runs on a machine that has your agents installed, pulls tasks from the hub and runs them.
+
+The worker only makes outbound requests, so the machine never needs to be reachable from the internet.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import signal
+import subprocess
+import tempfile
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .net import context_for
+
+log = logging.getLogger("dispatchbox.worker")
+
+HEARTBEAT_EVERY = 20
+OUTPUT_LIMIT = 200_000
+
+
+class HubGone(Exception):
+    """The hub no longer considers this run ours (it was reaped or finished elsewhere)."""
+
+
+@dataclass
+class WorkerConfig:
+    hub: str
+    name: str
+    token: str
+    slots: int = 2
+    wait: int = 50
+    allow_dirs: list[str] = field(default_factory=lambda: [os.getcwd()])
+    runners: list[str] = field(default_factory=lambda: ["claude", "codex"])
+    claude_bin: str = "claude"
+    codex_bin: str = "codex"
+
+
+class HubClient:
+    def __init__(self, base: str, token: str):
+        self.base = base.rstrip("/")
+        self.token = token
+
+    def _req(self, method: str, path: str, body: Any = None, timeout: float = 30) -> tuple[int, Any]:
+        data = json.dumps(body).encode() if body is not None else None
+        req = urllib.request.Request(self.base + path, data=data, method=method, headers={
+            "Authorization": f"Bearer {self.token}", "Content-Type": "application/json"})
+        try:
+            with urllib.request.urlopen(req, timeout=timeout, context=context_for(req.full_url)) as resp:
+                raw = resp.read()
+                return resp.status, (json.loads(raw) if raw else None)
+        except urllib.error.HTTPError as e:
+            raw = e.read()
+            try:
+                payload = json.loads(raw) if raw else None
+            except json.JSONDecodeError:
+                payload = raw.decode(errors="replace")
+            return e.code, payload
+
+    def claim(self, worker: str, wait: int, runners: list[str]) -> dict[str, Any] | None:
+        q = urllib.parse.urlencode({"worker": worker, "wait": wait, "runners": ",".join(runners)})
+        status, body = self._req("GET", f"/claim?{q}", timeout=wait + 15)
+        if status == 204:
+            return None
+        if status != 200:
+            raise RuntimeError(f"claim failed: {status} {body}")
+        return body
+
+    def heartbeat(self, run_id: int, worker: str) -> None:
+        status, body = self._req("POST", f"/runs/{run_id}/heartbeat", {"worker": worker})
+        if status == 409 or status == 404:
+            raise HubGone(str(body))
+        if status != 200:
+            raise RuntimeError(f"heartbeat failed: {status} {body}")
+
+    def result(self, run_id: int, worker: str, result: dict[str, Any]) -> None:
+        delay = 2.0
+        deadline = time.monotonic() + 600
+        while True:
+            try:
+                status, body = self._req("POST", f"/runs/{run_id}/result", {"worker": worker, **result})
+                if status == 200:
+                    return
+                if status in (404, 409):
+                    log.warning("hub refused result for run %s: %s", run_id, body)
+                    return
+                err = f"{status} {body}"
+            except (urllib.error.URLError, OSError) as e:
+                err = str(e)
+            if time.monotonic() > deadline:
+                log.error("giving up reporting run %s: %s", run_id, err)
+                return
+            log.warning("reporting run %s failed (%s), retrying in %.0fs", run_id, err, delay)
+            time.sleep(delay)
+            delay = min(delay * 2, 60)
+
+
+def resolve_cwd(raw: str, allow_dirs: list[str]) -> Path:
+    path = Path(os.path.expanduser(raw or ".")).resolve()
+    allowed = [Path(os.path.expanduser(d)).resolve() for d in allow_dirs]
+    if not any(path == a or a in path.parents for a in allowed):
+        raise PermissionError(f"{path} is outside the worker's allowed directories")
+    if not path.is_dir():
+        raise FileNotFoundError(f"{path} does not exist")
+    return path
+
+
+def build_command(task: dict[str, Any], cfg: WorkerConfig, tmpdir: str) -> tuple[list[str], str | None]:
+    """Return (argv, stdin text). Prompts go through stdin so they never show up in `ps`."""
+    runner = task["runner"]
+    extra = [str(a) for a in task.get("args") or []]
+    model = task.get("model")
+    resume = task.get("resume_session")
+    if runner == "claude":
+        cmd = [cfg.claude_bin, "-p", "--output-format", "json"]
+        if model:
+            cmd += ["--model", model]
+        if resume:
+            cmd += ["--resume", resume]
+        return cmd + extra, task["prompt"]
+    if runner == "codex":
+        last = os.path.join(tmpdir, "last_message.txt")
+        cmd = [cfg.codex_bin, "exec"]
+        if resume:
+            cmd += ["resume", resume]
+        cmd += ["--json", "-o", last]
+        if model:
+            cmd += ["-m", model]
+        return cmd + extra + ["-"], task["prompt"]
+    if runner == "shell":
+        return ["/bin/sh", "-c", task["command"]], None
+    raise ValueError(f"unknown runner {runner!r}")
+
+
+def parse_output(runner: str, stdout: str, stderr: str, tmpdir: str) -> dict[str, Any]:
+    """Pull the agent's final message and session id out of the raw process output."""
+    if runner == "claude":
+        try:
+            data = json.loads(stdout.strip().splitlines()[-1]) if stdout.strip() else {}
+        except (json.JSONDecodeError, IndexError):
+            data = {}
+        if isinstance(data, dict) and "result" in data:
+            out = {"output": str(data.get("result") or ""), "session_id": data.get("session_id")}
+            if data.get("is_error"):
+                out["agent_error"] = True
+            if data.get("total_cost_usd") is not None:
+                out["cost_usd"] = data["total_cost_usd"]
+            usage = data.get("usage") or {}
+            if isinstance(usage, dict) and usage:
+                # Claude reports cache reads and writes separately from fresh input; count all of it as input.
+                out["input_tokens"] = sum(int(usage.get(k) or 0) for k in
+                                          ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"))
+                out["output_tokens"] = int(usage.get("output_tokens") or 0)
+            return out
+        return {"output": (stdout + ("\n" + stderr if stderr else "")).strip()}
+    if runner == "codex":
+        session = None
+        tokens_in = tokens_out = 0
+        saw_usage = False
+        errors: list[str] = []
+        for line in stdout.splitlines():
+            try:
+                ev = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(ev, dict):
+                continue
+            kind = ev.get("type")
+            if kind in {"thread.started", "session.started"}:
+                session = ev.get("thread_id") or ev.get("session_id") or session
+            elif kind == "turn.completed" and isinstance(ev.get("usage"), dict):
+                # Codex's input_tokens already includes cached_input_tokens.
+                saw_usage = True
+                tokens_in += int(ev["usage"].get("input_tokens") or 0)
+                tokens_out += int(ev["usage"].get("output_tokens") or 0)
+            elif kind == "turn.failed":
+                errors.append(str((ev.get("error") or {}).get("message", "turn failed")))
+            elif kind == "error":
+                errors.append(str(ev.get("message", "error")))
+        last = Path(tmpdir, "last_message.txt")
+        text = last.read_text(errors="replace").strip() if last.exists() else ""
+        out: dict[str, Any] = {"session_id": session}
+        if errors and not text:
+            out["output"] = "codex failed: " + errors[-1][:2000]
+            out["agent_error"] = True
+        else:
+            out["output"] = text or stderr.strip()[-4000:]
+        if saw_usage:
+            out["input_tokens"], out["output_tokens"] = tokens_in, tokens_out
+        return out
+    return {"output": (stdout + ("\n" + stderr if stderr else "")).strip()}
+
+
+def run_task(task: dict[str, Any], cfg: WorkerConfig, client: HubClient) -> dict[str, Any]:
+    run_id = task["run_id"]
+    try:
+        cwd = resolve_cwd(task.get("cwd", "."), cfg.allow_dirs)
+    except (PermissionError, FileNotFoundError) as e:
+        return {"exit_code": 126, "output": f"dispatchbox worker refused to run: {e}", "summary": str(e)}
+
+    with tempfile.TemporaryDirectory(prefix=f"dispatchbox-{run_id}-") as tmpdir:
+        events_file = Path(tmpdir, "events.json")
+        events_file.write_text(json.dumps(task.get("events", []), ensure_ascii=False, indent=1))
+        env = {**os.environ, "DISPATCHBOX_RUN_ID": str(run_id), "DISPATCHBOX_RULE": task["rule"],
+               "DISPATCHBOX_EVENTS_FILE": str(events_file), "DISPATCHBOX_HUB": cfg.hub}
+        env.pop("DISPATCHBOX_TOKEN", None)  # agents never see the hub token
+        try:
+            cmd, stdin = build_command(task, cfg, tmpdir)
+        except (ValueError, KeyError) as e:
+            return {"exit_code": 2, "output": str(e), "summary": str(e)}
+
+        log.info("run #%s %s: %s in %s", run_id, task["rule"], cmd[0], cwd)
+        try:
+            proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdin=subprocess.PIPE if stdin is not None else subprocess.DEVNULL,
+                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True)
+        except FileNotFoundError as e:
+            return {"exit_code": 127, "output": str(e), "summary": f"{cmd[0]} is not installed on this worker"}
+
+        stop = threading.Event()
+        gone = threading.Event()
+
+        def beat() -> None:
+            while not stop.wait(HEARTBEAT_EVERY):
+                try:
+                    client.heartbeat(run_id, cfg.name)
+                except HubGone:
+                    log.warning("hub dropped run #%s, stopping it", run_id)
+                    gone.set()
+                    _kill(proc)
+                    return
+                except Exception as e:
+                    log.warning("heartbeat for run #%s failed: %s", run_id, e)
+
+        threading.Thread(target=beat, daemon=True).start()
+        timed_out = False
+        try:
+            stdout, stderr = proc.communicate(input=stdin, timeout=task.get("timeout", 1800))
+        except subprocess.TimeoutExpired:
+            timed_out = True
+            _kill(proc)
+            stdout, stderr = proc.communicate()
+        finally:
+            stop.set()
+
+        result = parse_output(task["runner"], stdout[-OUTPUT_LIMIT:], stderr[-OUTPUT_LIMIT:], tmpdir)
+        code = proc.returncode
+        if timed_out:
+            code = 124
+            result["summary"] = f"timed out after {task.get('timeout')}s"
+        elif result.pop("agent_error", False) and code == 0:
+            code = 1
+        result["exit_code"] = code
+        if gone.is_set():
+            result["summary"] = "stopped because the hub dropped this run"
+        return result
+
+
+def _kill(proc: subprocess.Popen) -> None:
+    for sig in (signal.SIGTERM, signal.SIGKILL):
+        try:
+            os.killpg(proc.pid, sig)
+        except ProcessLookupError:
+            return
+        try:
+            proc.wait(timeout=10)
+            return
+        except subprocess.TimeoutExpired:
+            continue
+
+
+def serve_worker(cfg: WorkerConfig, stop: threading.Event | None = None) -> None:
+    stop = stop or threading.Event()
+    client = HubClient(cfg.hub, cfg.token)
+    slots = threading.Semaphore(cfg.slots)
+    backoff = 2.0
+    log.info("worker %s polling %s (slots=%d, runners=%s, dirs=%s)", cfg.name, cfg.hub, cfg.slots,
+             ",".join(cfg.runners), ", ".join(cfg.allow_dirs))
+
+    def work(task: dict[str, Any]) -> None:
+        try:
+            result = run_task(task, cfg, client)
+            client.result(task["run_id"], cfg.name, result)
+            log.info("run #%s finished with exit %s", task["run_id"], result.get("exit_code"))
+        except Exception:
+            log.exception("run #%s crashed", task["run_id"])
+            client.result(task["run_id"], cfg.name, {"exit_code": 1, "output": "worker crashed", "summary": "worker crashed"})
+        finally:
+            slots.release()
+
+    while not stop.is_set():
+        if not slots.acquire(timeout=1):
+            continue
+        try:
+            task = client.claim(cfg.name, cfg.wait, cfg.runners)
+            backoff = 2.0
+        except Exception as e:
+            slots.release()
+            log.warning("cannot reach hub (%s), retrying in %.0fs", e, backoff)
+            stop.wait(backoff)
+            backoff = min(backoff * 2, 60)
+            continue
+        if task is None:
+            slots.release()
+            continue
+        threading.Thread(target=work, args=(task,), name=f"run-{task['run_id']}", daemon=True).start()