promigence 0.1.0-beta.12

package/dist/runners.js

@@ -0,0 +1,1458 @@
+import { randomBytes } from "node:crypto";
+import { chmodSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+import { configDir, PromigenceError } from "./sdk/index.js";
+import { bool, buildPriceOf, client, nexts, str } from "./commands.js";
+import { fitLine } from "./capacity.js";
+import { TIERS } from "./generated.js";
+import { buildPriceIn, buildPriceWords, COMPILED_NOTE, livePricing } from "./pricing.js";
+import { emit, errorEnvelope, errorTextLines, okEnvelope, parseCount, parseDuration, parseUsd, usageError, write } from "./io.js";
+import { escapeLine, jsonLine, sh } from "./term.js";
+import { GITHUB_CONFIG_PATH, GITHUB_LAUNCHER, GITLAB_CONFIG_PATH, GITLAB_LAUNCHER, RUNNER_TIERS, isRunnerTier, runnerAlias, runnerBuildSteps, runnerImageSpec, } from "./runners-spec.js";
+import * as gh from "./runners-github.js";
+import * as gl from "./runners-gitlab.js";
+export const TICK_MS = 2500;
+export const SELF_END_GRACE_S = 60;
+const IDLE_READ_MS = 2500;
+const BUSY_WAIT_MS = 25_000;
+export const IDLE_TIMEOUT_DEFAULT_MS = 90_000;
+const CANCEL_GRACE_MS = 20_000;
+const QUIT_GRACE_MS = 60_000;
+const SANDBOX_READY_MS = 120_000;
+const BUILD_WAIT_MS = 30 * 60_000;
+const DEFAULT_MAX = 10;
+const CEILING_STEP_TICKS = 12;
+const UNSERVABLE_MS = 10 * 60_000;
+const UNHANDED_FLOOR_MS = 60_000;
+const listPerSecond = (tier) => TIERS.find((t) => t.name === tier)?.per_second_usd ?? 0;
+const shape = (tier) => {
+    const t = TIERS.find((x) => x.name === tier);
+    return t ? `${t.vcpus} vCPU, ${t.mem_gib} GiB, ${t.disk_gib} GiB disk` : tier;
+};
+const usd = (n) => `$${n < 0.01 && n > 0 ? n.toFixed(4) : n.toFixed(2)}`;
+const secs = (ms) => {
+    const s = Math.max(0, Math.round(ms / 1000));
+    return s < 60 ? `${s}s` : s < 3600 ? `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}s` : `${Math.floor(s / 3600)}h${String(Math.floor((s % 3600) / 60)).padStart(2, "0")}m`;
+};
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+export function sanitizeRunnerLine(line, secrets) {
+    return gh.redact(line, secrets)
+        .replace(/\b(runner|token|jitconfig)=\S+/gi, "$1=[redacted]")
+        .replace(/\bwith token \S+/gi, "with token [redacted]")
+        .replace(/\b(glrt-|glpat-|ghp_|gho_|ghs_|github_pat_)[A-Za-z0-9_-]+/g, "[redacted]");
+}
+export function runnerArgv(launcher, graceS) {
+    const g = Math.max(1, Math.round(graceS));
+    return [
+        "/bin/sh",
+        "-c",
+        `trap : HUP INT QUIT TERM; "$0"; rc=$?; ` +
+            `setsid sh -c 'sleep ${g}; echo b > /proc/sysrq-trigger' </dev/null >/dev/null 2>&1 & exit $rc`,
+        launcher,
+    ];
+}
+const poolsDir = () => join(configDir(process.env), "runner-pools");
+export function registerPool(e) {
+    try {
+        const dir = poolsDir();
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+        chmodSync(dir, 0o700);
+        const tmp = join(dir, `.${e.pool_id}.${process.pid}.tmp`);
+        writeFileSync(tmp, JSON.stringify(e) + "\n", { mode: 0o600 });
+        renameSync(tmp, join(dir, `${e.pool_id}.json`));
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export function unregisterPool(poolId) {
+    try {
+        rmSync(join(poolsDir(), `${poolId}.json`), { force: true });
+    }
+    catch {
+    }
+}
+const pidAlive = (pid) => {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (e) {
+        return e?.code === "EPERM";
+    }
+};
+export function deadPools(scope, self) {
+    let names;
+    try {
+        names = readdirSync(poolsDir());
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const n of names) {
+        if (!/^[0-9a-f]{8}\.json$/.test(n))
+            continue;
+        try {
+            const e = JSON.parse(readFileSync(join(poolsDir(), n), "utf8"));
+            if (e.pool_id !== n.slice(0, -5) || e.pool_id === self || e.scope !== scope || !Number.isInteger(e.pid))
+                continue;
+            if (!pidAlive(e.pid))
+                out.push(e);
+        }
+        catch {
+        }
+    }
+    return out;
+}
+class Pool {
+    ctx;
+    c;
+    opts;
+    provider;
+    runners = [];
+    jobs = [];
+    tasks = new Set();
+    stopping;
+    killing = false;
+    fatal;
+    notes = [];
+    queued = 0;
+    #nextIndex = 0;
+    #lastLine = "";
+    #tableLines = 0;
+    #wake;
+    startedAt = Date.now();
+    idleCost = 0;
+    runnersStarted = 0;
+    taken = new Map();
+    lastItems = [];
+    failStreak = 0;
+    startsPausedUntil = 0;
+    ceiling;
+    ceilingAt = 0;
+    rateStreak = 0;
+    unservable = new Map();
+    constructor(ctx, c, opts, provider) {
+        this.ctx = ctx;
+        this.c = c;
+        this.opts = opts;
+        this.provider = provider;
+        this.ceiling = opts.max;
+    }
+    limit() {
+        return Math.min(this.opts.max, this.ceiling);
+    }
+    live() {
+        return this.runners.filter((s) => s.phase === "starting" || s.phase === "idle" || s.phase === "busy" || s.phase === "stopping");
+    }
+    get tty() {
+        return this.ctx.mode === "text" && process.stderr.isTTY === true;
+    }
+    async event(ev, text) {
+        if (this.ctx.mode === "json") {
+            await write(process.stdout, jsonLine(process.stdout, { event: ev["event"], at: new Date().toISOString(), ...ev }));
+            return;
+        }
+        if (text === undefined)
+            return;
+        this.clearTable();
+        await write(process.stdout, escapeLine(text) + "\n");
+        this.drawTable();
+    }
+    note(s) {
+        if (this.notes[this.notes.length - 1] === s)
+            return;
+        this.notes.push(s);
+        if (this.notes.length > 3)
+            this.notes.shift();
+        if (this.ctx.mode === "json")
+            void this.event({ event: "note", message: s }, undefined);
+        else if (!this.tty)
+            void write(process.stdout, `note: ${escapeLine(s)}\n`);
+    }
+    clearTable() {
+        if (!this.tty || this.#tableLines === 0)
+            return;
+        process.stderr.write(`\x1b[${this.#tableLines}A\r\x1b[J`);
+        this.#tableLines = 0;
+    }
+    drawTable() {
+        if (!this.tty)
+            return;
+        this.clearTable();
+        const cols = process.stderr.columns ?? 120;
+        const live = this.live();
+        const running = live.filter((s) => s.phase === "busy").length;
+        const done = this.jobs.length;
+        const spent = this.jobs.reduce((a, j) => a + j.cost_usd, 0) + this.idleCost;
+        const head = `${this.opts.kind} ${this.opts.scopeLabel} · queued ${this.queued} · running ${running} · done ${done} · ${usd(spent)}` +
+            (this.stopping ? (this.killing ? " · cancelling" : " · stopping (Ctrl-C again cancels running jobs)") : " · Ctrl-C stops");
+        const rows = [head];
+        if (live.length)
+            rows.push(`  ${"RUNNER".padEnd(12)}${"TIER".padEnd(8)}${"STATE".padEnd(10)}${"TIME".padEnd(8)}${"COST".padEnd(10)}JOB`);
+        for (const s of live.slice(0, 20)) {
+            const since = s.jobStartedAt ?? s.startedAt;
+            const cost = this.opts.perSecond(s.tier) * Math.max(0, Date.now() - s.startedAt) / 1000;
+            const state = s.phase === "starting" ? "starting" : s.phase === "idle" ? "idle" : s.phase === "busy" ? "running" : "ending";
+            rows.push(`  ${s.id.padEnd(12)}${s.tier.padEnd(8)}${state.padEnd(10)}${secs(Date.now() - since).padEnd(8)}${("~" + usd(cost)).padEnd(10)}${s.job ?? "-"}`);
+        }
+        if (live.length > 20)
+            rows.push(`  … ${live.length - 20} more`);
+        for (const n of this.notes)
+            rows.push(`  note: ${n}`);
+        const out = rows.map((r) => fitLine(escapeLine(r), cols));
+        process.stderr.write(out.join("\n") + "\n");
+        this.#tableLines = out.length;
+    }
+    async plainStatus() {
+        if (this.ctx.mode !== "text" || this.tty)
+            return;
+        const live = this.live();
+        const line = `status: queued ${this.queued} · starting ${live.filter((s) => s.phase === "starting").length} · idle ${live.filter((s) => s.phase === "idle").length} · running ${live.filter((s) => s.phase === "busy").length} · done ${this.jobs.length}`;
+        if (line !== this.#lastLine) {
+            this.#lastLine = line;
+            await write(process.stdout, line + "\n");
+        }
+    }
+    wake() {
+        this.#wake?.();
+    }
+    async nap(ms) {
+        await new Promise((r) => {
+            const t = setTimeout(r, ms);
+            this.#wake = () => { clearTimeout(t); r(); };
+        });
+        this.#wake = undefined;
+    }
+    async run() {
+        while (true) {
+            if (this.stopping || this.fatal) {
+                this.stopping ??= "SIGTERM";
+                await this.drain();
+                if (this.live().length === 0)
+                    break;
+            }
+            else {
+                await this.tick();
+            }
+            this.drawTable();
+            await this.plainStatus();
+            if (this.live().length === 0 && (this.stopping || this.fatal))
+                break;
+            await this.nap(this.opts.pollMs);
+        }
+        await Promise.allSettled([...this.tasks]);
+        this.clearTable();
+    }
+    async tick() {
+        const items = await this.provider.demand((s) => this.note(s)).catch((e) => {
+            this.fatal = e instanceof PromigenceError ? e : new PromigenceError({ code: "internal", message: String(e?.message ?? e), fix: "Run it again; if it repeats, tell support@promigence.ai.", exitCode: 1 });
+            return undefined;
+        });
+        if (this.fatal)
+            return;
+        const now = Date.now();
+        for (const [id, until] of this.taken)
+            if (until < now)
+                this.taken.delete(id);
+        if (items)
+            for (const id of this.unservable.keys())
+                if (!items.some((i) => i.id === id))
+                    this.unservable.delete(id);
+        const fresh = items?.filter((i) => !this.taken.has(i.id) && !((this.unservable.get(i.id)?.until ?? 0) > now));
+        if (items)
+            this.lastItems = items;
+        const live = this.live();
+        const idle = live.filter((s) => (s.phase === "starting" || s.phase === "idle") && !s.retiring);
+        const reserved = new Set();
+        const unmatched = [];
+        if (fresh) {
+            this.queued = fresh.length;
+            for (const item of fresh) {
+                const r = idle.find((s) => !reserved.has(s) && this.provider.covers(s, item));
+                if (r) {
+                    reserved.add(r);
+                    r.lastDemandAt = now;
+                    if (r.reservedFor !== item.id) {
+                        r.reservedFor = item.id;
+                        r.reservedSince = now;
+                    }
+                }
+                else {
+                    unmatched.push(item);
+                }
+            }
+            for (const s of idle)
+                if (!reserved.has(s)) {
+                    delete s.reservedFor;
+                    delete s.reservedSince;
+                }
+        }
+        if (this.ceiling < this.opts.max && now - this.ceilingAt >= CEILING_STEP_TICKS * this.opts.tickMs) {
+            this.ceiling += 1;
+            this.ceilingAt = now;
+        }
+        let room = this.limit() - live.length;
+        if (now < this.startsPausedUntil)
+            room = 0;
+        for (const item of unmatched) {
+            if (room <= 0) {
+                if (now >= this.startsPausedUntil) {
+                    const at = this.ceiling < this.opts.max ? `${this.ceiling} at once (the org's sandbox limit; --max ${this.opts.max})` : `--max ${this.opts.max}`;
+                    this.note(`at ${at}: ${unmatched.length} job${unmatched.length === 1 ? "" : "s"} wait for a free runner`);
+                }
+                break;
+            }
+            const route = this.provider.route(item);
+            if (this.opts.timeoutFor(route.tier) < 60_000) {
+                this.note(`${item.title} asks for promigence-${route.tier}: --cap buys less than a minute of it, so it is left queued (raise --cap)`);
+                continue;
+            }
+            if (!this.opts.snapshots.has(route.tier)) {
+                const id = await this.opts.ensureSnapshot(route.tier);
+                if (id === undefined)
+                    continue;
+            }
+            this.launch(route, "demand", item.id);
+            room -= 1;
+        }
+        if (this.opts.keep > 0 && now >= this.startsPausedUntil) {
+            const keepers = this.live().filter((s) => s.purpose === this.opts.keepPurpose &&
+                (this.opts.keepPurpose === "pool" ? true : s.phase === "starting" || s.phase === "idle"));
+            let want = this.opts.keep - keepers.length;
+            while (want > 0 && this.live().length < this.limit()) {
+                this.launch(this.provider.keepRoute(), this.opts.keepPurpose);
+                want -= 1;
+            }
+        }
+        for (const s of idle) {
+            if (s.purpose !== "warm" || reserved.has(s) || s.phase !== "idle")
+                continue;
+            if (now - s.startedAt > this.opts.timeoutFor(s.tier) / 2)
+                void this.retire(s, "recycled before its sandbox's end of life");
+        }
+        for (const s of idle) {
+            if (s.purpose !== "demand" || s.phase !== "idle" || this.provider.exitsByItself)
+                continue;
+            const listening = s.listeningAt ?? s.startedAt;
+            if (reserved.has(s)) {
+                const floor = Math.max(this.opts.idleTimeoutMs, UNHANDED_FLOOR_MS * this.opts.tickMs / TICK_MS);
+                if (s.reservedFor === undefined || s.listeningAt === undefined || now - Math.max(s.listeningAt, s.reservedSince ?? now) < floor)
+                    continue;
+                const item = fresh?.find((i) => i.id === s.reservedFor);
+                this.markUnservable(s.reservedFor, item?.title);
+                void this.retire(s, "it was not handed the job it was started for");
+                continue;
+            }
+            if (now - Math.max(s.lastDemandAt, listening) < this.opts.idleTimeoutMs)
+                continue;
+            void this.retire(s, "no job for it");
+        }
+    }
+    markUnservable(id, title) {
+        const prev = this.unservable.get(id);
+        const times = (prev?.times ?? 0) + 1;
+        const wait = Math.min(60 * 60_000, UNSERVABLE_MS * 2 ** (times - 1));
+        const name = title ?? prev?.title ?? `job ${id}`;
+        this.unservable.set(id, { until: Date.now() + wait, times, title: name });
+        this.note(`${name}: ${this.provider.unassignedHint}; no runner is started for it for ${secs(wait)}`);
+    }
+    async drain() {
+        for (const s of this.live()) {
+            if (this.killing && s.quitSentAt !== undefined && s.phase !== "busy")
+                s.retireNow = true;
+            if (s.phase === "starting" || s.phase === "idle")
+                void this.retire(s, "stopping");
+            if (s.phase === "busy" && (this.killing || this.opts.killOnExit) && s.cancelSentAt === undefined && s.execId && s.sandboxId) {
+                s.cancelSentAt = Date.now();
+                void this.c.sandboxes.execs.signal(s.sandboxId, s.execId, "SIGTERM").catch(() => undefined);
+            }
+        }
+    }
+    async retire(s, why) {
+        if (s.retiring || s.retireNow)
+            return;
+        s.retiring = true;
+        let how = "no";
+        try {
+            how = s.phase === "starting" ? "now" : await this.provider.retire(s);
+        }
+        catch {
+            how = "no";
+        }
+        if (how === "graceful") {
+            const asked = s.sandboxId && s.execId
+                ? await this.c.sandboxes.execs.signal(s.sandboxId, s.execId, "SIGQUIT").then((r) => r.delivered !== false).catch(() => false)
+                : false;
+            if (asked)
+                s.quitSentAt = Date.now();
+            else
+                how = "now";
+        }
+        if (how === "no") {
+            s.retiring = false;
+            return;
+        }
+        if (how === "now")
+            s.retireNow = true;
+        if (why !== "stopping")
+            this.note(`${s.id}: stopped (${why})`);
+    }
+    async reapOrphans() {
+        const c = this.c;
+        for (const dead of deadPools(this.opts.scopeKey, this.opts.poolId)) {
+            let rows;
+            try {
+                rows = await c.sandboxes.list({ labels: { ci: this.opts.kind, "ci-pool": dead.pool_id }, state: ["ready", "running"], limit: 200 });
+            }
+            catch {
+                continue;
+            }
+            const ended = [];
+            const running = [];
+            for (const sb of rows) {
+                if (sb.state !== "ready" && sb.state !== "running")
+                    continue;
+                let idle = false;
+                try {
+                    const execs = await c.sandboxes.execs.list(sb.sandbox_id);
+                    idle = !execs.some((e) => e.state === "running");
+                    const name = sb.labels?.["ci-runner"];
+                    if (!idle && name)
+                        idle = await this.provider.orphanIdle(name);
+                }
+                catch {
+                    idle = false;
+                }
+                if (!idle) {
+                    running.push(sb.sandbox_id);
+                    continue;
+                }
+                if (await this.killSure(() => c.sandboxes.kill(sb.sandbox_id)))
+                    ended.push(sb.sandbox_id);
+                else
+                    running.push(sb.sandbox_id);
+            }
+            if (running.length === 0)
+                unregisterPool(dead.pool_id);
+            if (ended.length === 0 && running.length === 0)
+                continue;
+            await this.event({ event: "orphans", pool_id: dead.pool_id, ended, running }, `earlier pool ${dead.pool_id} (no longer running) left ${ended.length + running.length} runner sandbox${ended.length + running.length === 1 ? "" : "es"}: ` +
+                `${ended.length} ended now` + (running.length ? `; ${running.length} running a job, each ends itself after its job (${sh `promigence sandbox list --label ci-pool=${dead.pool_id}`})` : ""));
+        }
+    }
+    launch(route, purpose, forItem) {
+        const runner = {
+            id: `ci-${randomBytes(4).toString("hex")}`,
+            index: this.#nextIndex++,
+            tier: route.tier,
+            labels: route.labels,
+            key: route.key,
+            purpose,
+            phase: "starting",
+            startedAt: Date.now(),
+            offsets: {},
+            partial: {},
+            lastDemandAt: Date.now(),
+            tail: [],
+            ...(forItem === undefined ? {} : { forItem, reservedFor: forItem, reservedSince: Date.now() }),
+        };
+        this.runners.push(runner);
+        this.runnersStarted += 1;
+        const t = this.lifecycle(runner).finally(() => { this.tasks.delete(t); this.wake(); });
+        this.tasks.add(t);
+    }
+    async lifecycle(runner) {
+        const c = this.c;
+        try {
+            const snapshot = this.opts.snapshots.get(runner.tier);
+            if (snapshot === undefined)
+                throw new PromigenceError({ code: "not_found", message: `no ${runner.tier} runner snapshot`, fix: "Build it first with --prepare.", exitCode: 8 });
+            const timeoutMs = this.opts.timeoutFor(runner.tier);
+            const sandbox = (async () => {
+                const started = await c.runs.start({
+                    snapshot, count: 1, timeoutMs, spendCap: this.opts.capFor(runner.tier),
+                    labels: { ci: this.opts.kind, "ci-pool": this.opts.poolId, "ci-runner": runner.id },
+                    waitMs: 20_000,
+                }).catch((e) => {
+                    const status = e instanceof PromigenceError ? e.status : undefined;
+                    if (status === undefined || status >= 500)
+                        runner.startUnknown = true;
+                    throw e;
+                });
+                runner.runId = started.run_id;
+                this.rateStreak = 0;
+                let sb = started.sandboxes?.find((s) => s.state === "ready" || s.state === "running");
+                if (!sb) {
+                    const w = await c.runs.waitReady(started.run_id, { waitTimeoutMs: SANDBOX_READY_MS });
+                    sb = w.sandboxes.find((s) => s.state === "ready" || s.state === "running");
+                    if (!sb) {
+                        throw new PromigenceError({
+                            code: w.timedOut ? "timeout" : "env_failed",
+                            message: w.timedOut ? `no sandbox within ${SANDBOX_READY_MS / 1000} s` : "the sandbox did not start",
+                            fix: `Read ${sh `promigence run report ${started.run_id}`}; a sandbox that did not start is not billed.`,
+                            exitCode: w.timedOut ? 11 : 3,
+                        });
+                    }
+                }
+                runner.sandboxId = sb.sandbox_id;
+                return sb.sandbox_id;
+            })();
+            const creds = this.provider.credentials(runner);
+            const [sbR, crR] = await Promise.allSettled([sandbox, creds]);
+            if (sbR.status === "rejected")
+                throw sbR.reason;
+            if (crR.status === "rejected")
+                throw crR.reason;
+            const sid = sbR.value;
+            const cred = crR.value;
+            if (runner.retireNow || this.stopping) {
+                runner.phase = "stopping";
+                return;
+            }
+            await c.sandboxes.files.put(sid, cred.path, cred.content, { mode: 0o600 });
+            const rec = await c.sandboxes.execBackground(sid, {
+                argv: runnerArgv(this.provider.launcher, this.opts.selfEndGraceS),
+                timeoutMs,
+                ...(cred.env ? { env: cred.env } : {}),
+            });
+            runner.execId = rec.exec_id;
+            runner.phase = "idle";
+            await this.event({ event: "runner_started", runner: runner.id, tier: runner.tier, labels: runner.labels, purpose: runner.purpose, sandbox_id: sid, run_id: runner.runId, ready_ms: Date.now() - runner.startedAt }, undefined);
+            await this.watch(runner);
+        }
+        catch (e) {
+            const err = e instanceof PromigenceError ? e : e instanceof gh.GitHubError ? githubFailure(e, this.provider.secrets()) : new PromigenceError({ code: "internal", message: String(e?.message ?? e), fix: "Run it again.", exitCode: 1 });
+            runner.error = gh.redact(err.message, this.provider.secrets());
+            if (e instanceof gh.GitHubError && (e.status === 401 || e.status === 403))
+                this.fatal ??= err;
+            this.onStartFailure(runner, err);
+        }
+        finally {
+            await this.end(runner);
+        }
+    }
+    onStartFailure(runner, err) {
+        const msg = gh.redact(err.message, this.provider.secrets());
+        if (err.code === "concurrency_limit") {
+            runner.refused = true;
+            const others = this.live().filter((s) => s !== runner).length;
+            this.ceiling = Math.max(1, Math.min(this.ceiling, others));
+            this.ceilingAt = Date.now();
+            if (others === 0)
+                this.startsPausedUntil = Math.max(this.startsPausedUntil, Date.now() + CEILING_STEP_TICKS * this.opts.tickMs);
+            this.note(`the org's limit on sandboxes at once refused a runner: up to ${this.ceiling} at once for now, rising back to --max ${this.opts.max} a step at a time`);
+            return;
+        }
+        if (err.code === "rate_limited") {
+            runner.refused = true;
+            this.rateStreak += 1;
+            const wait = Math.min(60_000, Math.max(err.retryAfterMs ?? 0, 4 * this.opts.tickMs * 2 ** (this.rateStreak - 1)));
+            this.startsPausedUntil = Math.max(this.startsPausedUntil, Date.now() + wait);
+            this.note(`runner starts were rate limited: next start in ${secs(wait)}`);
+            return;
+        }
+        if (err.cliExit === 5 || err.code === "cap_exceeded" || err.code === "credits_exhausted" || err.code === "auth" || err.cliExit === 7) {
+            this.fatal ??= err;
+            return;
+        }
+        if (runner.jobStartedAt === undefined)
+            this.note(`${runner.id}: ${msg.slice(0, 160)}`);
+    }
+    async watch(runner) {
+        const c = this.c;
+        const sid = runner.sandboxId;
+        const eid = runner.execId;
+        const decoders = new Map();
+        const idleRead = Math.min(IDLE_READ_MS, this.opts.tickMs);
+        let failures = 0;
+        const gone = (e) => {
+            failures += 1;
+            if (e instanceof PromigenceError && (e.code === "not_found" || e.code === "sandbox_ended" || e.cliExit === 8))
+                failures += 2;
+            return undefined;
+        };
+        while (true) {
+            let ended = false;
+            if (runner.phase === "busy" && !runner.retireNow) {
+                const r = await c.sandboxes.execs.get(sid, eid, { waitMs: BUSY_WAIT_MS }).catch(gone);
+                if (r && r.state !== "running")
+                    ended = true;
+            }
+            else {
+                for (let waited = 0; waited < idleRead && !runner.retireNow; waited += Math.min(250, idleRead))
+                    await sleep(Math.min(250, idleRead));
+            }
+            const readAll = async () => {
+                for (const stream of this.provider.streams) {
+                    const r = await c.sandboxes.execs.read(sid, eid, stream, { offset: runner.offsets[stream] ?? 0 }).catch(gone);
+                    if (!r)
+                        continue;
+                    failures = 0;
+                    if (r.data.length) {
+                        runner.offsets[stream] = (runner.offsets[stream] ?? 0) + r.data.length;
+                        const dec = decoders.get(stream) ?? new TextDecoder();
+                        decoders.set(stream, dec);
+                        const text = (runner.partial[stream] ?? "") + dec.decode(r.data, { stream: true });
+                        const lines = text.split("\n");
+                        runner.partial[stream] = lines.pop() ?? "";
+                        for (const line of lines)
+                            this.onLine(runner, line);
+                    }
+                    if (r.record.state !== "running")
+                        ended = true;
+                }
+            };
+            await readAll();
+            if (ended)
+                await readAll();
+            if (ended || failures >= 3) {
+                for (const stream of this.provider.streams)
+                    if (runner.partial[stream])
+                        this.onLine(runner, runner.partial[stream]);
+                if (!ended)
+                    runner.error ??= "the sandbox stopped answering";
+                return;
+            }
+            if (runner.retireNow)
+                return;
+            if (runner.cancelSentAt !== undefined && Date.now() - runner.cancelSentAt > CANCEL_GRACE_MS)
+                return;
+            if (runner.quitSentAt !== undefined && runner.phase !== "busy" && Date.now() - runner.quitSentAt > QUIT_GRACE_MS) {
+                runner.retireNow = true;
+                return;
+            }
+            if (runner.phase === "busy" && (this.killing || this.opts.killOnExit) && this.stopping && runner.cancelSentAt === undefined) {
+                runner.cancelSentAt = Date.now();
+                await c.sandboxes.execs.signal(sid, eid, "SIGTERM").catch(() => undefined);
+            }
+        }
+    }
+    onLine(runner, raw) {
+        const line = raw.replace(/\r$/, "");
+        if (line.trim()) {
+            runner.tail.push(sanitizeRunnerLine(line, this.provider.secrets()).slice(0, 300));
+            if (runner.tail.length > 4)
+                runner.tail.shift();
+        }
+        const was = runner.phase;
+        this.provider.onLine(runner, line);
+        if (was !== "busy" && runner.phase === "busy") {
+            const mine = this.lastItems.find((i) => i.id === runner.forItem && (runner.job === undefined || i.name === undefined || i.name === runner.job))
+                ?? this.lastItems.find((i) => i.name !== undefined && i.name === runner.job && !this.taken.has(i.id));
+            if (mine)
+                this.taken.set(mine.id, Date.now() + 15_000);
+            void this.event({ event: "job_started", runner: runner.id, job: runner.job, ...(runner.jobId === undefined ? {} : { job_id: runner.jobId }), tier: runner.tier, sandbox_id: runner.sandboxId, pickup_ms: (runner.jobStartedAt ?? Date.now()) - runner.startedAt }, `started  ${runner.id}  ${runner.tier.padEnd(6)}  ${runner.job ?? "a job"}`);
+        }
+    }
+    async end(runner) {
+        if (runner.phase === "done" || runner.phase === "failed")
+            return;
+        const wasBusy = runner.jobStartedAt !== undefined;
+        runner.phase = "stopping";
+        const c = this.c;
+        if (runner.sandboxId && runner.execId && !wasBusy && !runner.retireNow) {
+            for (const stream of ["stderr", "stdout"]) {
+                if (this.provider.streams.includes(stream))
+                    continue;
+                const r = await c.sandboxes.execs.read(runner.sandboxId, runner.execId, stream, { offset: 0 }).catch(() => undefined);
+                for (const l of new TextDecoder().decode(r?.data ?? new Uint8Array()).split("\n").filter((x) => x.trim()).slice(-3)) {
+                    runner.tail.push(sanitizeRunnerLine(l, this.provider.secrets()).slice(0, 300));
+                }
+                if (runner.tail.length > 6)
+                    runner.tail.splice(0, runner.tail.length - 6);
+            }
+        }
+        const waitedFor = runner.reservedFor ?? runner.forItem;
+        const waited = Date.now() - (runner.listeningAt ?? runner.reservedSince ?? Date.now()) >= this.opts.idleTimeoutMs / 2;
+        const leftUnhanded = runner.tail.some((l) => /has not received a job in/.test(l)) ||
+            (this.provider.exitsByItself && runner.listeningAt !== undefined && waited);
+        const unhandedFor = !wasBusy && leftUnhanded && waited && runner.purpose === "demand" && waitedFor !== undefined &&
+            !runner.retireNow && !this.stopping && this.lastItems.some((i) => i.id === waitedFor) && !this.taken.has(waitedFor)
+            ? waitedFor : undefined;
+        if (wasBusy)
+            this.failStreak = 0;
+        else if (unhandedFor !== undefined) {
+            this.markUnservable(unhandedFor, this.lastItems.find((i) => i.id === unhandedFor)?.title);
+        }
+        else if (!runner.retireNow && !runner.refused && runner.quitSentAt === undefined && !this.stopping && Date.now() - runner.startedAt < 120_000 &&
+            !leftUnhanded) {
+            this.failStreak += 1;
+            if (this.failStreak >= 3) {
+                const wait = Math.min(300_000, 4 * this.opts.tickMs * 2 ** (this.failStreak - 3));
+                this.startsPausedUntil = Date.now() + wait;
+                this.note(`runners keep ending without a job (${this.failStreak} in a row; last: ${runner.error ?? (runner.tail[runner.tail.length - 1] ?? "no output")}); next start in ${secs(wait)}`);
+            }
+        }
+        await this.endSandbox(runner);
+        await this.provider.cleanup(runner).catch(() => undefined);
+        const { cost, billable, estimated } = await this.costOf(runner);
+        runner.cost = cost;
+        runner.billableS = billable;
+        runner.costEstimated = estimated;
+        runner.endedAt = Date.now();
+        runner.phase = runner.error && !wasBusy ? "failed" : "done";
+        if (wasBusy) {
+            const end = runner.jobEndedAt ?? runner.endedAt;
+            const rec = {
+                runner: runner.id,
+                job: runner.job ?? "(unnamed job)",
+                ...(runner.jobId === undefined ? {} : { job_id: runner.jobId }),
+                tier: runner.tier,
+                result: runner.result ?? (runner.cancelSentAt !== undefined ? "cancelled" : "unknown"),
+                duration_s: Math.round((end - (runner.jobStartedAt ?? end)) / 100) / 10,
+                sandbox_s: Math.round((runner.endedAt - runner.startedAt) / 100) / 10,
+                cost_usd: cost,
+                ...(estimated ? { cost_estimated: true } : {}),
+                ...(runner.killFailed ? { sandbox_not_ended: true } : {}),
+                ...(runner.sandboxId ? { sandbox_id: runner.sandboxId } : {}),
+                ...(runner.runId ? { run_id: runner.runId } : {}),
+            };
+            this.jobs.push(rec);
+            await this.event({ event: "job_done", ...rec }, `${rec.result.padEnd(9)} ${runner.id}  ${runner.tier.padEnd(6)}  ${rec.job}  ${rec.duration_s}s  ${estimated ? "~" : ""}${usd(cost)}`);
+        }
+        else {
+            this.idleCost += cost;
+            await this.event({
+                event: "runner_ended", runner: runner.id, tier: runner.tier, purpose: runner.purpose, cost_usd: cost,
+                ...(estimated ? { cost_estimated: true } : {}),
+                ...(runner.killFailed ? { sandbox_not_ended: true } : {}),
+                reason: runner.error ? "error" : runner.retireNow || runner.quitSentAt !== undefined ? "stopped" : "exited",
+                ...(runner.error ? { error: runner.error } : {}),
+                ...(runner.tail.length && !runner.retireNow ? { last_output: runner.tail } : {}),
+                ...(runner.sandboxId ? { sandbox_id: runner.sandboxId } : {}),
+            }, runner.error ? `runner   ${runner.id}  failed to start: ${runner.error}` : undefined);
+        }
+    }
+    async endSandbox(runner) {
+        const c = this.c;
+        if (!runner.sandboxId && !runner.runId && runner.startUnknown) {
+            let found;
+            for (let i = 0; i < 3; i++) {
+                found = await c.sandboxes.list({ labels: { "ci-runner": runner.id }, limit: 10 }).catch(() => undefined);
+                if (found !== undefined && found.length > 0)
+                    break;
+                if (i < 2)
+                    await sleep(2000);
+            }
+            const look = sh `promigence sandbox list --label ci-runner=${runner.id}`;
+            if (found === undefined) {
+                this.note(`${runner.id}: its start's answer was lost and the sandbox list could not be read: ${look}`);
+            }
+            else if (found.length === 0) {
+                this.note(`${runner.id}: its start's answer was lost and no sandbox of it was found; one may still start: ${look}, and kill it`);
+            }
+            for (const sb of found ?? []) {
+                if (sb.state === "ended")
+                    continue;
+                if (runner.sandboxId === undefined) {
+                    runner.sandboxId = sb.sandbox_id;
+                    if (sb.run_id)
+                        runner.runId = sb.run_id;
+                }
+                else {
+                    await this.killSure(() => c.sandboxes.kill(sb.sandbox_id));
+                }
+            }
+        }
+        const target = runner.sandboxId ?? runner.runId;
+        if (target === undefined)
+            return;
+        const ok = runner.sandboxId ? await this.killSure(() => c.sandboxes.kill(runner.sandboxId)) : await this.killSure(() => c.runs.kill(runner.runId));
+        if (!ok) {
+            runner.killFailed = true;
+            this.note(`${runner.id}: its sandbox ${target} could not be ended; it ends at its timeout (${sh `promigence sandbox kill ${target}`})`);
+        }
+    }
+    async killSure(kill) {
+        for (let i = 0; i < 4; i++) {
+            try {
+                await kill();
+                return true;
+            }
+            catch (e) {
+                if (e instanceof PromigenceError && (e.code === "not_found" || e.code === "sandbox_ended"))
+                    return true;
+            }
+            if (i < 3)
+                await sleep(1000 * 2 ** i);
+        }
+        return false;
+    }
+    async costOf(runner) {
+        const est = () => {
+            const s = Math.ceil(((runner.endedAt ?? Date.now()) - (runner.startedAt)) / 1000);
+            return { cost: runner.sandboxId ? Math.round(s * this.opts.perSecond(runner.tier) * 1e6) / 1e6 : 0, billable: runner.sandboxId ? s : 0, estimated: true };
+        };
+        if (runner.killFailed) {
+            const s = Math.ceil(this.opts.timeoutFor(runner.tier) / 1000);
+            return { cost: this.opts.capFor(runner.tier), billable: s, estimated: true };
+        }
+        if (!runner.runId || !runner.sandboxId)
+            return { cost: 0, billable: 0, estimated: false };
+        for (let i = 0; i < 6; i++) {
+            const rep = await this.c.runs.report(runner.runId).catch(() => undefined);
+            const row = rep?.sandboxes?.find((s) => s.sandbox_id === runner.sandboxId);
+            if (row && typeof row.cost === "number" && row.state === "ended") {
+                return { cost: row.cost, billable: row.billable_seconds ?? 0, estimated: false };
+            }
+            await sleep(i < 2 ? 500 : 1000);
+        }
+        return est();
+    }
+}
+const GITHUB_PERMISSIONS_FIX = "The token needs, with --repo, Administration: read and write and Actions: read on the repository (fine-grained), or the classic `repo` scope; " +
+    "with --org, Self-hosted runners: read and write on the organization and Actions: read on its repositories (classic: `admin:org` and `repo`).";
+function githubFailure(e, secrets) {
+    const msg = gh.redact(e.message, secrets);
+    if (e.status === 401) {
+        return new PromigenceError({ code: "usage", message: `GitHub refused the token (401): ${msg}`, fix: "The token is wrong, expired or revoked: make a new one (see `promigence runners github --help` for the permissions) and export it again.", exitCode: 2 });
+    }
+    if (e.status === 403) {
+        return new PromigenceError({ code: "usage", message: `GitHub refused the request (403): ${msg}`, fix: GITHUB_PERMISSIONS_FIX, exitCode: 2 });
+    }
+    if (e.status === 404) {
+        return new PromigenceError({ code: "usage", message: `GitHub answered 404: ${msg}`, fix: "Check --repo OWNER/REPO (or --org): it must exist, and the token must be able to see it.", exitCode: 2 });
+    }
+    return new PromigenceError({ code: "unreachable", message: `GitHub API: ${msg || `HTTP ${e.status}`}`, fix: "GitHub did not answer as expected; this is retried on the next poll. If it repeats, check https://www.githubstatus.com.", exitCode: 9 });
+}
+function tierFlag(ctx) {
+    const t = str(ctx, "--tier") ?? "small";
+    if (!isRunnerTier(t))
+        throw usageError(`--tier: '${t}' is not a runner tier`, `Pass --tier ${RUNNER_TIERS.join(" | ")} (small: ${shape("small")}).`);
+    return t;
+}
+function tokenFrom(ctx, flag, defaults, what, example) {
+    const named = str(ctx, flag);
+    const names = named === undefined ? defaults : [named];
+    for (const n of names) {
+        const v = process.env[n];
+        if (v !== undefined && v.trim() !== "")
+            return { value: v.trim(), name: n };
+    }
+    throw usageError(named === undefined ? `no ${what}: ${defaults.join(" and ")} ${defaults.length > 1 ? "are" : "is"} unset` : `${flag} ${named}: the variable is empty or unset`, `Export it first (${example}), or name another variable with ${flag} VAR. The value is read from the environment only: never pass it on the command line.`);
+}
+async function confirm(question) {
+    const rl = createInterface({ input: process.stdin, output: process.stderr });
+    try {
+        const answer = await new Promise((resolve) => {
+            rl.question(question, resolve);
+            rl.on("close", () => resolve(""));
+        });
+        return /^y(es)?$/i.test(answer.trim());
+    }
+    finally {
+        rl.close();
+    }
+}
+async function commonSetup(ctx) {
+    const c = client(ctx);
+    const me = await c.me();
+    const prices = await livePricing(c);
+    const perSecond = (tier) => prices.tiers.find((t) => t.name === tier)?.per_second_usd ?? listPerSecond(tier);
+    const planMaxS = me.limits?.max_session_s ?? undefined;
+    const flag = str(ctx, "--timeout");
+    const timeoutMs = flag === undefined ? (planMaxS ? planMaxS * 1000 : 3_600_000) : parseDuration(flag, "--timeout");
+    if (timeoutMs < 60_000)
+        throw usageError("--timeout: a job needs at least 1m", "Pass --timeout 30m (the longest a job may run).");
+    if (planMaxS && timeoutMs > planMaxS * 1000) {
+        throw usageError(`--timeout ${flag}: your plan's sandboxes live at most ${secs(planMaxS * 1000)}`, `Pass --timeout ${secs(planMaxS * 1000)} or less (a job longer than that needs another plan).`);
+    }
+    const capFlag = str(ctx, "--cap");
+    const cap = capFlag === undefined ? undefined : parseUsd(capFlag, "--cap");
+    let capNote;
+    const timeoutFor = (tier) => cap === undefined ? timeoutMs : Math.min(timeoutMs, Math.floor(cap / perSecond(tier)) * 1000);
+    const capFor = (tier) => {
+        const full = Math.ceil(perSecond(tier) * (timeoutFor(tier) / 1000) * 1e6) / 1e6;
+        return cap === undefined ? full : Math.min(cap, full);
+    };
+    if (cap !== undefined) {
+        const small = perSecond(tierFlag(ctx));
+        if (timeoutFor(tierFlag(ctx)) < 60_000)
+            throw usageError(`--cap ${capFlag} buys less than a minute of a ${tierFlag(ctx)} runner`, `Pass --cap ${usd(small * 600)} or more (ten minutes), or leave --cap off: each job is then capped at its timeout's cost.`);
+        const cut = RUNNER_TIERS.filter((t) => timeoutFor(t) < timeoutMs);
+        if (cut.length)
+            capNote = `--cap ${usd(cap)} limits each job by size: ${cut.map((t) => `${t} ${timeoutFor(t) < 60_000 ? "not run" : secs(timeoutFor(t))}`).join(", ")}`;
+    }
+    return { c, timeoutMs: timeoutFor(tierFlag(ctx)), timeoutFor, capFor, prices, perSecond, capNote, maxVcpus: me.limits?.max_sandbox_vcpus ?? undefined, maxConcurrency: me.limits?.max_concurrency ?? undefined };
+}
+function clampMax(max, keep, keepFlag, setup) {
+    const plan = setup.maxConcurrency;
+    if (!plan || plan <= 0 || max <= plan)
+        return { max, note: undefined };
+    if (keep > plan) {
+        throw usageError(`${keepFlag} ${keep} is more than your plan's ${plan} sandboxes at once`, `Pass ${keepFlag} ${plan} or less (\`promigence me\` shows the limit).`);
+    }
+    return { max: plan, note: `--max ${max} is over your plan's ${plan} sandboxes at once: up to ${plan} run at once` };
+}
+async function ensureSnapshot(ctx, c, kind, tier, mid) {
+    const s = runnerImageSpec(kind, tier);
+    try {
+        const rec = await c.snapshots.get(s.alias);
+        if (rec?.snapshot_id && !rec.unavailable)
+            return { id: rec.snapshot_id, built: false };
+    }
+    catch (e) {
+        if (!(e instanceof PromigenceError) || e.code !== "not_found")
+            throw e;
+    }
+    const prices = await livePricing(c);
+    const most = buildPriceIn(prices, tier);
+    const price = `up to ${usd(most.most_usd)}, billed per second while it builds`;
+    const listed = prices.source === "compiled" ? ` (${COMPILED_NOTE})` : "";
+    const yes = bool(ctx, "--yes");
+    const what = `the ${kind === "github" ? "GitHub Actions" : "GitLab"} runner snapshot for ${tier} (${shape(tier)})`;
+    if (!yes) {
+        const interactive = process.stdin.isTTY === true && process.stderr.isTTY === true && ctx.mode === "text" && !mid;
+        if (!interactive) {
+            if (mid)
+                return undefined;
+            throw usageError(`${what} is not built yet: building it once costs ${price} (about two minutes)${listed}`, `Pass --yes to build it, or build it first: ${sh `promigence runners ${kind} --prepare --tier ${tier} --yes`}`);
+        }
+        const ok = await confirm(`${what} is not built yet. Build it now? One time, ${price}${listed}, about two minutes. [y/N] `);
+        if (!ok)
+            return undefined;
+    }
+    const b = await c.snapshots.create({ image: s.image, setup: runnerBuildSteps(s), verify: { cmd: s.verify }, tier, alias: s.alias });
+    const charged = b.deduped || b.joined ? 0 : typeof b.build_price === "number" ? b.build_price : most.most_usd;
+    const said = buildPriceOf(b);
+    await write(process.stderr, ctx.mode === "json"
+        ? jsonLine(process.stderr, {
+            hint: "build_price", build_price: charged, unit: "USD", tier, alias: s.alias, build_id: b.build_id,
+            ...(charged > 0 && said ? { per_second_usd: said.per_second_usd, max_seconds: said.max_s } : {}),
+        })
+        : `building ${s.alias}: ${charged === 0 ? `${usd(0)} (an identical build exists)` : said ? buildPriceWords(said) : usd(charged)}, build ${b.build_id}; jobs start once it is verified\n`);
+    let build = b;
+    if (!(b.deduped || b.state === "verified")) {
+        const w = await c.snapshots.waitBuild(b.build_id, { waitTimeoutMs: BUILD_WAIT_MS });
+        build = w.build;
+        if (w.timedOut) {
+            throw new PromigenceError({ code: "timeout", message: `the runner snapshot build ${b.build_id} is still running after ${secs(BUILD_WAIT_MS)}`, fix: `Wait for it (${sh `promigence snapshot status ${b.build_id} --wait`}), then start again.`, exitCode: 11 });
+        }
+    }
+    if (build.state !== "verified" || !build.snapshot_id) {
+        const log = await c.snapshots.buildLog(b.build_id, 15).catch(() => []);
+        throw new PromigenceError({
+            code: "env_failed",
+            message: `the runner snapshot build ${b.build_id} ended ${build.state}${build.error?.message ? `: ${build.error.message}` : ""}${log.length ? ` (last log line: ${log[log.length - 1]})` : ""}`,
+            fix: `${typeof build.cost_usd === "number" && build.cost_usd > 0 ? `Nothing was stored; the time it ran was billed (${usd(build.cost_usd)}).` : "Nothing was stored or charged."} Read ${sh `promigence snapshot status ${b.build_id} --log 80`}, and tell support@promigence.ai if it is not your network.`,
+            exitCode: 3,
+        });
+    }
+    if (build.alias !== s.alias)
+        await c.snapshots.alias(build.snapshot_id, s.alias).catch(() => undefined);
+    return { id: build.snapshot_id, built: true, buildId: b.build_id, price: charged };
+}
+async function prepare(ctx, kind) {
+    const c = client(ctx);
+    const tier = tierFlag(ctx);
+    const r = await ensureSnapshot(ctx, c, kind, tier, false);
+    if (!r)
+        throw usageError("not built: the build was declined", `Run it again and answer y, or pass --yes.`);
+    const alias = runnerAlias(kind, tier);
+    const next = nexts([kind === "github" ? "promigence runners github --repo OWNER/REPO" : "promigence runners gitlab --url https://gitlab.com", "start taking jobs"]);
+    return emit(ctx.mode, okEnvelope(ctx.cmd.name, { kind, tier, alias, snapshot_id: r.id, built: r.built, ...(r.buildId ? { build_id: r.buildId, build_price: r.price } : {}) }, next), [
+        `${alias}: ${r.built ? "built" : "already built"} (${r.id})`,
+        `  next: ${next[0].cmd}`,
+    ], 0, ctx.pointer);
+}
+async function runPool(ctx, pool, intro, introFields) {
+    const onSignal = (sig) => {
+        if (!pool.stopping) {
+            pool.stopping = sig;
+            pool.note(pool.opts.killOnExit ? "stopping: cancelling running jobs" : "stopping: running jobs finish first (Ctrl-C again cancels them)");
+        }
+        else if (!pool.killing) {
+            pool.killing = true;
+            pool.note("cancelling running jobs");
+        }
+        else {
+            void Promise.allSettled(pool.live().filter((s) => s.sandboxId).map((s) => pool.c.sandboxes.kill(s.sandboxId)))
+                .then(() => process.exit(130));
+        }
+        pool.wake();
+    };
+    const handlers = ["SIGINT", "SIGTERM", "SIGHUP"].map((sig) => [sig, () => onSignal(sig)]);
+    for (const [sig, h] of handlers)
+        process.on(sig, h);
+    await pool.event({ event: "pool_started", ...introFields }, undefined);
+    if (ctx.mode === "text")
+        await write(process.stdout, intro.map((l) => escapeLine(l)).join("\n") + "\n");
+    const registered = registerPool({ pool_id: pool.opts.poolId, pid: process.pid, scope: pool.opts.scopeKey, started_at: new Date().toISOString() });
+    await pool.reapOrphans();
+    await pool.run();
+    if (registered && !pool.runners.some((r) => r.killFailed))
+        unregisterPool(pool.opts.poolId);
+    for (const [sig, h] of handlers)
+        process.off(sig, h);
+    const jobs = pool.jobs;
+    const by = (r) => jobs.filter((j) => j.result.toLowerCase() === r).length;
+    const jobCost = jobs.reduce((a, j) => a + j.cost_usd, 0);
+    const total = Math.round((jobCost + pool.idleCost) * 1e6) / 1e6;
+    const summary = {
+        stopped_by: pool.fatal ? "error" : pool.stopping ?? "signal",
+        jobs: { total: jobs.length, succeeded: by("succeeded") + by("success"), failed: by("failed"), cancelled: by("cancelled") + by("canceled"), other: 0 },
+        job_seconds: Math.round(jobs.reduce((a, j) => a + j.duration_s, 0) * 10) / 10,
+        cost_usd: total,
+        jobs_cost_usd: Math.round(jobCost * 1e6) / 1e6,
+        idle_cost_usd: Math.round(pool.idleCost * 1e6) / 1e6,
+        runners_started: pool.runnersStarted,
+        job_list: jobs,
+    };
+    summary.jobs.other = jobs.length - summary.jobs.succeeded - summary.jobs.failed - summary.jobs.cancelled;
+    const lines = [
+        `runners ${pool.opts.kind}: stopped${pool.fatal ? " on an error" : ""}. ${jobs.length} job${jobs.length === 1 ? "" : "s"}: ${summary.jobs.succeeded} succeeded, ${summary.jobs.failed} failed, ${summary.jobs.cancelled} cancelled` +
+            (summary.jobs.other ? `, ${summary.jobs.other} other` : ""),
+        `  ${secs(summary.job_seconds * 1000)} of jobs · ${usd(total)} (jobs ${usd(jobCost)}, idle runners ${usd(pool.idleCost)}) · ${pool.runnersStarted} sandbox${pool.runnersStarted === 1 ? "" : "es"}`,
+    ];
+    if (pool.fatal) {
+        const err = pool.fatal;
+        const next = nexts(["promigence sandbox list --label ci-pool=" + pool.opts.poolId + " --state ready,running", "anything this pool left running"]);
+        const env = { ...errorEnvelope(ctx.cmd.name, err, next), summary };
+        return emit(ctx.mode, env, [...errorTextLines(ctx.cmd.name, err, err.cliExit, next), ...lines], err.cliExit, ctx.pointer);
+    }
+    const next = nexts([sh `promigence sandbox list --label ci-pool=${pool.opts.poolId}`, "the sandboxes this pool used"]);
+    return emit(ctx.mode, okEnvelope(ctx.cmd.name, summary, next), lines, 130, ctx.pointer);
+}
+class GithubProvider {
+    api;
+    scope;
+    policy;
+    groupId;
+    kind = "github";
+    launcher = GITHUB_LAUNCHER;
+    streams = ["stdout"];
+    exitsByItself = false;
+    unassignedHint = "GitHub did not hand it to the runner started for it (may the repository use this runner group? with --org, the Default group does not take public repositories)";
+    #repos = [];
+    #reposAt = 0;
+    constructor(api, scope, policy, groupId) {
+        this.api = api;
+        this.scope = scope;
+        this.policy = policy;
+        this.groupId = groupId;
+        if (scope.kind === "repo")
+            this.#repos = [scope.name];
+    }
+    secrets() {
+        return [this.api.secret];
+    }
+    async demand(note) {
+        try {
+            if (this.scope.kind === "org" && Date.now() - this.#reposAt > 5 * 60_000) {
+                const list = await gh.orgRepos(this.api, this.scope.name);
+                this.#repos = list.repos;
+                this.#reposAt = Date.now();
+                if (list.more)
+                    note(`${this.scope.name} has more than 100 repositories: only the 100 most recently pushed are watched`);
+            }
+            const jobs = await gh.queuedJobs(this.api, this.#repos, this.scope.kind === "org"
+                ? (repo) => {
+                    this.#repos = this.#repos.filter((r) => r !== repo);
+                    note(`${repo}: GitHub answered 404 (deleted, moved, or not visible to the token): not watched until the repository list is read again`);
+                }
+                : undefined);
+            const items = [];
+            for (const j of jobs) {
+                const route = gh.routeJob(j.labels, this.policy);
+                if (!route)
+                    continue;
+                items.push({
+                    id: j.id, tier: route.tier, labels: route.labels, key: route.key,
+                    title: `${j.repo}: ${j.workflow_name ? `${j.workflow_name} / ` : ""}${j.name}`, name: j.name,
+                    ...(j.created_at ? { createdAt: j.created_at } : {}),
+                });
+            }
+            items.sort((a, b) => (a.createdAt ?? "").localeCompare(b.createdAt ?? ""));
+            return items;
+        }
+        catch (e) {
+            if (e instanceof gh.GitHubError && (e.status === 429 || e.status === 0 || e.status >= 500)) {
+                note(gh.redact(e.message, [this.api.secret]));
+                return undefined;
+            }
+            if (e instanceof gh.GitHubError)
+                throw githubFailure(e, [this.api.secret]);
+            throw e;
+        }
+    }
+    async credentials(runner) {
+        const jit = await gh.mintJitConfig(this.api, this.scope, { name: runner.id, labels: runner.labels, groupId: this.groupId });
+        runner.ghRunnerId = jit.runnerId;
+        return { path: GITHUB_CONFIG_PATH, content: jit.encoded };
+    }
+    onLine(runner, line) {
+        const at = /^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\dZ):/.exec(line);
+        const when = at ? Date.parse(at[1].replace(" ", "T")) : Date.now();
+        if (/Listening for Jobs/.test(line))
+            runner.listeningAt ??= Date.now();
+        const run = /Running job: (.+)$/.exec(line);
+        if (run) {
+            runner.phase = "busy";
+            runner.job = run[1].trim();
+            runner.jobStartedAt = Number.isFinite(when) ? when : Date.now();
+        }
+        const done = /Job (.+) completed with result: (\w+)/.exec(line);
+        if (done) {
+            runner.result = done[2];
+            runner.jobEndedAt = Number.isFinite(when) ? when : Date.now();
+            runner.job ??= done[1].trim();
+        }
+    }
+    covers(runner, item) {
+        return gh.covers(runner.labels, item.labels);
+    }
+    async retire(runner) {
+        if (runner.jobStartedAt !== undefined)
+            return "no";
+        if (runner.ghRunnerId === undefined)
+            return "now";
+        const r = await gh.getRunner(this.api, this.scope, runner.ghRunnerId);
+        if (r?.busy)
+            return "no";
+        await gh.deleteRunner(this.api, this.scope, runner.ghRunnerId);
+        delete runner.ghRunnerId;
+        return "now";
+    }
+    async orphanIdle(runnerName) {
+        const r = await gh.findRunnerByName(this.api, this.scope, runnerName);
+        if (!r || r.busy)
+            return false;
+        await gh.deleteRunner(this.api, this.scope, r.id);
+        return true;
+    }
+    async cleanup(runner) {
+        if (runner.ghRunnerId !== undefined && runner.jobStartedAt === undefined) {
+            await gh.deleteRunner(this.api, this.scope, runner.ghRunnerId).catch(() => undefined);
+        }
+    }
+    route(item) {
+        return { tier: item.tier, labels: item.labels, key: item.key };
+    }
+    keepRoute() {
+        const labels = gh.warmLabels(this.policy);
+        return { tier: this.policy.defaultTier, labels, key: labels.join(",") };
+    }
+}
+const runnersGithub = async (ctx) => {
+    const tier = tierFlag(ctx);
+    if (bool(ctx, "--prepare"))
+        return prepare(ctx, "github");
+    const repo = str(ctx, "--repo");
+    const org = str(ctx, "--org");
+    if ((repo === undefined) === (org === undefined)) {
+        throw usageError("pass --repo OWNER/REPO or --org ORG (one of them)", "e.g. promigence runners github --repo acme/app: runners for one repository (--org: for every repository of an organization).");
+    }
+    if (repo !== undefined && !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) {
+        throw usageError(`--repo: '${repo}' is not OWNER/REPO`, "Pass --repo acme/app (the part of the URL after github.com/).");
+    }
+    if (org !== undefined && !/^[A-Za-z0-9-]+$/.test(org))
+        throw usageError(`--org: '${org}' is not an organization name`, "Pass --org acme.");
+    const group = str(ctx, "--runner-group");
+    if (group !== undefined && org === undefined)
+        throw usageError("--runner-group needs --org", "Runner groups belong to an organization: pass --org ORG with it.");
+    const pool = gh.parsePoolLabels(str(ctx, "--labels"));
+    const bad = pool.find((l) => !gh.validLabel(l) || /^promigence-(small|medium|large)$/.test(l) || gh.DEFAULT_RUNNER_LABELS.includes(l));
+    if (pool.length === 0 || bad !== undefined) {
+        throw usageError(`--labels: '${bad ?? ""}' cannot be a pool label`, "Pass --labels with your own names (letters, digits, . _ -), e.g. --labels promigence,ci; the tier labels promigence-small|medium|large and self-hosted/linux/x64 are always understood.");
+    }
+    const warm = str(ctx, "--warm") === undefined ? 0 : parseCountOrZero(str(ctx, "--warm"), "--warm");
+    let max = str(ctx, "--max") === undefined ? DEFAULT_MAX : parseCount(str(ctx, "--max"), "--max");
+    if (warm > max)
+        throw usageError(`--warm ${warm} is more than --max ${max}`, "Keep --warm at or below --max.");
+    const idleTimeoutMs = str(ctx, "--idle-timeout") === undefined ? IDLE_TIMEOUT_DEFAULT_MS : parseDuration(str(ctx, "--idle-timeout"), "--idle-timeout");
+    const pollFlag = str(ctx, "--poll");
+    const tickMs = pollFlag === undefined ? TICK_MS : Math.max(100, parseDuration(pollFlag, "--poll"));
+    const pollMs = tickMs;
+    const token = tokenFrom(ctx, "--token-env", ["GH_TOKEN", "GITHUB_TOKEN"], "GitHub token", "export GH_TOKEN=…");
+    const apiUrl = str(ctx, "--github-api") ?? gh.GITHUB_API_DEFAULT;
+    if (!/^https?:\/\//.test(apiUrl))
+        throw usageError(`--github-api: '${apiUrl}' is not a URL`, "Pass the API's URL, e.g. https://github.example.com/api/v3 (GitHub Enterprise Server).");
+    const setup = await commonSetup(ctx);
+    const c = setup.c;
+    const clamped = clampMax(max, warm, "--warm", setup);
+    max = clamped.max;
+    const api = new gh.GitHubApi({ apiUrl, token: token.value });
+    const scope = repo !== undefined ? { kind: "repo", name: repo } : { kind: "org", name: org };
+    let groupId = 1;
+    try {
+        await gh.checkAccess(api, scope);
+        if (scope.kind === "org")
+            groupId = await gh.runnerGroupId(api, scope.name, group);
+    }
+    catch (e) {
+        if (e instanceof gh.GitHubError)
+            throw githubFailure(e, [token.value]);
+        throw e;
+    }
+    const snapshots = new Map();
+    const first = await ensureSnapshot(ctx, c, "github", tier, false);
+    if (!first)
+        throw usageError("not started: the runner snapshot was not built", "Run it again and answer y, or pass --yes.");
+    snapshots.set(tier, first.id);
+    for (const t of RUNNER_TIERS) {
+        if (t === tier)
+            continue;
+        const rec = await c.snapshots.get(runnerAlias("github", t)).catch(() => undefined);
+        if (rec?.snapshot_id)
+            snapshots.set(t, rec.snapshot_id);
+    }
+    const policy = { pool, defaultTier: tier };
+    const provider = new GithubProvider(api, scope, policy, groupId);
+    const poolId = randomBytes(4).toString("hex");
+    const building = new Set();
+    const opts = {
+        kind: "github", scopeLabel: scope.name, snapshots, max, keep: warm, keepPurpose: "warm",
+        timeoutFor: setup.timeoutFor, capFor: setup.capFor, perSecond: setup.perSecond, tickMs, pollMs, idleTimeoutMs,
+        killOnExit: bool(ctx, "--kill-on-exit"), poolId, selfEndGraceS: graceFor(tickMs, TICK_MS),
+        scopeKey: `github ${c.apiUrl} ${apiUrl} ${scope.kind}:${scope.name}`,
+        ensureSnapshot: async (t) => {
+            if (snapshots.has(t))
+                return snapshots.get(t);
+            if (!bool(ctx, "--yes")) {
+                poolRef?.note(`a job asks for promigence-${t}: its runner snapshot is not built (${sh `promigence runners github --prepare --tier ${t} --yes`}, once: up to ${usd(buildPriceIn(setup.prices, t).most_usd)}, billed per second)`);
+                return undefined;
+            }
+            if (building.has(t))
+                return undefined;
+            building.add(t);
+            void ensureSnapshot(ctx, c, "github", t, true).then((r) => { if (r)
+                snapshots.set(t, r.id); })
+                .catch((e) => poolRef?.note(`building the ${t} runner snapshot failed: ${String(e?.message ?? e).slice(0, 160)}`))
+                .finally(() => building.delete(t));
+            return undefined;
+        },
+    };
+    const poolObj = new Pool(ctx, c, opts, provider);
+    poolRef = poolObj;
+    const rate = setup.perSecond(tier);
+    const intro = [
+        `runners github: ${scope.kind === "repo" ? "repository" : "organization"} ${scope.name}${scope.kind === "org" && group ? ` (runner group ${group})` : ""}`,
+        `  in your workflow: runs-on: ${pool[0]}   (a size: runs-on: promigence-medium | promigence-large)`,
+        `  default tier ${tier}: ${shape(tier)}, ${usd(rate * 3600)}/h billed per second; runner snapshots: ${[...snapshots.keys()].join(", ")}`,
+        `  up to ${max} job${max === 1 ? "" : "s"} at once, each at most ${secs(setup.timeoutMs)} and ${usd(setup.capFor(tier))} (${tier})` +
+            (warm ? `; ${warm} warm runner${warm === 1 ? "" : "s"} kept waiting (they bill while idle: ${usd(rate * 3600 * warm)}/h)` : "; runners start on demand (nothing bills while no job is queued)"),
+        ...(setup.capNote ? [`  note: ${setup.capNote}`] : []),
+        ...(setup.prices.source === "compiled" ? [`  note: prices: ${COMPILED_NOTE}`] : []),
+        ...(clamped.note ? [`  note: ${clamped.note}`] : []),
+        "  Ctrl-C stops taking jobs and waits for running ones; a second Ctrl-C cancels them.",
+    ];
+    return runPool(ctx, poolObj, intro, {
+        kind: "github", scope: scope.kind, name: scope.name, labels: pool, tier, tiers_ready: [...snapshots.keys()],
+        max, warm, timeout_s: setup.timeoutMs / 1000, cap_usd: setup.capFor(tier), price_source: setup.prices.source, pool_id: poolId, token_env: token.name,
+    });
+};
+let poolRef;
+function graceFor(tickMs, defaultTick) {
+    return Math.max(1, Math.round(SELF_END_GRACE_S * Math.min(1, tickMs / defaultTick)));
+}
+function parseCountOrZero(v, flag) {
+    if (v.trim() === "0")
+        return 0;
+    return parseCount(v, flag);
+}
+class GitlabProvider {
+    cfg;
+    tier;
+    seed;
+    api;
+    waitTimeoutS;
+    kind = "gitlab";
+    launcher = GITLAB_LAUNCHER;
+    streams = ["stderr", "stdout"];
+    exitsByItself = true;
+    unassignedHint = "GitLab did not hand it to the runner started for it (a protected branch the runner may not run, or a paused runner?)";
+    #runner;
+    #projects = [];
+    #projectsAt = 0;
+    constructor(cfg, tier, seed, api, waitTimeoutS) {
+        this.cfg = cfg;
+        this.tier = tier;
+        this.seed = seed;
+        this.api = api;
+        this.waitTimeoutS = waitTimeoutS;
+    }
+    secrets() {
+        return [this.cfg.token, this.api?.secret];
+    }
+    async init() {
+        if (!this.api)
+            return undefined;
+        this.#runner = await this.api.runner(this.cfg.token);
+        return this.#runner;
+    }
+    async demand(note) {
+        if (!this.api || !this.#runner)
+            return [];
+        try {
+            if (Date.now() - this.#projectsAt > 5 * 60_000) {
+                const ps = new Set(this.#runner.projects);
+                for (const g of this.#runner.groups)
+                    for (const p of await this.api.groupProjects(g))
+                        ps.add(p);
+                this.#projects = [...ps];
+                this.#projectsAt = Date.now();
+            }
+            const items = [];
+            for (const p of this.#projects) {
+                for (const j of await this.api.pendingJobs(p)) {
+                    if (!gl.runnerTakes(this.#runner, j))
+                        continue;
+                    items.push({ id: j.id, tier: this.tier, labels: [], key: this.tier, title: `${j.name} (#${j.id})` });
+                }
+            }
+            return items;
+        }
+        catch (e) {
+            if (e instanceof gh.GitHubError && (e.status === 429 || e.status === 0 || e.status >= 500)) {
+                note(gh.redact(e.message, this.secrets()));
+                return undefined;
+            }
+            if (e instanceof gh.GitHubError) {
+                throw new PromigenceError({ code: "usage", message: `GitLab API ${e.status}: ${gh.redact(e.message, this.secrets())}`, fix: "The API token needs read_api on the runner's projects (a personal, group or project access token).", exitCode: 2 });
+            }
+            throw e;
+        }
+    }
+    async credentials(runner) {
+        return {
+            path: GITLAB_CONFIG_PATH,
+            content: gl.gitlabConfigToml(this.cfg),
+            env: {
+                CI_SYSTEM_ID: gl.systemId(this.seed, runner.index % 64),
+                ...(this.waitTimeoutS === undefined ? {} : { CI_WAIT_TIMEOUT: String(this.waitTimeoutS) }),
+            },
+        };
+    }
+    onLine(runner, line) {
+        if (/Starting runner for/.test(line) || /Checking for jobs/.test(line))
+            runner.listeningAt ??= Date.now();
+        if (/Checking for jobs\.\.\.\s+received/.test(line) || (/received/.test(line) && /\bjob=\d+/.test(line))) {
+            runner.phase = "busy";
+            runner.jobStartedAt ??= Date.now();
+            const id = gl.receivedJobId(line);
+            if (id !== undefined)
+                runner.jobId = id;
+            const name = /job_name=("(?:[^"\\]|\\.)*"|\S+)/.exec(line);
+            const n = name ? (name[1].startsWith('"') ? safeJson(name[1]) : name[1]) : undefined;
+            runner.job = `${n ?? "job"}${id !== undefined ? ` (#${id})` : ""}`;
+        }
+        if (/Job succeeded/.test(line)) {
+            runner.result = "success";
+            runner.jobEndedAt ??= Date.now();
+        }
+        else if (/Job failed/.test(line)) {
+            runner.result = /canceled|cancelled/.test(line) ? "cancelled" : "failed";
+            runner.jobEndedAt ??= Date.now();
+        }
+    }
+    covers() {
+        return true;
+    }
+    async retire(runner) {
+        return runner.jobStartedAt === undefined ? "graceful" : "no";
+    }
+    async cleanup() { }
+    async orphanIdle() {
+        return false;
+    }
+    route() {
+        return { tier: this.tier, labels: [], key: this.tier };
+    }
+    keepRoute() {
+        return this.route();
+    }
+}
+function safeJson(s) {
+    try {
+        return JSON.parse(s);
+    }
+    catch {
+        return undefined;
+    }
+}
+export const GITLAB_TOKEN_NOTICE = "jobs on this runner can read its token (glrt-…): run it for projects whose CI code you trust";
+const runnersGitlab = async (ctx) => {
+    const tier = tierFlag(ctx);
+    if (bool(ctx, "--prepare"))
+        return prepare(ctx, "gitlab");
+    const url = (str(ctx, "--url") ?? gl.GITLAB_URL_DEFAULT).replace(/\/+$/, "");
+    if (!/^https?:\/\/[^\s/]+/.test(url))
+        throw usageError(`--url: '${url}' is not a GitLab URL`, "Pass --url https://gitlab.com or your instance's address.");
+    const executor = str(ctx, "--executor") ?? "docker";
+    if (!gl.GITLAB_EXECUTORS.includes(executor)) {
+        throw usageError(`--executor: '${executor}' is not one this runs`, "Pass --executor docker (jobs in containers, the default) or shell (jobs directly in the sandbox).");
+    }
+    const defaultImage = str(ctx, "--default-image") ?? gl.GITLAB_DEFAULT_IMAGE;
+    const token = tokenFrom(ctx, "--token-env", ["GITLAB_RUNNER_TOKEN"], "runner authentication token", "export GITLAB_RUNNER_TOKEN=glrt-…, from the runner you created in GitLab with its tags");
+    const apiTokenName = str(ctx, "--api-token-env");
+    const apiToken = apiTokenName === undefined ? undefined : tokenFrom(ctx, "--api-token-env", [apiTokenName], "GitLab API token", `export ${apiTokenName}=… (read_api)`);
+    const poolSize = str(ctx, "--pool") === undefined ? (apiToken ? 0 : 1) : parseCountOrZero(str(ctx, "--pool"), "--pool");
+    let max = str(ctx, "--max") === undefined ? Math.max(DEFAULT_MAX, poolSize) : parseCount(str(ctx, "--max"), "--max");
+    if (poolSize > max)
+        throw usageError(`--pool ${poolSize} is more than --max ${max}`, "Keep --pool at or below --max.");
+    if (poolSize === 0 && !apiToken)
+        throw usageError("--pool 0 without --api-token-env would never take a job", "Pass --pool N (N runners always up), or --api-token-env VAR to start runners only when jobs are pending.");
+    const idleTimeoutMs = str(ctx, "--idle-timeout") === undefined ? IDLE_TIMEOUT_DEFAULT_MS : parseDuration(str(ctx, "--idle-timeout"), "--idle-timeout");
+    const tickMs = str(ctx, "--poll") === undefined ? TICK_MS * 2 : Math.max(100, parseDuration(str(ctx, "--poll"), "--poll"));
+    const pollMs = tickMs;
+    const setup = await commonSetup(ctx);
+    const c = setup.c;
+    const clamped = clampMax(max, poolSize, "--pool", setup);
+    max = clamped.max;
+    const api = apiToken ? new gl.GitlabApi(url, apiToken.value) : undefined;
+    const cfg = { url, token: token.value, executor: executor, defaultImage };
+    const seed = randomBytes(6).toString("hex");
+    const provider = new GitlabProvider(cfg, tier, seed, api, api ? Math.max(30, Math.round(idleTimeoutMs / 1000)) : Math.max(60, Math.floor(setup.timeoutMs / 2000)));
+    let info;
+    try {
+        info = await provider.init();
+    }
+    catch (e) {
+        if (e instanceof gh.GitHubError) {
+            throw new PromigenceError({ code: "usage", message: `GitLab refused the runner check (${e.status}): ${gh.redact(e.message, provider.secrets())}`, fix: "Check --url, the runner authentication token (glrt-…) and that the API token can read the runner (read_api).", exitCode: 2 });
+        }
+        throw e;
+    }
+    if (api && info && info.type === "instance_type") {
+        throw usageError("on-demand runners need a project or group runner", "An instance runner's pending jobs cannot be listed: drop --api-token-env and keep a pool (--pool N).");
+    }
+    const first = await ensureSnapshot(ctx, c, "gitlab", tier, false);
+    if (!first)
+        throw usageError("not started: the runner snapshot was not built", "Run it again and answer y, or pass --yes.");
+    const snapshots = new Map([[tier, first.id]]);
+    const poolId = randomBytes(4).toString("hex");
+    const opts = {
+        kind: "gitlab", scopeLabel: url.replace(/^https?:\/\//, ""), snapshots, max, keep: poolSize, keepPurpose: "pool",
+        timeoutFor: setup.timeoutFor, capFor: setup.capFor, perSecond: setup.perSecond, tickMs, pollMs, idleTimeoutMs,
+        killOnExit: bool(ctx, "--kill-on-exit"), poolId, ensureSnapshot: async (t) => snapshots.get(t),
+        selfEndGraceS: graceFor(tickMs, TICK_MS * 2), scopeKey: `gitlab ${c.apiUrl} ${url}`,
+    };
+    const pool = new Pool(ctx, c, opts, provider);
+    poolRef = pool;
+    const rate = setup.perSecond(tier);
+    const intro = [
+        `runners gitlab: ${url} · executor ${executor}${executor === "docker" ? ` (default image ${defaultImage})` : ""}`,
+        `  note: ${GITLAB_TOKEN_NOTICE}`,
+        `  in your .gitlab-ci.yml: tags: [the tags you gave this runner in GitLab]${info ? ` (it has: ${info.tags.join(", ") || "none"}${info.runUntagged ? ", takes untagged jobs" : ""})` : ""}`,
+        `  tier ${tier}: ${shape(tier)}, ${usd(rate * 3600)}/h billed per second; each job at most ${secs(setup.timeoutMs)} and ${usd(setup.capFor(tier))}`,
+        poolSize > 0
+            ? `  ${poolSize} runner${poolSize === 1 ? "" : "s"} kept up (each takes one job, then a fresh one replaces it; they bill while idle: ${usd(rate * 3600 * poolSize)}/h)`
+            : `  runners start when GitLab has a pending job for this runner (up to ${max} at once; nothing bills while none is pending)`,
+        ...(setup.capNote ? [`  note: ${setup.capNote}`] : []),
+        ...(setup.prices.source === "compiled" ? [`  note: prices: ${COMPILED_NOTE}`] : []),
+        ...(clamped.note ? [`  note: ${clamped.note}`] : []),
+        "  Ctrl-C stops taking jobs and waits for running ones; a second Ctrl-C cancels them.",
+    ];
+    return runPool(ctx, pool, intro, {
+        kind: "gitlab", url, executor, tier, pool: poolSize, max, on_demand: Boolean(api), timeout_s: setup.timeoutMs / 1000,
+        cap_usd: setup.capFor(tier), price_source: setup.prices.source, pool_id: poolId, token_env: token.name, notice: GITLAB_TOKEN_NOTICE, ...(info ? { runner_tags: info.tags } : {}),
+    });
+};
+export const RUNNERS_HANDLERS = {
+    "runners github": runnersGithub,
+    "runners gitlab": runnersGitlab,
+};