withruntime 0.9.0 → 0.10.0

package/dist/client.js

@@ -1,15 +1,17 @@
 import { WAIT_FOR_TIMEOUT_SECONDS } from "./api-defaults.js";
 import { RuntimeError } from "./errors.js";
-import { clientFactories } from "./products/index.js";
+import { clientFactories, clientProductAliases } from "./products/index.js";
+import { usageExport as makeUsageExport } from "./products/observability.js";
 import { Sandbox, deleteSandbox, sandboxPage } from "./sandbox.js";
 import { Snapshots } from "./snapshots.js";
 import { Transport, missingKey } from "./transport.js";
-/** The key `runtime login` saved for this machine, found on first use. */
-const savedKey = async () => {
+/** The key `runtime login` saved for this machine and this API origin, found
+ * on first use. A constructor override must never receive another API's key. */
+const savedKey = async (apiOrigin) => {
     if (typeof process === "undefined" || !process.versions?.node)
         throw missingKey();
     const { resolveCredential } = await import("./credentials.js");
-    return resolveCredential(process.env);
+    return resolveCredential({ ...process.env, RUNTIME_API_URL: apiOrigin });
 };
 const env = (name) => typeof process === "undefined"
     ? undefined
@@ -17,32 +19,42 @@ const env = (name) => typeof process === "undefined"
 // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
 export class Runtime {
     transport;
+    /** The sandbox product; `sandboxes` remains the same client for compatibility. */
+    sandbox;
     sandboxes;
+    snapshot;
     snapshots;
     feedback;
     support;
     account;
+    /** Date-range settlements, paged with exact charges and the server's CSV. */
+    usageExport;
     constructor(options = {}) {
         this.transport = new Transport({
             // A key given, then RUNTIME_API_KEY, then (in Node and Bun) the key
             // `runtime login` saved for this machine, read on the first call.
-            apiKey: options.apiKey ?? env("RUNTIME_API_KEY") ?? savedKey,
+            apiKey: options.apiKey ?? env("RUNTIME_API_KEY") ?? (() => savedKey(this.transport.baseUrl)),
             baseUrl: options.baseUrl ?? env("RUNTIME_API_URL"),
             ...(options.fetch ? { fetch: options.fetch } : {}),
-            ...(options.timeoutMs ? { timeoutMs: options.timeoutMs } : {}),
+            ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
             ...(options.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}),
-            ...(options.maxConnections ? { maxConnections: options.maxConnections } : {}),
+            ...(options.maxConnections !== undefined ? { maxConnections: options.maxConnections } : {}),
             ...(options.waitForCapacityMs !== undefined
                 ? { waitForCapacityMs: options.waitForCapacityMs }
                 : {}),
         });
         this.sandboxes = new Sandboxes(this.transport);
+        this.sandbox = this.sandboxes;
         this.snapshots = new Snapshots(this.transport);
+        this.snapshot = this.snapshots;
         this.feedback = new FeedbackApi(this.transport);
         this.support = new SupportApi(this.transport);
         this.account = new AccountApi(this.transport);
+        this.usageExport = makeUsageExport(this.transport);
         for (const [name, make] of clientFactories())
             Object.defineProperty(this, name, { value: make(this.transport), enumerable: true });
+        for (const [alias, name] of Object.entries(clientProductAliases))
+            Object.defineProperty(this, alias, { value: this[name], enumerable: true });
     }
     /** Who this key is: organization, agent and credential. */
     me(options = {}) {
@@ -61,6 +73,17 @@ export class Runtime {
             ...options,
         });
     }
+    /** This account's authenticated API answers, in hourly windows. Counts are
+     * exact decimal strings, and an empty window has a null error percentage.
+     * A key needs usage permission. */
+    usageRequests(range = "24h", options = {}) {
+        return this.transport.json({
+            method: "GET",
+            path: "/v1/usage/requests",
+            query: { range },
+            ...options,
+        });
+    }
 }
 export class Sandboxes {
     t;
@@ -84,7 +107,12 @@ export class Sandboxes {
         });
         const sandbox = new Sandbox(this.t, info);
         if (wait !== false && info.state !== "running") {
-            await sandbox.waitFor("running", { timeoutSeconds: WAIT_FOR_TIMEOUT_SECONDS });
+            await sandbox.waitFor("running", {
+                timeoutSeconds: WAIT_FOR_TIMEOUT_SECONDS,
+                signal: options.signal,
+                timeoutMs: options.timeoutMs,
+                idempotencyKey: options.idempotencyKey,
+            });
             if (sandbox.state !== "running")
                 throw new RuntimeError({
                     message: `Sandbox ${info.id} is ${sandbox.state}, not running.`,
@@ -114,8 +142,9 @@ export class Sandboxes {
     delete(id, options = {}) {
         return deleteSandbox(this.t, id, options);
     }
-    /** Live sandboxes, oldest first. Await for a page, or `for await` over all. */
-    async list(filter = {}) {
+    /** Live sandboxes, oldest first. Await for a page, or `for await` over all.
+     * Request options apply to the initial request and every subsequent page. */
+    async list(filter = {}, options = {}) {
         const query = {
             ...(filter.state ? { state: filter.state } : {}),
             ...(filter.includeStopped ? { includeStopped: true } : {}),
@@ -125,7 +154,7 @@ export class Sandboxes {
             ...(filter.name ? { name: filter.name } : {}),
             ...(filter.limit ? { limit: filter.limit } : {}),
         };
-        return sandboxPage(this.t, await this.t.json({ method: "GET", path: "/v1/sandboxes", query }), query);
+        return sandboxPage(this.t, await this.t.json({ method: "GET", path: "/v1/sandboxes", query, ...options }), query, options);
     }
 }
 export class FeedbackApi {

package/dist/cloudflare/identity.d.ts

@@ -0,0 +1,7 @@
+/** A Durable Object ID includes its namespace; a display name does not. */
+export declare function workerSandboxIdentity(objectId: string, stored: {
+    nativeSandboxId?: string;
+    sandboxName?: {
+        name: string;
+    };
+}): string;

package/dist/cloudflare/identity.js

@@ -0,0 +1,13 @@
+/** A Durable Object ID includes its namespace; a display name does not. */
+export function workerSandboxIdentity(objectId, stored) {
+    if (stored.nativeSandboxId !== undefined) {
+        if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(stored.nativeSandboxId))
+            throw new Error("The stored Runtime sandbox identity is invalid");
+        return stored.nativeSandboxId;
+    }
+    if (stored.sandboxName)
+        throw new Error("This legacy Durable Object has no namespace-safe Runtime identity. Retain its existing sandbox and migrate its identity explicitly before using it.");
+    if (!/^[a-f0-9]{64}$/.test(objectId))
+        throw new Error("The Durable Object identity is invalid");
+    return `cf-do-${objectId}`;
+}

package/dist/cloudflare/index.d.ts

@@ -70,6 +70,7 @@ export interface ProcessOptions {
     onStart?: (process: Process) => void;
     onError?: (error: Error) => void;
 }
+export declare function validateSandboxConfiguration(options: Pick<SandboxOptions, "sleepAfter" | "keepAlive">): void;
 /** Worker bindings are deliberately not inspected or invoked. Supply Runtime credentials
  * through options.runtime in Workers; Node also supports RUNTIME_API_KEY/login. */
 export declare function getSandbox(namespace: unknown, id: string, options?: SandboxOptions): Sandbox;
@@ -77,9 +78,17 @@ export declare class Sandbox {
     readonly runtime: Runtime;
     private readonly options;
     private pending?;
+    private existing?;
     readonly id: string;
     constructor(runtime: Runtime, id: string, options?: SandboxOptions);
     configure(options: Pick<SandboxOptions, "sleepAfter" | "keepAlive">): Promise<void>;
+    /** Only a successfully initialized native handle can supply a recovery id. */
+    existingNativeId(): string | undefined;
+    /** Reconcile an uncertain update from an authoritative read, without allocation. */
+    reconcileConfiguration(nativeId?: string | undefined): Promise<{
+        sleepAfter: number;
+        keepAlive: boolean;
+    }>;
     native(): Promise<NativeSandbox>;
     private session;
     exec(command: string, options?: ExecOptions): Promise<ExecResult>;

package/dist/cloudflare/index.js

@@ -22,12 +22,24 @@ export function sanitizeSandboxId(id) {
 }
 const cleanEnv = (env = {}) => Object.fromEntries(Object.entries(env).filter((entry) => entry[1] !== undefined));
 function sleepSeconds(value = 600) {
+    let seconds;
     if (typeof value === "number")
-        return value;
-    const match = /^(\d+)(s|m|h)$/.exec(value);
-    if (!match)
-        throw new TypeError("sleepAfter must be seconds or a duration such as 5m");
-    return Number(match[1]) * ({ s: 1, m: 60, h: 3600 }[match[2]] ?? 1);
+        seconds = value;
+    else {
+        const match = /^(\d+)(s|m|h)$/.exec(value);
+        if (!match)
+            throw new TypeError("sleepAfter must be seconds or a duration such as 5m");
+        seconds = Number(match[1]) * ({ s: 1, m: 60, h: 3600 }[match[2]] ?? 1);
+    }
+    if (!Number.isSafeInteger(seconds) || seconds < 0 || seconds > 86_400)
+        throw new CompatibilityError("Cloudflare", "sleepAfter outside whole seconds from 0 to 86400");
+    return seconds;
+}
+export function validateSandboxConfiguration(options) {
+    if (options.sleepAfter !== undefined)
+        sleepSeconds(options.sleepAfter);
+    if (options.keepAlive !== undefined && typeof options.keepAlive !== "boolean")
+        throw new TypeError("keepAlive must be a boolean");
 }
 /** Worker bindings are deliberately not inspected or invoked. Supply Runtime credentials
  * through options.runtime in Workers; Node also supports RUNTIME_API_KEY/login. */
@@ -41,25 +53,50 @@ export class Sandbox {
     runtime;
     options;
     pending;
+    existing;
     id;
     constructor(runtime, id, options = {}) {
         this.runtime = runtime;
         this.options = options;
+        validateSandboxConfiguration(options);
         this.id = id;
     }
     async configure(options) {
-        if (options.sleepAfter !== undefined)
-            sleepSeconds(options.sleepAfter);
+        only("Cloudflare configure", options, ["sleepAfter", "keepAlive"]);
+        validateSandboxConfiguration(options);
         const changed = (options.sleepAfter !== undefined && options.sleepAfter !== this.options.sleepAfter) ||
             (options.keepAlive !== undefined && options.keepAlive !== this.options.keepAlive);
-        Object.assign(this.options, options);
+        const next = { ...this.options, ...options };
         if (changed && this.pending) {
             const s = await this.pending;
             await s.update({
-                idlePauseSeconds: this.options.keepAlive ? 0 : sleepSeconds(this.options.sleepAfter),
+                idlePauseSeconds: next.keepAlive ? 0 : sleepSeconds(next.sleepAfter),
                 ...(options.keepAlive !== undefined ? { persistent: options.keepAlive } : {}),
             });
         }
+        Object.assign(this.options, options);
+    }
+    /** Only a successfully initialized native handle can supply a recovery id. */
+    existingNativeId() {
+        return this.existing?.id;
+    }
+    /** Reconcile an uncertain update from an authoritative read, without allocation. */
+    async reconcileConfiguration(nativeId = this.existingNativeId()) {
+        if (!nativeId)
+            throw new Error("The uncertain Runtime configuration has no existing sandbox identity.");
+        const sandbox = await this.runtime.sandboxes.get(nativeId);
+        const { idlePauseSeconds, persistent } = sandbox.info;
+        if (typeof idlePauseSeconds !== "number" ||
+            !Number.isSafeInteger(idlePauseSeconds) ||
+            idlePauseSeconds < 0 ||
+            idlePauseSeconds > 86_400 ||
+            typeof persistent !== "boolean")
+            throw new Error("Runtime did not return authoritative idle and persistence settings.");
+        const configuration = { sleepAfter: idlePauseSeconds, keepAlive: persistent };
+        Object.assign(this.options, configuration);
+        this.existing = sandbox;
+        this.pending = Promise.resolve(sandbox);
+        return configuration;
     }
     async native() {
         if (!this.pending) {
@@ -82,6 +119,7 @@ export class Sandbox {
                         throw e;
                     }
                 }
+                this.existing = s;
                 return s;
             })();
             this.pending.catch(() => {

package/dist/cloudflare/worker.d.ts

@@ -14,12 +14,16 @@ type Configuration = {
         name: string;
         normalizeId?: boolean;
     };
+    nativeSandboxId?: string;
     sleepAfter?: string | number;
     keepAlive?: boolean;
 };
 export declare class Sandbox<Env extends RuntimeWorkerEnv = RuntimeWorkerEnv> extends DurableObject<Env> {
     private configuration;
+    private configurationIntent?;
     private facade?;
+    private configurationTail;
+    private configurationOperation;
     private previewTail;
     private previewOperation;
     private readonly initialized;

package/dist/cloudflare/worker.js

@@ -4,8 +4,47 @@
 import { DurableObject } from "cloudflare:workers";
 import { only, CompatibilityError } from "../compat/core.js";
 import { Runtime } from "../client.js";
-import { Sandbox as RuntimeSandbox, sanitizeSandboxId, } from "./index.js";
+import { workerSandboxIdentity } from "./identity.js";
+import { Sandbox as RuntimeSandbox, sanitizeSandboxId, validateSandboxConfiguration, } from "./index.js";
 export * from "./index.js";
+const CONFIGURATION_KEY = "runtime.configuration";
+const CONFIGURATION_INTENT_KEY = "runtime.configuration.pending";
+/** Preserve the original failure even when durable recovery also fails. */
+function configurationFailure(error, uncertain, cleanup) {
+    if (error !== null && (typeof error === "object" || typeof error === "function")) {
+        try {
+            if (uncertain)
+                Object.defineProperty(error, "configurationUncertain", { value: true, configurable: true });
+            if (cleanup !== undefined)
+                Object.defineProperty(error, "cause" in error && error.cause !== undefined ? "cleanupError" : "cause", {
+                    value: cleanup,
+                    configurable: true,
+                });
+        }
+        catch {
+            // A frozen exception still keeps its original identity and type.
+        }
+    }
+    return error;
+}
+/** Queue waits share the caller's existing signal and never reset its deadline. */
+async function waitForConfiguration(pending, signal) {
+    signal?.throwIfAborted();
+    if (!signal)
+        return pending;
+    let cancel;
+    const aborted = new Promise((_resolve, reject) => {
+        // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors -- rejects with the caller's abort reason as is, as the upstream SDK does
+        cancel = () => reject(signal.reason);
+        signal.addEventListener("abort", cancel, { once: true });
+    });
+    try {
+        return await Promise.race([pending, aborted]);
+    }
+    finally {
+        signal.removeEventListener("abort", cancel);
+    }
+}
 /** The official Worker SDK returns plain records with callable RPC properties,
  * so ids/status are values rather than remote-property promises. */
 function sessionTarget(session) {
@@ -48,7 +87,14 @@ function processTarget(process) {
 }
 export class Sandbox extends DurableObject {
     configuration = {};
+    configurationIntent;
     facade;
+    configurationTail = Promise.resolve();
+    configurationOperation(operation) {
+        const result = this.configurationTail.catch(() => undefined).then(operation);
+        this.configurationTail = result;
+        return result;
+    }
     previewTail = Promise.resolve();
     previewOperation(operation) {
         const result = this.previewTail.catch(() => undefined).then(operation);
@@ -59,29 +105,133 @@ export class Sandbox extends DurableObject {
     constructor(ctx, env) {
         super(ctx, env);
         this.initialized = ctx.blockConcurrencyWhile(async () => {
-            this.configuration = (await ctx.storage.get("runtime.configuration")) ?? {};
+            this.configuration = (await ctx.storage.get(CONFIGURATION_KEY)) ?? {};
+            this.configurationIntent =
+                await ctx.storage.get(CONFIGURATION_INTENT_KEY);
         });
     }
-    async sandbox() {
-        await this.initialized;
+    async sandbox(signal) {
+        await waitForConfiguration(this.initialized, signal);
+        // A staged intent is transient while an update is still running. Wait for
+        // the queued update before deciding whether uncertainty was retained.
+        let configurationTail;
+        do {
+            configurationTail = this.configurationTail;
+            await waitForConfiguration(configurationTail.catch(() => undefined), signal);
+        } while (configurationTail !== this.configurationTail);
+        signal?.throwIfAborted();
+        if (this.configurationIntent)
+            throw configurationFailure(new Error("Runtime's configuration is uncertain. Call configure() to reconcile it before use."), true);
         if (!this.configuration.sandboxName)
             throw new Error("Call getSandbox(namespace, id) before using the Durable Object");
         if (!this.env.RUNTIME_API_KEY?.startsWith("rtcloud_"))
             throw new Error("Set the RUNTIME_API_KEY Worker secret to a Runtime API key");
-        this.facade ??= new RuntimeSandbox(new Runtime({ apiKey: this.env.RUNTIME_API_KEY, baseUrl: this.env.RUNTIME_API_URL }), this.configuration.sandboxName.name, this.configuration);
+        this.facade ??= new RuntimeSandbox(new Runtime({ apiKey: this.env.RUNTIME_API_KEY, baseUrl: this.env.RUNTIME_API_URL }), workerSandboxIdentity(this.ctx.id.toString(), this.configuration), { ...this.configuration });
         return this.facade;
     }
     async configure(configuration) {
         only("Cloudflare Worker configure", configuration, ["sandboxName", "sleepAfter", "keepAlive"]);
-        await this.initialized;
-        if (this.configuration.sandboxName &&
-            configuration.sandboxName &&
-            this.configuration.sandboxName.name !== configuration.sandboxName.name)
-            throw new Error("A Durable Object cannot change sandbox identity");
-        this.configuration = { ...this.configuration, ...configuration };
-        await this.ctx.storage.put("runtime.configuration", this.configuration);
-        if (this.facade)
-            await this.facade.configure(configuration);
+        validateSandboxConfiguration(configuration);
+        if (configuration.sandboxName !== undefined) {
+            only("Cloudflare Worker sandboxName", configuration.sandboxName, ["name", "normalizeId"]);
+            sanitizeSandboxId(configuration.sandboxName.name);
+            if (configuration.sandboxName.normalizeId !== undefined &&
+                typeof configuration.sandboxName.normalizeId !== "boolean")
+                throw new TypeError("normalizeId must be a boolean");
+        }
+        const requested = {
+            ...(configuration.sandboxName === undefined
+                ? {}
+                : { sandboxName: { ...configuration.sandboxName } }),
+            ...(configuration.sleepAfter === undefined ? {} : { sleepAfter: configuration.sleepAfter }),
+            ...(configuration.keepAlive === undefined ? {} : { keepAlive: configuration.keepAlive }),
+        };
+        return this.configurationOperation(async () => {
+            await this.initialized;
+            if (this.configuration.sandboxName &&
+                requested.sandboxName &&
+                this.configuration.sandboxName.name !== requested.sandboxName.name)
+                throw new Error("A Durable Object cannot change sandbox identity");
+            if (this.configurationIntent) {
+                const facade = this.facade ??
+                    new RuntimeSandbox(new Runtime({ apiKey: this.env.RUNTIME_API_KEY, baseUrl: this.env.RUNTIME_API_URL }), workerSandboxIdentity(this.ctx.id.toString(), this.configuration), { ...this.configuration });
+                try {
+                    // Recovery uses the saved UUID even after a Durable Object restart.
+                    const actual = await facade.reconcileConfiguration(this.configurationIntent.sandboxId);
+                    const reconciled = { ...this.configuration, ...actual };
+                    await this.ctx.storage.put(CONFIGURATION_KEY, reconciled);
+                    await this.ctx.storage.delete(CONFIGURATION_INTENT_KEY);
+                    this.configuration = reconciled;
+                    this.facade = facade;
+                    this.configurationIntent = undefined;
+                }
+                catch (error) {
+                    throw configurationFailure(error, true);
+                }
+            }
+            const previous = this.configuration;
+            const stored = await this.ctx.storage.get(CONFIGURATION_KEY);
+            const nativeSandboxId = workerSandboxIdentity(this.ctx.id.toString(), previous);
+            const next = { ...previous, ...requested, nativeSandboxId };
+            if (!this.facade) {
+                await this.ctx.storage.put(CONFIGURATION_KEY, next);
+                this.configuration = next;
+                return;
+            }
+            this.configurationIntent = { sandboxId: this.facade.existingNativeId(), previous, next };
+            try {
+                await this.ctx.storage.put(CONFIGURATION_INTENT_KEY, this.configurationIntent);
+            }
+            catch (error) {
+                let cleanup;
+                try {
+                    // Native mutation has not started. A failed write may still have
+                    // persisted the intent, so clear this operation's staging record.
+                    await this.ctx.storage.delete(CONFIGURATION_INTENT_KEY);
+                    this.configurationIntent = undefined;
+                }
+                catch (failure) {
+                    cleanup = failure;
+                }
+                throw configurationFailure(error, this.configurationIntent !== undefined, cleanup);
+            }
+            let nativeAttempted = false;
+            try {
+                await this.ctx.storage.put(CONFIGURATION_KEY, next);
+                nativeAttempted = true;
+                await this.facade.configure({
+                    ...(requested.sleepAfter === undefined ? {} : { sleepAfter: requested.sleepAfter }),
+                    ...(requested.keepAlive === undefined ? {} : { keepAlive: requested.keepAlive }),
+                });
+            }
+            catch (error) {
+                let cleanup;
+                try {
+                    if (stored === undefined)
+                        await this.ctx.storage.delete(CONFIGURATION_KEY);
+                    else
+                        await this.ctx.storage.put(CONFIGURATION_KEY, stored);
+                    if (!nativeAttempted) {
+                        await this.ctx.storage.delete(CONFIGURATION_INTENT_KEY);
+                        this.configurationIntent = undefined;
+                    }
+                }
+                catch (failure) {
+                    cleanup = failure;
+                }
+                // A rejected native request may have applied remotely. Keep its intent
+                // until an authoritative read succeeds; rollback is only local state.
+                throw configurationFailure(error, this.configurationIntent !== undefined, cleanup);
+            }
+            this.configuration = next;
+            try {
+                await this.ctx.storage.delete(CONFIGURATION_INTENT_KEY);
+                this.configurationIntent = undefined;
+            }
+            catch (error) {
+                throw configurationFailure(error, true);
+            }
+        });
     }
     setSandboxName(name, normalizeId = false) {
         return this.configure({
@@ -95,10 +245,10 @@ export class Sandbox extends DurableObject {
         return this.configure({ keepAlive });
     }
     async exec(command, options) {
-        return (await this.sandbox()).exec(command, options);
+        return (await this.sandbox(options?.signal)).exec(command, options);
     }
     async execStream(command, options) {
-        return (await this.sandbox()).execStream(command, options);
+        return (await this.sandbox(options?.signal)).execStream(command, options);
     }
     async startProcess(command, options) {
         return processTarget(await (await this.sandbox()).startProcess(command, options));

package/dist/codesandbox/guest-service.js

@@ -30,15 +30,35 @@ def settled(run):
   # Keep terminal failure readable over the socket even when storage is full.
   run['exitCode']=125
   run['error']=(run.get('error','')+'; ' if run.get('error') else '')+'Cannot persist task result: '+str(e)
+def exited(proc):
+ if proc.returncode is not None: return True
+ # Leave the leader waitable: its PID cannot be reused before group cleanup.
+ try: return os.waitid(os.P_PID,proc.pid,os.WEXITED|os.WNOHANG|os.WNOWAIT) is not None
+ except ChildProcessError: return proc.returncode is not None
+def kill_group(proc,signum):
+ # Callers hold the run lock, so no other thread can reap this owned leader.
+ if proc.returncode is not None: return
+ try: os.killpg(proc.pid,signum)
+ except ProcessLookupError: pass
+ except PermissionError:
+  # Darwin reports EPERM for a group containing only zombies. Keep real
+  # permission failures observable; only accept a verified empty/live-free
+  # group while our unreaped leader still prevents group-ID reuse.
+  if sys.platform!='darwin' or not exited(proc): raise
+  states=subprocess.run(['ps','-o','stat=','-g',str(proc.pid)],capture_output=True,text=True,timeout=2)
+  if states.returncode not in (0,1) or any(not state.startswith('Z') for state in states.stdout.split()): raise
 def complete(run):
  fd=run['fd']
  failed=False
  try:
   with open(os.path.join(root,run['id']+'.log'),'ab',buffering=0) as out:
    while True:
+    with run['_lock']:
+     done=exited(run['process'])
+     if done: kill_group(run['process'],signal.SIGKILL)
     ready,_,_=select.select([fd],[],[],0.05)
     if not ready:
-     if run['process'].poll() is not None: break
+     if done: break
      continue
     try: data=os.read(fd,65536)
     except OSError as e:
@@ -49,27 +69,35 @@ def complete(run):
  except BaseException as e:
   failed=True
   with run['_lock']: run['error']=str(e)
-  try: os.killpg(run['process'].pid,signal.SIGKILL)
-  except ProcessLookupError: pass
+  with run['_lock']: kill_group(run['process'],signal.SIGKILL)
  finally:
-  code=run['process'].wait()
-  with run['_lock']:
-   try: os.close(fd)
-   except OSError: pass
-   run['fd']=None
-   run['exitCode']=125 if failed else code
-   run['state']='exited'
-   settled(run)
+  # A terminal can report EIO as its leader exits, before the polling branch
+  # observes that exit. Cleanup must also precede this final reap. Do not hold
+  # the run lock while a still-running leader waits, so stop remains usable.
+  while True:
+   with run['_lock']:
+    if exited(run['process']):
+     kill_group(run['process'],signal.SIGKILL)
+     code=run['process'].wait()
+     try: os.close(fd)
+     except OSError: pass
+     run['fd']=None
+     run['exitCode']=125 if failed else code
+     run['state']='exited'
+     settled(run)
+     break
+   time.sleep(0.02)
 def stop(run):
  if run.get('state')!='running': return
  proc=run.get('process')
- if proc is not None and proc.poll() is None:
-  try: os.killpg(proc.pid,signal.SIGTERM)
-  except ProcessLookupError: pass
-  try: proc.wait(timeout=2)
-  except subprocess.TimeoutExpired:
-   try: os.killpg(proc.pid,signal.SIGKILL)
-   except ProcessLookupError: pass
+ if proc is not None:
+  with run['_lock']:
+   kill_group(proc,signal.SIGTERM)
+   deadline=time.monotonic()+2
+   while not exited(proc) and time.monotonic()<deadline: time.sleep(0.02)
+   # The leader can exit on TERM while a descendant ignores it. Kill the
+   # remaining group before reaping, even when the leader already exited.
+   kill_group(proc,signal.SIGKILL)
    proc.wait()
  elif proc is None and run.get('pid') and run.get('identity') and identity(run['pid'])==run['identity']:
   # On supervisor recovery, fence against PID reuse before touching a process.
@@ -235,10 +263,19 @@ def boot():
    while True:
     try:
      with open(progress) as f: result=json.load(f)
-    except FileNotFoundError: return
+    except FileNotFoundError: raise RuntimeError('Setup exited without terminal progress')
     if result.get('state')!='IN_PROGRESS': break
+    # A second runner may have won the setup lock. Wait only while it owns
+    # that lock; an exited runner with stale progress cannot ever finish.
+    with open(os.path.join(setup_root,'lock'),'a') as active:
+     try: fcntl.flock(active,fcntl.LOCK_EX|fcntl.LOCK_NB)
+     except BlockingIOError: pass
+     else:
+      with open(progress) as f: result=json.load(f)
+      if result.get('state')=='IN_PROGRESS': raise RuntimeError('Setup exited without terminal progress')
+      break
     time.sleep(0.1)
-   if result.get('state')!='FINISHED' or result.get('config')!=setup['config']: return
+   if result.get('state')!='FINISHED' or result.get('config')!=setup['config']: raise RuntimeError('Setup exited without matching terminal progress')
  # The service owns automatic tasks independently of SDK connections.
  with mutex:
   for task,definition in cfg['tasks'].items():