smolmachines 1.27.0 → 1.27.2

package/README.md

@@ -360,6 +360,23 @@ const { id } = await base.checkpoint();
 const a = await Machine.restoreCheckpoint(id, "project-a", undefined, { cacheDisk: { cache: "project-a" } });
 ```
 
+## Jobs: long commands that survive a dropped connection (cloud)
+
+`exec` holds one HTTP request open for the whole command, so a network blip or
+a long build can lose it. A job starts the command detached and returns its id
+at once. Its output and exit code stay in the machine, and the machine does not
+auto-stop while it runs:
+
+```ts
+const job = await m.startJob(["make", "test"], { workdir: "/src", timeout: 3600 });
+const done = await job.wait();                  // polls; { state: "exited", exitCode }
+const log = await job.readAll();                // all of stdout, byte-exact
+const page = await job.output({ stream: "stderr", offset: 0 }); // then page.nextOffset
+```
+
+`state` is `"lost"` if the machine stopped under the job. `job.cancel()` stops
+it (exit 143), and `m.job(id)` picks a job up again from another process.
+
 ## Building from source
 
 This package's native core lives alongside it (Rust, `src/*.rs`) and links the

package/dist/errors.d.ts

@@ -4,9 +4,30 @@
  *  prefixed with a bracketed code, e.g. `"[KVM_UNAVAILABLE] …"`. We parse that
  *  back into a typed hierarchy so callers can branch on `err.code` /`instanceof`.
  */
+/** What the cloud API said about a failed request, beyond its message. */
+export interface CloudErrorDetail {
+    /** HTTP status of the response. */
+    status: number;
+    /** The server's stable problem code, e.g. `machine_interrupted`. */
+    problemCode?: string | undefined;
+    /** Whether sending the same request again can succeed. */
+    retryable: boolean;
+    /** How long the server asked callers to wait before retrying (ms). */
+    retryAfterMs?: number | undefined;
+}
 export declare class SmolError extends Error {
     readonly code: string;
-    constructor(code: string, message: string);
+    /** HTTP status, for an error the cloud API returned. */
+    readonly status?: number | undefined;
+    /** The server's stable problem code, for an error the cloud API returned. */
+    readonly problemCode?: string | undefined;
+    /** Whether sending the same request again can succeed. A command whose
+     *  machine went away (`MACHINE_INTERRUPTED`) is retryable, but it may have
+     *  partly run, so the SDK leaves that retry to the caller. */
+    readonly retryable: boolean;
+    /** How long the server asked callers to wait before retrying (ms). */
+    readonly retryAfterMs?: number | undefined;
+    constructor(code: string, message: string, detail?: CloudErrorDetail);
 }
 /** The active backend can't serve this operation (e.g. volumes on local). */
 export declare class NotSupportedError extends SmolError {

package/dist/errors.js

@@ -9,10 +9,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ExecutionError = exports.InvalidConfigError = exports.NotSupportedError = exports.SmolError = void 0;
 exports.wrapNativeError = wrapNativeError;
 class SmolError extends Error {
-    constructor(code, message) {
+    constructor(code, message, detail) {
         super(message);
         this.name = new.target.name;
         this.code = code;
+        this.status = detail?.status;
+        this.problemCode = detail?.problemCode;
+        this.retryable = detail?.retryable ?? false;
+        this.retryAfterMs = detail?.retryAfterMs;
     }
 }
 exports.SmolError = SmolError;

package/dist/index.d.ts

@@ -16,10 +16,10 @@
  * Runs against the local embedded engine (default, no server) or the smolfleet
  * cloud — same API, backend selected via ConnectOptions / SMOL_CLOUD_TOKEN.
  */
-export { Machine, Episode, CacheDisk } from './machine';
+export { Machine, Episode, CacheDisk, Job } from './machine';
 export { RolloutClient, RolloutError, adapterSha256 } from './rollout';
 export type { BuildRolloutJobOptions, EnsureRolloutExecutorOptions, RolloutBatchItem, RolloutClientOptions, RolloutCompletion, RolloutExecutorInfo, RolloutGenerateResponse, RolloutJob, RolloutPolicyInfo, RolloutPrompt, RolloutSampling, } from './rollout';
 export { SmolError, NotSupportedError, InvalidConfigError, ExecutionError } from './errors';
 export type { CloudCheckpointInfo } from './transport';
 export type { LocalAvailability, LocalUnavailableCode } from './availability';
-export type { MachineConfig, CredentialSpec, CacheDiskRef, CacheDiskInfo, CacheDiskVersion, CreateCacheDiskOptions, PublishedCacheDisk, NetworkPolicy, RestoreCheckpointOptions, MachineState, ResourceSpec, MountSpec, PortSpec, BranchOptions, BranchBatchOptions, CheckpointOptions, ResizeOptions, MachineResources, ForkOptions, ForkBatchOptions, AssignOptions, ExecOptions, ExecResult, MachineUsageReport, ShareLink, PortableCheckpointInfo, MachineUsageTotals, MachineCostBreakdown, ImageInfo, MachineSummary, ListOptions, ExecEvent, ConnectOptions, EgressInterceptor, StartOptions, ResumeOptions, WaitReadyOptions, PortEndpoint, } from './types';
+export type { MachineConfig, CredentialSpec, CacheDiskRef, CacheDiskInfo, CacheDiskVersion, CreateCacheDiskOptions, PublishedCacheDisk, NetworkPolicy, RestoreCheckpointOptions, MachineState, ResourceSpec, MountSpec, PortSpec, BranchOptions, BranchBatchOptions, CheckpointOptions, ResizeOptions, MachineResources, ForkOptions, ForkBatchOptions, AssignOptions, ExecOptions, JobInfo, JobOptions, JobOutput, JobOutputOptions, JobState, JobWaitOptions, ExecResult, MachineUsageReport, ShareLink, PortableCheckpointInfo, MachineUsageTotals, MachineCostBreakdown, ImageInfo, MachineSummary, ListOptions, ExecEvent, ConnectOptions, EgressInterceptor, StartOptions, ResumeOptions, WaitReadyOptions, PortEndpoint, } from './types';

package/dist/index.js

@@ -18,11 +18,12 @@
  * cloud — same API, backend selected via ConnectOptions / SMOL_CLOUD_TOKEN.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExecutionError = exports.InvalidConfigError = exports.NotSupportedError = exports.SmolError = exports.adapterSha256 = exports.RolloutError = exports.RolloutClient = exports.CacheDisk = exports.Episode = exports.Machine = void 0;
+exports.ExecutionError = exports.InvalidConfigError = exports.NotSupportedError = exports.SmolError = exports.adapterSha256 = exports.RolloutError = exports.RolloutClient = exports.Job = exports.CacheDisk = exports.Episode = exports.Machine = void 0;
 var machine_1 = require("./machine");
 Object.defineProperty(exports, "Machine", { enumerable: true, get: function () { return machine_1.Machine; } });
 Object.defineProperty(exports, "Episode", { enumerable: true, get: function () { return machine_1.Episode; } });
 Object.defineProperty(exports, "CacheDisk", { enumerable: true, get: function () { return machine_1.CacheDisk; } });
+Object.defineProperty(exports, "Job", { enumerable: true, get: function () { return machine_1.Job; } });
 var rollout_1 = require("./rollout");
 Object.defineProperty(exports, "RolloutClient", { enumerable: true, get: function () { return rollout_1.RolloutClient; } });
 Object.defineProperty(exports, "RolloutError", { enumerable: true, get: function () { return rollout_1.RolloutError; } });

package/dist/machine.d.ts

@@ -9,7 +9,7 @@
  */
 import { type LocalAvailability } from "./availability";
 import { type CloudCheckpointInfo, type Transport } from "./transport";
-import type { CacheDiskInfo, CreateCacheDiskOptions, PublishedCacheDisk, MachineResources, ResizeOptions, AssignOptions, BranchBatchOptions, BranchOptions, CheckpointOptions, ConnectOptions, ExecEvent, ExecOptions, ExecResult, ForkBatchOptions, ForkOptions, ImageInfo, ListOptions, MachineConfig, MachineSummary, NetworkPolicy, RestoreCheckpointOptions, StartOptions, ResumeOptions, MachineUsageReport, ShareLink, PortableCheckpointInfo, PortEndpoint, PortSpec, WaitReadyOptions } from "./types";
+import type { CacheDiskInfo, CreateCacheDiskOptions, PublishedCacheDisk, MachineResources, ResizeOptions, AssignOptions, BranchBatchOptions, BranchOptions, CheckpointOptions, ConnectOptions, ExecEvent, ExecOptions, ExecResult, ForkBatchOptions, ForkOptions, ImageInfo, JobInfo, JobOptions, JobOutput, JobOutputOptions, JobWaitOptions, ListOptions, MachineConfig, MachineSummary, NetworkPolicy, RestoreCheckpointOptions, StartOptions, ResumeOptions, MachineUsageReport, ShareLink, PortableCheckpointInfo, PortEndpoint, PortSpec, WaitReadyOptions } from "./types";
 export declare class Machine {
     private readonly transport;
     private constructor();
@@ -101,6 +101,21 @@ export declare class Machine {
      *  namespace, re-deployable anywhere. Distinct from a portable checkpoint,
      *  which also keeps live memory. Cloud only. */
     exportArtifact(): Promise<Record<string, unknown>>;
+    /** Start `command` as a detached job and return at once. Its output and exit
+     *  code are kept in the machine, so the caller can disconnect and come back:
+     *  poll with {@link Job.wait}, read output from an offset with
+     *  {@link Job.output}. The machine does not auto-stop while the job runs.
+     *  A string runs through `sh -c`; the image needs `sh`. Cloud only.
+     *
+     *  Unlike `exec`, a job survives a dropped connection, so prefer it for
+     *  anything that runs longer than a few minutes. */
+    startJob(command: string[] | string, opts?: JobOptions): Promise<Job>;
+    /** This machine's jobs, newest first, as last recorded (up to 100). Use
+     *  {@link Job.info} for a job's live state. Cloud only. */
+    jobs(): Promise<JobInfo[]>;
+    /** A handle to an existing job by id, for example one started by another
+     *  process. Makes no request. Cloud only. */
+    job(id: string): Job;
     /** Whether the machine is READY to do work. `state()` becoming "started"
      *  means only that the VM process launched — the guest is still booting and
      *  is NOT yet usable. `ready` becomes true once the in-VM agent is reachable
@@ -258,6 +273,27 @@ export declare class Machine {
      *  @returns the grader's `ExecResult` (use `.success` for pass/fail, `.stdout` for a score) */
     rewardFork(command: string[], opts?: ExecOptions, name?: string): Promise<ExecResult>;
 }
+/** A detached command in a cloud machine, from {@link Machine.startJob}. */
+export declare class Job {
+    #private;
+    readonly id: string;
+    constructor(transport: Transport, id: string);
+    /** The job's live state. A running job's machine is asked whether it has
+     *  finished; `lost` means the machine stopped or restarted under it. */
+    info(): Promise<JobInfo>;
+    /** A slice of stdout (default) or stderr from `offset`. Pass the returned
+     *  `nextOffset` back to read on; `eof` is true once the job has finished and
+     *  the slice reaches the end. */
+    output(opts?: JobOutputOptions): Promise<JobOutput>;
+    /** Everything written to `stream` so far. On a finished job, that is all
+     *  of it. */
+    readAll(stream?: "stdout" | "stderr"): Promise<Buffer>;
+    /** Stop the job with SIGTERM. It is recorded as exited with code 143. */
+    cancel(): Promise<JobInfo>;
+    /** Poll until the job is no longer running and return its final state:
+     *  `exited` with `exitCode`, or `lost`. */
+    wait(opts?: JobWaitOptions): Promise<JobInfo>;
+}
 /** A leased RL episode: a freshly-provisioned clone plus its lease. Created by
  *  `Machine.assign()`. Always `complete()` it (e.g. in a `finally`) so the clone
  *  is reclaimed — the RL "one clean worker per task, guaranteed reclaimed"

package/dist/machine.js

@@ -19,9 +19,9 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
     if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
     return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
 };
-var _Episode_machine, _Episode_ownerToken, _Episode_leaseTransport, _Episode_completed;
+var _Job_transport, _Episode_machine, _Episode_ownerToken, _Episode_leaseTransport, _Episode_completed;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CacheDisk = exports.Episode = exports.Machine = void 0;
+exports.CacheDisk = exports.Episode = exports.Job = exports.Machine = void 0;
 const node_crypto_1 = require("node:crypto");
 const node_path_1 = require("node:path");
 const errors_1 = require("./errors");
@@ -200,6 +200,34 @@ class Machine {
             throw new errors_1.NotSupportedError("exportArtifact is cloud-only.");
         return this.transport.exportArtifact();
     }
+    /** Start `command` as a detached job and return at once. Its output and exit
+     *  code are kept in the machine, so the caller can disconnect and come back:
+     *  poll with {@link Job.wait}, read output from an offset with
+     *  {@link Job.output}. The machine does not auto-stop while the job runs.
+     *  A string runs through `sh -c`; the image needs `sh`. Cloud only.
+     *
+     *  Unlike `exec`, a job survives a dropped connection, so prefer it for
+     *  anything that runs longer than a few minutes. */
+    async startJob(command, opts = {}) {
+        if (!this.transport.startJob)
+            throw new errors_1.NotSupportedError("startJob is cloud-only.");
+        const info = await this.transport.startJob(command, opts);
+        return new Job(this.transport, info.id);
+    }
+    /** This machine's jobs, newest first, as last recorded (up to 100). Use
+     *  {@link Job.info} for a job's live state. Cloud only. */
+    jobs() {
+        if (!this.transport.listJobs)
+            throw new errors_1.NotSupportedError("jobs is cloud-only.");
+        return this.transport.listJobs();
+    }
+    /** A handle to an existing job by id, for example one started by another
+     *  process. Makes no request. Cloud only. */
+    job(id) {
+        if (!this.transport.getJob)
+            throw new errors_1.NotSupportedError("job is cloud-only.");
+        return new Job(this.transport, id);
+    }
     /** Whether the machine is READY to do work. `state()` becoming "started"
      *  means only that the VM process launched — the guest is still booting and
      *  is NOT yet usable. `ready` becomes true once the in-VM agent is reachable
@@ -438,6 +466,75 @@ class Machine {
     }
 }
 exports.Machine = Machine;
+/** A detached command in a cloud machine, from {@link Machine.startJob}. */
+class Job {
+    constructor(transport, id) {
+        _Job_transport.set(this, void 0);
+        __classPrivateFieldSet(this, _Job_transport, transport, "f");
+        this.id = id;
+    }
+    /** The job's live state. A running job's machine is asked whether it has
+     *  finished; `lost` means the machine stopped or restarted under it. */
+    info() {
+        return __classPrivateFieldGet(this, _Job_transport, "f").getJob(this.id);
+    }
+    /** A slice of stdout (default) or stderr from `offset`. Pass the returned
+     *  `nextOffset` back to read on; `eof` is true once the job has finished and
+     *  the slice reaches the end. */
+    output(opts = {}) {
+        return __classPrivateFieldGet(this, _Job_transport, "f").jobOutput(this.id, opts);
+    }
+    /** Everything written to `stream` so far. On a finished job, that is all
+     *  of it. */
+    async readAll(stream = "stdout") {
+        const chunks = [];
+        let offset = 0;
+        for (;;) {
+            const out = await this.output({ stream, offset, limit: 1024 * 1024 });
+            chunks.push(out.data);
+            offset = out.nextOffset;
+            if (out.eof || out.data.length === 0)
+                break;
+        }
+        return Buffer.concat(chunks);
+    }
+    /** Stop the job with SIGTERM. It is recorded as exited with code 143. */
+    cancel() {
+        return __classPrivateFieldGet(this, _Job_transport, "f").cancelJob(this.id);
+    }
+    /** Poll until the job is no longer running and return its final state:
+     *  `exited` with `exitCode`, or `lost`. */
+    async wait(opts = {}) {
+        const pollMs = opts.pollMs ?? 1000;
+        const deadline = opts.timeoutMs === undefined ? Infinity : Date.now() + opts.timeoutMs;
+        for (;;) {
+            opts.signal?.throwIfAborted();
+            const info = await this.info();
+            if (info.state !== "running")
+                return info;
+            const left = deadline - Date.now();
+            if (left <= 0) {
+                throw new errors_1.SmolError("TIMEOUT", `job ${this.id} still running after ${opts.timeoutMs}ms`);
+            }
+            await sleep(Math.min(pollMs, left), opts.signal);
+        }
+    }
+}
+exports.Job = Job;
+_Job_transport = new WeakMap();
+function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(timer);
+            reject(signal?.reason);
+        };
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}
 /** A leased RL episode: a freshly-provisioned clone plus its lease. Created by
  *  `Machine.assign()`. Always `complete()` it (e.g. in a `finally`) so the clone
  *  is reclaimed — the RL "one clean worker per task, guaranteed reclaimed"

package/dist/transport.d.ts

@@ -7,7 +7,7 @@
  *  Cloud-only/local-only capability gaps surface as `NotSupportedError`.
  */
 import { type NativeMachineConfig } from "./native";
-import type { CheckpointOptions, CacheDiskInfo, CreateCacheDiskOptions, PublishedCacheDisk, MachineResources, ResizeOptions, ConnectOptions, NetworkPolicy, RestoreCheckpointOptions, ExecEvent, ExecOptions, EgressInterceptor, AssignOptions, ForkBatchOptions, ForkOptions, ImageInfo, ListOptions, MachineConfig, MachineSummary, MachineUsageReport, ShareLink, PortableCheckpointInfo, PortEndpoint, PortSpec, WaitReadyOptions } from "./types";
+import type { CheckpointOptions, JobInfo, JobOptions, JobOutput, JobOutputOptions, CacheDiskInfo, CreateCacheDiskOptions, PublishedCacheDisk, MachineResources, ResizeOptions, ConnectOptions, NetworkPolicy, RestoreCheckpointOptions, ExecEvent, ExecOptions, EgressInterceptor, AssignOptions, ForkBatchOptions, ForkOptions, ImageInfo, ListOptions, MachineConfig, MachineSummary, MachineUsageReport, ShareLink, PortableCheckpointInfo, PortEndpoint, PortSpec, WaitReadyOptions } from "./types";
 /** Raw exec result (the ergonomic wrapper is added in machine.ts). */
 export interface RawExec {
     /** The detached process's pid, for a background exec. */
@@ -31,6 +31,16 @@ export interface Transport {
     logs?(tail: number): Promise<string>;
     /** Cloud only: export the stopped machine as a `.smolmachine` in the registry. */
     exportArtifact?(): Promise<Record<string, unknown>>;
+    /** Cloud only: start a detached job. */
+    startJob?(command: string[] | string, opts: JobOptions): Promise<JobInfo>;
+    /** Cloud only: the machine's jobs, newest first. */
+    listJobs?(): Promise<JobInfo[]>;
+    /** Cloud only: one job's live state. */
+    getJob?(jobId: string): Promise<JobInfo>;
+    /** Cloud only: a slice of a job's stdout or stderr. */
+    jobOutput?(jobId: string, opts: JobOutputOptions): Promise<JobOutput>;
+    /** Cloud only: stop a job with SIGTERM. */
+    cancelJob?(jobId: string): Promise<JobInfo>;
     state(): Promise<string>;
     ready(): Promise<boolean>;
     readyAt(): Promise<string | null>;
@@ -122,6 +132,8 @@ interface CloudConn {
     /** The caller's `fetch` (ConnectOptions.fetch), when it supplied one. */
     fetch?: typeof fetch;
     readResponseBytes?: (response: Response) => Promise<Uint8Array>;
+    /** ConnectOptions.maxRetries, when the caller set it. */
+    maxRetries?: number;
 }
 /** The largest file the cloud's file routes accept. */
 export declare const CLOUD_MAX_FILE_BYTES: number;
@@ -140,6 +152,12 @@ export declare function errorSentence(body: string): string;
  *  files route is a wildcard (`/files/<path>`), so slashes are meaningful while
  *  spaces/?/#/% in a filename must be escaped. */
 export declare function encodePath(p: string): string;
+/** Parse a `Retry-After` header (delta-seconds or an HTTP date) into ms. */
+export declare function retryAfterMs(header: string | null, now?: number): number | undefined;
+/** Whether the SDK may resend a request on its own. A rate-limited request
+ *  never ran, so it is always safe; an unavailable one may have, so only a
+ *  call that is safe to repeat is sent again. */
+export declare function sdkMayRetry(status: number, method: string, idempotent: boolean): boolean;
 /** Does this request target the cloud?
  *
  * Only an EXPLICIT credential may select cloud: `conn.apiKey` and

package/dist/transport.js

@@ -20,6 +20,8 @@ exports.transferTimeoutMs = transferTimeoutMs;
 exports.checkCloudFileSize = checkCloudFileSize;
 exports.errorSentence = errorSentence;
 exports.encodePath = encodePath;
+exports.retryAfterMs = retryAfterMs;
+exports.sdkMayRetry = sdkMayRetry;
 exports.selectsCloud = selectsCloud;
 exports.makeTransport = makeTransport;
 exports.listMachines = listMachines;
@@ -702,6 +704,7 @@ function cloudConnOf(conn, baseUrl, apiKey) {
         apiKey,
         ...(conn.fetch ? { fetch: conn.fetch } : {}),
         ...(conn.readResponseBytes ? { readResponseBytes: conn.readResponseBytes } : {}),
+        ...(conn.maxRetries !== undefined ? { maxRetries: conn.maxRetries } : {}),
     };
 }
 const DEFAULT_CLOUD_URL = "https://api.smolmachines.com";
@@ -765,7 +768,110 @@ function errorSentence(body) {
 function encodePath(p) {
     return p.split("/").map(encodeURIComponent).join("/");
 }
+/** The typed error for a failed cloud response, carrying what the problem
+ *  body and headers say about retrying. */
+async function cloudError(res, what) {
+    const raw = await res.text().catch(() => "");
+    const text = errorSentence(raw);
+    const problem = problemOf(raw);
+    // Surface the server's correlation id (every response carries `x-request-id`)
+    // in the error message — clients see the error body but not headers, so
+    // without this the id is invisible and support can't correlate the call.
+    const rid = res.headers.get("x-request-id");
+    const code = problem.code === "machine_interrupted"
+        ? "MACHINE_INTERRUPTED"
+        : res.status === 404
+            ? "NOT_FOUND"
+            : res.status === 401
+                ? "UNAUTHORIZED"
+                : "SMOLVM_ERROR";
+    return new errors_1.SmolError(code, `${what} → ${res.status}${text ? `: ${text}` : ""}${rid ? ` [request id: ${rid}]` : ""}`, {
+        status: res.status,
+        problemCode: problem.code,
+        retryable: problem.retryable ?? retryableStatus(res.status),
+        retryAfterMs: retryAfterMs(res.headers.get("retry-after")),
+    });
+}
+/** Retries for a retryable response when the caller sets none. */
+const DEFAULT_CLOUD_RETRIES = 3;
+/** Longest single wait between retries (ms), whatever `Retry-After` asks. */
+const MAX_RETRY_WAIT_MS = 30_000;
+/** Parse a `Retry-After` header (delta-seconds or an HTTP date) into ms. */
+function retryAfterMs(header, now = Date.now()) {
+    if (!header)
+        return undefined;
+    const trimmed = header.trim();
+    if (/^\d+$/.test(trimmed))
+        return Number(trimmed) * 1_000;
+    const at = Date.parse(trimmed);
+    return Number.isNaN(at) ? undefined : Math.max(0, at - now);
+}
+/** What a failed response's problem body says, beyond its sentence. */
+function problemOf(body) {
+    try {
+        const parsed = JSON.parse(body.trim());
+        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+            const fields = parsed;
+            return {
+                ...(typeof fields.code === "string" ? { code: fields.code } : {}),
+                ...(typeof fields.retryable === "boolean" ? { retryable: fields.retryable } : {}),
+            };
+        }
+    }
+    catch {
+        // Not a problem document.
+    }
+    return {};
+}
+/** Statuses the server uses for "try again": rate limited or briefly unavailable. */
+function retryableStatus(status) {
+    return status === 408 || status === 429 || status === 502 || status === 503 || status === 504;
+}
+/** Whether the SDK may resend a request on its own. A rate-limited request
+ *  never ran, so it is always safe; an unavailable one may have, so only a
+ *  call that is safe to repeat is sent again. */
+function sdkMayRetry(status, method, idempotent) {
+    if (status === 429)
+        return true;
+    if (!retryableStatus(status))
+        return false;
+    return idempotent || method === "GET" || method === "HEAD" || method === "PUT" || method === "DELETE";
+}
 async function cloudFetch(conn, method, path, opts = {}) {
+    const retries = Math.max(0, conn.maxRetries ?? DEFAULT_CLOUD_RETRIES);
+    for (let attempt = 0;; attempt++) {
+        try {
+            return await cloudFetchOnce(conn, method, path, opts);
+        }
+        catch (error) {
+            if (!(error instanceof errors_1.SmolError) ||
+                attempt >= retries ||
+                error.status === undefined ||
+                !sdkMayRetry(error.status, method, opts.idempotent ?? false)) {
+                throw error;
+            }
+            const backoff = 500 * 2 ** attempt;
+            const wait = Math.min(error.retryAfterMs ?? backoff, MAX_RETRY_WAIT_MS);
+            await sleepUnlessAborted(wait, opts.signal);
+        }
+    }
+}
+/** Sleep `ms`, rejecting with the abort reason if `signal` fires first. */
+function sleepUnlessAborted(ms, signal) {
+    signal?.throwIfAborted();
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(timer);
+            reject(signal?.reason);
+        };
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}
+async function cloudFetchOnce(conn, method, path, opts) {
     const headers = {
         authorization: `Bearer ${conn.apiKey}`,
     };
@@ -808,18 +914,8 @@ async function cloudFetch(conn, method, path, opts = {}) {
         clearTimeout(timer);
         opts.signal?.removeEventListener("abort", onAbort);
     }
-    if (!res.ok) {
-        const text = errorSentence(await res.text().catch(() => ""));
-        // Surface the server's correlation id (every response carries `x-request-id`)
-        // in the error message — clients see the error body but not headers, so
-        // without this the id is invisible and support can't correlate the call.
-        const rid = res.headers.get("x-request-id");
-        throw new errors_1.SmolError(res.status === 404
-            ? "NOT_FOUND"
-            : res.status === 401
-                ? "UNAUTHORIZED"
-                : "SMOLVM_ERROR", `cloud ${method} ${path} → ${res.status}${text ? `: ${text}` : ""}${rid ? ` [request id: ${rid}]` : ""}`);
-    }
+    if (!res.ok)
+        throw await cloudError(res, `cloud ${method} ${path}`);
     if (opts.accept === "response")
         return res;
     if (opts.accept === "text")
@@ -1080,15 +1176,8 @@ class CloudTransport {
                 throw signal.reason;
             throw new errors_1.SmolError("CONNECTION", `cloud exec/stream failed: ${e.message}`);
         }
-        if (!res.ok) {
-            const text = errorSentence(await res.text().catch(() => ""));
-            const rid = res.headers.get("x-request-id");
-            throw new errors_1.SmolError(res.status === 404
-                ? "NOT_FOUND"
-                : res.status === 401
-                    ? "UNAUTHORIZED"
-                    : "SMOLVM_ERROR", `cloud POST /v1/machines/${this.id}/exec/stream → ${res.status}${text ? `: ${text}` : ""}${rid ? ` [request id: ${rid}]` : ""}`);
-        }
+        if (!res.ok)
+            throw await cloudError(res, `cloud POST /v1/machines/${this.id}/exec/stream`);
         if (!res.body)
             throw new errors_1.SmolError("SMOLVM_ERROR", "exec/stream returned no response body");
         // Parse the server's SSE stream: each event is `event: <kind>` + one or more
@@ -1242,13 +1331,16 @@ class CloudTransport {
         throw new errors_1.NotSupportedError("setNetworkPolicy() is local-only.");
     }
     async stop() {
-        await cloudFetch(this.conn, "POST", `/v1/machines/${this.id}/stop`);
+        await cloudFetch(this.conn, "POST", `/v1/machines/${this.id}/stop`, { idempotent: true });
     }
     async pause() {
         await cloudFetch(this.conn, "POST", `/v1/machines/${this.id}/pause`, { timeoutMs: CLOUD_START_TIMEOUT_MS });
     }
     async resume(waitUntilReady = true) {
-        await cloudFetch(this.conn, "POST", `/v1/machines/${this.id}/resume`, { timeoutMs: CLOUD_START_TIMEOUT_MS });
+        await cloudFetch(this.conn, "POST", `/v1/machines/${this.id}/resume`, {
+            timeoutMs: CLOUD_START_TIMEOUT_MS,
+            idempotent: true,
+        });
         if (waitUntilReady)
             await waitForReady(this.conn, this.id);
     }
@@ -1258,6 +1350,7 @@ class CloudTransport {
         // Resume a stopped machine, then wait for its agent so the handle is usable.
         await cloudFetch(this.conn, "POST", `/v1/machines/${this.id}/start`, {
             timeoutMs: CLOUD_START_TIMEOUT_MS,
+            idempotent: true,
         });
         if (waitUntilReady)
             await waitForReady(this.conn, this.id);
@@ -1276,6 +1369,45 @@ class CloudTransport {
             timeoutMs: CLOUD_START_TIMEOUT_MS,
         });
     }
+    async startJob(command, opts) {
+        // Not idempotent: a resend could start the command twice.
+        return cloudFetch(this.conn, "POST", `/v1/machines/${this.id}/jobs`, {
+            json: {
+                command,
+                cwd: opts.workdir,
+                env: opts.env ?? {},
+                timeoutSeconds: opts.timeout,
+                user: opts.user,
+            },
+            timeoutMs: CLOUD_START_TIMEOUT_MS,
+        });
+    }
+    async listJobs() {
+        const out = await cloudFetch(this.conn, "GET", `/v1/machines/${this.id}/jobs`);
+        return out.jobs;
+    }
+    async getJob(jobId) {
+        return cloudFetch(this.conn, "GET", `/v1/machines/${this.id}/jobs/${encodeURIComponent(jobId)}`);
+    }
+    async jobOutput(jobId, opts) {
+        const q = new URLSearchParams({ stream: opts.stream ?? "stdout", offset: String(opts.offset ?? 0) });
+        if (opts.limit !== undefined)
+            q.set("limit", String(opts.limit));
+        const out = await cloudFetch(this.conn, "GET", `/v1/machines/${this.id}/jobs/${encodeURIComponent(jobId)}/output?${q}`);
+        const data = out.dataB64 !== undefined ? Buffer.from(out.dataB64, "base64") : Buffer.from(out.data);
+        return {
+            stream: out.stream,
+            offset: out.offset,
+            nextOffset: out.nextOffset,
+            data,
+            text: out.data,
+            eof: out.eof,
+        };
+    }
+    async cancelJob(jobId) {
+        // Cancelling twice is harmless, so a 503 may be resent.
+        return cloudFetch(this.conn, "POST", `/v1/machines/${this.id}/jobs/${encodeURIComponent(jobId)}/cancel`, { idempotent: true });
+    }
     async delete() {
         await cloudFetch(this.conn, "DELETE", `/v1/machines/${this.id}`);
     }

package/dist/types.d.ts

@@ -527,6 +527,12 @@ export interface ConnectOptions {
     /** Cloud only: read a binary response body, for a `fetch` whose responses
      *  lack a working `arrayBuffer()`. */
     readResponseBytes?: (response: Response) => Promise<Uint8Array>;
+    /** Cloud only: how many times a request the server marks retryable is sent
+     *  again, waiting as its `Retry-After` asks. A 429 is retried for any call;
+     *  a 502/503/504 only for calls that are safe to repeat (reads, deletes,
+     *  start/stop/resume), never for exec or create, which could run twice.
+     *  Default 3; 0 turns retries off. */
+    maxRetries?: number;
 }
 /** Per-launch trusted host service. Keep the token out of logs and persistent config. */
 export interface EgressInterceptor {
@@ -596,3 +602,66 @@ export interface PublishedCacheDisk {
     cacheDisk: CacheDiskInfo;
     version: CacheDiskVersion;
 }
+/** Options for {@link Machine.startJob}. Cloud only. */
+export interface JobOptions {
+    /** Environment variables. */
+    env?: Record<string, string>;
+    /** Working directory inside the machine. */
+    workdir?: string;
+    /** Stop the job with SIGTERM after this many **seconds**; it then exits 143. */
+    timeout?: number;
+    /** Run the job as this user: a name from the image or a numeric `uid[:gid]`. */
+    user?: string;
+}
+/** `running`; `exited` (see `exitCode`); or `lost`, when the machine stopped or
+ *  restarted while the job ran, so it never finished. */
+export type JobState = "running" | "exited" | "lost";
+/** A job and what is known about it. */
+export interface JobInfo {
+    id: string;
+    machineId: string;
+    /** The command as it was run (argv). */
+    command: string[];
+    state: JobState;
+    /** Set once `exited`; 143 when the job was cancelled or hit its timeout. */
+    exitCode?: number;
+    createdAt: string;
+    finishedAt?: string;
+    /** Bytes of stdout written so far, when the machine was asked. */
+    stdoutBytes?: number;
+    /** Bytes of stderr written so far, when the machine was asked. */
+    stderrBytes?: number;
+}
+/** Which slice of a job's output to read. */
+export interface JobOutputOptions {
+    /** Default `"stdout"`. */
+    stream?: "stdout" | "stderr";
+    /** Byte offset to start at. Default 0. */
+    offset?: number;
+    /** Most bytes to return. The server defaults to 256 KiB and caps at 1 MiB. */
+    limit?: number;
+}
+/** A slice of a job's stdout or stderr. */
+export interface JobOutput {
+    stream: "stdout" | "stderr";
+    /** Byte offset this slice starts at. */
+    offset: number;
+    /** Byte offset to read from next. */
+    nextOffset: number;
+    /** The slice, byte-exact. */
+    data: Buffer;
+    /** The slice as lossy UTF-8. */
+    text: string;
+    /** True once the job has finished and this slice reaches the end of the
+     *  stream: nothing more will be written. */
+    eof: boolean;
+}
+/** How {@link Job.wait} polls. */
+export interface JobWaitOptions {
+    /** Delay between polls. Default 1000 ms. */
+    pollMs?: number;
+    /** Give up with a `TIMEOUT` error after this long. Default: wait forever. */
+    timeoutMs?: number;
+    /** Stop waiting; rejects with `signal.reason`. The job keeps running. */
+    signal?: AbortSignal;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smolmachines",
-  "version": "1.27.0",
+  "version": "1.27.2",
   "description": "Embed isolated microVM sandboxes directly in your code — no server to run.",
   "license": "Apache-2.0",
   "author": "smol-machines",
@@ -64,8 +64,8 @@
     "typescript": "^5.6.0"
   },
   "optionalDependencies": {
-    "smolmachines-darwin-arm64": "1.27.0",
-    "smolmachines-linux-x64-gnu": "1.27.0",
-    "smolmachines-linux-arm64-gnu": "1.27.0"
+    "smolmachines-darwin-arm64": "1.27.2",
+    "smolmachines-linux-x64-gnu": "1.27.2",
+    "smolmachines-linux-arm64-gnu": "1.27.2"
   }
 }