promigence 0.1.0-beta.15 → 0.1.0-beta.16

package/dist/account.js

@@ -1,11 +1,28 @@
-import { bool, client, KEPT_BY_HOLDERS, list, nexts, optimizeForFlag, positional, required, secretListLines, str, } from "./commands.js";
+import { bool, client, KEPT_BY_HOLDERS, keyOriginText, list, nexts, optimizeForFlag, positional, required, secretListLines, str, } from "./commands.js";
 import { queuedLines } from "./capacity.js";
 import { duration } from "./report.js";
-import { emit, ms, okEnvelope, overMost, parseCount, parseDuration, parseUsd, usageError } from "./io.js";
+import { emit, errorEnvelope, errorTextLines, ms, okEnvelope, overMost, parseCount, parseDuration, parseUsd, usageError } from "./io.js";
+import { ERRORS_DOCS_URL, PromigenceError, blake3Hex } from "./sdk/index.js";
 import { sh } from "./term.js";
 import { autoReloadShort, creditsLine, resumeAllPausedForCredits } from "./billing.js";
 import { dollars } from "./money.js";
-const usd = (n) => n === undefined ? "—" : `$${n.toFixed(n < 0.01 && n > 0 ? 4 : 2)}`;
+import { livePricing } from "./pricing.js";
+const usd = (n) => n === undefined ? "—" : n < 0 ? `-${usd(-n)}` : `$${n.toFixed(n < 0.01 && n > 0 ? 4 : 2)}`;
+export function keyAccess(prefix) {
+    if (!prefix)
+        return undefined;
+    return prefix.startsWith("prom_read_") ? "read" : prefix.startsWith("prom_run_") ? "run" : prefix.startsWith("prom_api_") ? "full" : undefined;
+}
+export const KIND_WORDS = {
+    sandbox: "sandboxes", build: "builds", egress: "data out", storage: "storage",
+    failure_credit: "credited back for failures of ours",
+};
+export const kindWords = (k) => KIND_WORDS[k] ?? k.replace(/_/g, " ");
+export function spendLine(spend, byKind, monthTotal) {
+    const parts = Object.entries(byKind ?? {}).filter(([, v]) => v !== 0).sort(([, a], [, b]) => b - a);
+    const month = monthTotal === undefined ? "" : `this month ${usd(monthTotal)} (the bill: promigence billing usage) · `;
+    return `spent: ${month}all time ${usd(spend)}${parts.length ? ` (${parts.map(([k, v]) => `${kindWords(k)} ${usd(v)}`).join(" · ")})` : ""}`;
+}
 export const PAYMENT_METHOD_NAMES = {
     apple_pay: "Apple Pay",
     google_pay: "Google Pay",
@@ -48,6 +65,13 @@ export async function concurrencyOf(c, me) {
 export function concurrencyText(c) {
     return `${c.in_use}${c.at_least ? "+" : ""} of ${c.limit} in use (paused sandboxes are not counted)`;
 }
+export function planDisplay(me, b) {
+    const name = b?.plan_status?.name;
+    return typeof name === "string" && name ? name : planName(me, b);
+}
+export function sessionLimit(seconds) {
+    return seconds % 3600 === 0 ? `${seconds / 3600} h per sandbox` : `${span(seconds)} per sandbox`;
+}
 export function planName(me, b) {
     if (typeof me?.plan === "string" && me.plan)
         return me.plan;
@@ -66,6 +90,34 @@ function span(seconds) {
         return `${Math.floor(s / 60)}m${s % 60 ? `${s % 60}s` : ""}`;
     return `${s}s`;
 }
+function clock(at) {
+    const d = new Date(at);
+    if (Number.isNaN(d.getTime()))
+        return { hm: at, day: "" };
+    const MON = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+    return { hm: d.toISOString().slice(11, 16), day: `${d.getUTCDate()} ${MON[d.getUTCMonth()]}` };
+}
+export function checksSummary(raw) {
+    if (raw === null || typeof raw !== "object")
+        return undefined;
+    const c = raw;
+    if (typeof c.success_pct !== "number" || typeof c.checks !== "number")
+        return undefined;
+    const lines = [`  checks, last 24 h: ${c.success_pct.toFixed(1)}% passed (${typeof c.ok === "number" ? c.ok : "?"} of ${c.checks})`];
+    const failures = (Array.isArray(c.failures) ? c.failures : [])
+        .map((f) => (f && typeof f === "object" ? String(f.at ?? "") : ""))
+        .filter(Boolean)
+        .sort();
+    if (failures.length) {
+        const [a, b] = [clock(failures[0]), clock(failures[failures.length - 1])];
+        const span = a.day === b.day ? `${a.hm} to ${b.hm} UTC on ${a.day}` : `${a.hm} UTC ${a.day} to ${b.hm} UTC ${b.day}`;
+        lines.push(`  failed checks: ${failures.length}, ${failures.length === 1 ? `at ${a.hm} UTC on ${a.day}` : span}`);
+    }
+    const lastFailed = c.last?.ok === false;
+    if (lastFailed)
+        lines.push("  the last check failed: new sandboxes may not start right now");
+    return { lines, lastFailed };
+}
 const status = async (ctx) => {
     const c = client(ctx);
     let failure;
@@ -88,12 +140,17 @@ const status = async (ctx) => {
         ...(conc ? { concurrency: conc } : {}),
         ...(me?.spend === undefined ? {} : { spend: me.spend }),
     }, next);
-    const lines = [`promigence status: ${health?.ok ?? svc ? "up" : "no answer"}${health?.version ? ` (api ${health.version})` : ""}`];
+    const checks = checksSummary(svc?.checks);
+    const up = health?.ok ?? Boolean(svc);
+    const head = !up ? "no answer" : checks?.lastFailed ? "up, but the last check failed" : "up";
+    const lines = [`promigence status: ${head}${health?.version ? ` (api ${health.version})` : ""}`];
     for (const s of services) {
         lines.push(`  ${s.name}: ${s.state ?? "?"}`);
     }
     if (!services.length)
         lines.push("  (no service reported)");
+    if (checks)
+        lines.push(...checks.lines);
     if (conc)
         lines.push(`  your concurrency: ${concurrencyText(conc)}`);
     else if (!c.authenticated)
@@ -104,28 +161,40 @@ const status = async (ctx) => {
 };
 const me = async (ctx) => {
     const c = client(ctx);
-    const [info, outcomes] = await Promise.all([c.me(), c.outcomes().catch(() => undefined)]);
+    const [info, outcomes, bill, keys] = await Promise.all([
+        c.me(), c.outcomes().catch(() => undefined), c.billing.usage().catch(() => undefined),
+        c.request({ method: "GET", path: "/keys" })
+            .catch(() => undefined),
+    ]);
     const conc = await concurrencyOf(c, info);
     const plan = planName(info);
     const trial = info.trial;
     const next = nexts(["promigence billing usage", "this month's spend by tier, and the credit that paid it"], ["promigence status", "is the service up, and how much of your limit is in use"]);
     const org = info.org && info.org !== info.org_id ? `${info.org} (${info.org_id})` : (info.org_id ?? "org");
-    const lines = [`promigence me: ${org}${info.key?.key_id ? ` · key ${info.key.key_id}` : ""}`];
+    const from = keyOriginText(c.keyOrigin);
+    const row = keys?.keys?.find((k) => k.current === true || (info.key?.key_id !== undefined && k.key_id === info.key.key_id));
+    const access = row?.access ?? keyAccess(info.key?.prefix);
+    const expires = row === undefined ? undefined : (row.expires_at ?? null);
+    const keyWords = [access ? `${access} access` : "", expires === undefined ? "" : `expires ${expires ?? "never"}`].filter(Boolean).join(", ");
+    const lines = [`promigence me: ${org}${info.key?.key_id ? ` · key ${info.key.key_id}${keyWords ? ` (${keyWords})` : ""}` : ""}${from ? ` from ${from}` : ""}`];
     if (info.user && typeof info.user === "object" && info.user.email)
         lines.push(`  signed up as ${info.user.email}`);
-    if (plan)
-        lines.push(`  plan: ${plan}${trial?.on_trial ? " (trial)" : ""}`);
+    const shown = planDisplay(info, bill) ?? plan;
+    if (shown)
+        lines.push(`  plan: ${shown}${trial?.on_trial ? " (trial)" : ""}`);
     if (conc)
         lines.push(`  concurrency: ${concurrencyText(conc)}`);
-    if (trial) {
+    const billCredits = creditsLine(bill?.credits);
+    if (billCredits)
+        lines.push(`  ${billCredits}`);
+    else if (trial) {
         const available = trial.credits_available ?? trial.credits_remaining;
         const held = trial.credits_reserved ?? 0;
         lines.push(`  credits: ${dollars(available)} available${held > 0 ? ` (${dollars(held)} held by running work)` : ""}`);
     }
     if (typeof info.spend === "number") {
-        const kinds = Object.entries(info.spend_detail?.by_kind ?? {}).filter(([, v]) => v > 0);
-        const kindName = { sandbox: "sandboxes", build: "builds", egress: "data out", storage: "storage" };
-        lines.push(`  spent: ${usd(info.spend)}${kinds.length ? ` (${kinds.map(([k, v]) => `${kindName[k] ?? k} ${usd(v)}`).join(" · ")})` : ""}`);
+        const month = bill?.usage?.total;
+        lines.push(`  ${spendLine(info.spend, info.spend_detail?.by_kind, typeof month === "number" ? month : undefined)}`);
     }
     if (info.spend === undefined && info.trial === undefined) {
         lines.push("  spend and credit: shown to an owner's key of the default project (or `promigence billing usage --ledger` for this key's project)");
@@ -137,7 +206,7 @@ const me = async (ctx) => {
     const lim = info.limits ?? {};
     const shape = [
         typeof lim.max_sandbox_vcpus === "number" ? `sandboxes up to ${lim.max_sandbox_vcpus} vCPU` : "",
-        typeof lim.max_session_s === "number" ? `sessions up to ${span(lim.max_session_s)}` : "",
+        typeof lim.max_session_s === "number" ? sessionLimit(lim.max_session_s) : "",
     ].filter(Boolean);
     if (shape.length)
         lines.push(`  limits: ${shape.join(" · ")}`);
@@ -146,12 +215,22 @@ const me = async (ctx) => {
         lines.push(`  ${head}:`, ...rest.map((l) => `  ${l}`));
     }
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, {
-        me: info,
+        me: { ...info, default_spend_cap: typeof cap === "number" && cap > 0 ? cap : null },
+        ...(c.keyOrigin ? { credential: { ...c.keyOrigin, from: from ?? null } } : {}),
+        ...(info.key?.key_id ? { key: { key_id: info.key.key_id, access: access ?? null, ...(expires === undefined ? {} : { expires_at: expires }) } } : {}),
         ...(outcomes ? { outcomes } : {}),
         ...(plan ? { plan } : {}),
         ...(conc ? { concurrency: { in_use: conc.in_use, limit: conc.limit, source: conc.source, ...(conc.at_least ? { at_least: true } : {}) } } : {}),
     }, next), lines, 0, ctx.pointer);
 };
+export function spentLine(total, byKind) {
+    const name = {
+        sandbox: "sandboxes", build: "builds", egress: "data out", storage: "storage", failure_credit: "credited back",
+    };
+    const parts = Object.entries(byKind ?? {}).filter(([, v]) => typeof v === "number" && Math.abs(v) >= 0.005);
+    const money = (v) => (v < 0 ? `-${usd(-v)}` : usd(v));
+    return `${usd(total)}${parts.length ? ` (${parts.map(([k, v]) => `${name[k] ?? k.replace(/_/g, " ")} ${money(v)}`).join(" · ")})` : ""}`;
+}
 export function outcomeLines(o) {
     const usd = (n) => `$${n.toFixed(2)}`;
     const lines = [`this month (since ${o.since.slice(0, 10)})`];
@@ -161,7 +240,7 @@ export function outcomeLines(o) {
     }
     lines.push(`  runs                ${o.runs.toLocaleString("en-US")}`);
     const ready = o.ready_ms_p50 ?? null;
-    lines.push(`  sandboxes           ${o.sandboxes.toLocaleString("en-US")}${ready === null ? "" : `, ready in ${ms(ready)} (median)`}`);
+    lines.push(`  sandboxes           ${o.sandboxes.toLocaleString("en-US")}${ready === null ? "" : `, ready in ${ms(ready)} on our side (median, from the request to a ready sandbox)`}`);
     if (o.failures_ours > 0)
         lines.push(`  failures of ours    ${o.failures_ours}, not billed${o.failures_ours_usd > 0 ? ` (${usd(o.failures_ours_usd)} given back)` : ""}`);
     const speedup = o.batch_speedup ?? null;
@@ -177,6 +256,15 @@ export function outcomeLines(o) {
         lines.push(`  paused, not billed  ${hours.toFixed(1)} h${paidNot === null ? "" : ` (${usd(paidNot)} you did not pay)`}`);
     return lines;
 }
+export function belowList(prices, tier, row) {
+    const rate = prices?.tiers.find((t) => t.name === tier)?.per_second_usd;
+    if (rate === undefined || row.amount === undefined || !row.seconds)
+        return "";
+    const list = row.seconds * rate;
+    if (row.amount >= list - 0.005)
+        return "";
+    return ` (${usd(list)} at this tier's rate; part of it billed at a lower rate, such as preemptible)`;
+}
 const billingUsage = async (ctx) => {
     const limit = limitOf(ctx, 20, 1000);
     const wantLedger = bool(ctx, "--ledger");
@@ -184,13 +272,14 @@ const billingUsage = async (ctx) => {
     if (cursor !== undefined && !wantLedger)
         throw usageError("--cursor pages the ledger", "Pass it with --ledger.");
     const c = client(ctx);
-    const [b, me] = await Promise.all([
+    const [b, me, prices] = await Promise.all([
         c.billing.usage().catch((e) => {
             if (wantLedger && e?.code === "forbidden")
                 return undefined;
             throw e;
         }),
         c.me().catch(() => undefined),
+        livePricing(c).catch(() => undefined),
     ]);
     const page = wantLedger ? await c.billing.ledgerPage({ limit, ...(cursor === undefined ? {} : { cursor }) }) : undefined;
     const entries = page?.items ?? [];
@@ -233,9 +322,9 @@ const billingUsage = async (ctx) => {
         ...(wantLedger ? { ledger: entries, next_cursor: page?.nextCursor ?? null } : {}),
     }, next);
     const period = b.period?.start ? `${b.period.start.slice(0, 10)} → ${(b.period.end ?? "").slice(0, 10)}` : "this month";
-    const lines = [`promigence billing usage: ${usd(usage.total)} (${period}, plan ${plan ?? "?"}${onTrial ? ", on the trial" : ""})`];
+    const lines = [`promigence billing usage: ${usd(usage.total)} (${period}, plan ${planDisplay(me, b) ?? "?"}${onTrial ? ", on the trial" : ""})`];
     const detail = (row) => `${usd(row.amount)}${row.seconds === undefined ? "" : ` · ${row.seconds}s`}${row.count === undefined ? "" : ` · ${row.count}`}` +
-        `${row.gib === undefined ? "" : ` · ${row.gib} GiB`}${row.entries === undefined ? "" : ` · ${row.entries} entries`}` +
+        `${row.gib === undefined ? "" : ` · ${row.gib} GiB`}${row.entries === undefined ? "" : ` · ${row.entries} ${row.entries === 1 ? "entry" : "entries"}`}` +
         `${row.episodes === undefined ? "" : ` · ${row.episodes} episodes`}`;
     for (const [k, v] of Object.entries(usage)) {
         if (k === "total" || v === null || typeof v !== "object")
@@ -248,7 +337,8 @@ const billingUsage = async (ctx) => {
                 const r = inner;
                 if (!r.amount && !r.seconds && !r.count && !r.entries && !r.episodes)
                     continue;
-                lines.push(`  ${k} ${name}: ${detail(r)}`);
+                const label = k === "other" ? kindWords(name) : `${k} ${name}`;
+                lines.push(`  ${label}: ${detail(r)}${k === "sandboxes" ? belowList(prices, name, r) : ""}`);
             }
             continue;
         }
@@ -492,22 +582,37 @@ const snapshotAlias = async (ctx) => {
     if (clear && name) {
         throw usageError("--clear takes no name", `Use either ${sh `promigence snapshot alias ${arg} NAME`} or ${sh `promigence snapshot alias ${arg} --clear`}.`);
     }
-    const res = await client(ctx).snapshots.alias(arg, clear ? null : name);
+    const moveAlias = bool(ctx, "--move-alias");
+    if (moveAlias && clear) {
+        throw usageError("--move-alias takes a name, not --clear", `Use ${sh `promigence snapshot alias ${arg} NAME --move-alias`}.`);
+    }
+    const res = await client(ctx).snapshots.alias(arg, clear ? null : name, moveAlias ? { moveAlias: true } : {});
     const next = nexts(["promigence snapshot list", "your snapshots and their names"]);
-    return emit(ctx.mode, okEnvelope(ctx.cmd.name, { snapshot_id: res.snapshot_id, alias: res.alias ?? null }, next), [`snapshot ${res.snapshot_id}: ${clear ? "alias cleared" : `alias ${res.alias}`}`], 0, ctx.pointer);
+    const said = res.warnings;
+    const warnings = Array.isArray(said) ? said.filter((w) => typeof w === "string") : [];
+    return emit(ctx.mode, okEnvelope(ctx.cmd.name, { snapshot_id: res.snapshot_id, alias: res.alias ?? null, ...(warnings.length ? { warnings } : {}) }, next), [`snapshot ${res.snapshot_id}: ${clear ? "alias cleared" : `alias ${res.alias}`}`, ...warnings.map((w) => `  warning: ${w}`)], 0, ctx.pointer);
 };
 const runList = async (ctx) => {
     const wanted = stateFlag(ctx, RUN_STATES);
+    const labels = Object.entries(labelsOf(ctx));
     const limit = limitOf(ctx, 20, 500);
     const cursor = str(ctx, "--cursor");
-    const page = await client(ctx).runs.listPage({ limit: wanted === undefined ? limit : 500, ...(cursor === undefined ? {} : { cursor }) });
-    const matched = wanted === undefined ? page.items : page.items.filter((r) => r.state === wanted);
+    const filtering = wanted !== undefined || labels.length > 0;
+    const page = await client(ctx).runs.listPage({ limit: filtering ? 500 : limit, ...(cursor === undefined ? {} : { cursor }) });
+    const labelsOfRun = (r) => (r.labels ?? {});
+    const labelled = (r) => labels.every(([k, v]) => labelsOfRun(r)[k] === v);
+    const matched = page.items.filter((r) => (wanted === undefined || r.state === wanted) && labelled(r));
     const runs = matched.slice(0, limit);
     const more = runs.length === matched.length ? page.nextCursor : null;
-    const next = nexts(...(more ? [[`${sh `promigence run list --limit ${String(limit)}`}${wanted ? sh ` --state ${wanted}` : ""}${sh ` --cursor ${more}`}`, "the next page"]] : []), ...(runs.length ? [[sh `promigence run report ${runs[0].run_id}`, "the newest run's validity report"]] : []));
+    const labelArgs = labels.map(([k, v]) => sh ` --label ${`${k}=${v}`}`).join("");
+    const next = nexts(...(more ? [[`${sh `promigence run list --limit ${String(limit)}`}${wanted ? sh ` --state ${wanted}` : ""}${labelArgs}${sh ` --cursor ${more}`}`, "the next page"]] : []), ...(runs.length ? [[sh `promigence run report ${runs[0].run_id}`, "the newest run's validity report"]] : []));
+    const labelText = (r) => {
+        const own = Object.entries(labelsOfRun(r));
+        return own.length ? `  ${own.map(([k, v]) => `${k}=${v}`).join(",")}` : "";
+    };
     const lines = [
-        `promigence run list: ${runs.length}${wanted ? ` in state ${wanted}` : ""}`,
-        ...runs.map((r) => `  ${r.run_id}  ${r.state}  ${r.count ?? "?"}×  ${r.snapshot_id || r.snapshot || "—"}  ${r.started_at ?? ""}`),
+        `promigence run list: ${runs.length}${wanted ? ` in state ${wanted}` : ""}${labels.length ? ` with ${labels.map(([k, v]) => `${k}=${v}`).join(", ")}` : ""}`,
+        ...runs.map((r) => `  ${r.run_id}  ${r.state}  ${r.count ?? "?"}×  ${r.snapshot_id || r.snapshot || "—"}  ${r.started_at ?? ""}${labelText(r)}`),
         ...(runs.length ? [] : ["  (no runs)"]),
         ...(more ? [`  more: --cursor ${more}`] : []),
     ];
@@ -531,7 +636,14 @@ function labelsOf(ctx) {
     }
     return out;
 }
-const HELD_STATES = ["queued", "placing", "ready", "running", "paused"];
+const HELD_STATES = ["queued", "ready", "running", "paused"];
+export function sandboxListFlags(f) {
+    return [
+        ...(f.limit === undefined ? [] : [sh `--limit ${String(f.limit)}`]),
+        ...(f.states?.length ? [sh `--state ${f.states.join(",")}`] : []),
+        ...Object.entries(f.labels ?? {}).map(([k, v]) => sh `--label ${`${k}=${v}`}`),
+    ].join(" ");
+}
 export function pauseReasonText(reason) {
     if (reason === "spend_cap")
         return "spend cap reached";
@@ -543,8 +655,9 @@ const sandboxList = async (ctx) => {
     const labels = labelsOf(ctx);
     const states = list(ctx, "--state").flatMap((s) => s.split(",")).map((s) => s.trim()).filter(Boolean);
     const cursor = str(ctx, "--cursor");
+    const limit = limitOf(ctx, 50);
     const page = await client(ctx).sandboxes.listPage({
-        limit: limitOf(ctx, 50),
+        limit,
         ...(Object.keys(labels).length ? { labels } : {}),
         ...(states.length ? { state: states } : {}),
         ...(cursor === undefined ? {} : { cursor }),
@@ -564,10 +677,10 @@ const sandboxList = async (ctx) => {
                 held = { count, atLeast: live.nextCursor !== null };
         }
     }
-    const heldCmd = sh `promigence sandbox list --state ${HELD_STATES.join(",")}`;
+    const heldCmd = ["promigence sandbox list", sandboxListFlags({ states: HELD_STATES, labels })].filter(Boolean).join(" ");
     const firstLive = sandboxes.find((s) => s.state !== "ended");
     const next = nexts(...(held ? [[heldCmd, "the sandboxes still held (live or paused) that this page does not show"]] : []), ...(sandboxes.length ? [[sh `promigence sandbox info ${sandboxes[0].sandbox_id}`, "one sandbox: its run, its state and the execs it has run"]] : []), ...(page.nextCursor
-        ? [[sh `promigence sandbox list --cursor ${page.nextCursor}`, "the next page"]]
+        ? [[["promigence sandbox list", sandboxListFlags({ limit, states, labels }), sh `--cursor ${page.nextCursor}`].filter(Boolean).join(" "), "the next page"]]
         : []), ...(firstLive ? [["promigence sandbox kill SID", "end one: put its id from this list in place of SID"]] : []));
     const shown = Object.entries(labels).map(([k, v]) => `${k}=${v}`).join(" ");
     const lines = [
@@ -577,7 +690,7 @@ const sandboxList = async (ctx) => {
             return `  ${s.sandbox_id}  ${s.state}${why ? ` (${why})` : ""}${s.outcome ? ` (${s.outcome})` : ""}  ${s.run_id ?? ""}`;
         }),
         ...(sandboxes.length ? [] : ["  (no sandboxes)"]),
-        ...(page.nextCursor ? [`  more: --cursor ${page.nextCursor}`] : []),
+        ...(page.nextCursor ? [`  more: ${sandboxListFlags({ limit, states, labels })} --cursor ${page.nextCursor}`] : []),
         ...(held ? [`  ${held.atLeast ? "at least " : ""}${held.count} live or paused not on this page: ${heldCmd}`] : []),
     ];
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, {
@@ -642,10 +755,29 @@ export function sandboxFactLines(s, facts, nowMs = Date.now()) {
     if (elapsed !== undefined && facts.perSecond !== undefined) {
         const secs = Math.ceil(elapsed / 1000);
         const rate = `$${facts.perSecond.toFixed(9).replace(/0+$/, "").replace(/\.$/, "")}`;
-        out.push(`  cost so far: about ${usd(secs * facts.perSecond)} (${secs} s at ${rate}/s)`);
+        const billed = typeof s["cost_usd"] === "number" && s["state"] !== "ended" ? `; billed so far ${usd(s["cost_usd"])}` : "";
+        out.push(`  cost so far: about ${usd(secs * facts.perSecond)} (${secs} s at ${rate}/s${billed})`);
     }
     return out;
 }
+export function timeLeftMs(s) {
+    if (s.state === "ended" || typeof s.timeout_ms !== "number" || typeof s.elapsed_ms !== "number")
+        return undefined;
+    return Math.max(0, s.timeout_ms - s.elapsed_ms);
+}
+export function pausedLines(sid, s) {
+    if (s.state !== "paused" || pauseReasonText(s.pause_reason))
+        return [];
+    const wakes = Number(s.auto_pause_s ?? 0) > 0;
+    return [
+        wakes
+            ? "  it paused itself when idle: the next request (exec, files, ssh) wakes it"
+            : `  it stays paused until \`${sh `promigence sandbox resume ${sid}`}\` (it has no idle rule, so a request does not wake it)`,
+        s.pause_expires_at
+            ? `  its files and state are kept until ${s.pause_expires_at} unless it is resumed before then`
+            : "  its files and state are kept at least 30 days from the pause; resuming starts them again",
+    ];
+}
 export function supportStopLines(s) {
     if (s.outcome !== "stopped_by_support")
         return [];
@@ -668,11 +800,15 @@ const sandboxInfo = async (ctx) => {
             : [];
     const next = nexts(...limitNext, ...(execs.length ? [[sh `promigence exec output ${execs[execs.length - 1].exec_id} --sandbox ${sid}`, "what its last command printed"]] : []), ...(ended
         ? (s.run_id ? [[sh `promigence run report ${s.run_id} --sandbox ${sid}`, "how it ended, in its run's report"]] : [])
-        : why ? [] : [[sh `promigence exec --sandbox ${sid} -- CMD`, "run something in it"]]));
+        : why ? []
+            : s.state === "paused" && !(Number(s.auto_pause_s ?? 0) > 0)
+                ? [[sh `promigence sandbox resume ${sid}`, "bring it back; a request does not wake it"]]
+                : [[sh `promigence exec --sandbox ${sid} -- CMD`, "run something in it"]]));
     const lines = [
         `sandbox ${sid}: ${s.state}${why ? ` (${why})` : ""}${s.outcome ? ` (${s.outcome})` : ""}${s.run_id ? ` · run ${s.run_id}` : ""}`,
         ...supportStopLines(s),
         ...(why ? ["  its files and state are kept, and paused time is not billed"] : []),
+        ...pausedLines(sid, s),
         ...queuedLines([s]),
         ...(s.snapshot_id ? [`  snapshot ${s.snapshot_id}`] : []),
         ...sandboxFactLines(s, facts),
@@ -682,15 +818,22 @@ const sandboxInfo = async (ctx) => {
         ...(Number(s.auto_pause_s ?? 0) > 0 ? [`  pauses itself after ${s.auto_pause_s} s idle`] : []),
         ...execs.map((e) => `  ${e.exec_id}  exit ${e.exit_code ?? "—"}  ${(e.argv ?? []).join(" ")}`),
         ...(execs.length ? [] : ["  (no execs recorded)"]),
+        ...(ended ? [] : [`  background commands (exec --background): ${sh `promigence exec ps --sandbox ${sid}`}`]),
     ];
+    const left = timeLeftMs(s);
+    const billed = typeof s["cost_usd"] === "number" ? s["cost_usd"] : undefined;
+    const costSoFar = ended
+        ? billed
+        : facts.perSecond !== undefined && typeof s.elapsed_ms === "number"
+            ? Math.round(Math.ceil(s.elapsed_ms / 1000) * facts.perSecond * 1e6) / 1e6
+            : undefined;
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, {
         sandbox: s,
         ...(facts.tier ? { tier: facts.tier } : {}),
-        ...(facts.perSecond !== undefined && typeof s.elapsed_ms === "number"
-            ? { cost_so_far_usd: Math.round(Math.ceil(s.elapsed_ms / 1000) * facts.perSecond * 1e6) / 1e6 }
-            : {}),
-        ...(ended && typeof s["cost_usd"] === "number"
-            ? { cost_usd: s["cost_usd"], billable_seconds: s["billable_seconds"] }
+        ...(left === undefined ? {} : { time_left_ms: left }),
+        ...(costSoFar === undefined ? {} : { cost_so_far_usd: costSoFar }),
+        ...(ended && billed !== undefined
+            ? { cost_usd: billed, billable_seconds: s["billable_seconds"] }
             : {}),
     }, next), lines, 0, ctx.pointer);
 };
@@ -705,7 +848,7 @@ const sandboxPause = async (ctx) => {
         `sandbox ${sid}: paused${res.already_paused ? " (already was)" : ""}`,
         "  it holds no capacity and bills no seconds while parked",
         ...(took === undefined ? [] : [`  took ${ms(took)}`]),
-        ...(bytes?.allocated === undefined ? [] : [`  parked image: ${Math.round(bytes.allocated / 1048576)} MiB`]),
+        ...(bytes?.allocated === undefined ? [] : [`  holds ${bytes.allocated >= 1073741824 ? `${(bytes.allocated / 1073741824).toFixed(1)} GiB` : `${Math.round(bytes.allocated / 1048576)} MiB`} of storage while paused`]),
         ...(killed > 0 ? [`  ended ${killed} background process${killed === 1 ? "" : "es"} (started with exec --background)`] : []),
         ...(res.warnings ?? []).map((w) => `  warning: ${w}`),
     ];
@@ -789,7 +932,7 @@ const sandboxAutopause = async (ctx) => {
         throw usageError("--mode and --off ask for opposite things", "--off removes the idle rule; --mode says what idle means under one.");
     }
     const rule = (s, m) => s > 0
-        ? `pauses itself after ${s} s idle${m === "activity" ? " (activity: no CPU, no bytes, no open connection)" : ""}`
+        ? `pauses itself after ${s} s idle${m === "activity" ? " (activity: also CPU under 5% of one core, no network bytes, no outside connection)" : ""}`
         : "never pauses itself on idle";
     const next = nexts([sh `promigence sandbox info ${sid}`, "the sandbox as it stands now"]);
     if (after === undefined && !off && mode === undefined) {
@@ -818,18 +961,122 @@ const sandboxAutopause = async (ctx) => {
     ];
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, { ...res, sandbox_id: sid, auto_pause_s: secs, auto_pause_mode: m, ...(pausedNow ? { paused: true } : {}) }, pausedNow ? nexts([sh `promigence sandbox resume ${sid}`, "bring it back: with the rule off, a request no longer wakes it"]) : next), lines, 0, ctx.pointer);
 };
+export const PROBE_DIGEST_MAX_BYTES = 16 * 1024 * 1024;
+export async function pathNow(c, sid, path) {
+    try {
+        const { stat } = await c.sandboxes.files.stat(sid, path);
+        if (stat.is_dir)
+            return { kind: "dir" };
+        if (stat.is_symlink)
+            return { kind: "symlink" };
+        if (stat.size > PROBE_DIGEST_MAX_BYTES)
+            return { kind: "file", size: stat.size };
+        const bytes = await c.sandboxes.files.get(sid, path);
+        return { kind: "file", size: bytes.length, digest: `blake3:${blake3Hex(bytes)}` };
+    }
+    catch (e) {
+        return e?.code === "not_found" ? { kind: "missing" } : undefined;
+    }
+}
+export function ownDigests(paths) {
+    const out = new Map();
+    for (const p of paths) {
+        const at = p.indexOf("=");
+        if (at > 0)
+            out.set(p.slice(0, at), p.slice(at + 1));
+    }
+    return out;
+}
+export function ownPathOf(bad, own) {
+    for (const p of own.keys()) {
+        const root = p.length > 1 ? p.replace(/\/+$/, "") : p;
+        if (bad === p || bad === root || bad.startsWith(root === "/" ? "/" : `${root}/`))
+            return p;
+    }
+    return undefined;
+}
+function nowWords(now) {
+    if (!now)
+        return undefined;
+    if (now.kind === "missing")
+        return "it does not exist";
+    if (now.kind === "dir")
+        return "it is a directory";
+    if (now.kind === "symlink")
+        return "it is a symbolic link";
+    return now.digest ? `it is ${now.digest} now` : `it is a file of ${now.size} bytes`;
+}
+async function probeRefusal(ctx, c, sid, e) {
+    if (!(e instanceof PromigenceError) || e.code !== "usage")
+        throw e;
+    const later = (path, digest) => sh `promigence sandbox probe ${sid} --path ${`${path}=${digest}`}`;
+    if (/has nothing to compare against/.test(e.message)) {
+        throw usageError(e.message, `Name the files to check: ${sh `promigence sandbox probe ${sid} --path /work/FILE`} answers with the file's digest now, ` +
+            "and --path /work/FILE=<that digest> checks later that it has not changed.");
+    }
+    const m = /^(.+): the snapshot recorded no digest for this path\b/.exec(e.message);
+    if (!m)
+        throw e;
+    const path = m[1];
+    const now = await pathNow(c, sid, path);
+    const digest = now?.kind === "file" ? now.digest : undefined;
+    const fix = digest
+        ? `Its digest now is ${digest}. To check later that it has not changed: ${later(path, digest)}`
+        : now?.kind === "missing"
+            ? `It does not exist in sandbox ${sid}: check the path (${sh `promigence files ls / --sandbox ${sid}`}).`
+            : now?.kind === "file"
+                ? `It is over ${PROBE_DIGEST_MAX_BYTES / 1048576} MiB, too large to read back here: name it as PATH=blake3:<hex> with a digest you computed (b3sum).`
+                : `Name it as PATH=blake3:<hex> (${nowWords(now) ?? "the digest of the file's bytes"}).`;
+    const err = usageError(e.message, fix);
+    const next = digest ? nexts([later(path, digest), "check later that the file has not changed"]) : nexts([sh `promigence sandbox info ${sid}`, "the sandbox and its snapshot"]);
+    return emit(ctx.mode, {
+        ...errorEnvelope(ctx.cmd.name, err, next),
+        sandbox_id: sid,
+        ...(digest ? { digests: [{ path, digest }] } : {}),
+    }, errorTextLines(ctx.cmd.name, err, 2, next), 2, ctx.pointer);
+}
 const sandboxProbe = async (ctx) => {
     const sid = positional(ctx, "a sandbox id", "promigence sandbox probe sbx_01");
     const paths = list(ctx, "--path");
-    const res = await client(ctx).sandboxes.probe(sid, ...(paths.length ? [{ paths }] : []));
+    const own = ownDigests(paths);
+    const c = client(ctx);
+    const res = await c.sandboxes.probe(sid, ...(paths.length ? [{ paths }] : [])).catch((e) => probeRefusal(ctx, c, sid, e));
     const ok = res.ok !== false;
-    const env = { ...okEnvelope(ctx.cmd.name, { ...res, sandbox_id: sid }, nexts([sh `promigence sandbox info ${sid}`, "the sandbox, its run and the commands it ran"])), ok };
-    const lines = [
-        `sandbox ${sid}: ${ok ? "matches the verified environment" : "DIFFERS from the verified environment"}`,
-        ...(res.first_bad_path ? [`  first difference at ${res.first_bad_path}`] : []),
-        ...(res.ms === undefined ? [] : [`  checked in ${ms(res.ms)}`]),
-    ];
-    return emit(ctx.mode, env, lines, ok ? 0 : 3, ctx.pointer);
+    const infoNext = nexts([sh `promigence sandbox info ${sid}`, "the sandbox, its run and the commands it ran"]);
+    const took = res.ms === undefined ? [] : [`  checked in ${ms(res.ms)}`];
+    if (ok) {
+        const allOwn = paths.length > 0 && own.size === paths.length;
+        return emit(ctx.mode, okEnvelope(ctx.cmd.name, { ...res, sandbox_id: sid }, infoNext), [
+            `sandbox ${sid}: ${allOwn ? "every path matches the digest you gave" : "matches the verified environment"}`,
+            ...took,
+        ], 0, ctx.pointer);
+    }
+    const bad = res.first_bad_path ?? undefined;
+    const mine = bad === undefined ? undefined : ownPathOf(bad, own);
+    const now = bad === undefined ? undefined : await pathNow(c, sid, bad);
+    const words = nowWords(now);
+    const where = bad === undefined ? "" : bad;
+    const err = mine !== undefined
+        ? new PromigenceError({
+            code: "digest_mismatch", exitCode: 4, docs: `${ERRORS_DOCS_URL}#digest_mismatch`,
+            message: now?.kind === "missing"
+                ? `${where} does not exist in sandbox ${sid}, so it cannot match the digest you gave${mine !== bad ? ` for ${mine}` : ""}`
+                : `${mine !== bad ? `${mine} does not match the digest you gave, first at ${where}` : `${where} does not match the digest you gave`} (${own.get(mine)})${words ? `: ${words}` : ""}`,
+            fix: now?.kind === "missing"
+                ? `Nothing failed on our side. Check the path (${sh `promigence files ls / --sandbox ${sid}`}), or what should have made the file.`
+                : "Nothing failed on our side: the file is not what that digest says. Check the digest (blake3 of the file's bytes) and the path, or what produced the file.",
+        })
+        : new PromigenceError({
+            code: "integrity_mismatch", exitCode: 3, docs: `${ERRORS_DOCS_URL}#integrity_mismatch`,
+            message: `sandbox ${sid} differs from its verified snapshot${where ? ` at ${where}` : ""}${words ? ` (${words})` : ""}`,
+            fix: "If a command you ran changed it, that is expected. If nothing did, this sandbox is not the verified environment: " +
+                "start a new one from the same snapshot and probe it; if that one differs too, rebuild the snapshot and tell support@promigence.ai the sandbox id.",
+        });
+    return emit(ctx.mode, {
+        ...errorEnvelope(ctx.cmd.name, err, infoNext),
+        sandbox_id: sid, checked: res.checked, ms: res.ms, first_bad_path: bad ?? null,
+        difference: { path: bad ?? null, ...(mine === undefined ? { against: "snapshot" } : { against: "your_digest", digest_given: own.get(mine) }), now: now ?? null },
+    }, [...errorTextLines(ctx.cmd.name, err, err.cliExit, infoNext), ...took], err.cliExit, ctx.pointer);
 };
 const filesRm = async (ctx) => {
     const path = positional(ctx, "a path", "promigence files rm /work/out.log --sandbox sbx_01");
@@ -839,8 +1086,14 @@ const filesRm = async (ctx) => {
     const sid = required(ctx, "--sandbox", "Pass --sandbox sbx_<id> — the sandbox the path is in.");
     const recursive = bool(ctx, "--recursive");
     const res = await client(ctx).sandboxes.files.delete(sid, path, { recursive });
-    const next = nexts([sh `promigence files ls --sandbox ${sid}`, "what is left"]);
-    return emit(ctx.mode, okEnvelope(ctx.cmd.name, { sandbox_id: sid, path, deleted: res.deleted, existed: res.existed, recursive }, next), [`${sid}:${path}: ${res.deleted ? "deleted" : "not deleted"}${res.existed === false ? " (it was not there)" : ""}`], 0, ctx.pointer);
+    const gone = res.existed === false;
+    const dir = path.replace(/\/+$/, "").replace(/\/[^/]*$/, "") || "/";
+    const next = gone
+        ? nexts([sh `promigence files ls ${dir} --sandbox ${sid}`, "check the path: what is in its directory"])
+        : nexts([sh `promigence files ls --sandbox ${sid}`, "what is left"]);
+    return emit(ctx.mode, okEnvelope(ctx.cmd.name, { sandbox_id: sid, path, deleted: res.deleted, existed: res.existed, recursive }, next), [gone
+            ? `${sid}:${path}: not there, nothing was deleted (check the path)`
+            : `${sid}:${path}: ${res.deleted ? "deleted" : "not deleted"}`], 0, ctx.pointer);
 };
 const secretsList = async (ctx) => {
     const { secrets } = await client(ctx).secrets.list();

package/dist/agent.js

@@ -1,6 +1,6 @@
 import { DEFAULT_SNAPSHOT, PromigenceError, checkImageRef, checkRepoUrl, imageSnapshot, } from "./sdk/index.js";
 import { snapshotArg } from "./bundle.js";
-import { bool, client, nexts, spendCap, str, waitBound, } from "./commands.js";
+import { bool, client, list, nexts, spendCap, str, waitBound, } from "./commands.js";
 import { emit, errorEnvelope, errorTextLines, okEnvelope, parseDuration, usageError, write, } from "./io.js";
 import { findCommand } from "./surface.js";
 import { LIVE_HANDLERS } from "./live.js";
@@ -75,7 +75,7 @@ const OPENCODE_PROVIDER_VAR = {
 const OWN_PROVIDER = {
     claude: { id: "anthropic", name: "Anthropic" }, codex: { id: "openai", name: "OpenAI" },
 };
-function withModel(which, agent, model) {
+function withModel(which, agent, model, keySecret) {
     if (model === undefined)
         return { agent };
     if (!/^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$/.test(model)) {
@@ -95,7 +95,11 @@ function withModel(which, agent, model) {
     if (slash <= 0 || slash === model.length - 1) {
         throw usageError(`opencode takes --model as provider/model, got '${model}'`, "Name the provider first, e.g. --model xai/grok-4-0709 or --model anthropic/claude-sonnet-4-5.");
     }
-    const v = OPENCODE_PROVIDER_VAR[model.slice(0, slash).toLowerCase()];
+    const provider = model.slice(0, slash).toLowerCase();
+    const v = OPENCODE_PROVIDER_VAR[provider];
+    if (v === undefined && keySecret === undefined) {
+        throw usageError(`opencode's provider '${provider}' (--model ${model}) is not one whose key variable is known here`, `Name its key: --key-secret NAME=VAR, the stored secret and the variable opencode reads for ${provider}; or a provider known here: ${Object.keys(OPENCODE_PROVIDER_VAR).join(", ")}.`);
+    }
     return { agent: v ? { ...agent, defaultSecret: v, keyVars: [v] } : agent, model };
 }
 const an = (word) => (/^[AEIOU]/.test(word) ? "an" : "a");
@@ -220,7 +224,7 @@ const agentStart = async (ctx) => {
     if (known === undefined) {
         throw usageError(`unknown agent '${which}'`, `Pick one of ${AGENT_NAMES.join(" | ")}.`);
     }
-    const { agent, model } = withModel(which, known, str(ctx, "--model"));
+    const { agent, model } = withModel(which, known, str(ctx, "--model"), str(ctx, "--key-secret"));
     const key = keyBindings(agent, str(ctx, "--key-secret"), which);
     const fromEnv = str(ctx, "--key-from-env");
     let keyValue;
@@ -231,6 +235,15 @@ const agentStart = async (ctx) => {
         }
     }
     checkSource(ctx);
+    const labels = {};
+    for (const l of list(ctx, "--label")) {
+        const i = l.indexOf("=");
+        if (i < 1)
+            throw usageError(`--label: '${l}' is not NAME=VALUE`, "Pass --label name=value (repeatable).");
+        if (l.slice(0, i) === "agent")
+            throw usageError("--label agent=… is the agent's own label", "Name another label: agent start sets agent=NAME itself.");
+        labels[l.slice(0, i)] = l.slice(i + 1);
+    }
     const repo = str(ctx, "--repo");
     if (repo !== undefined)
         checkRepo(repo);
@@ -254,7 +267,7 @@ const agentStart = async (ctx) => {
     const snapshot = await snapshotFor(ctx, c);
     const started = await c.runs.start({
         snapshot, count: 1, timeoutMs, spendCap: await spendCap(ctx, c), secrets: key.bindings,
-        labels: { agent: which },
+        labels: { ...labels, agent: which },
     });
     const runId = started.run_id;
     const bound = waitBound(ctx);