promigence 0.1.0-beta.15 → 0.1.0-beta.17

package/dist/account.js

@@ -1,11 +1,28 @@
-import { bool, client, KEPT_BY_HOLDERS, list, nexts, optimizeForFlag, positional, required, secretListLines, str, } from "./commands.js";
+import { bool, client, KEPT_BY_HOLDERS, keyOriginText, list, nexts, optimizeForFlag, positional, required, secretListLines, str, } from "./commands.js";
 import { queuedLines } from "./capacity.js";
 import { duration } from "./report.js";
-import { emit, ms, okEnvelope, overMost, parseCount, parseDuration, parseUsd, usageError } from "./io.js";
+import { emit, errorEnvelope, errorTextLines, ms, okEnvelope, overMost, parseCount, parseDuration, parseUsd, usageError } from "./io.js";
+import { ERRORS_DOCS_URL, PromigenceError, blake3Hex } from "./sdk/index.js";
 import { sh } from "./term.js";
 import { autoReloadShort, creditsLine, resumeAllPausedForCredits } from "./billing.js";
 import { dollars } from "./money.js";
-const usd = (n) => n === undefined ? "—" : `$${n.toFixed(n < 0.01 && n > 0 ? 4 : 2)}`;
+import { livePricing } from "./pricing.js";
+const usd = (n) => n === undefined ? "—" : n < 0 ? `-${usd(-n)}` : `$${n.toFixed(n < 0.01 && n > 0 ? 4 : 2)}`;
+export function keyAccess(prefix) {
+    if (!prefix)
+        return undefined;
+    return prefix.startsWith("prom_read_") ? "read" : prefix.startsWith("prom_run_") ? "run" : prefix.startsWith("prom_api_") ? "full" : undefined;
+}
+export const KIND_WORDS = {
+    sandbox: "sandboxes", build: "builds", egress: "data out", storage: "storage",
+    failure_credit: "credited back for failures of ours",
+};
+export const kindWords = (k) => KIND_WORDS[k] ?? k.replace(/_/g, " ");
+export function spendLine(spend, byKind, monthTotal) {
+    const parts = Object.entries(byKind ?? {}).filter(([, v]) => v !== 0).sort(([, a], [, b]) => b - a);
+    const month = monthTotal === undefined ? "" : `this month ${usd(monthTotal)} (the bill: promigence billing usage) · `;
+    return `spent: ${month}all time ${usd(spend)}${parts.length ? ` (${parts.map(([k, v]) => `${kindWords(k)} ${usd(v)}`).join(" · ")})` : ""}`;
+}
 export const PAYMENT_METHOD_NAMES = {
     apple_pay: "Apple Pay",
     google_pay: "Google Pay",
@@ -48,6 +65,28 @@ export async function concurrencyOf(c, me) {
 export function concurrencyText(c) {
     return `${c.in_use}${c.at_least ? "+" : ""} of ${c.limit} in use (paused sandboxes are not counted)`;
 }
+export function planDisplay(me, b) {
+    const name = b?.plan_status?.name;
+    return typeof name === "string" && name ? name : planName(me, b);
+}
+function gbText(bytes) {
+    const r = Math.round((bytes / 2 ** 30) * 10) / 10;
+    return Number.isInteger(r) ? String(r) : r.toFixed(1);
+}
+export function storageLine(me) {
+    const s = me?.storage;
+    if (!s || typeof s.used_bytes !== "number")
+        return undefined;
+    const lim = me?.limits ?? {};
+    const of = (used, max) => `${gbText(used)} GB${typeof max === "number" ? ` of ${max} GB` : ""}`;
+    const capMax = lim.max_capture_storage_gb;
+    const captures = typeof s.captures_bytes === "number" ? s.captures_bytes : 0;
+    return `storage: ${of(s.used_bytes, lim.max_storage_gb)} used · saved failures ${of(captures, capMax)}` +
+        (typeof capMax === "number" ? ", the oldest removed at that limit" : "");
+}
+export function sessionLimit(seconds) {
+    return seconds % 3600 === 0 ? `${seconds / 3600} h per sandbox` : `${span(seconds)} per sandbox`;
+}
 export function planName(me, b) {
     if (typeof me?.plan === "string" && me.plan)
         return me.plan;
@@ -66,6 +105,34 @@ function span(seconds) {
         return `${Math.floor(s / 60)}m${s % 60 ? `${s % 60}s` : ""}`;
     return `${s}s`;
 }
+function clock(at) {
+    const d = new Date(at);
+    if (Number.isNaN(d.getTime()))
+        return { hm: at, day: "" };
+    const MON = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+    return { hm: d.toISOString().slice(11, 16), day: `${d.getUTCDate()} ${MON[d.getUTCMonth()]}` };
+}
+export function checksSummary(raw) {
+    if (raw === null || typeof raw !== "object")
+        return undefined;
+    const c = raw;
+    if (typeof c.success_pct !== "number" || typeof c.checks !== "number")
+        return undefined;
+    const lines = [`  checks, last 24 h: ${c.success_pct.toFixed(1)}% passed (${typeof c.ok === "number" ? c.ok : "?"} of ${c.checks})`];
+    const failures = (Array.isArray(c.failures) ? c.failures : [])
+        .map((f) => (f && typeof f === "object" ? String(f.at ?? "") : ""))
+        .filter(Boolean)
+        .sort();
+    if (failures.length) {
+        const [a, b] = [clock(failures[0]), clock(failures[failures.length - 1])];
+        const span = a.day === b.day ? `${a.hm} to ${b.hm} UTC on ${a.day}` : `${a.hm} UTC ${a.day} to ${b.hm} UTC ${b.day}`;
+        lines.push(`  failed checks: ${failures.length}, ${failures.length === 1 ? `at ${a.hm} UTC on ${a.day}` : span}`);
+    }
+    const lastFailed = c.last?.ok === false;
+    if (lastFailed)
+        lines.push("  the last check failed: new sandboxes may not start right now");
+    return { lines, lastFailed };
+}
 const status = async (ctx) => {
     const c = client(ctx);
     let failure;
@@ -88,12 +155,17 @@ const status = async (ctx) => {
         ...(conc ? { concurrency: conc } : {}),
         ...(me?.spend === undefined ? {} : { spend: me.spend }),
     }, next);
-    const lines = [`promigence status: ${health?.ok ?? svc ? "up" : "no answer"}${health?.version ? ` (api ${health.version})` : ""}`];
+    const checks = checksSummary(svc?.checks);
+    const up = health?.ok ?? Boolean(svc);
+    const head = !up ? "no answer" : checks?.lastFailed ? "up, but the last check failed" : "up";
+    const lines = [`promigence status: ${head}${health?.version ? ` (api ${health.version})` : ""}`];
     for (const s of services) {
         lines.push(`  ${s.name}: ${s.state ?? "?"}`);
     }
     if (!services.length)
         lines.push("  (no service reported)");
+    if (checks)
+        lines.push(...checks.lines);
     if (conc)
         lines.push(`  your concurrency: ${concurrencyText(conc)}`);
     else if (!c.authenticated)
@@ -104,28 +176,40 @@ const status = async (ctx) => {
 };
 const me = async (ctx) => {
     const c = client(ctx);
-    const [info, outcomes] = await Promise.all([c.me(), c.outcomes().catch(() => undefined)]);
+    const [info, outcomes, bill, keys] = await Promise.all([
+        c.me(), c.outcomes().catch(() => undefined), c.billing.usage().catch(() => undefined),
+        c.request({ method: "GET", path: "/keys" })
+            .catch(() => undefined),
+    ]);
     const conc = await concurrencyOf(c, info);
     const plan = planName(info);
     const trial = info.trial;
     const next = nexts(["promigence billing usage", "this month's spend by tier, and the credit that paid it"], ["promigence status", "is the service up, and how much of your limit is in use"]);
     const org = info.org && info.org !== info.org_id ? `${info.org} (${info.org_id})` : (info.org_id ?? "org");
-    const lines = [`promigence me: ${org}${info.key?.key_id ? ` · key ${info.key.key_id}` : ""}`];
+    const from = keyOriginText(c.keyOrigin);
+    const row = keys?.keys?.find((k) => k.current === true || (info.key?.key_id !== undefined && k.key_id === info.key.key_id));
+    const access = row?.access ?? keyAccess(info.key?.prefix);
+    const expires = row === undefined ? undefined : (row.expires_at ?? null);
+    const keyWords = [access ? `${access} access` : "", expires === undefined ? "" : `expires ${expires ?? "never"}`].filter(Boolean).join(", ");
+    const lines = [`promigence me: ${org}${info.key?.key_id ? ` · key ${info.key.key_id}${keyWords ? ` (${keyWords})` : ""}` : ""}${from ? ` from ${from}` : ""}`];
     if (info.user && typeof info.user === "object" && info.user.email)
         lines.push(`  signed up as ${info.user.email}`);
-    if (plan)
-        lines.push(`  plan: ${plan}${trial?.on_trial ? " (trial)" : ""}`);
+    const shown = planDisplay(info, bill) ?? plan;
+    if (shown)
+        lines.push(`  plan: ${shown}${trial?.on_trial ? " (trial)" : ""}`);
     if (conc)
         lines.push(`  concurrency: ${concurrencyText(conc)}`);
-    if (trial) {
+    const billCredits = creditsLine(bill);
+    if (billCredits)
+        lines.push(`  ${billCredits}`);
+    else if (trial) {
         const available = trial.credits_available ?? trial.credits_remaining;
         const held = trial.credits_reserved ?? 0;
         lines.push(`  credits: ${dollars(available)} available${held > 0 ? ` (${dollars(held)} held by running work)` : ""}`);
     }
     if (typeof info.spend === "number") {
-        const kinds = Object.entries(info.spend_detail?.by_kind ?? {}).filter(([, v]) => v > 0);
-        const kindName = { sandbox: "sandboxes", build: "builds", egress: "data out", storage: "storage" };
-        lines.push(`  spent: ${usd(info.spend)}${kinds.length ? ` (${kinds.map(([k, v]) => `${kindName[k] ?? k} ${usd(v)}`).join(" · ")})` : ""}`);
+        const month = bill?.usage?.total;
+        lines.push(`  ${spendLine(info.spend, info.spend_detail?.by_kind, typeof month === "number" ? month : undefined)}`);
     }
     if (info.spend === undefined && info.trial === undefined) {
         lines.push("  spend and credit: shown to an owner's key of the default project (or `promigence billing usage --ledger` for this key's project)");
@@ -137,21 +221,34 @@ const me = async (ctx) => {
     const lim = info.limits ?? {};
     const shape = [
         typeof lim.max_sandbox_vcpus === "number" ? `sandboxes up to ${lim.max_sandbox_vcpus} vCPU` : "",
-        typeof lim.max_session_s === "number" ? `sessions up to ${span(lim.max_session_s)}` : "",
+        typeof lim.max_session_s === "number" ? sessionLimit(lim.max_session_s) : "",
     ].filter(Boolean);
     if (shape.length)
         lines.push(`  limits: ${shape.join(" · ")}`);
+    const stored = storageLine(info);
+    if (stored)
+        lines.push(`  ${stored}`);
     if (outcomes && outcomes.sandboxes > 0) {
         const [head, ...rest] = outcomeLines(outcomes);
         lines.push(`  ${head}:`, ...rest.map((l) => `  ${l}`));
     }
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, {
-        me: info,
+        me: { ...info, default_spend_cap: typeof cap === "number" && cap > 0 ? cap : null },
+        ...(c.keyOrigin ? { credential: { ...c.keyOrigin, from: from ?? null } } : {}),
+        ...(info.key?.key_id ? { key: { key_id: info.key.key_id, access: access ?? null, ...(expires === undefined ? {} : { expires_at: expires }) } } : {}),
         ...(outcomes ? { outcomes } : {}),
         ...(plan ? { plan } : {}),
         ...(conc ? { concurrency: { in_use: conc.in_use, limit: conc.limit, source: conc.source, ...(conc.at_least ? { at_least: true } : {}) } } : {}),
     }, next), lines, 0, ctx.pointer);
 };
+export function spentLine(total, byKind) {
+    const name = {
+        sandbox: "sandboxes", build: "builds", egress: "data out", storage: "storage", failure_credit: "credited back",
+    };
+    const parts = Object.entries(byKind ?? {}).filter(([, v]) => typeof v === "number" && Math.abs(v) >= 0.005);
+    const money = (v) => (v < 0 ? `-${usd(-v)}` : usd(v));
+    return `${usd(total)}${parts.length ? ` (${parts.map(([k, v]) => `${name[k] ?? k.replace(/_/g, " ")} ${money(v)}`).join(" · ")})` : ""}`;
+}
 export function outcomeLines(o) {
     const usd = (n) => `$${n.toFixed(2)}`;
     const lines = [`this month (since ${o.since.slice(0, 10)})`];
@@ -161,7 +258,7 @@ export function outcomeLines(o) {
     }
     lines.push(`  runs                ${o.runs.toLocaleString("en-US")}`);
     const ready = o.ready_ms_p50 ?? null;
-    lines.push(`  sandboxes           ${o.sandboxes.toLocaleString("en-US")}${ready === null ? "" : `, ready in ${ms(ready)} (median)`}`);
+    lines.push(`  sandboxes           ${o.sandboxes.toLocaleString("en-US")}${ready === null ? "" : `, ready in ${ms(ready)} on our side (median, from the request to a ready sandbox)`}`);
     if (o.failures_ours > 0)
         lines.push(`  failures of ours    ${o.failures_ours}, not billed${o.failures_ours_usd > 0 ? ` (${usd(o.failures_ours_usd)} given back)` : ""}`);
     const speedup = o.batch_speedup ?? null;
@@ -177,6 +274,15 @@ export function outcomeLines(o) {
         lines.push(`  paused, not billed  ${hours.toFixed(1)} h${paidNot === null ? "" : ` (${usd(paidNot)} you did not pay)`}`);
     return lines;
 }
+export function belowList(prices, tier, row) {
+    const rate = prices?.tiers.find((t) => t.name === tier)?.per_second_usd;
+    if (rate === undefined || row.amount === undefined || !row.seconds)
+        return "";
+    const list = row.seconds * rate;
+    if (row.amount >= list - 0.005)
+        return "";
+    return ` (${usd(list)} at this tier's rate; part of it billed at a lower rate, such as preemptible)`;
+}
 const billingUsage = async (ctx) => {
     const limit = limitOf(ctx, 20, 1000);
     const wantLedger = bool(ctx, "--ledger");
@@ -184,13 +290,14 @@ const billingUsage = async (ctx) => {
     if (cursor !== undefined && !wantLedger)
         throw usageError("--cursor pages the ledger", "Pass it with --ledger.");
     const c = client(ctx);
-    const [b, me] = await Promise.all([
+    const [b, me, prices] = await Promise.all([
         c.billing.usage().catch((e) => {
             if (wantLedger && e?.code === "forbidden")
                 return undefined;
             throw e;
         }),
         c.me().catch(() => undefined),
+        livePricing(c).catch(() => undefined),
     ]);
     const page = wantLedger ? await c.billing.ledgerPage({ limit, ...(cursor === undefined ? {} : { cursor }) }) : undefined;
     const entries = page?.items ?? [];
@@ -233,9 +340,9 @@ const billingUsage = async (ctx) => {
         ...(wantLedger ? { ledger: entries, next_cursor: page?.nextCursor ?? null } : {}),
     }, next);
     const period = b.period?.start ? `${b.period.start.slice(0, 10)} → ${(b.period.end ?? "").slice(0, 10)}` : "this month";
-    const lines = [`promigence billing usage: ${usd(usage.total)} (${period}, plan ${plan ?? "?"}${onTrial ? ", on the trial" : ""})`];
+    const lines = [`promigence billing usage: ${usd(usage.total)} (${period}, plan ${planDisplay(me, b) ?? "?"}${onTrial ? ", on the trial" : ""})`];
     const detail = (row) => `${usd(row.amount)}${row.seconds === undefined ? "" : ` · ${row.seconds}s`}${row.count === undefined ? "" : ` · ${row.count}`}` +
-        `${row.gib === undefined ? "" : ` · ${row.gib} GiB`}${row.entries === undefined ? "" : ` · ${row.entries} entries`}` +
+        `${row.gib === undefined ? "" : ` · ${row.gib} GiB`}${row.entries === undefined ? "" : ` · ${row.entries} ${row.entries === 1 ? "entry" : "entries"}`}` +
         `${row.episodes === undefined ? "" : ` · ${row.episodes} episodes`}`;
     for (const [k, v] of Object.entries(usage)) {
         if (k === "total" || v === null || typeof v !== "object")
@@ -248,7 +355,8 @@ const billingUsage = async (ctx) => {
                 const r = inner;
                 if (!r.amount && !r.seconds && !r.count && !r.entries && !r.episodes)
                     continue;
-                lines.push(`  ${k} ${name}: ${detail(r)}`);
+                const label = k === "other" ? kindWords(name) : `${k} ${name}`;
+                lines.push(`  ${label}: ${detail(r)}${k === "sandboxes" ? belowList(prices, name, r) : ""}`);
             }
             continue;
         }
@@ -260,7 +368,7 @@ const billingUsage = async (ctx) => {
         lines.push(`  paid from credits: ${usd(usage["credits_applied"])}`);
     }
     const trial = b.trial;
-    const credits = creditsLine(b.credits) ?? (trial?.credits_remaining === undefined ? undefined : `credits: ${dollars(trial.credits_remaining)} available`);
+    const credits = creditsLine(b) ?? (trial?.credits_remaining === undefined ? undefined : `credits: ${dollars(trial.credits_remaining)} available`);
     if (credits)
         lines.push(`  ${credits}`);
     if (b.auto_reload)
@@ -461,20 +569,54 @@ function stateFlag(ctx, states) {
 }
 const BUILD_STATES = ["queued", "running", "verified", "built", "rejected", "failed"];
 const RUN_STATES = ["running", "completed", "cap_killed", "aborted"];
+export const FILTER_PAGES = 10;
+export async function filteredPage(page, keep, limit, cursor) {
+    const rows = [];
+    const from = (at) => (at === undefined ? {} : { cursor: at });
+    let at = cursor;
+    let read = 0;
+    for (let n = 0; n < FILTER_PAGES; n++) {
+        const p = await page({ limit: 500, ...from(at) });
+        for (let i = 0; i < p.items.length; i++) {
+            const row = p.items[i];
+            if (!keep(row))
+                continue;
+            if (rows.length < limit) {
+                rows.push(row);
+                continue;
+            }
+            read += i;
+            const nextCursor = i === 0 ? at ?? null : (await page({ limit: i, ...from(at) })).nextCursor ?? at ?? null;
+            return { rows, nextCursor, read, bounded: false };
+        }
+        read += p.items.length;
+        if (!p.nextCursor)
+            return { rows, nextCursor: null, read, bounded: false };
+        at = p.nextCursor;
+    }
+    return { rows, nextCursor: at ?? null, read, bounded: true };
+}
+function moreLines(more, filtered, rows, limit, read, what, bounded = false) {
+    if (!more)
+        return [];
+    return [filtered && (bounded || rows < limit) ? `  the newest ${read} ${what} were read; older ones: --cursor ${more}` : `  more: --cursor ${more}`];
+}
 const snapshotBuilds = async (ctx) => {
     const wanted = stateFlag(ctx, BUILD_STATES);
     const limit = limitOf(ctx, 20, 500);
     const cursor = str(ctx, "--cursor");
-    const page = await client(ctx).snapshots.buildsPage({ limit: wanted === undefined ? limit : 500, ...(cursor === undefined ? {} : { cursor }) });
-    const matched = wanted === undefined ? page.items : page.items.filter((b) => b.state === wanted);
-    const builds = matched.slice(0, limit);
-    const more = builds.length === matched.length ? page.nextCursor : null;
+    const c = client(ctx);
+    const found = wanted === undefined
+        ? await c.snapshots.buildsPage({ limit, ...(cursor === undefined ? {} : { cursor }) }).then((p) => ({ rows: p.items, nextCursor: p.nextCursor, read: p.items.length, bounded: false }))
+        : await filteredPage((o) => c.snapshots.buildsPage(o), (b) => b.state === wanted, limit, cursor);
+    const builds = found.rows;
+    const more = found.nextCursor;
     const next = nexts(...(more ? [[`${sh `promigence snapshot builds --limit ${String(limit)}`}${wanted ? sh ` --state ${wanted}` : ""}${sh ` --cursor ${more}`}`, "the next page"]] : []), ...(builds.length ? [[sh `promigence snapshot status ${builds[0].build_id} --log 40`, "the newest build, with the tail of its log"]] : []));
     const lines = [
         `promigence snapshot builds: ${builds.length}${wanted ? ` in state ${wanted}` : ""}`,
         ...builds.map((b) => `  ${b.build_id}  ${b.state}  ${b.snapshot_id ?? "—"}${buildAliasNote(b)}${b.error?.code ? `  ${b.error.code}` : ""}`),
         ...(builds.length ? [] : ["  (no builds)"]),
-        ...(more ? [`  more: --cursor ${more}`] : []),
+        ...moreLines(more, wanted !== undefined, builds.length, limit, found.read, "builds", found.bounded),
     ];
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, { builds, count: builds.length, next_cursor: more }, next), lines, 0, ctx.pointer);
 };
@@ -492,24 +634,41 @@ const snapshotAlias = async (ctx) => {
     if (clear && name) {
         throw usageError("--clear takes no name", `Use either ${sh `promigence snapshot alias ${arg} NAME`} or ${sh `promigence snapshot alias ${arg} --clear`}.`);
     }
-    const res = await client(ctx).snapshots.alias(arg, clear ? null : name);
+    const moveAlias = bool(ctx, "--move-alias");
+    if (moveAlias && clear) {
+        throw usageError("--move-alias takes a name, not --clear", `Use ${sh `promigence snapshot alias ${arg} NAME --move-alias`}.`);
+    }
+    const res = await client(ctx).snapshots.alias(arg, clear ? null : name, moveAlias ? { moveAlias: true } : {});
     const next = nexts(["promigence snapshot list", "your snapshots and their names"]);
-    return emit(ctx.mode, okEnvelope(ctx.cmd.name, { snapshot_id: res.snapshot_id, alias: res.alias ?? null }, next), [`snapshot ${res.snapshot_id}: ${clear ? "alias cleared" : `alias ${res.alias}`}`], 0, ctx.pointer);
+    const said = res.warnings;
+    const warnings = Array.isArray(said) ? said.filter((w) => typeof w === "string") : [];
+    return emit(ctx.mode, okEnvelope(ctx.cmd.name, { snapshot_id: res.snapshot_id, alias: res.alias ?? null, ...(warnings.length ? { warnings } : {}) }, next), [`snapshot ${res.snapshot_id}: ${clear ? "alias cleared" : `alias ${res.alias}`}`, ...warnings.map((w) => `  warning: ${w}`)], 0, ctx.pointer);
 };
 const runList = async (ctx) => {
     const wanted = stateFlag(ctx, RUN_STATES);
+    const labels = Object.entries(labelsOf(ctx));
     const limit = limitOf(ctx, 20, 500);
     const cursor = str(ctx, "--cursor");
-    const page = await client(ctx).runs.listPage({ limit: wanted === undefined ? limit : 500, ...(cursor === undefined ? {} : { cursor }) });
-    const matched = wanted === undefined ? page.items : page.items.filter((r) => r.state === wanted);
-    const runs = matched.slice(0, limit);
-    const more = runs.length === matched.length ? page.nextCursor : null;
-    const next = nexts(...(more ? [[`${sh `promigence run list --limit ${String(limit)}`}${wanted ? sh ` --state ${wanted}` : ""}${sh ` --cursor ${more}`}`, "the next page"]] : []), ...(runs.length ? [[sh `promigence run report ${runs[0].run_id}`, "the newest run's validity report"]] : []));
+    const filtering = wanted !== undefined || labels.length > 0;
+    const labelsOfRun = (r) => (r.labels ?? {});
+    const labelled = (r) => labels.every(([k, v]) => labelsOfRun(r)[k] === v);
+    const c = client(ctx);
+    const found = filtering
+        ? await filteredPage((o) => c.runs.listPage(o), (r) => (wanted === undefined || r.state === wanted) && labelled(r), limit, cursor)
+        : await c.runs.listPage({ limit, ...(cursor === undefined ? {} : { cursor }) }).then((p) => ({ rows: p.items, nextCursor: p.nextCursor, read: p.items.length, bounded: false }));
+    const runs = found.rows;
+    const more = found.nextCursor;
+    const labelArgs = labels.map(([k, v]) => sh ` --label ${`${k}=${v}`}`).join("");
+    const next = nexts(...(more ? [[`${sh `promigence run list --limit ${String(limit)}`}${wanted ? sh ` --state ${wanted}` : ""}${labelArgs}${sh ` --cursor ${more}`}`, "the next page"]] : []), ...(runs.length ? [[sh `promigence run report ${runs[0].run_id}`, "the newest run's validity report"]] : []));
+    const labelText = (r) => {
+        const own = Object.entries(labelsOfRun(r));
+        return own.length ? `  ${own.map(([k, v]) => `${k}=${v}`).join(",")}` : "";
+    };
     const lines = [
-        `promigence run list: ${runs.length}${wanted ? ` in state ${wanted}` : ""}`,
-        ...runs.map((r) => `  ${r.run_id}  ${r.state}  ${r.count ?? "?"}×  ${r.snapshot_id || r.snapshot || "—"}  ${r.started_at ?? ""}`),
+        `promigence run list: ${runs.length}${wanted ? ` in state ${wanted}` : ""}${labels.length ? ` with ${labels.map(([k, v]) => `${k}=${v}`).join(", ")}` : ""}`,
+        ...runs.map((r) => `  ${r.run_id}  ${r.state}  ${r.count ?? "?"}×  ${r.snapshot_id || r.snapshot || "—"}  ${r.started_at ?? ""}${labelText(r)}`),
         ...(runs.length ? [] : ["  (no runs)"]),
-        ...(more ? [`  more: --cursor ${more}`] : []),
+        ...moreLines(more, filtering, runs.length, limit, found.read, "runs", found.bounded),
     ];
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, { runs, count: runs.length, next_cursor: more }, next), lines, 0, ctx.pointer);
 };
@@ -531,7 +690,14 @@ function labelsOf(ctx) {
     }
     return out;
 }
-const HELD_STATES = ["queued", "placing", "ready", "running", "paused"];
+const HELD_STATES = ["queued", "ready", "running", "paused"];
+export function sandboxListFlags(f) {
+    return [
+        ...(f.limit === undefined ? [] : [sh `--limit ${String(f.limit)}`]),
+        ...(f.states?.length ? [sh `--state ${f.states.join(",")}`] : []),
+        ...Object.entries(f.labels ?? {}).map(([k, v]) => sh `--label ${`${k}=${v}`}`),
+    ].join(" ");
+}
 export function pauseReasonText(reason) {
     if (reason === "spend_cap")
         return "spend cap reached";
@@ -543,8 +709,9 @@ const sandboxList = async (ctx) => {
     const labels = labelsOf(ctx);
     const states = list(ctx, "--state").flatMap((s) => s.split(",")).map((s) => s.trim()).filter(Boolean);
     const cursor = str(ctx, "--cursor");
+    const limit = limitOf(ctx, 50);
     const page = await client(ctx).sandboxes.listPage({
-        limit: limitOf(ctx, 50),
+        limit,
         ...(Object.keys(labels).length ? { labels } : {}),
         ...(states.length ? { state: states } : {}),
         ...(cursor === undefined ? {} : { cursor }),
@@ -564,10 +731,10 @@ const sandboxList = async (ctx) => {
                 held = { count, atLeast: live.nextCursor !== null };
         }
     }
-    const heldCmd = sh `promigence sandbox list --state ${HELD_STATES.join(",")}`;
+    const heldCmd = ["promigence sandbox list", sandboxListFlags({ states: HELD_STATES, labels })].filter(Boolean).join(" ");
     const firstLive = sandboxes.find((s) => s.state !== "ended");
     const next = nexts(...(held ? [[heldCmd, "the sandboxes still held (live or paused) that this page does not show"]] : []), ...(sandboxes.length ? [[sh `promigence sandbox info ${sandboxes[0].sandbox_id}`, "one sandbox: its run, its state and the execs it has run"]] : []), ...(page.nextCursor
-        ? [[sh `promigence sandbox list --cursor ${page.nextCursor}`, "the next page"]]
+        ? [[["promigence sandbox list", sandboxListFlags({ limit, states, labels }), sh `--cursor ${page.nextCursor}`].filter(Boolean).join(" "), "the next page"]]
         : []), ...(firstLive ? [["promigence sandbox kill SID", "end one: put its id from this list in place of SID"]] : []));
     const shown = Object.entries(labels).map(([k, v]) => `${k}=${v}`).join(" ");
     const lines = [
@@ -577,7 +744,7 @@ const sandboxList = async (ctx) => {
             return `  ${s.sandbox_id}  ${s.state}${why ? ` (${why})` : ""}${s.outcome ? ` (${s.outcome})` : ""}  ${s.run_id ?? ""}`;
         }),
         ...(sandboxes.length ? [] : ["  (no sandboxes)"]),
-        ...(page.nextCursor ? [`  more: --cursor ${page.nextCursor}`] : []),
+        ...(page.nextCursor ? [`  more: ${sandboxListFlags({ limit, states, labels })} --cursor ${page.nextCursor}`] : []),
         ...(held ? [`  ${held.atLeast ? "at least " : ""}${held.count} live or paused not on this page: ${heldCmd}`] : []),
     ];
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, {
@@ -642,10 +809,29 @@ export function sandboxFactLines(s, facts, nowMs = Date.now()) {
     if (elapsed !== undefined && facts.perSecond !== undefined) {
         const secs = Math.ceil(elapsed / 1000);
         const rate = `$${facts.perSecond.toFixed(9).replace(/0+$/, "").replace(/\.$/, "")}`;
-        out.push(`  cost so far: about ${usd(secs * facts.perSecond)} (${secs} s at ${rate}/s)`);
+        const billed = typeof s["cost_usd"] === "number" && s["state"] !== "ended" ? `; billed so far ${usd(s["cost_usd"])}` : "";
+        out.push(`  cost so far: about ${usd(secs * facts.perSecond)} (${secs} s at ${rate}/s${billed})`);
     }
     return out;
 }
+export function timeLeftMs(s) {
+    if (s.state === "ended" || typeof s.timeout_ms !== "number" || typeof s.elapsed_ms !== "number")
+        return undefined;
+    return Math.max(0, s.timeout_ms - s.elapsed_ms);
+}
+export function pausedLines(sid, s) {
+    if (s.state !== "paused" || pauseReasonText(s.pause_reason))
+        return [];
+    const wakes = Number(s.auto_pause_s ?? 0) > 0;
+    return [
+        wakes
+            ? "  it has an idle rule, so the next request (exec, files, ssh) wakes it, whether it paused itself when idle or was paused by hand"
+            : `  it stays paused until \`${sh `promigence sandbox resume ${sid}`}\` (it has no idle rule, so a request does not wake it)`,
+        s.pause_expires_at
+            ? `  its files and state are kept until ${s.pause_expires_at} unless it is resumed before then`
+            : "  its files and state are kept at least 30 days from the pause; resuming starts them again",
+    ];
+}
 export function supportStopLines(s) {
     if (s.outcome !== "stopped_by_support")
         return [];
@@ -668,11 +854,15 @@ const sandboxInfo = async (ctx) => {
             : [];
     const next = nexts(...limitNext, ...(execs.length ? [[sh `promigence exec output ${execs[execs.length - 1].exec_id} --sandbox ${sid}`, "what its last command printed"]] : []), ...(ended
         ? (s.run_id ? [[sh `promigence run report ${s.run_id} --sandbox ${sid}`, "how it ended, in its run's report"]] : [])
-        : why ? [] : [[sh `promigence exec --sandbox ${sid} -- CMD`, "run something in it"]]));
+        : why ? []
+            : s.state === "paused" && !(Number(s.auto_pause_s ?? 0) > 0)
+                ? [[sh `promigence sandbox resume ${sid}`, "bring it back; a request does not wake it"]]
+                : [[sh `promigence exec --sandbox ${sid} -- CMD`, "run something in it"]]));
     const lines = [
         `sandbox ${sid}: ${s.state}${why ? ` (${why})` : ""}${s.outcome ? ` (${s.outcome})` : ""}${s.run_id ? ` · run ${s.run_id}` : ""}`,
         ...supportStopLines(s),
         ...(why ? ["  its files and state are kept, and paused time is not billed"] : []),
+        ...pausedLines(sid, s),
         ...queuedLines([s]),
         ...(s.snapshot_id ? [`  snapshot ${s.snapshot_id}`] : []),
         ...sandboxFactLines(s, facts),
@@ -682,15 +872,22 @@ const sandboxInfo = async (ctx) => {
         ...(Number(s.auto_pause_s ?? 0) > 0 ? [`  pauses itself after ${s.auto_pause_s} s idle`] : []),
         ...execs.map((e) => `  ${e.exec_id}  exit ${e.exit_code ?? "—"}  ${(e.argv ?? []).join(" ")}`),
         ...(execs.length ? [] : ["  (no execs recorded)"]),
+        ...(ended ? [] : [`  background commands (exec --background): ${sh `promigence exec ps --sandbox ${sid}`}`]),
     ];
+    const left = timeLeftMs(s);
+    const billed = typeof s["cost_usd"] === "number" ? s["cost_usd"] : undefined;
+    const costSoFar = ended
+        ? billed
+        : facts.perSecond !== undefined && typeof s.elapsed_ms === "number"
+            ? Math.round(Math.ceil(s.elapsed_ms / 1000) * facts.perSecond * 1e6) / 1e6
+            : undefined;
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, {
         sandbox: s,
         ...(facts.tier ? { tier: facts.tier } : {}),
-        ...(facts.perSecond !== undefined && typeof s.elapsed_ms === "number"
-            ? { cost_so_far_usd: Math.round(Math.ceil(s.elapsed_ms / 1000) * facts.perSecond * 1e6) / 1e6 }
-            : {}),
-        ...(ended && typeof s["cost_usd"] === "number"
-            ? { cost_usd: s["cost_usd"], billable_seconds: s["billable_seconds"] }
+        ...(left === undefined ? {} : { time_left_ms: left }),
+        ...(costSoFar === undefined ? {} : { cost_so_far_usd: costSoFar }),
+        ...(ended && billed !== undefined
+            ? { cost_usd: billed, billable_seconds: s["billable_seconds"] }
             : {}),
     }, next), lines, 0, ctx.pointer);
 };
@@ -705,7 +902,7 @@ const sandboxPause = async (ctx) => {
         `sandbox ${sid}: paused${res.already_paused ? " (already was)" : ""}`,
         "  it holds no capacity and bills no seconds while parked",
         ...(took === undefined ? [] : [`  took ${ms(took)}`]),
-        ...(bytes?.allocated === undefined ? [] : [`  parked image: ${Math.round(bytes.allocated / 1048576)} MiB`]),
+        ...(bytes?.allocated === undefined ? [] : [`  holds ${bytes.allocated >= 1073741824 ? `${(bytes.allocated / 1073741824).toFixed(1)} GiB` : `${Math.round(bytes.allocated / 1048576)} MiB`} of storage while paused`]),
         ...(killed > 0 ? [`  ended ${killed} background process${killed === 1 ? "" : "es"} (started with exec --background)`] : []),
         ...(res.warnings ?? []).map((w) => `  warning: ${w}`),
     ];
@@ -789,7 +986,7 @@ const sandboxAutopause = async (ctx) => {
         throw usageError("--mode and --off ask for opposite things", "--off removes the idle rule; --mode says what idle means under one.");
     }
     const rule = (s, m) => s > 0
-        ? `pauses itself after ${s} s idle${m === "activity" ? " (activity: no CPU, no bytes, no open connection)" : ""}`
+        ? `pauses itself after ${s} s idle${m === "activity" ? " (activity: also CPU under 5% of one core, no network bytes, no outside connection)" : ""}`
         : "never pauses itself on idle";
     const next = nexts([sh `promigence sandbox info ${sid}`, "the sandbox as it stands now"]);
     if (after === undefined && !off && mode === undefined) {
@@ -818,18 +1015,122 @@ const sandboxAutopause = async (ctx) => {
     ];
     return emit(ctx.mode, okEnvelope(ctx.cmd.name, { ...res, sandbox_id: sid, auto_pause_s: secs, auto_pause_mode: m, ...(pausedNow ? { paused: true } : {}) }, pausedNow ? nexts([sh `promigence sandbox resume ${sid}`, "bring it back: with the rule off, a request no longer wakes it"]) : next), lines, 0, ctx.pointer);
 };
+export const PROBE_DIGEST_MAX_BYTES = 16 * 1024 * 1024;
+export async function pathNow(c, sid, path) {
+    try {
+        const { stat } = await c.sandboxes.files.stat(sid, path);
+        if (stat.is_dir)
+            return { kind: "dir" };
+        if (stat.is_symlink)
+            return { kind: "symlink" };
+        if (stat.size > PROBE_DIGEST_MAX_BYTES)
+            return { kind: "file", size: stat.size };
+        const bytes = await c.sandboxes.files.get(sid, path);
+        return { kind: "file", size: bytes.length, digest: `blake3:${blake3Hex(bytes)}` };
+    }
+    catch (e) {
+        return e?.code === "not_found" ? { kind: "missing" } : undefined;
+    }
+}
+export function ownDigests(paths) {
+    const out = new Map();
+    for (const p of paths) {
+        const at = p.indexOf("=");
+        if (at > 0)
+            out.set(p.slice(0, at), p.slice(at + 1));
+    }
+    return out;
+}
+export function ownPathOf(bad, own) {
+    for (const p of own.keys()) {
+        const root = p.length > 1 ? p.replace(/\/+$/, "") : p;
+        if (bad === p || bad === root || bad.startsWith(root === "/" ? "/" : `${root}/`))
+            return p;
+    }
+    return undefined;
+}
+function nowWords(now) {
+    if (!now)
+        return undefined;
+    if (now.kind === "missing")
+        return "it does not exist";
+    if (now.kind === "dir")
+        return "it is a directory";
+    if (now.kind === "symlink")
+        return "it is a symbolic link";
+    return now.digest ? `it is ${now.digest} now` : `it is a file of ${now.size} bytes`;
+}
+async function probeRefusal(ctx, c, sid, e) {
+    if (!(e instanceof PromigenceError) || e.code !== "usage")
+        throw e;
+    const later = (path, digest) => sh `promigence sandbox probe ${sid} --path ${`${path}=${digest}`}`;
+    if (/has nothing to compare against/.test(e.message)) {
+        throw usageError(e.message, `Name the files to check: ${sh `promigence sandbox probe ${sid} --path /work/FILE`} answers with the file's digest now, ` +
+            "and --path /work/FILE=<that digest> checks later that it has not changed.");
+    }
+    const m = /^(.+): the snapshot recorded no digest for this path\b/.exec(e.message);
+    if (!m)
+        throw e;
+    const path = m[1];
+    const now = await pathNow(c, sid, path);
+    const digest = now?.kind === "file" ? now.digest : undefined;
+    const fix = digest
+        ? `Its digest now is ${digest}. To check later that it has not changed: ${later(path, digest)}`
+        : now?.kind === "missing"
+            ? `It does not exist in sandbox ${sid}: check the path (${sh `promigence files ls / --sandbox ${sid}`}).`
+            : now?.kind === "file"
+                ? `It is over ${PROBE_DIGEST_MAX_BYTES / 1048576} MiB, too large to read back here: name it as PATH=blake3:<hex> with a digest you computed (b3sum).`
+                : `Name it as PATH=blake3:<hex> (${nowWords(now) ?? "the digest of the file's bytes"}).`;
+    const err = usageError(e.message, fix);
+    const next = digest ? nexts([later(path, digest), "check later that the file has not changed"]) : nexts([sh `promigence sandbox info ${sid}`, "the sandbox and its snapshot"]);
+    return emit(ctx.mode, {
+        ...errorEnvelope(ctx.cmd.name, err, next),
+        sandbox_id: sid,
+        ...(digest ? { digests: [{ path, digest }] } : {}),
+    }, errorTextLines(ctx.cmd.name, err, 2, next), 2, ctx.pointer);
+}
 const sandboxProbe = async (ctx) => {
     const sid = positional(ctx, "a sandbox id", "promigence sandbox probe sbx_01");
     const paths = list(ctx, "--path");
-    const res = await client(ctx).sandboxes.probe(sid, ...(paths.length ? [{ paths }] : []));
+    const own = ownDigests(paths);
+    const c = client(ctx);
+    const res = await c.sandboxes.probe(sid, ...(paths.length ? [{ paths }] : [])).catch((e) => probeRefusal(ctx, c, sid, e));
     const ok = res.ok !== false;
-    const env = { ...okEnvelope(ctx.cmd.name, { ...res, sandbox_id: sid }, nexts([sh `promigence sandbox info ${sid}`, "the sandbox, its run and the commands it ran"])), ok };
-    const lines = [
-        `sandbox ${sid}: ${ok ? "matches the verified environment" : "DIFFERS from the verified environment"}`,
-        ...(res.first_bad_path ? [`  first difference at ${res.first_bad_path}`] : []),
-        ...(res.ms === undefined ? [] : [`  checked in ${ms(res.ms)}`]),
-    ];
-    return emit(ctx.mode, env, lines, ok ? 0 : 3, ctx.pointer);
+    const infoNext = nexts([sh `promigence sandbox info ${sid}`, "the sandbox, its run and the commands it ran"]);
+    const took = res.ms === undefined ? [] : [`  checked in ${ms(res.ms)}`];
+    if (ok) {
+        const allOwn = paths.length > 0 && own.size === paths.length;
+        return emit(ctx.mode, okEnvelope(ctx.cmd.name, { ...res, sandbox_id: sid }, infoNext), [
+            `sandbox ${sid}: ${allOwn ? "every path matches the digest you gave" : "matches the verified environment"}`,
+            ...took,
+        ], 0, ctx.pointer);
+    }
+    const bad = res.first_bad_path ?? undefined;
+    const mine = bad === undefined ? undefined : ownPathOf(bad, own);
+    const now = bad === undefined ? undefined : await pathNow(c, sid, bad);
+    const words = nowWords(now);
+    const where = bad === undefined ? "" : bad;
+    const err = mine !== undefined
+        ? new PromigenceError({
+            code: "digest_mismatch", exitCode: 4, docs: `${ERRORS_DOCS_URL}#digest_mismatch`,
+            message: now?.kind === "missing"
+                ? `${where} does not exist in sandbox ${sid}, so it cannot match the digest you gave${mine !== bad ? ` for ${mine}` : ""}`
+                : `${mine !== bad ? `${mine} does not match the digest you gave, first at ${where}` : `${where} does not match the digest you gave`} (${own.get(mine)})${words ? `: ${words}` : ""}`,
+            fix: now?.kind === "missing"
+                ? `Nothing failed on our side. Check the path (${sh `promigence files ls / --sandbox ${sid}`}), or what should have made the file.`
+                : "Nothing failed on our side: the file is not what that digest says. Check the digest (blake3 of the file's bytes) and the path, or what produced the file.",
+        })
+        : new PromigenceError({
+            code: "integrity_mismatch", exitCode: 3, docs: `${ERRORS_DOCS_URL}#integrity_mismatch`,
+            message: `sandbox ${sid} differs from its verified snapshot${where ? ` at ${where}` : ""}${words ? ` (${words})` : ""}`,
+            fix: "If a command you ran changed it, that is expected. If nothing did, this sandbox is not the verified environment: " +
+                "start a new one from the same snapshot and probe it; if that one differs too, rebuild the snapshot and tell support@promigence.ai the sandbox id.",
+        });
+    return emit(ctx.mode, {
+        ...errorEnvelope(ctx.cmd.name, err, infoNext),
+        sandbox_id: sid, checked: res.checked, ms: res.ms, first_bad_path: bad ?? null,
+        difference: { path: bad ?? null, ...(mine === undefined ? { against: "snapshot" } : { against: "your_digest", digest_given: own.get(mine) }), now: now ?? null },
+    }, [...errorTextLines(ctx.cmd.name, err, err.cliExit, infoNext), ...took], err.cliExit, ctx.pointer);
 };
 const filesRm = async (ctx) => {
     const path = positional(ctx, "a path", "promigence files rm /work/out.log --sandbox sbx_01");
@@ -839,8 +1140,14 @@ const filesRm = async (ctx) => {
     const sid = required(ctx, "--sandbox", "Pass --sandbox sbx_<id> — the sandbox the path is in.");
     const recursive = bool(ctx, "--recursive");
     const res = await client(ctx).sandboxes.files.delete(sid, path, { recursive });
-    const next = nexts([sh `promigence files ls --sandbox ${sid}`, "what is left"]);
-    return emit(ctx.mode, okEnvelope(ctx.cmd.name, { sandbox_id: sid, path, deleted: res.deleted, existed: res.existed, recursive }, next), [`${sid}:${path}: ${res.deleted ? "deleted" : "not deleted"}${res.existed === false ? " (it was not there)" : ""}`], 0, ctx.pointer);
+    const gone = res.existed === false;
+    const dir = path.replace(/\/+$/, "").replace(/\/[^/]*$/, "") || "/";
+    const next = gone
+        ? nexts([sh `promigence files ls ${dir} --sandbox ${sid}`, "check the path: what is in its directory"])
+        : nexts([sh `promigence files ls --sandbox ${sid}`, "what is left"]);
+    return emit(ctx.mode, okEnvelope(ctx.cmd.name, { sandbox_id: sid, path, deleted: res.deleted, existed: res.existed, recursive }, next), [gone
+            ? `${sid}:${path}: not there, nothing was deleted (check the path)`
+            : `${sid}:${path}: ${res.deleted ? "deleted" : "not deleted"}`], 0, ctx.pointer);
 };
 const secretsList = async (ctx) => {
     const { secrets } = await client(ctx).secrets.list();