rightsize 0.7.8 → 0.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/backend-docker/backend.d.ts +4 -1
  2. package/dist/backend-docker/backend.d.ts.map +1 -1
  3. package/dist/backend-docker/backend.js +4 -0
  4. package/dist/backend-docker/backend.js.map +1 -1
  5. package/dist/backend-msb/backend.d.ts +618 -68
  6. package/dist/backend-msb/backend.d.ts.map +1 -1
  7. package/dist/backend-msb/backend.js +1236 -133
  8. package/dist/backend-msb/backend.js.map +1 -1
  9. package/dist/backend-msb/commands.d.ts +160 -16
  10. package/dist/backend-msb/commands.d.ts.map +1 -1
  11. package/dist/backend-msb/commands.js +197 -31
  12. package/dist/backend-msb/commands.js.map +1 -1
  13. package/dist/backend-msb/provisioner.d.ts +1 -1
  14. package/dist/backend-msb/provisioner.d.ts.map +1 -1
  15. package/dist/backend-msb/provisioner.js +1 -1
  16. package/dist/backend-msb/provisioner.js.map +1 -1
  17. package/dist/backend-msb/restore-access-denied.d.ts +50 -0
  18. package/dist/backend-msb/restore-access-denied.d.ts.map +1 -0
  19. package/dist/backend-msb/restore-access-denied.js +52 -0
  20. package/dist/backend-msb/restore-access-denied.js.map +1 -0
  21. package/dist/backend-msb/restore-broker.d.ts +181 -0
  22. package/dist/backend-msb/restore-broker.d.ts.map +1 -0
  23. package/dist/backend-msb/restore-broker.js +264 -0
  24. package/dist/backend-msb/restore-broker.js.map +1 -0
  25. package/dist/backend-msb/sandbox-already-exists.d.ts +32 -0
  26. package/dist/backend-msb/sandbox-already-exists.d.ts.map +1 -0
  27. package/dist/backend-msb/sandbox-already-exists.js +35 -0
  28. package/dist/backend-msb/sandbox-already-exists.js.map +1 -0
  29. package/dist/backend-msb/snapshot-create.d.ts +22 -0
  30. package/dist/backend-msb/snapshot-create.d.ts.map +1 -0
  31. package/dist/backend-msb/snapshot-create.js +33 -0
  32. package/dist/backend-msb/snapshot-create.js.map +1 -0
  33. package/dist/backend-msb/snapshot-import.d.ts +30 -14
  34. package/dist/backend-msb/snapshot-import.d.ts.map +1 -1
  35. package/dist/backend-msb/snapshot-import.js +35 -17
  36. package/dist/backend-msb/snapshot-import.js.map +1 -1
  37. package/dist/backend-msb/snapshot-rm.d.ts +28 -0
  38. package/dist/backend-msb/snapshot-rm.d.ts.map +1 -0
  39. package/dist/backend-msb/snapshot-rm.js +30 -0
  40. package/dist/backend-msb/snapshot-rm.js.map +1 -0
  41. package/dist/core/backend.d.ts +53 -22
  42. package/dist/core/backend.d.ts.map +1 -1
  43. package/dist/core/checkpoint/api.d.ts +2 -2
  44. package/dist/core/checkpoint/api.js +2 -2
  45. package/dist/core/checkpoint/registry.d.ts +34 -0
  46. package/dist/core/checkpoint/registry.d.ts.map +1 -1
  47. package/dist/core/checkpoint/registry.js +20 -1
  48. package/dist/core/checkpoint/registry.js.map +1 -1
  49. package/dist/core/errors.d.ts +49 -0
  50. package/dist/core/errors.d.ts.map +1 -1
  51. package/dist/core/errors.js +61 -0
  52. package/dist/core/errors.js.map +1 -1
  53. package/dist/core/generic-container.d.ts +58 -10
  54. package/dist/core/generic-container.d.ts.map +1 -1
  55. package/dist/core/generic-container.js +222 -27
  56. package/dist/core/generic-container.js.map +1 -1
  57. package/dist/core/model.d.ts +3 -2
  58. package/dist/core/model.d.ts.map +1 -1
  59. package/dist/core/sandbox-name.d.ts +3 -0
  60. package/dist/core/sandbox-name.d.ts.map +1 -0
  61. package/dist/core/sandbox-name.js +19 -0
  62. package/dist/core/sandbox-name.js.map +1 -0
  63. package/dist/index.d.ts +1 -1
  64. package/dist/index.d.ts.map +1 -1
  65. package/dist/index.js +1 -1
  66. package/dist/index.js.map +1 -1
  67. package/dist/modules/minio.d.ts +5 -3
  68. package/dist/modules/minio.d.ts.map +1 -1
  69. package/dist/modules/minio.js +14 -4
  70. package/dist/modules/minio.js.map +1 -1
  71. package/package.json +1 -1
  72. package/dist/backend-msb/snapshot-list.d.ts +0 -43
  73. package/dist/backend-msb/snapshot-list.d.ts.map +0 -1
  74. package/dist/backend-msb/snapshot-list.js +0 -53
  75. package/dist/backend-msb/snapshot-list.js.map +0 -1
@@ -2,7 +2,10 @@ import { spawn, spawnSync } from "node:child_process";
2
2
  import * as readline from "node:readline";
3
3
  import * as fs from "node:fs/promises";
4
4
  import * as path from "node:path";
5
- import { BackendError, PortBindConflictError, UnsupportedByBackendError, TmpfsRootCheckpointError } from "../core/errors.js";
5
+ import { BackendError, PortBindConflictError, UnsupportedByBackendError, TmpfsRootCheckpointError, CheckpointWorkloadCommandMissingError, } from "../core/errors.js";
6
+ import { cacheDir } from "../core/cache-dir.js";
7
+ import { nextSandboxName } from "../core/sandbox-name.js";
8
+ import { trackSandbox, untrackSandbox } from "../core/reaper/init.js";
6
9
  import { MsbCommands } from "./commands.js";
7
10
  import { runningNames, statusOf } from "./ls-json.js";
8
11
  import { hasSandboxStartedMarker } from "./fast-exit.js";
@@ -12,12 +15,16 @@ import { isImageCacheCorruption } from "./image-cache.js";
12
15
  import { isMsbStateDbError } from "./state-db.js";
13
16
  import { isAgentEndpointNotReady } from "./agent-endpoint.js";
14
17
  import { isSnapshotNotFoundError } from "./snapshot-not-found.js";
15
- import { isSnapshotAlreadyExistsError, parseImportedDigestDirName } from "./snapshot-import.js";
18
+ import { isSnapshotAlreadyExistsError, parseImportedArtifactPath } from "./snapshot-import.js";
16
19
  import { isSnapshotSaveAccessDeniedFailure, salvageStagedArchive } from "./snapshot-save-fsync.js";
17
- import { parseSnapshotList, confirmDigestDirNamePresent } from "./snapshot-list.js";
20
+ import { parseSnapshotCreateArtifactPath } from "./snapshot-create.js";
21
+ import { isSnapshotHeadRemovalRefused } from "./snapshot-rm.js";
18
22
  import { undeliveredLines } from "./follow-replay.js";
19
23
  import { requireNoDuplicateGuestPorts, requireAliasesAreValid, hostsAliasScript } from "./network-links.js";
20
24
  import { ExecTunnel } from "./exec-tunnel.js";
25
+ import { isRestoreAccessDeniedFailure } from "./restore-access-denied.js";
26
+ import { isSandboxAlreadyExistsFailure } from "./sandbox-already-exists.js";
27
+ import { realRestoreBroker, } from "./restore-broker.js";
21
28
  const FIRST_RUN_PULL_TIMEOUT_MS = 600_000; // a cold pull can be slow
22
29
  const READINESS_POLL_MS = 300;
23
30
  const STOP_TIMEOUT_MS = 60_000;
@@ -44,6 +51,51 @@ const COPY_TIMEOUT_MS = 120_000;
44
51
  // already confirmed no longer Running. Never a wait-for-content budget: a
45
52
  // stopped sandbox's log cannot grow, so the first successful fetch is final.
46
53
  const TERMINAL_FETCH_FAILURE_BUDGET_MS = 10_000;
54
+ // How long `reviveWorkload` gives a freshly-spawned workload-revival exec
55
+ // child to prove it isn't an immediate boot failure before treating it as
56
+ // the ordinary long-lived case. Unlike `bootRunOnce`'s own attached child,
57
+ // there is no separate "reached Running" signal to poll for here — the
58
+ // SANDBOX already reports Running regardless of whether this exec succeeds
59
+ // — so a short settle window is what that same exit-vs-success race
60
+ // collapses to when the only observable signal left is the child's own
61
+ // exit. A few multiples of the ordinary poll cadence is plenty: a failing
62
+ // workload (bad command, missing binary, an immediate usage error) exits
63
+ // within milliseconds of spawn, while a genuine long-lived server never
64
+ // exits on its own at all.
65
+ const EXEC_REVIVE_SETTLE_MS = READINESS_POLL_MS * 3;
66
+ // Bounded retry policy for `msb restore` hitting a Windows access-denied
67
+ // failure on its own just-written snapshot artifact (see
68
+ // `isRestoreAccessDeniedFailure`) — a brief file-handle release lag that
69
+ // normally clears within one retry, so several short-backoff attempts cover
70
+ // the real cases without masking a genuinely stuck lock.
71
+ const RESTORE_ACCESS_DENIED_RETRY_LIMIT = 3;
72
+ const RESTORE_ACCESS_DENIED_RETRY_DELAY_MS = 500;
73
+ /**
74
+ * How long `createCheckpoint`'s own reboot step keeps retrying msb's
75
+ * "sandbox already exists" refusal (see `isSandboxAlreadyExistsFailure`) and
76
+ * its Windows access-denied refusal on the just-written snapshot artifact
77
+ * (see `isRestoreAccessDeniedFailure`), and the pause between attempts — the
78
+ * same install-lock-poll shape `INSTALL_LOCK_RETRY_BUDGET_MS`/
79
+ * `INSTALL_LOCK_RETRY_DELAY_MS` already use. EMPIRICALLY VERIFIED against a
80
+ * real msb 0.7.1 binary (an isolated `MSB_HOME` testbed): `msb restore
81
+ * --name X` validates the artifact FIRST — an integrity failure exits 1 and
82
+ * leaves NO sandbox record — but a failure AFTER validation (matching either
83
+ * classified signature above) leaves `X` behind as a STOPPED SANDBOX RECORD
84
+ * visible in `msb ls`, and msb's own restore-time collision check
85
+ * (`existing.is_some() || dir_exists`, `prepare_create_target` in
86
+ * `sdk/rust/lib/backend/local/sandbox/create.rs`) then refuses any retry
87
+ * under that SAME name — on Windows CI this refusal has been observed
88
+ * holding for the whole span of a same-name retry loop under load. Each
89
+ * retry attempt therefore mints a BRAND NEW name rather than reusing the one
90
+ * that just failed (see `rebootUnderFreshName`), which sidesteps that exact
91
+ * collision structurally — a name nothing else has ever used cannot collide
92
+ * with a record or a lingering on-disk directory belonging to a name nothing
93
+ * will ever restore under again. This budget bounds how long that
94
+ * fresh-name retry loop as a whole keeps advancing before giving up, not a
95
+ * single name's own collision.
96
+ */
97
+ const CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS = 30_000;
98
+ const CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_DELAY_MS = 2_000;
47
99
  /**
48
100
  * The boot failure `start()` heals and retries — carries the `msb run`
49
101
  * child's combined output for the second-failure diagnostic. Internal to the
@@ -106,6 +158,45 @@ class InstallLockActiveError extends Error {
106
158
  this.output = output;
107
159
  }
108
160
  }
161
+ /**
162
+ * Restore-path classified failure for `isRestoreAccessDeniedFailure` — a
163
+ * Windows-only `msb restore` transient against its own just-written
164
+ * snapshot artifact (see that function's own doc). Internal to the boot
165
+ * path, like its siblings above; `bootClassified` owns the bounded-retry
166
+ * policy. Only ever thrown from `bootRestoreOnce`, since it is a restore
167
+ * invocation's own failure signature — an ordinary `run` boot never touches
168
+ * a snapshot artifact at all.
169
+ */
170
+ class RestoreAccessDeniedError extends Error {
171
+ output;
172
+ constructor(output) {
173
+ super(`msb restore access-denied on its own snapshot artifact:\n${output}`);
174
+ this.output = output;
175
+ }
176
+ }
177
+ /**
178
+ * Restore-path classified failure for `isSandboxAlreadyExistsFailure` — msb
179
+ * refusing `restore` because a sandbox under this name already exists (its
180
+ * database record, its on-disk directory, or both — see that function's own
181
+ * doc). Internal to the boot path, like its siblings above; only ever
182
+ * thrown from `bootRestoreOnce`. `bootClassified` itself never retries this
183
+ * one (unlike its siblings) — it simply propagates, so the ordinary
184
+ * `start()` path (a `GenericContainer.fromCheckpoint(cp).start()` restore
185
+ * of a name that turns out to still be live) surfaces it immediately, a
186
+ * real error. Only `createCheckpoint`'s own reboot step, via
187
+ * `rebootUnderFreshName`, retries it — under a NEWLY minted name each time,
188
+ * never the one that just collided (see that method's own doc) — a caller
189
+ * reusing a live name is never this backend's own race to hide, but the
190
+ * checkpoint cycle's own historical `rm`-then-restore of the SAME name was
191
+ * exactly that race.
192
+ */
193
+ class SandboxAlreadyExistsError extends Error {
194
+ output;
195
+ constructor(output) {
196
+ super(`msb restore refused — a sandbox with this name already exists:\n${output}`);
197
+ this.output = output;
198
+ }
199
+ }
109
200
  /**
110
201
  * How long to wait before retrying a boot that hit msb's state-database
111
202
  * error — enough for a winning concurrent invocation's migration transaction
@@ -192,16 +283,17 @@ function describeHeal(heal) {
192
283
  }
193
284
  /**
194
285
  * True when `ref` (an absolute path ref) looks like a checkpoint artifact
195
- * this backend itself would have written — a directory named `rz-ckpt-*`
196
- * (`checkpointRef`'s own prefix) containing a `snapshot.json` file, the same
197
- * shape `hasCheckpoint` checks for. `removeCheckpoint`'s recursive delete is
198
- * gated on this: `ref` is caller-supplied (a corrupt or hand-edited registry
199
- * entry), and skipping the shape check would let an arbitrary path get
200
- * `fs.rm(..., { recursive: true })`'d just because it happened to be passed
201
- * in as a "ref".
286
+ * this backend itself would have written — a directory named `snap_<hex>`
287
+ * (msb's own snapshot-store naming since 0.7.1 — see
288
+ * `parseSnapshotCreateArtifactPath`) containing a `snapshot.json` file, the
289
+ * same shape `hasCheckpoint` checks for. `removeCheckpoint`'s recursive
290
+ * delete is gated on this: `ref` is caller-supplied (a corrupt or
291
+ * hand-edited registry entry), and skipping the shape check would let an
292
+ * arbitrary path get `fs.rm(..., { recursive: true })`'d just because it
293
+ * happened to be passed in as a "ref".
202
294
  */
203
295
  async function looksLikeCheckpointArtifactDir(ref) {
204
- if (!path.basename(ref).startsWith("rz-ckpt-")) {
296
+ if (!/^snap_[0-9a-f]+$/i.test(path.basename(ref))) {
205
297
  return false;
206
298
  }
207
299
  const stat = await fs.stat(ref).catch(() => undefined);
@@ -234,6 +326,90 @@ function drainTail(stream, tail) {
234
326
  rl.on("close", () => resolveDrain());
235
327
  });
236
328
  }
329
+ /**
330
+ * `drainTail`'s sibling for text this backend already has in hand rather
331
+ * than a live stream — a brokered restore's own combined output
332
+ * (`launchRestoreViaBroker`), or a diagnostic note about the broker itself.
333
+ * Same `TAIL_LINES` cap, same shift-oldest behavior, so a brokered attempt's
334
+ * pre-Running diagnostics tail looks exactly like a direct attempt's
335
+ * regardless of which path produced it.
336
+ */
337
+ function pushLinesToTail(tail, text) {
338
+ for (const line of text.split("\n")) {
339
+ tail.push(line);
340
+ if (tail.length > TAIL_LINES) {
341
+ tail.shift();
342
+ }
343
+ }
344
+ }
345
+ /**
346
+ * Busybox-ash-compatible: finds the first non-kernel child of PID 1 in the
347
+ * guest and prints its `/proc/<pid>/cmdline` RAW — NUL-separated with a
348
+ * trailing NUL, the kernel's own on-disk shape, never re-encoded by this
349
+ * script itself — to stdout, exit 0; exits 1 if none is found. Walks
350
+ * every `/proc/<pid>/stat` in NUMERIC pid order (`ls | sort -n`, since a bare
351
+ * shell glob sorts lexically — "10" before "2" — which is wrong for "the
352
+ * first child") and, for each, reads field 4 (`ppid`, 1-indexed per
353
+ * `/proc/pid/stat`'s own `pid (comm) state ppid ...` layout), skipping any
354
+ * whose `ppid` isn't `1`. `comm` is split out via the FIRST `(` and the LAST
355
+ * `)` (`${st#*(}` / `${comm%)*}`, both plain POSIX parameter expansion, no
356
+ * `extglob` needed) rather than a naive single-`(`/`)` split, since a
357
+ * process name may itself contain spaces (or, in principle, parens) — the
358
+ * same robustness a greedy sed capture between the first "(" and the last
359
+ * ")" would get. Two names are excluded even though
360
+ * their `ppid` may show `1`: the guest's own init (`init.krun`) and any
361
+ * bracketed kernel-thread name (`[kworker/0:1]`, `[ksoftirqd/0]`, ...) — the
362
+ * shape every kernel thread's `comm` renders as. Read by `MsbCliBackend`'s
363
+ * own `createCheckpoint`, BEFORE it stops the source sandbox — see that
364
+ * method's own doc — via a plain `exec`, never anything backend-specific
365
+ * beyond that: this is guest-side shell, not msb CLI surface.
366
+ */
367
+ const CAPTURE_WORKLOAD_CMDLINE_SCRIPT = [
368
+ // A marker comment, not functional shell — lets the msb fixture (and any
369
+ // future test double) recognize this exact exec call by its script
370
+ // content rather than guessing from argv shape alone.
371
+ "# rightsize:capture-workload-cmdline",
372
+ "for pid in $(ls /proc 2>/dev/null | grep -E '^[0-9]+$' | sort -n); do",
373
+ " st=$(cat /proc/$pid/stat 2>/dev/null) || continue",
374
+ " rest=${st##*) }",
375
+ " set -- $rest",
376
+ " ppid=$2",
377
+ " if [ \"$ppid\" != \"1\" ]; then continue; fi",
378
+ " comm=${st#*(}",
379
+ " comm=${comm%)*}",
380
+ " case \"$comm\" in",
381
+ " init.krun|\\[*\\]) continue ;;",
382
+ " esac",
383
+ " cat /proc/$pid/cmdline",
384
+ " exit 0",
385
+ "done",
386
+ "exit 1",
387
+ ].join("\n");
388
+ /**
389
+ * Parses `CAPTURE_WORKLOAD_CMDLINE_SCRIPT`'s stdout — the discovered
390
+ * process's raw `/proc/<pid>/cmdline` bytes, NUL-separated with a trailing
391
+ * NUL — into an argv. `captureGuestWorkloadCmdline` fetches this through the
392
+ * ordinary `exec()` (`invoke()`'s line-based reconstruction, not
393
+ * `fetchStdoutExact`'s byte-exact one — see that function's own doc), which
394
+ * appends a trailing `\n` to any non-empty output; since the guest script's
395
+ * own output never contains a real newline of its own (it is exactly the
396
+ * NUL-joined cmdline bytes), that single appended `\n` is always the whole
397
+ * of `invoke()`'s own last "line" and is stripped here before splitting,
398
+ * rather than being read back as a bogus trailing argv element. `undefined`
399
+ * on anything that isn't a clean, non-empty argv after that: empty output
400
+ * (the script found no matching child, exited 1, or the guest's own cmdline
401
+ * was itself empty) or content that splits into zero non-empty tokens. Never
402
+ * throws — see `captureGuestWorkloadCmdline`'s own doc on why a capture
403
+ * failure must never fail the checkpoint that triggered it.
404
+ */
405
+ function parseCapturedWorkloadCmdline(stdout) {
406
+ if (stdout.length === 0) {
407
+ return undefined;
408
+ }
409
+ const withoutTrailingNewline = stdout.endsWith("\n") ? stdout.slice(0, -1) : stdout;
410
+ const parts = withoutTrailingNewline.split("\0").filter((s) => s.length > 0);
411
+ return parts.length > 0 ? parts : undefined;
412
+ }
237
413
  /**
238
414
  * The attached-mode CLI driver: every sandbox this backend starts runs as a
239
415
  * held child process (`msb run`, no `-d`) because detached mode never
@@ -265,6 +441,10 @@ export class MsbCliBackend {
265
441
  /** The stop/snapshot/reboot cycle reboots the microVM — the workload restarts. */
266
442
  checkpointRestartsWorkload: true,
267
443
  };
444
+ // Both keyed by a sandbox's CURRENT name (== SandboxHandle.id) — normally
445
+ // stable for a handle's whole lifetime, except across createCheckpoint's
446
+ // own reboot, which re-keys both from the source sandbox's name to the
447
+ // fresh one it boots under (see that method's own doc).
268
448
  handles = new Map();
269
449
  startedNames = new Set();
270
450
  // Mirrors msbPathPromise's eventual value as soon as it settles, purely so
@@ -272,8 +452,27 @@ export class MsbCliBackend {
272
452
  // anything) has a best-effort synchronous read of it. Never written to
273
453
  // except by this one .then() below; never awaited anywhere else.
274
454
  resolvedMsbPath;
275
- constructor(msbPathPromise) {
455
+ // Test-only override seam for rebootUnderFreshName's own budget/delay
456
+ // — defaults to the real CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS/
457
+ // _DELAY_MS constants so production behavior is unchanged. Without this, a
458
+ // budget-exhaustion red-proof would mean a unit test actually blocking for
459
+ // the real ~30s; a test reaches these the same way it already reaches
460
+ // `handles`/`startedNames` elsewhere in this suite — an unsafe cast — to
461
+ // shrink them to milliseconds instead.
462
+ checkpointRebootAlreadyExistsRetryBudgetMs = CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS;
463
+ checkpointRebootAlreadyExistsRetryDelayMs = CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_DELAY_MS;
464
+ // POLICY v2's broker escalation — see `RestoreLaunchMode`/`restore-broker.ts`.
465
+ restoreBroker;
466
+ // Test-only override for the Windows-only escalation gate
467
+ // (`isWindowsPlatform`) — `undefined` in production, where the real
468
+ // `process.platform` always decides. Reached the same unsafe-cast way this
469
+ // suite already reaches `checkpointRebootAlreadyExistsRetryBudgetMs`
470
+ // above, so both the escalation trigger AND the "non-Windows never
471
+ // brokers" invariant are exercisable without an actual Windows host.
472
+ platformOverrideForTests = undefined;
473
+ constructor(msbPathPromise, options = {}) {
276
474
  this.msbPathPromise = msbPathPromise;
475
+ this.restoreBroker = options.restoreBroker ?? realRestoreBroker;
277
476
  this.msbPathPromise.then((p) => {
278
477
  this.resolvedMsbPath = p;
279
478
  }, () => {
@@ -282,11 +481,24 @@ export class MsbCliBackend {
282
481
  // when it awaits msbPathPromise itself.
283
482
  });
284
483
  }
484
+ /**
485
+ * Whether this process should treat itself as running on Windows for
486
+ * POLICY v2's broker-escalation gate — `process.platform === "win32"` in
487
+ * production, or `platformOverrideForTests` when a test has set it. The
488
+ * ONLY thing this gates is whether `rebootUnderFreshName`/
489
+ * `retryRestoreAfterAccessDenied` ever set `RestoreLaunchMode` to
490
+ * `"broker"`; `bootOnce`/`bootRestoreOnce` themselves stay platform-
491
+ * agnostic dispatchers on whatever mode they're handed, so the broker path
492
+ * itself is exercisable on any host once a test picks it explicitly.
493
+ */
494
+ isWindowsPlatform() {
495
+ return (this.platformOverrideForTests ?? process.platform) === "win32";
496
+ }
285
497
  async msbPath() {
286
498
  return this.msbPathPromise;
287
499
  }
288
500
  async create(spec) {
289
- this.handles.set(spec.name, { attached: undefined, attachedExited: false, resources: [], logTail: [] });
501
+ this.handles.set(spec.name, { attached: undefined, attachedExited: false, resources: [], logTail: [], capturedCommand: undefined });
290
502
  return { id: spec.name, spec };
291
503
  }
292
504
  /**
@@ -331,14 +543,48 @@ export class MsbCliBackend {
331
543
  * knows: the install-lock poll, the one-shot state-database retry, and the
332
544
  * one-shot image-cache heal. Both the ordinary `start()` path and the
333
545
  * checkpoint cycle's post-snapshot reboot come through here — a reboot
334
- * from a snapshot is as exposed to msb's transients as any other boot,
335
- * and skipping the classification there turned a passing install-lock
336
- * poll into an immediate checkpoint failure on a live Windows run.
546
+ * from a snapshot (now `msb restore ...`, see `bootRestoreOnce`) is
547
+ * as exposed to msb's transients as any other boot, and skipping the
548
+ * classification there turned a passing install-lock poll into an
549
+ * immediate checkpoint failure on a live Windows run. The image-cache heal
550
+ * targets `handle.spec.image`, which for a restore is the checkpoint ref
551
+ * GenericContainer.fromCheckpoint() threaded through as the builder's
552
+ * "image" — never a real OCI reference `msb image remove` can act on
553
+ * meaningfully; this predates the 0.7.1 migration (the same was true of
554
+ * `--from-snapshot` boots) and is unchanged here.
555
+ *
556
+ * `retryAccessDenied` (default `true`) gates ONLY the
557
+ * `RestoreAccessDeniedError` branch below — every other classified retry
558
+ * always runs regardless. The ordinary `start()` path (including a plain
559
+ * `GenericContainer.fromCheckpoint(cp).start()` restore) leaves it at the
560
+ * default, delegating to `retryRestoreAfterAccessDenied` — see that
561
+ * method's own doc for why it, like `createCheckpoint`'s own reboot, must
562
+ * mint a fresh name for every retry rather than reusing the one that just
563
+ * failed: the live-verified dossier this backend's fresh-naming behavior
564
+ * is built from says a restore that fails PAST msb's own artifact
565
+ * validation (this access-denied signature included) leaves the attempted
566
+ * name behind as a stopped sandbox record, so a same-name retry only ever
567
+ * collides with that record instead of retrying the actual transient.
568
+ * `createCheckpoint`'s own reboot (`rebootUnderFreshName`) passes `false`:
569
+ * it owns this error class itself, one layer up, via its own fresh-naming
570
+ * loop, and needs this error to propagate on the first hit rather than
571
+ * being retried here at all.
572
+ *
573
+ * `restoreLaunchMode` (default `"direct"`) is threaded straight down to
574
+ * every `bootOnce` call this method makes, including its own internal
575
+ * install-lock/state-db/image-cache retries — POLICY v2's broker
576
+ * escalation, once triggered by the caller (`rebootUnderFreshName`/
577
+ * `retryRestoreAfterAccessDenied`), applies to EVERY remaining attempt of
578
+ * that reboot, not just the ones those callers themselves loop over, so an
579
+ * install-lock hit mid-brokered-attempt must retry brokered too rather
580
+ * than silently reverting to a direct spawn. See `RestoreLaunchMode`'s own
581
+ * doc.
337
582
  */
338
- async bootClassified(msbPath, handle, state) {
583
+ async bootClassified(msbPath, handle, state, options = { retryAccessDenied: true }) {
584
+ const restoreLaunchMode = options.restoreLaunchMode ?? "direct";
339
585
  let firstOutput;
340
586
  try {
341
- await this.bootOnce(msbPath, handle, state);
587
+ await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
342
588
  return;
343
589
  }
344
590
  catch (first) {
@@ -355,7 +601,7 @@ export class MsbCliBackend {
355
601
  while (Date.now() < deadline) {
356
602
  await sleep(INSTALL_LOCK_RETRY_DELAY_MS);
357
603
  try {
358
- await this.bootOnce(msbPath, handle, state);
604
+ await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
359
605
  return;
360
606
  }
361
607
  catch (again) {
@@ -377,7 +623,7 @@ export class MsbCliBackend {
377
623
  // image-cache heal below.
378
624
  await sleep(STATE_DB_RETRY_DELAY_MS);
379
625
  try {
380
- await this.bootOnce(msbPath, handle, state);
626
+ await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
381
627
  return;
382
628
  }
383
629
  catch (second) {
@@ -390,6 +636,17 @@ export class MsbCliBackend {
390
636
  `host.\nfirst attempt:\n${first.output}\nafter retry:\n${second.output}`);
391
637
  }
392
638
  }
639
+ if (first instanceof RestoreAccessDeniedError) {
640
+ if (!options.retryAccessDenied) {
641
+ // The checkpoint reboot's own fresh-naming retry
642
+ // (`rebootUnderFreshName`) owns this error class itself, one layer
643
+ // up — see this method's own doc on `retryAccessDenied`. Propagate
644
+ // immediately rather than retrying it here at all.
645
+ throw first;
646
+ }
647
+ await this.retryRestoreAfterAccessDenied(msbPath, handle, state, first);
648
+ return;
649
+ }
393
650
  if (!(first instanceof ImageCacheCorruptionError)) {
394
651
  throw first;
395
652
  }
@@ -397,7 +654,7 @@ export class MsbCliBackend {
397
654
  }
398
655
  const heal = await invoke(msbPath, MsbCommands.imageRemove(handle.spec.image), STOP_TIMEOUT_MS).catch((e) => e);
399
656
  try {
400
- await this.bootOnce(msbPath, handle, state);
657
+ await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
401
658
  }
402
659
  catch (second) {
403
660
  if (!(second instanceof ImageCacheCorruptionError)) {
@@ -411,30 +668,50 @@ export class MsbCliBackend {
411
668
  }
412
669
  }
413
670
  /**
414
- * One boot attempt: spawns the attached `msb run` child and polls until the
415
- * sandbox reaches Running. `state.attached` and (for non-keepAlive specs)
416
- * `startedNames` are populated only on success; on any failure the child is reaped here (for
671
+ * One boot attempt, dispatched by shape: an ordinary spec (no
672
+ * `checkpointRef`) drives `run()`'s ATTACHED supervision model
673
+ * (`bootRunOnce`); a checkpoint-restore spec drives `msb restore`'s
674
+ * fundamentally different DETACHED shape (`bootRestoreOnce`) — see each
675
+ * method's own doc. Resets the per-attempt diagnostics tail before either:
676
+ * a retried boot must not blend its tail with the failed attempt's.
677
+ * `restoreLaunchMode` is passed straight through to `bootRestoreOnce`
678
+ * (irrelevant to `bootRunOnce`'s own ordinary `run` path, which never
679
+ * brokers — see `RestoreLaunchMode`'s own doc).
680
+ */
681
+ async bootOnce(msbPath, handle, state, restoreLaunchMode = "direct") {
682
+ state.logTail = [];
683
+ state.attachedExited = false;
684
+ if (handle.spec.checkpointRef !== undefined) {
685
+ await this.bootRestoreOnce(msbPath, handle, state, restoreLaunchMode);
686
+ return;
687
+ }
688
+ await this.bootRunOnce(msbPath, handle, state);
689
+ }
690
+ /**
691
+ * One RUN boot attempt: spawns ATTACHED `msb run`, which stays alive as
692
+ * the sandbox's own supervisor for its whole lifetime, and polls
693
+ * `msb ls --format json` until the name shows Running.
694
+ *
695
+ * `state.attached` and (for non-keepAlive specs) `startedNames` are
696
+ * populated only on success; on any failure the child is reaped here (for
417
697
  * the classified early-exit failures it has already exited; a readiness
418
698
  * timeout leaves it alive and it is hard-killed) so a failed attempt leaves
419
699
  * no live process or registered cleanup state behind — the caller owns
420
- * retry policy, never cleanup (an exit-0 fast-exit success is the one
421
- * exception: it IS success, so `state.attached`/`startedNames` ARE
422
- * populated for it — see below). An early exit is classified from the
423
- * child's combined output: the image-cache-corruption signature throws
424
- * `ImageCacheCorruptionError` (the one failure `start()` heals and
425
- * retries), a host-port bind conflict throws `PortBindConflictError`, and
426
- * otherwise — if the exit code was 0 — `isCompletedFastExit` gets a chance
427
- * to reclassify the exit as a workload that ran to completion before this
428
- * poll loop could ever observe Running (msb 0.6.16's convergent-lifecycle
429
- * rework; see that method's own doc); anything else surfaces the raw
430
- * output as an ordinary boot failure, unchanged.
700
+ * retry policy, never cleanup (an exit-0 success without a live child is
701
+ * the one exception: the fast-exit case below, see `isCompletedFastExit`'s
702
+ * own doc). An early exit is classified from the child's combined output:
703
+ * the image-cache-corruption signature throws `ImageCacheCorruptionError`
704
+ * (the one failure `start()` heals and retries), a host-port bind conflict
705
+ * throws `PortBindConflictError`, and otherwise — if the exit code was 0 —
706
+ * `isCompletedFastExit` gets a chance to reclassify the exit as a workload
707
+ * that ran to completion before this poll loop could ever observe Running
708
+ * (msb 0.6.16's convergent-lifecycle rework; see that method's own doc);
709
+ * anything else surfaces the raw output as an ordinary boot failure,
710
+ * unchanged.
431
711
  */
432
- async bootOnce(msbPath, handle, state) {
433
- // Fresh per-attempt diagnostics: a retried boot must not blend its tail
434
- // with the failed attempt's.
435
- state.logTail = [];
436
- state.attachedExited = false;
437
- const child = spawn(msbPath, MsbCommands.run(handle.spec), { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
712
+ async bootRunOnce(msbPath, handle, state) {
713
+ const argv = MsbCommands.run(handle.spec);
714
+ const child = spawn(msbPath, argv, { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
438
715
  // Merge stdout+stderr into one tail, kept only for the boot diagnostics
439
716
  // below: this pipe is the sole carrier of msb's own output (registry/pull
440
717
  // errors, crash output printed before the sandbox exists). It is not a
@@ -479,8 +756,9 @@ export class MsbCliBackend {
479
756
  }
480
757
  return;
481
758
  }
482
- throw new BackendError(`msb run for sandbox ${handle.id} exited (code ${exited.code ?? "unknown"}) before reaching ` +
483
- `Running — check the image entrypoint and 'msb run' output below:\n${output}`);
759
+ throw new BackendError(`msb run for sandbox ${handle.id} exited (code ${exited.code ?? "unknown"}) ` +
760
+ `before reaching Running — check the image entrypoint and 'msb run' ` +
761
+ `output below:\n${output}`);
484
762
  }
485
763
  if ((await this.runningSandboxNames(msbPath)).has(handle.id)) {
486
764
  state.attached = child;
@@ -501,9 +779,400 @@ export class MsbCliBackend {
501
779
  await sleep(READINESS_POLL_MS);
502
780
  }
503
781
  }
782
+ /**
783
+ * One RESTORE boot attempt. `msb restore` is not `run`'s attached
784
+ * supervision model at all — EMPIRICALLY VERIFIED against msb 0.7.1 (its
785
+ * own `restore.rs` doc: "Restore a snapshot into a new detached
786
+ * sandbox"), the restore CLI process exits — typically within seconds,
787
+ * often with little or no stdout — the INSTANT activation succeeds, while
788
+ * the sandbox keeps booting in the background and only reaches Running
789
+ * some time after that (live-confirmed: `msb ls` shows Running, and exec
790
+ * works, only once the restore process has already exited). A clean exit
791
+ * is therefore not itself a completed boot, and a nonzero exit is msb's
792
+ * own failure signal with the reason on stdout/stderr — the inverse of
793
+ * `run`'s attached child, whose exit before Running always means failure.
794
+ *
795
+ * Two phases follow directly from those two facts:
796
+ * 1. Spawn `msb restore ...` and wait for IT to exit, bounded by
797
+ * `FIRST_RUN_PULL_TIMEOUT_MS` — the same boot budget `bootRunOnce`
798
+ * polls against; a restore's own activation is exposed to the same
799
+ * "this could be a slow cold operation" uncertainty a pull is.
800
+ * Classified from the combined output the same way any boot's early
801
+ * exit is (install-lock, state-db, image-cache-corruption,
802
+ * port-bind-conflict — see `bootClassified`'s own doc on why a
803
+ * restore boot needs the identical classification an ordinary one
804
+ * gets) — a nonzero, unclassified exit throws an ordinary
805
+ * `BackendError` carrying the output, same shape as `bootRunOnce`'s.
806
+ * 2. Once it exits 0, poll `msb ls` for the sandbox to reach Running —
807
+ * the same `READINESS_POLL_MS` cadence and a fresh instance of the
808
+ * identical `FIRST_RUN_PULL_TIMEOUT_MS` budget the attached path's
809
+ * own poll uses (so a restore boot gets the identical readiness
810
+ * patience an ordinary one does, counted from when there is
811
+ * actually something to poll for rather than shaved out of phase
812
+ * 1's own budget). The sandbox settling on exactly `"Stopped"`, or
813
+ * its name disappearing from a `msb ls` listing that itself came
814
+ * back, is a definite failure the moment it is observed — never
815
+ * worth waiting out the rest of the budget for, unlike a status
816
+ * that simply hasn't reached Running yet.
817
+ * 3. The `ls` probe itself failing to run or return in time (spawn
818
+ * error, or the same `LOGS_TIMEOUT_MS` `invoke` enforces on every
819
+ * other probe) is a different thing entirely from the sandbox's name
820
+ * being missing from a listing that DID come back, even though both
821
+ * collapse to the same `undefined` once the probe's own promise is
822
+ * caught — see `isCompletedFastExit`'s own doc, whose sibling
823
+ * contract this mirrors: a probe failure must never be upgraded into
824
+ * a false, specific diagnosis ("disappeared from msb ls entirely" is
825
+ * exactly that kind of false claim when the truth is just that `ls`
826
+ * itself didn't answer in time). A failed probe is therefore treated
827
+ * as "not yet confirmed either way" and retried within budget —
828
+ * install-lock races and an overloaded msb daemon are the same
829
+ * transient shapes this file already documents for `run`/`restore`/
830
+ * `stop`, and one `ls` hiccup during the readiness window must not
831
+ * turn into a spurious restore failure.
832
+ *
833
+ * The `msb restore` CLI process itself is never held onto — it has already
834
+ * exited by the time phase 2 even starts, and msb itself (out-of-process)
835
+ * is the sandbox's own supervisor from here on, the same as it always was.
836
+ * But a restored sandbox reaches Running with ONLY its guest agent inside
837
+ * — the captured workload never re-executes on its own (EMPIRICALLY
838
+ * VERIFIED against msb 0.7.1) — so once Running is confirmed, this method
839
+ * calls `reviveWorkload` to start it itself BEFORE returning: THAT call is
840
+ * what populates `state.attached` (a workload-revival `msb exec` child,
841
+ * not the restore CLI process), and `stop()`'s ordinary attached-child
842
+ * handling (exit-based death detection, the SIGKILL escalation) applies to
843
+ * IT exactly as it always did for `bootRunOnce`'s own attached `msb run`
844
+ * child. See `reviveWorkload`'s own doc for the full revival contract,
845
+ * including the typed error a checkpoint predating workload-cmdline
846
+ * capture throws instead of booting silently idle.
847
+ *
848
+ * Phase 1 (spawning the restore itself and reading back its own exit code
849
+ * + combined output) is dispatched by `launchMode` to either
850
+ * `launchRestoreDirect` (the unconditional spawn this method has always
851
+ * used) or `launchRestoreViaBroker` (POLICY v2's Windows job-object
852
+ * escape hatch — see `RestoreLaunchMode`/`restore-broker.ts`'s own doc).
853
+ * Both return the exact same shape, classified identically right below —
854
+ * a brokered attempt's output goes through the SAME predicates
855
+ * (`isImageCacheCorruption`, `isRestoreAccessDeniedFailure`, ...) a direct
856
+ * attempt's always has, since the underlying `msb restore` invocation and
857
+ * its failure signatures are identical either way; only how its stdout/
858
+ * stderr and exit code got back to this process differs. A broker launch
859
+ * can also come back `"unconfirmed"` (its own `ecFile` never appeared
860
+ * within its bound) — see `RestoreBrokerUnconfirmed`'s own doc — which
861
+ * skips classification entirely and falls straight through to phase 2
862
+ * exactly as a confirmed exit 0 does, since that phase's own `msb ls` poll
863
+ * is activation-gated and settles the question on its own regardless.
864
+ */
865
+ async bootRestoreOnce(msbPath, handle, state, launchMode = "direct") {
866
+ const launch = launchMode === "broker"
867
+ ? await this.launchRestoreViaBroker(msbPath, handle, state)
868
+ : await this.launchRestoreDirect(msbPath, handle, state);
869
+ if (launch.kind === "completed") {
870
+ const { exitCode, output } = launch;
871
+ if (isImageCacheCorruption(output)) {
872
+ throw new ImageCacheCorruptionError(output);
873
+ }
874
+ if (isMsbStateDbError(output)) {
875
+ throw new StateDbError(output);
876
+ }
877
+ if (isMsbInstallLockActive(output)) {
878
+ throw new InstallLockActiveError(output);
879
+ }
880
+ if (isPortBindConflictOutput(output)) {
881
+ throw new PortBindConflictError(`msb restore for sandbox ${handle.id} could not bind a host port: ${output}`);
882
+ }
883
+ if (isRestoreAccessDeniedFailure(output)) {
884
+ throw new RestoreAccessDeniedError(output);
885
+ }
886
+ if (isSandboxAlreadyExistsFailure(output)) {
887
+ throw new SandboxAlreadyExistsError(output);
888
+ }
889
+ if (exitCode !== 0) {
890
+ throw new BackendError(`msb restore for sandbox ${handle.id} exited (code ${exitCode}) — check the snapshot ref and ` +
891
+ `'msb restore' output below:\n${output}`);
892
+ }
893
+ }
894
+ // Exit 0 (direct or brokered), or the broker's own "unconfirmed" outcome
895
+ // — the restore CLI's own task is done, or close enough to trust `msb
896
+ // ls` to settle it (see this method's own doc) — poll for Running the
897
+ // same way bootRunOnce does, under a fresh instance of the identical
898
+ // budget.
899
+ const readyDeadline = Date.now() + FIRST_RUN_PULL_TIMEOUT_MS;
900
+ let lastSeenStatus;
901
+ for (;;) {
902
+ const ls = await invoke(msbPath, MsbCommands.ls(), LOGS_TIMEOUT_MS).catch(() => undefined);
903
+ // `ls === undefined` here means the probe itself never came back
904
+ // (spawn error, or its own LOGS_TIMEOUT_MS timing out) — a DIFFERENT
905
+ // thing from a returned listing that genuinely omits the sandbox's
906
+ // name (see this method's own doc, point 3, and `isCompletedFastExit`'s
907
+ // sibling doc). Only a listing that actually came back gets to update
908
+ // `lastSeenStatus` or trigger the Stopped/disappeared fast-fail below;
909
+ // a failed probe falls straight through to the readiness-budget check
910
+ // and is retried, never asserted as "the sandbox disappeared."
911
+ if (ls !== undefined) {
912
+ lastSeenStatus = statusOf(ls.stdout, handle.id);
913
+ if (lastSeenStatus === "Running") {
914
+ // `msb restore` boots a restored sandbox with only its guest agent
915
+ // inside — the captured workload never re-runs on its own (see
916
+ // this method's own doc and `reviveWorkload`'s) — so this backend
917
+ // starts it itself before registering the boot as complete. A
918
+ // `reviveWorkload` failure propagates unclassified: never register
919
+ // `startedNames` for a sandbox whose workload never actually came
920
+ // up.
921
+ await this.reviveWorkload(msbPath, handle, state);
922
+ if (!handle.spec.keepAlive) {
923
+ this.startedNames.add(handle.id);
924
+ }
925
+ return;
926
+ }
927
+ if (lastSeenStatus === "Stopped" || lastSeenStatus === undefined) {
928
+ throw new BackendError(await this.restoreBootFailureMessage(msbPath, handle.id, lastSeenStatus, false));
929
+ }
930
+ }
931
+ if (Date.now() >= readyDeadline) {
932
+ throw new BackendError(await this.restoreBootFailureMessage(msbPath, handle.id, lastSeenStatus, true));
933
+ }
934
+ await sleep(READINESS_POLL_MS);
935
+ }
936
+ }
937
+ /**
938
+ * `bootRestoreOnce`'s own DIRECT launch — the unconditional child-process
939
+ * spawn every restore attempt used before POLICY v2's broker escalation
940
+ * existed, extracted unchanged from that method's own former body so the
941
+ * two `RestoreLaunchMode`s share one classification point (see
942
+ * `bootRestoreOnce`'s own doc). Bounded by the same `FIRST_RUN_PULL_TIMEOUT_MS`
943
+ * budget `bootRunOnce` polls against, and always resolves `"completed"` —
944
+ * a direct spawn has no analog of the broker's own `"unconfirmed"` outcome,
945
+ * since this process reads the child's real exit code directly.
946
+ */
947
+ async launchRestoreDirect(msbPath, handle, state) {
948
+ const argv = MsbCommands.restore(handle.spec);
949
+ const child = spawn(msbPath, argv, { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
950
+ const stdoutDone = drainTail(child.stdout, state.logTail);
951
+ const stderrDone = drainTail(child.stderr, state.logTail);
952
+ const exitCode = await new Promise((resolveExit, rejectExit) => {
953
+ let settled = false;
954
+ const timer = setTimeout(() => {
955
+ if (settled) {
956
+ return;
957
+ }
958
+ settled = true;
959
+ child.kill("SIGKILL");
960
+ rejectExit(new BackendError(`msb restore for sandbox ${handle.id} did not exit within ${FIRST_RUN_PULL_TIMEOUT_MS / 1000}s — ` +
961
+ `msb itself may be unresponsive; last output:\n${state.logTail.join("\n")}`));
962
+ }, FIRST_RUN_PULL_TIMEOUT_MS);
963
+ child.once("exit", (code) => {
964
+ if (settled) {
965
+ return;
966
+ }
967
+ settled = true;
968
+ clearTimeout(timer);
969
+ resolveExit(code ?? -1);
970
+ });
971
+ });
972
+ await Promise.all([stdoutDone, stderrDone]);
973
+ return { kind: "completed", exitCode, output: state.logTail.join("\n") };
974
+ }
975
+ /**
976
+ * `bootRestoreOnce`'s own BROKERED launch — POLICY v2's Windows job-object
977
+ * escape hatch (see `RestoreLaunchMode`/`restore-broker.ts`'s own module
978
+ * doc on the root cause and the live-validated WMI mitigation). Calls the
979
+ * injected `restoreBroker` seam with the exact same argv a direct attempt
980
+ * would spawn (`MsbCommands.restore`, unmodified) and the same
981
+ * `FIRST_RUN_PULL_TIMEOUT_MS` budget.
982
+ *
983
+ * The broker's own result is either passed straight through
984
+ * (`"completed"`/`"unconfirmed"`, both appended to `state.logTail` for the
985
+ * same pre-Running diagnostics purpose `drainTail` serves on the direct
986
+ * path) or, if the broker call itself REJECTS — broker infrastructure
987
+ * failing (a missing `powershell.exe`, a script-file write failure, WMI
988
+ * refusing to even create the process; see `RestoreBrokerLauncher`'s own
989
+ * doc on what counts as this vs. an ordinary brokered-restore failure) —
990
+ * caught here and turned into a one-attempt fallback to
991
+ * `launchRestoreDirect` instead (POLICY v2 point 5: the broker must never
992
+ * become a new single point of failure). That fallback is scoped to THIS
993
+ * attempt only; it does not touch the caller's own `RestoreLaunchMode` for
994
+ * whatever attempt comes after it.
995
+ */
996
+ async launchRestoreViaBroker(msbPath, handle, state) {
997
+ const argv = MsbCommands.restore(handle.spec);
998
+ let result;
999
+ try {
1000
+ result = await this.restoreBroker(msbPath, argv, FIRST_RUN_PULL_TIMEOUT_MS);
1001
+ }
1002
+ catch (err) {
1003
+ pushLinesToTail(state.logTail, `[restore broker] broker infrastructure failed for sandbox ${handle.id}, falling back to a direct ` +
1004
+ `attempt: ${err instanceof Error ? err.message : String(err)}`);
1005
+ return this.launchRestoreDirect(msbPath, handle, state);
1006
+ }
1007
+ if (result.kind === "unconfirmed") {
1008
+ pushLinesToTail(state.logTail, `[restore broker] its own exit-code file never appeared within its wait bound for sandbox ` +
1009
+ `${handle.id} — proceeding to the ls-poll phase rather than guessing an outcome.`);
1010
+ return { kind: "unconfirmed" };
1011
+ }
1012
+ pushLinesToTail(state.logTail, result.output);
1013
+ return { kind: "completed", exitCode: result.exitCode, output: result.output };
1014
+ }
1015
+ /**
1016
+ * Renders a restore boot's post-mortem message once its poll for Running
1017
+ * has definitely failed (a settled `"Stopped"`, the name disappearing
1018
+ * from `msb ls` outright, or the readiness budget itself running out).
1019
+ * `msb restore` itself already exited 0 by this point, so the CLI
1020
+ * process's own output (already surfaced for a nonzero exit, above) has
1021
+ * nothing left to add — `msb logs --source system` is the one channel
1022
+ * that can still show what happened to the sandbox's own background boot
1023
+ * (see `MsbCommands.systemLog`'s own doc and `isCompletedFastExit`, its
1024
+ * other caller). Best-effort: a failed probe falls back to a plain "no
1025
+ * output" note rather than letting a diagnostics-gathering failure itself
1026
+ * throw and mask the real one.
1027
+ */
1028
+ async restoreBootFailureMessage(msbPath, id, status, timedOut) {
1029
+ const systemLog = await invoke(msbPath, MsbCommands.systemLog(id), LOGS_TIMEOUT_MS).catch(() => undefined);
1030
+ const diagnostics = systemLog !== undefined && systemLog.stdout.trim() !== ""
1031
+ ? `'msb logs ${id} --source system' output:\n${systemLog.stdout}`
1032
+ : `'msb logs ${id} --source system' produced no output`;
1033
+ // `status` is the last CONFIRMED status a returned `msb ls` listing
1034
+ // actually reported — never a value fabricated from a failed probe (see
1035
+ // bootRestoreOnce's own doc, point 3): `undefined` here on the timedOut
1036
+ // branch means every `ls` probe across the whole readiness budget failed
1037
+ // to even come back, not that a listing came back without the name in
1038
+ // it (that is the non-timedOut, "disappeared" branch below, which is
1039
+ // only ever reached from a listing that DID come back).
1040
+ const reason = timedOut
1041
+ ? `did not reach Running within ${FIRST_RUN_PULL_TIMEOUT_MS / 1000}s of 'msb restore' exiting ` +
1042
+ `(last confirmed status: ${status ?? "none — 'msb ls' never returned a usable listing"})`
1043
+ : status === undefined
1044
+ ? `disappeared from 'msb ls' entirely`
1045
+ : `settled as '${status}' in 'msb ls'`;
1046
+ return `sandbox ${id} was restored ('msb restore' exited 0) but ${reason} before ever reaching Running — ${diagnostics}`;
1047
+ }
1048
+ /**
1049
+ * Starts the workload a restore itself never re-runs. EMPIRICALLY VERIFIED
1050
+ * against msb 0.7.1: a restored sandbox reaches Running with ONLY its
1051
+ * guest agent inside (`guest ps` shows `/init.krun` and kernel threads —
1052
+ * the captured workload command does not re-execute; `msb start`/`msb
1053
+ * logs` on such a sandbox are equally idle/empty). Only ever called from
1054
+ * `bootRestoreOnce`, once it has confirmed Running — see this class's own
1055
+ * doc on `bootRestoreOnce`.
1056
+ *
1057
+ * Spawns a LONG-LIVED, attached `msb exec [-e K=V]... <name> -- <argv>`
1058
+ * session (`MsbCommands.execWithEnv`, env from `handle.spec.env` — a
1059
+ * restore's own `msb restore` has no `-e`/`--env` flag at all, so this exec
1060
+ * is the one place a restored sandbox's guest ever sees it again) — this
1061
+ * becomes the sandbox's own workload from here on, and exec sessions are
1062
+ * exactly what msb's own log capture records (the primary session's
1063
+ * stdout/stderr land in `exec.log`, served by `msb logs`/`-f` — see
1064
+ * `MsbCommands.exec`'s own doc). This exec child slots into the EXACT SAME
1065
+ * `state.attached` role `bootRunOnce`'s own attached `msb run` child fills
1066
+ * for an ordinary boot: child-exit-based death detection, reap-on-stop
1067
+ * (`stop()`'s SIGKILL escalation), and every other attached-child teardown
1068
+ * semantic apply to it unchanged — the detached-restore round left that
1069
+ * slot merely optional, never removed it.
1070
+ *
1071
+ * `handle.spec.command` is ALREADY the fully-resolved workload argv by the
1072
+ * time this runs, in priority order: an explicit command the source
1073
+ * container carried, or — when it had none — the guest cmdline
1074
+ * `createCheckpoint` captured at checkpoint time, merged in by
1075
+ * `createCheckpoint` itself (for its own immediate reboot) or by
1076
+ * `fromCheckpointRegistryEntry` (for a registry-mediated restore, same or
1077
+ * later process — see both functions' own docs). `undefined` here means
1078
+ * NEITHER source exists — an old registry entry predating capture, or one
1079
+ * whose capture attempt itself failed — so this throws
1080
+ * `CheckpointWorkloadCommandMissingError` itself rather than depending on
1081
+ * a caller to have checked first: never boot a restored sandbox silently
1082
+ * idle.
1083
+ *
1084
+ * The exec child gets a brief settle window (`EXEC_REVIVE_SETTLE_MS`) to
1085
+ * prove it isn't an immediate boot failure before this returns success —
1086
+ * mirroring `bootRunOnce`'s own exit-vs-Running race, except a restore's
1087
+ * exec session has no separate "Running" signal of its own to poll for
1088
+ * (the SANDBOX already reports Running regardless of whether this exec
1089
+ * succeeds), so a settle window is what that race collapses to here. An
1090
+ * exit observed within the window is classified exactly like
1091
+ * `bootRunOnce`'s own early exit: exit 0 counts as success only when
1092
+ * `isCompletedFastExit` ALSO confirms it (the sandbox itself settled
1093
+ * Stopped with the boot-completion marker — the repo's existing fast-exit-
1094
+ * completion semantics, reused verbatim); any other exit — nonzero, or
1095
+ * exit 0 without that confirmation — throws a `BackendError` carrying the
1096
+ * exec child's own output, the same failure shape an attached run's early
1097
+ * exit already has.
1098
+ */
1099
+ async reviveWorkload(msbPath, handle, state) {
1100
+ const command = handle.spec.command;
1101
+ if (command === undefined) {
1102
+ throw new CheckpointWorkloadCommandMissingError(handle.spec.checkpointRef ?? handle.id);
1103
+ }
1104
+ state.logTail = [];
1105
+ const argv = MsbCommands.execWithEnv(handle.id, handle.spec.env, command);
1106
+ const child = spawn(msbPath, argv, { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
1107
+ const stdoutDone = drainTail(child.stdout, state.logTail);
1108
+ const stderrDone = drainTail(child.stderr, state.logTail);
1109
+ let exited;
1110
+ child.once("exit", (code) => {
1111
+ exited = { code };
1112
+ state.attachedExited = true;
1113
+ });
1114
+ const settleDeadline = Date.now() + EXEC_REVIVE_SETTLE_MS;
1115
+ while (exited === undefined && Date.now() < settleDeadline) {
1116
+ await sleep(READINESS_POLL_MS);
1117
+ }
1118
+ if (exited === undefined) {
1119
+ // Stayed up through the whole settle window — the ordinary long-lived
1120
+ // case. Slots into state.attached exactly like bootRunOnce's own
1121
+ // attached child.
1122
+ state.attached = child;
1123
+ return;
1124
+ }
1125
+ await Promise.all([stdoutDone, stderrDone]);
1126
+ const output = state.logTail.join("\n");
1127
+ if (exited.code === 0 && (await this.isCompletedFastExit(msbPath, handle.id))) {
1128
+ // Mirrors bootRunOnce's own fast-exit success case: the workload ran to
1129
+ // completion so quickly this settle window caught its natural exit
1130
+ // rather than a crash. No live child left to hold onto — state.attached
1131
+ // stays undefined, same as bootRunOnce's own fast-exit branch.
1132
+ return;
1133
+ }
1134
+ throw new BackendError(`msb exec for sandbox ${handle.id}'s revived workload exited (code ${exited.code ?? "unknown"}) before ` +
1135
+ `staying up — check the workload command and its output below:\n${output}`);
1136
+ }
1137
+ /**
1138
+ * Best-effort: execs `CAPTURE_WORKLOAD_CMDLINE_SCRIPT` in `handle`'s guest
1139
+ * and parses its stdout via `parseCapturedWorkloadCmdline`. Only ever
1140
+ * called from `createCheckpoint`, BEFORE it stops the sandbox (see that
1141
+ * method's own doc), and only when `handle.spec.command` is undefined — an
1142
+ * explicit command needs no capture. Never throws: an exec failure
1143
+ * (nonzero exit, the exec channel itself erroring) and unparseable output
1144
+ * both resolve `undefined` rather than failing the checkpoint — the
1145
+ * captured cmdline is a best-effort fallback, and its absence is fully
1146
+ * handled later, at restore time (`reviveWorkload` throws
1147
+ * `CheckpointWorkloadCommandMissingError` when neither an explicit nor a
1148
+ * captured command exists).
1149
+ */
1150
+ async captureGuestWorkloadCmdline(handle) {
1151
+ try {
1152
+ const result = await this.exec(handle, ["sh", "-c", CAPTURE_WORKLOAD_CMDLINE_SCRIPT]);
1153
+ if (result.exitCode !== 0) {
1154
+ return undefined;
1155
+ }
1156
+ return parseCapturedWorkloadCmdline(result.stdout);
1157
+ }
1158
+ catch {
1159
+ return undefined;
1160
+ }
1161
+ }
1162
+ /**
1163
+ * SPI implementation of `SandboxBackend.capturedWorkloadCommand` — see its
1164
+ * own doc. Reads back whatever `createCheckpoint` most recently stashed on
1165
+ * this sandbox's `HandleState` (keyed by name, so it survives the
1166
+ * stop/snapshot/reboot cycle's own handle-object churn); `undefined` if
1167
+ * this sandbox was never checkpointed, or its checkpoint needed no
1168
+ * capture, or the capture attempt failed.
1169
+ */
1170
+ capturedWorkloadCommand(handle) {
1171
+ return this.handles.get(handle.id)?.capturedCommand;
1172
+ }
504
1173
  /**
505
1174
  * The fast-exit post-mortem classification: only ever consulted from
506
- * `bootOnce` above, and only once the attached `msb run` child has already
1175
+ * `bootRunOnce` above, and only once the attached `msb run` child has already
507
1176
  * exited with code 0 before Running was observed. msb 0.6.16's
508
1177
  * convergent-lifecycle rework means a workload that finishes quickly is
509
1178
  * never observed `"Running"` at all — only `"Starting"`, then the attached
@@ -543,11 +1212,16 @@ export class MsbCliBackend {
543
1212
  }
544
1213
  await invoke(msbPath, MsbCommands.stop(handle.id), STOP_TIMEOUT_MS).catch(() => { });
545
1214
  const attached = state?.attached;
546
- // The attached `msb run` child is msb's own supervisor for this sandbox:
547
- // it stays alive for the sandbox's entire lifetime and only exits once
548
- // the `msb stop` call just above lands, so the common path here is
549
- // "attach a listener, then observe the exit that our own stop just
550
- // caused." state.attachedExited exists for the other case: if the child
1215
+ // The attached child is either an ordinary boot's `msb run` process —
1216
+ // msb's own supervisor for this sandbox, staying alive for its entire
1217
+ // lifetime — or, for a restored sandbox, the `msb exec` session
1218
+ // `reviveWorkload` spawned to revive its captured workload (msb itself,
1219
+ // out-of-process, is the actual supervisor there; this exec child is
1220
+ // just the workload session riding inside it). Either way it stays alive
1221
+ // until the sandbox itself stops, and only exits once the `msb stop`
1222
+ // call just above lands, so the common path here is "attach a listener,
1223
+ // then observe the exit that our own stop just caused." state.attachedExited
1224
+ // exists for the other case: if the child
551
1225
  // had already died before this method ever ran (crashed, or killed by
552
1226
  // something external), start()'s own listener already flipped it, and
553
1227
  // Node never replays a past "exit" event to a listener attached after
@@ -586,30 +1260,340 @@ export class MsbCliBackend {
586
1260
  this.startedNames.delete(handle.id);
587
1261
  this.handles.delete(handle.id);
588
1262
  }
1263
+ /**
1264
+ * `bootClassified`'s own retry for a `RestoreAccessDeniedError` hit on the
1265
+ * ordinary `start()`/`fromCheckpoint().start()` path (its default
1266
+ * `retryAccessDenied: true`) — the sibling of `rebootUnderFreshName`
1267
+ * (`createCheckpoint`'s own reboot retry) for the SAME classified failure,
1268
+ * built from the SAME live-verified policy: `msb restore --name X`
1269
+ * validates the artifact FIRST — an integrity failure exits 1 and leaves
1270
+ * NO sandbox record — but a failure AFTER validation (this access-denied
1271
+ * signature included) leaves `X` behind as a STOPPED SANDBOX RECORD, and
1272
+ * any retry of `restore --name X` then fails outright with msb's own
1273
+ * "already exists" refusal rather than hitting the transient a second
1274
+ * time. A same-name retry (what this method replaces) therefore only ever
1275
+ * proves that collision — never the actual transient — so this mints a
1276
+ * NEW name from `nextSandboxName()` for every retry instead, the same
1277
+ * generator `GenericContainer.start()`'s own boot loop and
1278
+ * `rebootUnderFreshName` both use, tracking it in the reaper ledger BEFORE
1279
+ * that attempt's restore runs and best-effort `msb rm`-ing (then
1280
+ * untracking) a failed attempt's own name before advancing — identical
1281
+ * per-attempt bookkeeping to `rebootUnderFreshName`, just owned one layer
1282
+ * lower.
1283
+ *
1284
+ * The difference from `rebootUnderFreshName` is entirely about WHO owns
1285
+ * the re-keying, not the retry policy itself: `createCheckpoint` calls
1286
+ * `rebootUnderFreshName` with no live sandbox left at all (the source was
1287
+ * already stopped and removed as part of the snapshot step), so EVERY
1288
+ * attempt — including the first — mints a fresh name, and the winning
1289
+ * handle is handed back for `createCheckpoint` itself to re-key. Here,
1290
+ * `handle` is already live under its own name when `first` (the initial
1291
+ * `RestoreAccessDeniedError`) is caught, and `bootClassified`'s caller
1292
+ * (`start()`) has no re-keying step of its own — it just awaits
1293
+ * `bootClassified` and returns — so this method re-keys
1294
+ * `this.handles`/`this.startedNames` AND mutates the caller's own `handle`
1295
+ * object IN PLACE itself on success, the same "one carve-out"
1296
+ * `SandboxHandle`'s own interface doc allows for `createCheckpoint`'s
1297
+ * reboot (see that method's own doc on `handle.id`/`handle.spec`
1298
+ * mutation).
1299
+ *
1300
+ * `handle`'s own original name was already tracked in the reaper ledger by
1301
+ * `GenericContainer.start()`'s own boot loop before `create()`/`start()`
1302
+ * ever ran (mirroring `rebootUnderFreshName`'s treatment of the checkpoint
1303
+ * reboot's own `originalName`) — this method best-effort `msb rm`'s it
1304
+ * once `first` is confirmed retryable, but deliberately leaves its own
1305
+ * ledger entry alone, for the same not-found-tolerant sweep to find,
1306
+ * rather than untracking a name this method never tracked itself.
1307
+ *
1308
+ * Retries up to `RESTORE_ACCESS_DENIED_RETRY_LIMIT` times,
1309
+ * `RESTORE_ACCESS_DENIED_RETRY_DELAY_MS` apart — unchanged from the
1310
+ * same-name retry this replaces. Each attempt goes through
1311
+ * `bootClassified` itself with `retryAccessDenied: false` (so a
1312
+ * `StateDbError`/`InstallLockActiveError`/`ImageCacheCorruptionError` hit
1313
+ * mid-retry still gets its own ordinary classified handling; only
1314
+ * `RestoreAccessDeniedError`'s own retry is owned here) and treats either
1315
+ * `RestoreAccessDeniedError` or `SandboxAlreadyExistsError` as retryable —
1316
+ * a freshly minted name should never collide, but `rebootUnderFreshName`'s
1317
+ * own live-CI evidence says a loaded host can still refuse one, and the
1318
+ * SAME fresh-naming response (rm the failed name, mint another) covers
1319
+ * both signatures identically. Anything else propagates immediately,
1320
+ * unretried.
1321
+ *
1322
+ * POLICY v2's broker escalation applies here too, on the exact same terms
1323
+ * as `rebootUnderFreshName`'s own (see that method's own doc): this
1324
+ * method is only ever entered once the reboot's first, always-direct
1325
+ * attempt has already hit `RestoreAccessDeniedError` (`first`), so on a
1326
+ * Windows host (`isWindowsPlatform()`) every retry THIS method itself
1327
+ * performs is, by construction, an attempt after that trigger — there is
1328
+ * no "first attempt still direct" sub-case to preserve inside this loop
1329
+ * the way `rebootUnderFreshName` has to across its own. `launchMode` is
1330
+ * therefore decided once, before the loop, not re-evaluated per attempt.
1331
+ */
1332
+ async retryRestoreAfterAccessDenied(msbPath, handle, state, first) {
1333
+ const originalName = handle.id;
1334
+ // Best-effort: `first` may have left a stopped sandbox record behind
1335
+ // under `originalName` (see this method's own doc) — cheap cleanup, and
1336
+ // correctness never depends on it, since no retry below ever reuses this
1337
+ // name. Its own ledger entry is deliberately left alone; see this
1338
+ // method's own doc.
1339
+ await invoke(msbPath, MsbCommands.rm(originalName), STOP_TIMEOUT_MS).catch(() => { });
1340
+ const launchMode = this.isWindowsPlatform() ? "broker" : "direct";
1341
+ let last = first;
1342
+ for (let attempt = 1; attempt <= RESTORE_ACCESS_DENIED_RETRY_LIMIT; attempt++) {
1343
+ await sleep(RESTORE_ACCESS_DENIED_RETRY_DELAY_MS);
1344
+ const name = nextSandboxName();
1345
+ if (!handle.spec.keepAlive) {
1346
+ await trackSandbox(name);
1347
+ }
1348
+ const candidate = { id: name, spec: { ...handle.spec, name } };
1349
+ try {
1350
+ await this.bootClassified(msbPath, candidate, state, { retryAccessDenied: false, restoreLaunchMode: launchMode });
1351
+ // Success under `name` — re-key this backend's own runtime
1352
+ // registries (see class doc on `handles`) and publish the new
1353
+ // identity onto the CALLER's own handle object, IN PLACE, exactly
1354
+ // like `createCheckpoint`'s own reboot does (see its own doc).
1355
+ this.handles.delete(originalName);
1356
+ this.handles.set(name, state);
1357
+ this.startedNames.delete(originalName);
1358
+ const mutableHandle = handle;
1359
+ mutableHandle.id = name;
1360
+ mutableHandle.spec = candidate.spec;
1361
+ return;
1362
+ }
1363
+ catch (err) {
1364
+ const classified = err instanceof RestoreAccessDeniedError || err instanceof SandboxAlreadyExistsError;
1365
+ if (!classified) {
1366
+ if (!handle.spec.keepAlive) {
1367
+ await untrackSandbox(name);
1368
+ }
1369
+ throw err;
1370
+ }
1371
+ // Best-effort, result ignored either way — see this method's own
1372
+ // doc on why correctness no longer depends on it.
1373
+ await invoke(msbPath, MsbCommands.rm(name), STOP_TIMEOUT_MS).catch(() => { });
1374
+ if (!handle.spec.keepAlive) {
1375
+ await untrackSandbox(name);
1376
+ }
1377
+ last = err;
1378
+ }
1379
+ }
1380
+ throw new BackendError(`msb restore for sandbox ${originalName} kept hitting msb's Windows access-denied/"sandbox already ` +
1381
+ `exists" refusal across ${RESTORE_ACCESS_DENIED_RETRY_LIMIT} retries, each under a freshly minted ` +
1382
+ `name — this is normally a brief file-handle release lag that clears within one retry, so a failure ` +
1383
+ `held this long looks like a genuinely stuck lock on this host.\n${last.output}`);
1384
+ }
1385
+ /**
1386
+ * `createCheckpoint`'s own reboot step. Mints a NEW sandbox name from
1387
+ * `nextSandboxName()` (the same generator `GenericContainer.start()`'s own
1388
+ * boot loop uses) for EVERY attempt, tracks it in the reaper ledger BEFORE
1389
+ * that attempt's restore runs (mirroring `GenericContainer.start()`'s own
1390
+ * `trackSandbox` call, keepAlive-excluded the same way), and retries two
1391
+ * classified failure signatures on that basis instead of surfacing either
1392
+ * immediately: msb's "sandbox already exists" refusal
1393
+ * (`SandboxAlreadyExistsError`, see `isSandboxAlreadyExistsFailure`'s own
1394
+ * doc) and its Windows access-denied refusal on the just-written snapshot
1395
+ * artifact (`RestoreAccessDeniedError`, see `isRestoreAccessDeniedFailure`'s
1396
+ * own doc — retried here rather than inside `bootClassified` itself, via
1397
+ * that method's own `retryAccessDenied: false`, see its doc).
1398
+ *
1399
+ * LIVE-VERIFIED POLICY this retry is built from: `msb restore --name X`
1400
+ * validates the artifact FIRST — an integrity failure exits 1 and leaves NO
1401
+ * sandbox record — but a failure AFTER validation (a block-device open
1402
+ * PermissionDenied on unix; `RestoreAccessDeniedError`'s own signature on
1403
+ * Windows) leaves `X` behind as a STOPPED SANDBOX RECORD visible in `msb
1404
+ * ls`, and any retry of `restore --name X` then fails with msb's own
1405
+ * "already exists" refusal — so a same-name retry of EITHER classified
1406
+ * failure only ever collides with itself. Reusing a failed attempt's name
1407
+ * is therefore never safe (`msb rm` alone is not reliably enough either: on
1408
+ * Windows it frees only the DB record, not the on-disk directory, which can
1409
+ * outlive it well past this budget — see the fresh-name reboot's own doc on
1410
+ * `createCheckpoint`) — the only universally safe policy is a NEW name per
1411
+ * attempt, same as this backend's ordinary `start()` boot loop already uses
1412
+ * for its own, unrelated port-conflict retries. On a classified failure
1413
+ * this best-effort `msb rm`'s the failed attempt's own name anyway (result
1414
+ * ignored — cheap cleanup, and correctness no longer depends on it now
1415
+ * that the next attempt never reuses that name) and untracks it from the
1416
+ * ledger, before advancing to a freshly minted name for the next attempt.
1417
+ *
1418
+ * `this.checkpointRebootAlreadyExistsRetryBudgetMs`/`_RetryDelayMs` back
1419
+ * this loop's overall deadline/delay rather than the bare
1420
+ * `CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS`/`_DELAY_MS` constants
1421
+ * directly, so a budget-exhaustion test can shrink them to milliseconds
1422
+ * instead of actually blocking for the real ~30s — see those fields' own
1423
+ * doc. Only `SandboxAlreadyExistsError`/`RestoreAccessDeniedError` are
1424
+ * retried here; any other failure `bootClassified` throws (on the first
1425
+ * attempt or a later one) propagates immediately, unretried, WITHOUT
1426
+ * advancing to a new name — `createCheckpoint`'s own caller untracks that
1427
+ * attempt's name on this path (see its own catch block). Not reached by
1428
+ * the ordinary `start()` path: a `GenericContainer.fromCheckpoint(cp)
1429
+ * .start()` restore's own `RestoreAccessDeniedError` is retried by its own
1430
+ * sibling, `retryRestoreAfterAccessDenied` (see that method's own doc) —
1431
+ * same fresh-naming policy, applied one layer lower since that path has no
1432
+ * separate caller left to hand a replacement handle to for re-keying.
1433
+ *
1434
+ * Returns the WINNING attempt's own `SandboxHandle` (a fresh object, never
1435
+ * `undefined`) — `createCheckpoint` re-keys its runtime registries and the
1436
+ * caller's live handle from this, not from a name minted up front.
1437
+ *
1438
+ * POLICY v2's broker escalation (see `RestoreLaunchMode`/`restore-broker.ts`'s
1439
+ * own module doc on the Windows job-object root cause and the live-
1440
+ * validated WMI mitigation) lives here: `launchMode` starts `"direct"` —
1441
+ * the first attempt of ANY reboot is always a direct spawn, unconditionally,
1442
+ * even on Windows — and flips to `"broker"` the moment an attempt hits
1443
+ * `RestoreAccessDeniedError` on a Windows host (`isWindowsPlatform()`);
1444
+ * every attempt from then on, for the rest of THIS reboot, launches
1445
+ * brokered instead, never downgrading back to direct even if a later
1446
+ * attempt's own failure is only `SandboxAlreadyExistsError` (still
1447
+ * retried, but no longer the trigger — only access-denied escalates).
1448
+ * Off Windows, `launchMode` never leaves `"direct"` at all — this reboot's
1449
+ * whole retry loop behaves exactly as it did before this policy existed.
1450
+ */
1451
+ async rebootUnderFreshName(msbPath, sourceSpec, effectiveRef, capturedCommand, state) {
1452
+ const deadline = Date.now() + this.checkpointRebootAlreadyExistsRetryBudgetMs;
1453
+ let attempt = 0;
1454
+ let last;
1455
+ let launchMode = "direct";
1456
+ for (;;) {
1457
+ attempt += 1;
1458
+ const name = nextSandboxName();
1459
+ if (!sourceSpec.keepAlive) {
1460
+ await trackSandbox(name);
1461
+ }
1462
+ const rebootHandle = {
1463
+ id: name,
1464
+ spec: { ...sourceSpec, name, checkpointRef: effectiveRef, command: sourceSpec.command ?? capturedCommand },
1465
+ };
1466
+ try {
1467
+ await this.bootClassified(msbPath, rebootHandle, state, { retryAccessDenied: false, restoreLaunchMode: launchMode });
1468
+ return rebootHandle;
1469
+ }
1470
+ catch (err) {
1471
+ const classified = err instanceof SandboxAlreadyExistsError || err instanceof RestoreAccessDeniedError;
1472
+ if (!classified) {
1473
+ if (!sourceSpec.keepAlive) {
1474
+ await untrackSandbox(name);
1475
+ }
1476
+ throw err;
1477
+ }
1478
+ if (err instanceof RestoreAccessDeniedError && this.isWindowsPlatform()) {
1479
+ // POLICY v2: escalate for every remaining attempt of this reboot —
1480
+ // see this method's own doc on `launchMode`.
1481
+ launchMode = "broker";
1482
+ }
1483
+ // Best-effort: ignore the result either way — a "not found" is as
1484
+ // fine as an actual removal, and this is cleanup, not something the
1485
+ // next attempt (a brand-new name) depends on for correctness.
1486
+ await invoke(msbPath, MsbCommands.rm(name), STOP_TIMEOUT_MS).catch(() => { });
1487
+ if (!sourceSpec.keepAlive) {
1488
+ await untrackSandbox(name);
1489
+ }
1490
+ last = err;
1491
+ if (Date.now() >= deadline) {
1492
+ throw new BackendError(`msb restore kept hitting msb's "sandbox already exists"/access-denied refusal for ` +
1493
+ `${this.checkpointRebootAlreadyExistsRetryBudgetMs / 1000}s across ${attempt} attempt` +
1494
+ `${attempt === 1 ? "" : "s"}, each under a freshly minted name — msb's own on-disk sandbox ` +
1495
+ `directory can lag its database record's own release on a loaded Windows host well past a short ` +
1496
+ `wait, but a refusal held this long looks like a genuinely stuck host rather than a release ` +
1497
+ `race.\nlast attempt ('${name}'):\n${last.output}`);
1498
+ }
1499
+ await sleep(this.checkpointRebootAlreadyExistsRetryDelayMs);
1500
+ }
1501
+ }
1502
+ }
589
1503
  /**
590
1504
  * The stop/snapshot/reboot cycle: `msb stop <name>` (reusing this
591
1505
  * backend's own `stop()`, which also quiesces the attached child and any
592
- * network-link tunnels), `msb snapshot create --from <name> <ref>`, then
593
- * `msb rm <name>` followed by a fresh ATTACHED boot of the SAME name from
594
- * that snapshot — never `msb start`. Upstream's detached-start path
595
- * (`Sandbox::start_detached`) passes `CREATE_BREAKAWAY_FROM_JOB` on
596
- * Windows, which `ERROR_ACCESS_DENIED`s outright whenever the msb CLI runs
597
- * inside a job object that doesn't grant breakaway rights — a Gradle/
598
- * cargo/node test runner on a hosted Windows runner, or any process that
599
- * embeds this library inside its own restrictive job object — and that
600
- * denial is deterministic, not transient, so no retry shape fixes it.
601
- * Attached `msb run` (this backend's normal boot, including `--from-snapshot`
602
- * boots) never hits it, which is why the reboot reuses `bootOnce` instead
603
- * of resuming in place: the stopped sandbox's disk state is already IN the
604
- * snapshot, so `rm`-ing it first and booting a fresh attached sandbox
605
- * under the same name/ports/env/memory (via a spec identical to
606
- * `handle.spec` except `checkpointRef` set to the new `ref`) reproduces
607
- * the exact same observable contract. `bootOnce` swaps `state.attached` to
608
- * the freshly spawned child itself; nothing about `this.handles`/
609
- * `startedNames` or the reaping ledger changes, since the name never
610
- * changed. Its workload restarts from scratch (the VM reboots), which is
611
- * why `capabilities.checkpointRestartsWorkload` is `true` here and the
612
- * generic layer re-runs the wait strategy after this returns.
1506
+ * network-link tunnels), `msb snapshot create --from-sandbox <name> <ref>`
1507
+ * (whose printed artifact path — never `ref` itself — becomes the
1508
+ * EFFECTIVE checkpoint ref this method returns; see
1509
+ * `parseSnapshotCreateArtifactPath`), then `msb rm <name>` followed by
1510
+ * `msb restore <effective-ref> --name <fresh-name>` of a FRESH name from
1511
+ * that snapshot (via `rebootUnderFreshName`/`bootRestoreOnce`, see
1512
+ * `MsbCommands.restore`'s own doc — no `--disk-only`, which a disk-scope
1513
+ * snapshot rejects) — never `msb start`. The fresh name — never the
1514
+ * original — is minted by `nextSandboxName()` (the same generator
1515
+ * `GenericContainer.start()`'s own ordinary boot loop uses; see
1516
+ * `core/sandbox-name.ts`), and a NEW one is minted for EVERY attempt, never
1517
+ * just once for the whole reboot: EMPIRICALLY VERIFIED against a real msb
1518
+ * 0.7.1 binary, a restore that fails PAST msb's own artifact validation
1519
+ * (its Windows access-denied signature, `RestoreAccessDeniedError`) leaves
1520
+ * its `--name` behind as a STOPPED SANDBOX RECORD, and any retry under that
1521
+ * SAME name then collides with msb's own restore-time collision check
1522
+ * (`existing.is_some() || dir_exists`) immediately — CONFIRMED on Windows
1523
+ * CI, where five checkpoint tests collided on their fresh reboot names for
1524
+ * the retry loop's ENTIRE budget once the first attempt hit that
1525
+ * access-denied failure (see `CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS`'s
1526
+ * own doc). A same-name retry therefore only ever retries through a race of
1527
+ * its own making; a NEW name per attempt sidesteps it structurally — a name
1528
+ * nothing else has ever used can never collide with a record or a lingering
1529
+ * directory left by a name nothing will ever restore under again (see
1530
+ * `rebootUnderFreshName`'s own doc for the full retry policy, including its
1531
+ * best-effort `msb rm` of each failed attempt's own name). On success, this method
1532
+ * mutates `handle.id` and `handle.spec` (name only — see below) IN PLACE
1533
+ * on the caller's own `handle`, so every subsequent operation against it
1534
+ * (exec/logs/stop/rm, and — one layer up, via the SAME `SandboxHandle`
1535
+ * reference — `GenericContainer.checkpoint()`'s own post-reboot
1536
+ * `installNetworkLinks`/wait-strategy re-run and `capturedWorkloadCommand()`
1537
+ * call) targets the sandbox actually running now. `SandboxBackend`'s own
1538
+ * interface doc calls `SandboxHandle` "immutable" — this is the one
1539
+ * carve-out, and only this method makes it: a checkpoint reboot is the one
1540
+ * operation that changes what sandbox a handle even refers to. `ports`,
1541
+ * `env`, and `memoryLimitMb` are untouched by the rename — same ports, env,
1542
+ * and memory ceiling as before the reboot, only the name differs.
1543
+ * Upstream's
1544
+ * detached-start path (`Sandbox::start_detached`) passes
1545
+ * `CREATE_BREAKAWAY_FROM_JOB` on Windows, which `ERROR_ACCESS_DENIED`s
1546
+ * outright whenever the msb CLI runs inside a job object that doesn't
1547
+ * grant breakaway rights — a Gradle/cargo/node test runner on a hosted
1548
+ * Windows runner, or any process that embeds this library inside its own
1549
+ * restrictive job object — and that denial is deterministic, not
1550
+ * transient, so no retry shape fixes it. Before 0.7.1, attached `msb run`
1551
+ * (including its `--from-snapshot` boots) never hit this, which is why the
1552
+ * reboot has always gone through the classified boot path (`bootClassified`)
1553
+ * rather than `msb start`. `msb restore` has no attached/detached
1554
+ * distinction of its own — its CLI process always calls `sandbox.detach()`
1555
+ * internally and exits once activation is confirmed (see
1556
+ * `bootRestoreOnce`'s own doc on why its exit is a SUCCESS signal, not
1557
+ * something to reap) — so whether it shares upstream's
1558
+ * `CREATE_BREAKAWAY_FROM_JOB` path on Windows is NOT verified here: this
1559
+ * backend never runs the real msb binary (see this repo's hard "never boot
1560
+ * a sandbox" rule), so this is a real open question for CI, not something
1561
+ * this migration could confirm either way. If it recurs, it will surface
1562
+ * as an ordinary unclassified `BackendError` out of `bootRestoreOnce`
1563
+ * (deterministic exit-code failure, not one of the classified transients
1564
+ * above), the same shape a genuine breakaway denial always took.
1565
+ * `rm`-ing the sandbox first and restoring a fresh one under a fresh
1566
+ * name, same ports/memory (via a spec identical to `handle.spec` except
1567
+ * `name`/`checkpointRef` set to the fresh name and the EFFECTIVE ref this
1568
+ * method discovers, below — env is no longer threaded through at all, see
1569
+ * `MsbCommands.restore`) reproduces the exact same observable contract
1570
+ * MODULO the name itself, which was always an implementation detail, not
1571
+ * part of what a checkpoint promises to preserve. This backend's own
1572
+ * `handles`/`startedNames` registries ARE re-keyed from the original name
1573
+ * to the WINNING attempt's name (see below), and each attempt's own fresh
1574
+ * name is tracked in the reaping ledger before that attempt's restore is
1575
+ * even attempted, exactly like an ordinary `create()` — a failed attempt's
1576
+ * name is untracked again by `rebootUnderFreshName` itself, while the
1577
+ * original name's own ledger entry is deliberately left alone throughout,
1578
+ * for the ledger's existing not-found-tolerant sweep to find (it was
1579
+ * already `msb rm`-ed above). Its workload restarts from scratch (the VM
1580
+ * reboots), which is why `capabilities.checkpointRestartsWorkload` is
1581
+ * `true` here and the generic layer re-runs the wait strategy after this
1582
+ * returns — AFTER `bootRestoreOnce` has already revived the workload
1583
+ * itself via `reviveWorkload` (see that method's own doc): `msb restore`
1584
+ * boots the reboot idle, only the guest agent inside, so `bootRestoreOnce`
1585
+ * no longer leaves `state.attached` untouched the way it did before this
1586
+ * revival step existed — it now carries the workload-revival exec child,
1587
+ * the same attached-child slot `bootRunOnce`'s own `msb run` child fills
1588
+ * for an ordinary boot.
1589
+ *
1590
+ * Before stopping the source sandbox, this also best-effort captures its
1591
+ * guest workload cmdline (`captureGuestWorkloadCmdline`) when
1592
+ * `handle.spec.command` is undefined — the image's own default entrypoint
1593
+ * was running, so there is no explicit command for `reviveWorkload` to
1594
+ * fall back on at either this method's own immediate reboot or a later,
1595
+ * registry-mediated restore. See that method's own doc for why a capture
1596
+ * failure never fails the checkpoint itself.
613
1597
  *
614
1598
  * If the snapshot step fails, the sandbox is left stopped — no
615
1599
  * best-effort restart, since that restart would itself be the broken `msb
@@ -621,6 +1605,22 @@ export class MsbCliBackend {
621
1605
  *
622
1606
  * Refuses outright, before any of the above, when `handle.spec.tmpfsRootMb`
623
1607
  * is set: a tmpfs root has nothing on disk for a snapshot to capture.
1608
+ *
1609
+ * `ref` is the WORKING ref this method is asked to checkpoint under — its
1610
+ * basename becomes the snapshot create call's `<name>` argument (still
1611
+ * meaningful: it lands in msb's own index, see `MsbCommands.snapshotCreate`'s
1612
+ * own doc) and, for a path ref, its dirname becomes `--dest-dir`. It is
1613
+ * NOT necessarily where the artifact ends up: EMPIRICALLY VERIFIED against
1614
+ * a real msb 0.7.1 binary, `snapshot create` always writes under
1615
+ * `<destDir-or-default>/<sandbox>/snap_<32-hex-digest>`, a path `name`
1616
+ * never determines. This method therefore parses that real artifact path
1617
+ * back out of the command's own stdout (last non-empty line, required to
1618
+ * be absolute — see `parseSnapshotCreateArtifactPath`) and returns THAT as
1619
+ * the EFFECTIVE ref — the one used for the reboot below and the one the
1620
+ * caller (`GenericContainer.checkpoint()`) stores in the registry and
1621
+ * hands back on the `Checkpoint` it returns. Malformed or unrecognizable
1622
+ * output (empty, no absolute last line) throws a `BackendError` quoting
1623
+ * the raw, unparsed output verbatim rather than guessing a ref.
624
1624
  */
625
1625
  async createCheckpoint(handle, ref) {
626
1626
  if (handle.spec.tmpfsRootMb !== undefined) {
@@ -630,58 +1630,146 @@ export class MsbCliBackend {
630
1630
  throw new TmpfsRootCheckpointError();
631
1631
  }
632
1632
  const msbPath = await this.msbPath();
1633
+ // Captured up front: `handle.id` itself is mutated in place, below, once
1634
+ // the reboot under the fresh name has actually succeeded — every
1635
+ // reference to the SOURCE sandbox's own name in this method (the
1636
+ // stop/snapshot/rm steps, the pre-mutation error messages) goes through
1637
+ // this local instead, never `handle.id` directly, so it stays correct
1638
+ // regardless of when that mutation happens.
1639
+ const originalName = handle.id;
1640
+ const state = this.handles.get(originalName);
1641
+ if (state === undefined) {
1642
+ throw new BackendError(`no handle state for sandbox '${originalName}' — create() was never called for it`);
1643
+ }
1644
+ // Guest cmdline capture — BEFORE stopping the source sandbox (see
1645
+ // captureGuestWorkloadCmdline's own doc) — only when there is no
1646
+ // explicit command for a restore to fall back on later. Best-effort: a
1647
+ // capture failure never fails the checkpoint itself (captureGuestWorkloadCmdline
1648
+ // already swallows it into `undefined`); its absence only surfaces
1649
+ // later, at restore time, as CheckpointWorkloadCommandMissingError.
1650
+ // Stashed on `state` (kept by reference across this method's own
1651
+ // re-keying below, so it survives the remove+reboot churn) for
1652
+ // `capturedWorkloadCommand()` to hand back to `GenericContainer.checkpoint()`
1653
+ // afterward.
1654
+ const capturedCommand = handle.spec.command === undefined ? await this.captureGuestWorkloadCmdline(handle) : undefined;
1655
+ state.capturedCommand = capturedCommand;
633
1656
  await this.stop(handle);
634
- // A path ref (see checkpoint/ref.ts) stores its artifact under the ref
635
- // itself: mkdirs the parent, then hands msb the ref's own basename as
636
- // the snapshot name and the parent as --dest-dir, so the artifact msb
637
- // writes lands at exactly <parent>/<basename> === ref. A bare-name ref
638
- // (pre-dest-dir checkpoints, still restorable) keeps going through
639
- // msb's own default snapshot store, unchanged.
1657
+ // A path ref (see checkpoint/ref.ts) hands msb the ref's own basename as
1658
+ // the snapshot NAME and the parent as --dest-dir — the parent directory
1659
+ // is honored, but (since msb 0.7.1) the artifact itself lands nested
1660
+ // under <parent>/<sandbox>/snap_<digest>, never literally at `ref`; see
1661
+ // this method's own doc. A bare-name ref (pre-dest-dir checkpoints,
1662
+ // still restorable) keeps going through msb's own default snapshot
1663
+ // store, unchanged.
640
1664
  const isPathRef = path.isAbsolute(ref);
641
1665
  if (isPathRef) {
642
1666
  await fs.mkdir(path.dirname(ref), { recursive: true });
643
1667
  }
644
1668
  const snapshotArgv = isPathRef
645
- ? MsbCommands.snapshotCreate(handle.id, path.basename(ref), path.dirname(ref))
646
- : MsbCommands.snapshotCreate(handle.id, ref);
1669
+ ? MsbCommands.snapshotCreate(originalName, path.basename(ref), path.dirname(ref))
1670
+ : MsbCommands.snapshotCreate(originalName, ref);
647
1671
  const snap = await invoke(msbPath, snapshotArgv, CHECKPOINT_TIMEOUT_MS);
648
1672
  if (snap.exitCode !== 0) {
649
- throw new BackendError(`msb snapshot create --from ${handle.id} ${ref} failed (exit ${snap.exitCode}): ${snap.stderr.trim()} — ` +
650
- `the sandbox is left stopped; run 'msb start ${handle.id}' by hand to bring it back up.`);
1673
+ throw new BackendError(`msb snapshot create --from ${originalName} ${ref} failed (exit ${snap.exitCode}): ${snap.stderr.trim()} — ` +
1674
+ `the sandbox is left stopped; run 'msb start ${originalName}' by hand to bring it back up.`);
651
1675
  }
652
- await invoke(msbPath, MsbCommands.rm(handle.id), STOP_TIMEOUT_MS).catch(() => { });
653
- const state = this.handles.get(handle.id);
654
- if (state === undefined) {
655
- throw new BackendError(`no handle state for sandbox '${handle.id}' — create() was never called for it`);
1676
+ const effectiveRef = parseSnapshotCreateArtifactPath(snap.stdout);
1677
+ if (effectiveRef === undefined) {
1678
+ throw new BackendError(`msb snapshot create --from ${originalName} ${ref} did not print a recognizable artifact path as its ` +
1679
+ `last line — the sandbox is left stopped; run 'msb start ${originalName}' by hand to bring it back up. ` +
1680
+ `Raw output:\n${snap.stdout}${snap.stderr}`);
656
1681
  }
657
- const rebootHandle = { id: handle.id, spec: { ...handle.spec, checkpointRef: ref } };
1682
+ await invoke(msbPath, MsbCommands.rm(originalName), STOP_TIMEOUT_MS).catch(() => { });
1683
+ // `command` resolves the same way `fromCheckpointRegistryEntry` resolves
1684
+ // it for a later, registry-mediated restore: the source's own explicit
1685
+ // command first, the guest cmdline just captured above as the fallback —
1686
+ // so `reviveWorkload`, inside the reboot this triggers next, already has
1687
+ // the fully-resolved workload argv without any restore-time lookup. This
1688
+ // merged `command` (and `checkpointRef`) live ONLY on each attempt's own
1689
+ // handle inside `rebootUnderFreshName`, never on the caller's own
1690
+ // `handle.spec` (mutated below) — that spec must keep reading back
1691
+ // exactly what the SOURCE container's own `spec.command` was (`undefined`
1692
+ // when there was none), since `GenericContainer.checkpoint()` reads
1693
+ // `handle.spec` again right after this returns to build the
1694
+ // named-checkpoint registry entry, which pins that same "explicit
1695
+ // command vs. captured fallback" distinction as two separate fields (see
1696
+ // `CheckpointRegistryEntry`'s own doc).
1697
+ //
1698
+ // `rebootUnderFreshName` owns minting a NEW name per attempt (never
1699
+ // `originalName`, never a prior failed attempt's own name), tracking each
1700
+ // one in the reaping ledger before that attempt's restore runs, and —
1701
+ // per attempt — best-effort `msb rm`-ing and untracking a failed name
1702
+ // before advancing to another fresh one (see that method's own doc for
1703
+ // the full policy). `originalName`'s own ledger entry is deliberately
1704
+ // left alone throughout: it was already `msb rm`-ed above, and the
1705
+ // ledger's sweep is already not-found-tolerant for exactly this shape (a
1706
+ // name the ledger still lists but msb itself has no record of).
1707
+ let rebootHandle;
658
1708
  try {
659
- await this.bootClassified(msbPath, rebootHandle, state);
1709
+ rebootHandle = await this.rebootUnderFreshName(msbPath, handle.spec, effectiveRef, capturedCommand, state);
660
1710
  }
661
1711
  catch (err) {
662
1712
  const detail = err instanceof Error ? err.message : String(err);
663
- throw new BackendError(`sandbox '${handle.id}' was removed after a successful checkpoint snapshot, but booting a fresh ` +
1713
+ throw new BackendError(`sandbox '${originalName}' was removed after a successful checkpoint snapshot, but booting a fresh ` +
664
1714
  `sandbox back up from that snapshot failed: ${detail} — the sandbox's disk state is preserved in ` +
665
- `checkpoint '${ref}', restorable via GenericContainer.fromCheckpoint().`);
1715
+ `checkpoint '${effectiveRef}', restorable via GenericContainer.fromCheckpoint().`);
666
1716
  }
1717
+ const freshName = rebootHandle.id;
1718
+ // The reboot succeeded under `freshName` — the WINNING attempt's own
1719
+ // name, not `originalName` — re-key this backend's own runtime
1720
+ // registries (see class doc on `handles`), then publish the new identity
1721
+ // onto the CALLER's own handle object, IN PLACE, so every subsequent
1722
+ // operation on it targets the sandbox actually running now (see this
1723
+ // method's own doc). Only `id`/`spec.name` change on the live handle —
1724
+ // `spec.command`/`spec.checkpointRef` stay exactly as they were on the
1725
+ // SOURCE spec; see the comment above for why.
1726
+ this.handles.delete(originalName);
1727
+ this.handles.set(freshName, state);
1728
+ this.startedNames.delete(originalName);
1729
+ const mutableHandle = handle;
1730
+ mutableHandle.id = freshName;
1731
+ mutableHandle.spec = { ...handle.spec, name: freshName };
1732
+ return effectiveRef;
667
1733
  }
668
1734
  /**
669
- * Best-effort `msb snapshot rm <basename(ref)>` — "not found" is success,
670
- * the same contract as `removeByName`. msb's own removal deletes both its
671
- * index entry and the dest-dir artifact for a path ref, but afterwards this
672
- * also best-effort recursively deletes the ref path itself: if msb's index
673
- * ever loses track of an artifact without deleting it, the directory would
674
- * otherwise linger under the cache dir forever. That recursive delete is
675
- * gated on `looksLikeCheckpointArtifactDir` first — a `ref` is caller-
676
- * supplied (a corrupt registry entry, a hand-edited env var, …), and a
677
- * `fs.rm(ref, { recursive: true })` on an unverified path would happily
678
- * wipe out an arbitrary directory that merely happens to share its name.
1735
+ * Best-effort `msb snapshot rm <ref> -f` — "not found" is success, the
1736
+ * same contract as `removeByName`. `ref` is passed FULL, never reduced to
1737
+ * `path.basename(ref)`: EMPIRICALLY VERIFIED against a real msb 0.7.1
1738
+ * binary, name-based removal does not resolve at all — the artifact PATH
1739
+ * is the only address that reliably works (see `MsbCommands.snapshotRemove`'s
1740
+ * own doc). Every real caller already hands this the EFFECTIVE ref
1741
+ * `createCheckpoint` returned (an absolute path since 0.7.1), so this is
1742
+ * also simply correct for the common case, not just a defensive choice.
1743
+ *
1744
+ * One failure shape is deliberately NOT swallowed alongside "not found":
1745
+ * msb refuses to remove a snapshot that is still the current HEAD of
1746
+ * older siblings from the same source sandbox (see
1747
+ * `isSnapshotHeadRemovalRefused`) — that refusal propagates as a
1748
+ * `BackendError` naming msb's own remedy, rather than silently doing
1749
+ * nothing, since automatic head rotation is out of scope for this method
1750
+ * (see the checkpoints guide's cleanup section for the documented
1751
+ * limitation).
1752
+ *
1753
+ * msb's own removal deletes both its index entry and the dest-dir
1754
+ * artifact for a path ref, but afterwards this also best-effort
1755
+ * recursively deletes the ref path itself: if msb's index ever loses
1756
+ * track of an artifact without deleting it, the directory would otherwise
1757
+ * linger under the cache dir forever. That recursive delete is gated on
1758
+ * `looksLikeCheckpointArtifactDir` first — a `ref` is caller-supplied (a
1759
+ * corrupt registry entry, a hand-edited env var, …), and a `fs.rm(ref, {
1760
+ * recursive: true })` on an unverified path would happily wipe out an
1761
+ * arbitrary directory that merely happens to share its name.
679
1762
  */
680
1763
  async removeCheckpoint(ref) {
681
1764
  const msbPath = await this.msbPath();
682
1765
  const isPathRef = path.isAbsolute(ref);
683
- const name = isPathRef ? path.basename(ref) : ref;
684
- await invoke(msbPath, MsbCommands.snapshotRemove(name), CHECKPOINT_TIMEOUT_MS).catch(() => { });
1766
+ const result = await invoke(msbPath, MsbCommands.snapshotRemove(ref), CHECKPOINT_TIMEOUT_MS).catch(() => undefined);
1767
+ if (result !== undefined && result.exitCode !== 0 && isSnapshotHeadRemovalRefused(result.stderr)) {
1768
+ throw new BackendError(`msb snapshot rm ${ref} -f was refused (exit ${result.exitCode}): ${result.stderr.trim()} — this ` +
1769
+ `checkpoint is still the newest snapshot of other, older ones from the same source sandbox; ` +
1770
+ `select another snapshot as head first ('msb snapshot head ...', see msb's own message above), ` +
1771
+ `or remove the older siblings first.`);
1772
+ }
685
1773
  if (isPathRef && (await looksLikeCheckpointArtifactDir(ref))) {
686
1774
  await fs.rm(ref, { recursive: true, force: true }).catch(() => { });
687
1775
  }
@@ -757,51 +1845,66 @@ export class MsbCliBackend {
757
1845
  }
758
1846
  }
759
1847
  /**
760
- * `msb snapshot load <archive>`, then resolves the EFFECTIVE ref: msb
761
- * writes the import under a digest-derived directory name it never lets
762
- * the caller choose (never the archive's own recorded `ref`), parsed from
763
- * the printed artifact path's basename (see `parseImportedDigestDirName`).
1848
+ * `msb snapshot load <archive> --dest <checkpointsDir>`, then resolves the
1849
+ * EFFECTIVE ref: EMPIRICALLY VERIFIED against a real msb 0.7.1 binary, a
1850
+ * successful load prints a `group msb-<hex>: head snap_<digest>
1851
+ * (Initialized)` line, a digest line, and finally the loaded artifact's
1852
+ * own absolute path as its LAST line — `<checkpointsDir>/<generated-
1853
+ * group>/snap_<digest>`, never the archive's own recorded `ref` — parsed
1854
+ * back out by `parseImportedArtifactPath` (the same last-non-empty-line,
1855
+ * must-be-absolute defensive pattern `parseSnapshotCreateArtifactPath`
1856
+ * uses for `snapshot create`). `checkpointsDir` is always this backend's
1857
+ * own checkpoints cache directory (`<cacheDir>/checkpoints`, the same
1858
+ * directory `checkpointRef()` mints created-checkpoint refs under) —
1859
+ * omitting `--dest` would import into msb's own global default snapshot
1860
+ * store instead, outside anywhere this library looks for or cleans up
1861
+ * checkpoints. Created fresh (mkdir -p) before the load, since nothing
1862
+ * guarantees a caller ever created a checkpoint locally first (an
1863
+ * imported-only machine has no reason to have this directory yet).
1864
+ *
1865
+ * This replaces the pre-0.7.1 shape entirely: `snapshot load` used to
1866
+ * write under a digest-derived directory name with no `--dest` of its
1867
+ * own, so this method had to parse out just that bare NAME and then
1868
+ * separately CONFIRM it via `msb snapshot list --format json` before
1869
+ * handing it back (the only way to get a usable ref at all, since the
1870
+ * bare name alone still had to resolve against msb's default store). 0.7.1
1871
+ * prints the full, directly-usable, already-`--dest`-scoped path as part
1872
+ * of `load`'s own output, so that whole list-and-confirm round trip is
1873
+ * gone — dead code that this migration deletes outright, not merely
1874
+ * bypassed.
1875
+ *
764
1876
  * An already-exists failure — msb's own content-addressed dedup — is
765
1877
  * treated as success, since the artifact is already present under that
766
- * digest either way; any OTHER import failure surfaces msb's own stderr.
767
- * The digest-dir basename is then CONFIRMED present via `msb snapshot
768
- * list --format json` (matching it against each entry's
769
- * `name`/`artifact_path`) and returned as-is — never the entry's `digest`
770
- * field. Live-verified against msb 0.6.8: the full `sha256:<64hex>`
771
- * digest does not resolve as a snapshot ref at all (`msb snapshot inspect
772
- * sha256:<full>` fails "snapshot not found"); only the digest-dir name
773
- * resolves for `inspect`/`rm`/`run --from-snapshot`, so it — not the full
774
- * digest — is the ref this must hand back for `hasCheckpoint` and every
775
- * other snapshot-ref call to keep working. `_ref` (the archive's own
776
- * recorded ref) is unused here — msb's importer never takes one, unlike
777
- * docker's, where the effective ref really is the ref passed in.
1878
+ * digest either way, and its stderr is parsed the same way as a success's
1879
+ * stdout; any OTHER import failure surfaces msb's own stderr in a
1880
+ * `BackendError`, and unparseable output (no recognizable absolute last
1881
+ * line — e.g. only the "group ... (Initialized)" line, msb's OWN wording
1882
+ * for a state that is not itself a path) throws a `BackendError` quoting
1883
+ * the raw output verbatim rather than misreading a status line as a ref.
1884
+ * `_ref` (the archive's own recorded ref) is unused here — msb's importer
1885
+ * never takes one, unlike docker's, where the effective ref really is the
1886
+ * ref passed in.
778
1887
  */
779
1888
  async importCheckpoint(srcFile, _ref) {
780
1889
  const msbPath = await this.msbPath();
781
- const imported = await invoke(msbPath, MsbCommands.snapshotImport(srcFile), CHECKPOINT_TIMEOUT_MS);
782
- let digestDirName;
1890
+ const checkpointsDir = path.join(cacheDir(), "checkpoints");
1891
+ await fs.mkdir(checkpointsDir, { recursive: true });
1892
+ const imported = await invoke(msbPath, MsbCommands.snapshotImport(srcFile, checkpointsDir), CHECKPOINT_TIMEOUT_MS);
1893
+ let artifactPath;
783
1894
  if (imported.exitCode === 0) {
784
- digestDirName = parseImportedDigestDirName(imported.stdout);
1895
+ artifactPath = parseImportedArtifactPath(imported.stdout);
785
1896
  }
786
1897
  else if (isSnapshotAlreadyExistsError(imported.stderr)) {
787
- digestDirName = parseImportedDigestDirName(imported.stderr);
1898
+ artifactPath = parseImportedArtifactPath(imported.stderr);
788
1899
  }
789
1900
  else {
790
- throw new BackendError(`msb snapshot load ${srcFile} failed (exit ${imported.exitCode}): ${imported.stderr.trim()}`);
791
- }
792
- if (digestDirName === undefined) {
793
- throw new BackendError(`msb snapshot load ${srcFile} did not print a recognizable artifact path — output:\n` +
794
- `${imported.stdout}${imported.stderr}`);
795
- }
796
- const list = await invoke(msbPath, MsbCommands.snapshotList(), CHECKPOINT_TIMEOUT_MS);
797
- if (list.exitCode !== 0) {
798
- throw new BackendError(`msb snapshot list failed (exit ${list.exitCode}): ${list.stderr.trim()}`);
1901
+ throw new BackendError(`msb snapshot load ${srcFile} --dest ${checkpointsDir} failed (exit ${imported.exitCode}): ${imported.stderr.trim()}`);
799
1902
  }
800
- const confirmedRef = confirmDigestDirNamePresent(parseSnapshotList(list.stdout), digestDirName);
801
- if (confirmedRef === undefined) {
802
- throw new BackendError(`imported snapshot '${digestDirName}' from ${srcFile} did not appear in 'msb snapshot list' — could not confirm the import`);
1903
+ if (artifactPath === undefined) {
1904
+ throw new BackendError(`msb snapshot load ${srcFile} --dest ${checkpointsDir} did not print a recognizable artifact path as its ` +
1905
+ `last line — output:\n${imported.stdout}${imported.stderr}`);
803
1906
  }
804
- return confirmedRef;
1907
+ return artifactPath;
805
1908
  }
806
1909
  async runningSandboxNames(msbPath) {
807
1910
  const result = await invoke(msbPath, MsbCommands.ls(), LOGS_TIMEOUT_MS);