rightsize 0.7.9 → 0.7.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +6 -4
  2. package/dist/backend-docker/backend.d.ts +29 -3
  3. package/dist/backend-docker/backend.d.ts.map +1 -1
  4. package/dist/backend-docker/backend.js +30 -3
  5. package/dist/backend-docker/backend.js.map +1 -1
  6. package/dist/backend-msb/backend.d.ts +657 -83
  7. package/dist/backend-msb/backend.d.ts.map +1 -1
  8. package/dist/backend-msb/backend.js +1319 -149
  9. package/dist/backend-msb/backend.js.map +1 -1
  10. package/dist/backend-msb/commands.d.ts +162 -16
  11. package/dist/backend-msb/commands.d.ts.map +1 -1
  12. package/dist/backend-msb/commands.js +228 -32
  13. package/dist/backend-msb/commands.js.map +1 -1
  14. package/dist/backend-msb/network-links.d.ts +86 -1
  15. package/dist/backend-msb/network-links.d.ts.map +1 -1
  16. package/dist/backend-msb/network-links.js +119 -4
  17. package/dist/backend-msb/network-links.js.map +1 -1
  18. package/dist/backend-msb/provisioner.d.ts +1 -1
  19. package/dist/backend-msb/provisioner.d.ts.map +1 -1
  20. package/dist/backend-msb/provisioner.js +1 -1
  21. package/dist/backend-msb/provisioner.js.map +1 -1
  22. package/dist/backend-msb/restore-access-denied.d.ts +50 -0
  23. package/dist/backend-msb/restore-access-denied.d.ts.map +1 -0
  24. package/dist/backend-msb/restore-access-denied.js +52 -0
  25. package/dist/backend-msb/restore-access-denied.js.map +1 -0
  26. package/dist/backend-msb/restore-broker.d.ts +181 -0
  27. package/dist/backend-msb/restore-broker.d.ts.map +1 -0
  28. package/dist/backend-msb/restore-broker.js +264 -0
  29. package/dist/backend-msb/restore-broker.js.map +1 -0
  30. package/dist/backend-msb/sandbox-already-exists.d.ts +32 -0
  31. package/dist/backend-msb/sandbox-already-exists.d.ts.map +1 -0
  32. package/dist/backend-msb/sandbox-already-exists.js +35 -0
  33. package/dist/backend-msb/sandbox-already-exists.js.map +1 -0
  34. package/dist/backend-msb/snapshot-create.d.ts +22 -0
  35. package/dist/backend-msb/snapshot-create.d.ts.map +1 -0
  36. package/dist/backend-msb/snapshot-create.js +33 -0
  37. package/dist/backend-msb/snapshot-create.js.map +1 -0
  38. package/dist/backend-msb/snapshot-import.d.ts +30 -14
  39. package/dist/backend-msb/snapshot-import.d.ts.map +1 -1
  40. package/dist/backend-msb/snapshot-import.js +35 -17
  41. package/dist/backend-msb/snapshot-import.js.map +1 -1
  42. package/dist/backend-msb/snapshot-rm.d.ts +28 -0
  43. package/dist/backend-msb/snapshot-rm.d.ts.map +1 -0
  44. package/dist/backend-msb/snapshot-rm.js +30 -0
  45. package/dist/backend-msb/snapshot-rm.js.map +1 -0
  46. package/dist/core/backend.d.ts +68 -23
  47. package/dist/core/backend.d.ts.map +1 -1
  48. package/dist/core/checkpoint/api.d.ts +2 -2
  49. package/dist/core/checkpoint/api.js +2 -2
  50. package/dist/core/checkpoint/registry.d.ts +46 -0
  51. package/dist/core/checkpoint/registry.d.ts.map +1 -1
  52. package/dist/core/checkpoint/registry.js +44 -3
  53. package/dist/core/checkpoint/registry.js.map +1 -1
  54. package/dist/core/errors.d.ts +49 -0
  55. package/dist/core/errors.d.ts.map +1 -1
  56. package/dist/core/errors.js +61 -0
  57. package/dist/core/errors.js.map +1 -1
  58. package/dist/core/free-ports.d.ts +18 -0
  59. package/dist/core/free-ports.d.ts.map +1 -1
  60. package/dist/core/free-ports.js +61 -0
  61. package/dist/core/free-ports.js.map +1 -1
  62. package/dist/core/generic-container.d.ts +101 -12
  63. package/dist/core/generic-container.d.ts.map +1 -1
  64. package/dist/core/generic-container.js +399 -71
  65. package/dist/core/generic-container.js.map +1 -1
  66. package/dist/core/model.d.ts +40 -3
  67. package/dist/core/model.d.ts.map +1 -1
  68. package/dist/core/network.d.ts +6 -2
  69. package/dist/core/network.d.ts.map +1 -1
  70. package/dist/core/network.js +4 -1
  71. package/dist/core/network.js.map +1 -1
  72. package/dist/core/reuse/hash.d.ts +12 -0
  73. package/dist/core/reuse/hash.d.ts.map +1 -1
  74. package/dist/core/reuse/hash.js +1 -0
  75. package/dist/core/reuse/hash.js.map +1 -1
  76. package/dist/core/reuse/registry.d.ts +10 -1
  77. package/dist/core/reuse/registry.d.ts.map +1 -1
  78. package/dist/core/reuse/registry.js +15 -1
  79. package/dist/core/reuse/registry.js.map +1 -1
  80. package/dist/core/sandbox-name.d.ts +3 -0
  81. package/dist/core/sandbox-name.d.ts.map +1 -0
  82. package/dist/core/sandbox-name.js +19 -0
  83. package/dist/core/sandbox-name.js.map +1 -0
  84. package/dist/index.d.ts +2 -2
  85. package/dist/index.d.ts.map +1 -1
  86. package/dist/index.js +1 -1
  87. package/dist/index.js.map +1 -1
  88. package/dist/modules/cassandra.d.ts +12 -12
  89. package/dist/modules/cassandra.js +13 -13
  90. package/dist/modules/cassandra.js.map +1 -1
  91. package/dist/modules/minio.d.ts +5 -3
  92. package/dist/modules/minio.d.ts.map +1 -1
  93. package/dist/modules/minio.js +14 -4
  94. package/dist/modules/minio.js.map +1 -1
  95. package/dist/modules/postgres.d.ts +2 -2
  96. package/dist/modules/postgres.d.ts.map +1 -1
  97. package/dist/modules/postgres.js +11 -9
  98. package/dist/modules/postgres.js.map +1 -1
  99. package/package.json +1 -1
  100. package/dist/backend-msb/snapshot-list.d.ts +0 -43
  101. package/dist/backend-msb/snapshot-list.d.ts.map +0 -1
  102. package/dist/backend-msb/snapshot-list.js +0 -53
  103. package/dist/backend-msb/snapshot-list.js.map +0 -1
@@ -2,7 +2,10 @@ import { spawn, spawnSync } from "node:child_process";
2
2
  import * as readline from "node:readline";
3
3
  import * as fs from "node:fs/promises";
4
4
  import * as path from "node:path";
5
- import { BackendError, PortBindConflictError, UnsupportedByBackendError, TmpfsRootCheckpointError } from "../core/errors.js";
5
+ import { BackendError, PortBindConflictError, UnsupportedByBackendError, TmpfsRootCheckpointError, CheckpointWorkloadCommandMissingError, } from "../core/errors.js";
6
+ import { cacheDir } from "../core/cache-dir.js";
7
+ import { nextSandboxName } from "../core/sandbox-name.js";
8
+ import { trackSandbox, untrackSandbox } from "../core/reaper/init.js";
6
9
  import { MsbCommands } from "./commands.js";
7
10
  import { runningNames, statusOf } from "./ls-json.js";
8
11
  import { hasSandboxStartedMarker } from "./fast-exit.js";
@@ -12,14 +15,26 @@ import { isImageCacheCorruption } from "./image-cache.js";
12
15
  import { isMsbStateDbError } from "./state-db.js";
13
16
  import { isAgentEndpointNotReady } from "./agent-endpoint.js";
14
17
  import { isSnapshotNotFoundError } from "./snapshot-not-found.js";
15
- import { isSnapshotAlreadyExistsError, parseImportedDigestDirName } from "./snapshot-import.js";
18
+ import { isSnapshotAlreadyExistsError, parseImportedArtifactPath } from "./snapshot-import.js";
16
19
  import { isSnapshotSaveAccessDeniedFailure, salvageStagedArchive } from "./snapshot-save-fsync.js";
17
- import { parseSnapshotList, confirmDigestDirNamePresent } from "./snapshot-list.js";
20
+ import { parseSnapshotCreateArtifactPath } from "./snapshot-create.js";
21
+ import { isSnapshotHeadRemovalRefused } from "./snapshot-rm.js";
18
22
  import { undeliveredLines } from "./follow-replay.js";
19
- import { requireNoDuplicateGuestPorts, requireAliasesAreValid, hostsAliasScript } from "./network-links.js";
23
+ import { requireNoDuplicateGuestPorts, requireAliasesAreValid, hostsAliasScript, udpForwarderProbeScript, installUdpForwarderScript, udpReadinessProbeScript, udpForwarderLogPath, } from "./network-links.js";
20
24
  import { ExecTunnel } from "./exec-tunnel.js";
25
+ import { isRestoreAccessDeniedFailure } from "./restore-access-denied.js";
26
+ import { isSandboxAlreadyExistsFailure } from "./sandbox-already-exists.js";
27
+ import { realRestoreBroker, } from "./restore-broker.js";
21
28
  const FIRST_RUN_PULL_TIMEOUT_MS = 600_000; // a cold pull can be slow
22
29
  const READINESS_POLL_MS = 300;
30
+ // A UDP link's forwarder prints nothing on bind (`nc -u -l` is silent), so
31
+ // readiness is polled against /proc/net/udp{,6} directly rather than
32
+ // inferred from the launch exec alone — short interval, short budget: the
33
+ // script's own listener loop starts within a handful of shell built-ins of
34
+ // launching, never a cold pull or a poll-driven state transition like the
35
+ // boot-readiness constants above.
36
+ const UDP_LINK_READINESS_POLL_MS = 100;
37
+ const UDP_LINK_READINESS_TIMEOUT_MS = 5_000;
23
38
  const STOP_TIMEOUT_MS = 60_000;
24
39
  const EXEC_TIMEOUT_MS = 120_000;
25
40
  // How long an exec keeps retrying while the guest agent's endpoint has not
@@ -44,6 +59,51 @@ const COPY_TIMEOUT_MS = 120_000;
44
59
  // already confirmed no longer Running. Never a wait-for-content budget: a
45
60
  // stopped sandbox's log cannot grow, so the first successful fetch is final.
46
61
  const TERMINAL_FETCH_FAILURE_BUDGET_MS = 10_000;
62
+ // How long `reviveWorkload` gives a freshly-spawned workload-revival exec
63
+ // child to prove it isn't an immediate boot failure before treating it as
64
+ // the ordinary long-lived case. Unlike `bootRunOnce`'s own attached child,
65
+ // there is no separate "reached Running" signal to poll for here — the
66
+ // SANDBOX already reports Running regardless of whether this exec succeeds
67
+ // — so a short settle window is what that same exit-vs-success race
68
+ // collapses to when the only observable signal left is the child's own
69
+ // exit. A few multiples of the ordinary poll cadence is plenty: a failing
70
+ // workload (bad command, missing binary, an immediate usage error) exits
71
+ // within milliseconds of spawn, while a genuine long-lived server never
72
+ // exits on its own at all.
73
+ const EXEC_REVIVE_SETTLE_MS = READINESS_POLL_MS * 3;
74
+ // Bounded retry policy for `msb restore` hitting a Windows access-denied
75
+ // failure on its own just-written snapshot artifact (see
76
+ // `isRestoreAccessDeniedFailure`) — a brief file-handle release lag that
77
+ // normally clears within one retry, so several short-backoff attempts cover
78
+ // the real cases without masking a genuinely stuck lock.
79
+ const RESTORE_ACCESS_DENIED_RETRY_LIMIT = 3;
80
+ const RESTORE_ACCESS_DENIED_RETRY_DELAY_MS = 500;
81
+ /**
82
+ * How long `createCheckpoint`'s own reboot step keeps retrying msb's
83
+ * "sandbox already exists" refusal (see `isSandboxAlreadyExistsFailure`) and
84
+ * its Windows access-denied refusal on the just-written snapshot artifact
85
+ * (see `isRestoreAccessDeniedFailure`), and the pause between attempts — the
86
+ * same install-lock-poll shape `INSTALL_LOCK_RETRY_BUDGET_MS`/
87
+ * `INSTALL_LOCK_RETRY_DELAY_MS` already use. EMPIRICALLY VERIFIED against a
88
+ * real msb 0.7.1 binary (an isolated `MSB_HOME` testbed): `msb restore
89
+ * --name X` validates the artifact FIRST — an integrity failure exits 1 and
90
+ * leaves NO sandbox record — but a failure AFTER validation (matching either
91
+ * classified signature above) leaves `X` behind as a STOPPED SANDBOX RECORD
92
+ * visible in `msb ls`, and msb's own restore-time collision check
93
+ * (`existing.is_some() || dir_exists`, `prepare_create_target` in
94
+ * `sdk/rust/lib/backend/local/sandbox/create.rs`) then refuses any retry
95
+ * under that SAME name — on Windows CI this refusal has been observed
96
+ * holding for the whole span of a same-name retry loop under load. Each
97
+ * retry attempt therefore mints a BRAND NEW name rather than reusing the one
98
+ * that just failed (see `rebootUnderFreshName`), which sidesteps that exact
99
+ * collision structurally — a name nothing else has ever used cannot collide
100
+ * with a record or a lingering on-disk directory belonging to a name nothing
101
+ * will ever restore under again. This budget bounds how long that
102
+ * fresh-name retry loop as a whole keeps advancing before giving up, not a
103
+ * single name's own collision.
104
+ */
105
+ const CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS = 30_000;
106
+ const CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_DELAY_MS = 2_000;
47
107
  /**
48
108
  * The boot failure `start()` heals and retries — carries the `msb run`
49
109
  * child's combined output for the second-failure diagnostic. Internal to the
@@ -106,6 +166,45 @@ class InstallLockActiveError extends Error {
106
166
  this.output = output;
107
167
  }
108
168
  }
169
+ /**
170
+ * Restore-path classified failure for `isRestoreAccessDeniedFailure` — a
171
+ * Windows-only `msb restore` transient against its own just-written
172
+ * snapshot artifact (see that function's own doc). Internal to the boot
173
+ * path, like its siblings above; `bootClassified` owns the bounded-retry
174
+ * policy. Only ever thrown from `bootRestoreOnce`, since it is a restore
175
+ * invocation's own failure signature — an ordinary `run` boot never touches
176
+ * a snapshot artifact at all.
177
+ */
178
+ class RestoreAccessDeniedError extends Error {
179
+ output;
180
+ constructor(output) {
181
+ super(`msb restore access-denied on its own snapshot artifact:\n${output}`);
182
+ this.output = output;
183
+ }
184
+ }
185
+ /**
186
+ * Restore-path classified failure for `isSandboxAlreadyExistsFailure` — msb
187
+ * refusing `restore` because a sandbox under this name already exists (its
188
+ * database record, its on-disk directory, or both — see that function's own
189
+ * doc). Internal to the boot path, like its siblings above; only ever
190
+ * thrown from `bootRestoreOnce`. `bootClassified` itself never retries this
191
+ * one (unlike its siblings) — it simply propagates, so the ordinary
192
+ * `start()` path (a `GenericContainer.fromCheckpoint(cp).start()` restore
193
+ * of a name that turns out to still be live) surfaces it immediately, a
194
+ * real error. Only `createCheckpoint`'s own reboot step, via
195
+ * `rebootUnderFreshName`, retries it — under a NEWLY minted name each time,
196
+ * never the one that just collided (see that method's own doc) — a caller
197
+ * reusing a live name is never this backend's own race to hide, but the
198
+ * checkpoint cycle's own historical `rm`-then-restore of the SAME name was
199
+ * exactly that race.
200
+ */
201
+ class SandboxAlreadyExistsError extends Error {
202
+ output;
203
+ constructor(output) {
204
+ super(`msb restore refused — a sandbox with this name already exists:\n${output}`);
205
+ this.output = output;
206
+ }
207
+ }
109
208
  /**
110
209
  * How long to wait before retrying a boot that hit msb's state-database
111
210
  * error — enough for a winning concurrent invocation's migration transaction
@@ -192,16 +291,17 @@ function describeHeal(heal) {
192
291
  }
193
292
  /**
194
293
  * True when `ref` (an absolute path ref) looks like a checkpoint artifact
195
- * this backend itself would have written — a directory named `rz-ckpt-*`
196
- * (`checkpointRef`'s own prefix) containing a `snapshot.json` file, the same
197
- * shape `hasCheckpoint` checks for. `removeCheckpoint`'s recursive delete is
198
- * gated on this: `ref` is caller-supplied (a corrupt or hand-edited registry
199
- * entry), and skipping the shape check would let an arbitrary path get
200
- * `fs.rm(..., { recursive: true })`'d just because it happened to be passed
201
- * in as a "ref".
294
+ * this backend itself would have written — a directory named `snap_<hex>`
295
+ * (msb's own snapshot-store naming since 0.7.1 — see
296
+ * `parseSnapshotCreateArtifactPath`) containing a `snapshot.json` file, the
297
+ * same shape `hasCheckpoint` checks for. `removeCheckpoint`'s recursive
298
+ * delete is gated on this: `ref` is caller-supplied (a corrupt or
299
+ * hand-edited registry entry), and skipping the shape check would let an
300
+ * arbitrary path get `fs.rm(..., { recursive: true })`'d just because it
301
+ * happened to be passed in as a "ref".
202
302
  */
203
303
  async function looksLikeCheckpointArtifactDir(ref) {
204
- if (!path.basename(ref).startsWith("rz-ckpt-")) {
304
+ if (!/^snap_[0-9a-f]+$/i.test(path.basename(ref))) {
205
305
  return false;
206
306
  }
207
307
  const stat = await fs.stat(ref).catch(() => undefined);
@@ -234,10 +334,97 @@ function drainTail(stream, tail) {
234
334
  rl.on("close", () => resolveDrain());
235
335
  });
236
336
  }
337
+ /**
338
+ * `drainTail`'s sibling for text this backend already has in hand rather
339
+ * than a live stream — a brokered restore's own combined output
340
+ * (`launchRestoreViaBroker`), or a diagnostic note about the broker itself.
341
+ * Same `TAIL_LINES` cap, same shift-oldest behavior, so a brokered attempt's
342
+ * pre-Running diagnostics tail looks exactly like a direct attempt's
343
+ * regardless of which path produced it.
344
+ */
345
+ function pushLinesToTail(tail, text) {
346
+ for (const line of text.split("\n")) {
347
+ tail.push(line);
348
+ if (tail.length > TAIL_LINES) {
349
+ tail.shift();
350
+ }
351
+ }
352
+ }
353
+ /**
354
+ * Busybox-ash-compatible: finds the first non-kernel child of PID 1 in the
355
+ * guest and prints its `/proc/<pid>/cmdline` RAW — NUL-separated with a
356
+ * trailing NUL, the kernel's own on-disk shape, never re-encoded by this
357
+ * script itself — to stdout, exit 0; exits 1 if none is found. Walks
358
+ * every `/proc/<pid>/stat` in NUMERIC pid order (`ls | sort -n`, since a bare
359
+ * shell glob sorts lexically — "10" before "2" — which is wrong for "the
360
+ * first child") and, for each, reads field 4 (`ppid`, 1-indexed per
361
+ * `/proc/pid/stat`'s own `pid (comm) state ppid ...` layout), skipping any
362
+ * whose `ppid` isn't `1`. `comm` is split out via the FIRST `(` and the LAST
363
+ * `)` (`${st#*(}` / `${comm%)*}`, both plain POSIX parameter expansion, no
364
+ * `extglob` needed) rather than a naive single-`(`/`)` split, since a
365
+ * process name may itself contain spaces (or, in principle, parens) — the
366
+ * same robustness a greedy sed capture between the first "(" and the last
367
+ * ")" would get. Two names are excluded even though
368
+ * their `ppid` may show `1`: the guest's own init (`init.krun`) and any
369
+ * bracketed kernel-thread name (`[kworker/0:1]`, `[ksoftirqd/0]`, ...) — the
370
+ * shape every kernel thread's `comm` renders as. Read by `MsbCliBackend`'s
371
+ * own `createCheckpoint`, BEFORE it stops the source sandbox — see that
372
+ * method's own doc — via a plain `exec`, never anything backend-specific
373
+ * beyond that: this is guest-side shell, not msb CLI surface.
374
+ */
375
+ const CAPTURE_WORKLOAD_CMDLINE_SCRIPT = [
376
+ // A marker comment, not functional shell — lets the msb fixture (and any
377
+ // future test double) recognize this exact exec call by its script
378
+ // content rather than guessing from argv shape alone.
379
+ "# rightsize:capture-workload-cmdline",
380
+ "for pid in $(ls /proc 2>/dev/null | grep -E '^[0-9]+$' | sort -n); do",
381
+ " st=$(cat /proc/$pid/stat 2>/dev/null) || continue",
382
+ " rest=${st##*) }",
383
+ " set -- $rest",
384
+ " ppid=$2",
385
+ " if [ \"$ppid\" != \"1\" ]; then continue; fi",
386
+ " comm=${st#*(}",
387
+ " comm=${comm%)*}",
388
+ " case \"$comm\" in",
389
+ " init.krun|\\[*\\]) continue ;;",
390
+ " esac",
391
+ " cat /proc/$pid/cmdline",
392
+ " exit 0",
393
+ "done",
394
+ "exit 1",
395
+ ].join("\n");
396
+ /**
397
+ * Parses `CAPTURE_WORKLOAD_CMDLINE_SCRIPT`'s stdout — the discovered
398
+ * process's raw `/proc/<pid>/cmdline` bytes, NUL-separated with a trailing
399
+ * NUL — into an argv. `captureGuestWorkloadCmdline` fetches this through the
400
+ * ordinary `exec()` (`invoke()`'s line-based reconstruction, not
401
+ * `fetchStdoutExact`'s byte-exact one — see that function's own doc), which
402
+ * appends a trailing `\n` to any non-empty output; since the guest script's
403
+ * own output never contains a real newline of its own (it is exactly the
404
+ * NUL-joined cmdline bytes), that single appended `\n` is always the whole
405
+ * of `invoke()`'s own last "line" and is stripped here before splitting,
406
+ * rather than being read back as a bogus trailing argv element. `undefined`
407
+ * on anything that isn't a clean, non-empty argv after that: empty output
408
+ * (the script found no matching child, exited 1, or the guest's own cmdline
409
+ * was itself empty) or content that splits into zero non-empty tokens. Never
410
+ * throws — see `captureGuestWorkloadCmdline`'s own doc on why a capture
411
+ * failure must never fail the checkpoint that triggered it.
412
+ */
413
+ function parseCapturedWorkloadCmdline(stdout) {
414
+ if (stdout.length === 0) {
415
+ return undefined;
416
+ }
417
+ const withoutTrailingNewline = stdout.endsWith("\n") ? stdout.slice(0, -1) : stdout;
418
+ const parts = withoutTrailingNewline.split("\0").filter((s) => s.length > 0);
419
+ return parts.length > 0 ? parts : undefined;
420
+ }
237
421
  /**
238
422
  * The attached-mode CLI driver: every sandbox this backend starts runs as a
239
- * held child process (`msb run`, no `-d`) because detached mode never
240
- * executes the image's own ENTRYPOINT/CMD — only attached mode does.
423
+ * held child process (`msb run`, no `-d`). Detached mode (`msb run -d`) runs
424
+ * the image's own ENTRYPOINT/CMD too, so this isn't about getting the
425
+ * workload to start — it's supervision: the held child gives this backend
426
+ * child-exit-based death detection and a place to capture pre-Running boot
427
+ * diagnostics, neither of which a detached boot provides on its own.
241
428
  * Readiness is never inferred from that child's own output; it comes from
242
429
  * polling `msb ls --format json` until the name shows `"Running"`. The
243
430
  * attached child's stdout/stderr carries msb's own boot diagnostics and is
@@ -265,6 +452,10 @@ export class MsbCliBackend {
265
452
  /** The stop/snapshot/reboot cycle reboots the microVM — the workload restarts. */
266
453
  checkpointRestartsWorkload: true,
267
454
  };
455
+ // Both keyed by a sandbox's CURRENT name (== SandboxHandle.id) — normally
456
+ // stable for a handle's whole lifetime, except across createCheckpoint's
457
+ // own reboot, which re-keys both from the source sandbox's name to the
458
+ // fresh one it boots under (see that method's own doc).
268
459
  handles = new Map();
269
460
  startedNames = new Set();
270
461
  // Mirrors msbPathPromise's eventual value as soon as it settles, purely so
@@ -272,8 +463,35 @@ export class MsbCliBackend {
272
463
  // anything) has a best-effort synchronous read of it. Never written to
273
464
  // except by this one .then() below; never awaited anywhere else.
274
465
  resolvedMsbPath;
275
- constructor(msbPathPromise) {
466
+ // Test-only override seam for rebootUnderFreshName's own budget/delay
467
+ // — defaults to the real CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS/
468
+ // _DELAY_MS constants so production behavior is unchanged. Without this, a
469
+ // budget-exhaustion red-proof would mean a unit test actually blocking for
470
+ // the real ~30s; a test reaches these the same way it already reaches
471
+ // `handles`/`startedNames` elsewhere in this suite — an unsafe cast — to
472
+ // shrink them to milliseconds instead.
473
+ checkpointRebootAlreadyExistsRetryBudgetMs = CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS;
474
+ checkpointRebootAlreadyExistsRetryDelayMs = CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_DELAY_MS;
475
+ // POLICY v2's broker escalation — see `RestoreLaunchMode`/`restore-broker.ts`.
476
+ restoreBroker;
477
+ // Test-only override for the Windows-only escalation gate
478
+ // (`isWindowsPlatform`) — `undefined` in production, where the real
479
+ // `process.platform` always decides. Reached the same unsafe-cast way this
480
+ // suite already reaches `checkpointRebootAlreadyExistsRetryBudgetMs`
481
+ // above, so both the escalation trigger AND the "non-Windows never
482
+ // brokers" invariant are exercisable without an actual Windows host.
483
+ platformOverrideForTests = undefined;
484
+ // Test-only override seam for installUdpLink's own readiness poll
485
+ // interval/budget — defaults to the real UDP_LINK_READINESS_POLL_MS/
486
+ // _TIMEOUT_MS constants so production behavior is unchanged. Reached the
487
+ // same unsafe-cast way this suite already reaches
488
+ // checkpointRebootAlreadyExistsRetryBudgetMs above, so a readiness-timeout
489
+ // red-proof runs in milliseconds rather than the real 5s.
490
+ udpLinkReadinessPollMs = UDP_LINK_READINESS_POLL_MS;
491
+ udpLinkReadinessTimeoutMs = UDP_LINK_READINESS_TIMEOUT_MS;
492
+ constructor(msbPathPromise, options = {}) {
276
493
  this.msbPathPromise = msbPathPromise;
494
+ this.restoreBroker = options.restoreBroker ?? realRestoreBroker;
277
495
  this.msbPathPromise.then((p) => {
278
496
  this.resolvedMsbPath = p;
279
497
  }, () => {
@@ -282,11 +500,24 @@ export class MsbCliBackend {
282
500
  // when it awaits msbPathPromise itself.
283
501
  });
284
502
  }
503
+ /**
504
+ * Whether this process should treat itself as running on Windows for
505
+ * POLICY v2's broker-escalation gate — `process.platform === "win32"` in
506
+ * production, or `platformOverrideForTests` when a test has set it. The
507
+ * ONLY thing this gates is whether `rebootUnderFreshName`/
508
+ * `retryRestoreAfterAccessDenied` ever set `RestoreLaunchMode` to
509
+ * `"broker"`; `bootOnce`/`bootRestoreOnce` themselves stay platform-
510
+ * agnostic dispatchers on whatever mode they're handed, so the broker path
511
+ * itself is exercisable on any host once a test picks it explicitly.
512
+ */
513
+ isWindowsPlatform() {
514
+ return (this.platformOverrideForTests ?? process.platform) === "win32";
515
+ }
285
516
  async msbPath() {
286
517
  return this.msbPathPromise;
287
518
  }
288
519
  async create(spec) {
289
- this.handles.set(spec.name, { attached: undefined, attachedExited: false, resources: [], logTail: [] });
520
+ this.handles.set(spec.name, { attached: undefined, attachedExited: false, resources: [], logTail: [], capturedCommand: undefined });
290
521
  return { id: spec.name, spec };
291
522
  }
292
523
  /**
@@ -331,14 +562,48 @@ export class MsbCliBackend {
331
562
  * knows: the install-lock poll, the one-shot state-database retry, and the
332
563
  * one-shot image-cache heal. Both the ordinary `start()` path and the
333
564
  * checkpoint cycle's post-snapshot reboot come through here — a reboot
334
- * from a snapshot is as exposed to msb's transients as any other boot,
335
- * and skipping the classification there turned a passing install-lock
336
- * poll into an immediate checkpoint failure on a live Windows run.
565
+ * from a snapshot (now `msb restore ...`, see `bootRestoreOnce`) is
566
+ * as exposed to msb's transients as any other boot, and skipping the
567
+ * classification there turned a passing install-lock poll into an
568
+ * immediate checkpoint failure on a live Windows run. The image-cache heal
569
+ * targets `handle.spec.image`, which for a restore is the checkpoint ref
570
+ * GenericContainer.fromCheckpoint() threaded through as the builder's
571
+ * "image" — never a real OCI reference `msb image remove` can act on
572
+ * meaningfully; this predates the 0.7.1 migration (the same was true of
573
+ * `--from-snapshot` boots) and is unchanged here.
574
+ *
575
+ * `retryAccessDenied` (default `true`) gates ONLY the
576
+ * `RestoreAccessDeniedError` branch below — every other classified retry
577
+ * always runs regardless. The ordinary `start()` path (including a plain
578
+ * `GenericContainer.fromCheckpoint(cp).start()` restore) leaves it at the
579
+ * default, delegating to `retryRestoreAfterAccessDenied` — see that
580
+ * method's own doc for why it, like `createCheckpoint`'s own reboot, must
581
+ * mint a fresh name for every retry rather than reusing the one that just
582
+ * failed: the live-verified dossier this backend's fresh-naming behavior
583
+ * is built from says a restore that fails PAST msb's own artifact
584
+ * validation (this access-denied signature included) leaves the attempted
585
+ * name behind as a stopped sandbox record, so a same-name retry only ever
586
+ * collides with that record instead of retrying the actual transient.
587
+ * `createCheckpoint`'s own reboot (`rebootUnderFreshName`) passes `false`:
588
+ * it owns this error class itself, one layer up, via its own fresh-naming
589
+ * loop, and needs this error to propagate on the first hit rather than
590
+ * being retried here at all.
591
+ *
592
+ * `restoreLaunchMode` (default `"direct"`) is threaded straight down to
593
+ * every `bootOnce` call this method makes, including its own internal
594
+ * install-lock/state-db/image-cache retries — POLICY v2's broker
595
+ * escalation, once triggered by the caller (`rebootUnderFreshName`/
596
+ * `retryRestoreAfterAccessDenied`), applies to EVERY remaining attempt of
597
+ * that reboot, not just the ones those callers themselves loop over, so an
598
+ * install-lock hit mid-brokered-attempt must retry brokered too rather
599
+ * than silently reverting to a direct spawn. See `RestoreLaunchMode`'s own
600
+ * doc.
337
601
  */
338
- async bootClassified(msbPath, handle, state) {
602
+ async bootClassified(msbPath, handle, state, options = { retryAccessDenied: true }) {
603
+ const restoreLaunchMode = options.restoreLaunchMode ?? "direct";
339
604
  let firstOutput;
340
605
  try {
341
- await this.bootOnce(msbPath, handle, state);
606
+ await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
342
607
  return;
343
608
  }
344
609
  catch (first) {
@@ -355,7 +620,7 @@ export class MsbCliBackend {
355
620
  while (Date.now() < deadline) {
356
621
  await sleep(INSTALL_LOCK_RETRY_DELAY_MS);
357
622
  try {
358
- await this.bootOnce(msbPath, handle, state);
623
+ await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
359
624
  return;
360
625
  }
361
626
  catch (again) {
@@ -377,7 +642,7 @@ export class MsbCliBackend {
377
642
  // image-cache heal below.
378
643
  await sleep(STATE_DB_RETRY_DELAY_MS);
379
644
  try {
380
- await this.bootOnce(msbPath, handle, state);
645
+ await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
381
646
  return;
382
647
  }
383
648
  catch (second) {
@@ -390,6 +655,17 @@ export class MsbCliBackend {
390
655
  `host.\nfirst attempt:\n${first.output}\nafter retry:\n${second.output}`);
391
656
  }
392
657
  }
658
+ if (first instanceof RestoreAccessDeniedError) {
659
+ if (!options.retryAccessDenied) {
660
+ // The checkpoint reboot's own fresh-naming retry
661
+ // (`rebootUnderFreshName`) owns this error class itself, one layer
662
+ // up — see this method's own doc on `retryAccessDenied`. Propagate
663
+ // immediately rather than retrying it here at all.
664
+ throw first;
665
+ }
666
+ await this.retryRestoreAfterAccessDenied(msbPath, handle, state, first);
667
+ return;
668
+ }
393
669
  if (!(first instanceof ImageCacheCorruptionError)) {
394
670
  throw first;
395
671
  }
@@ -397,7 +673,7 @@ export class MsbCliBackend {
397
673
  }
398
674
  const heal = await invoke(msbPath, MsbCommands.imageRemove(handle.spec.image), STOP_TIMEOUT_MS).catch((e) => e);
399
675
  try {
400
- await this.bootOnce(msbPath, handle, state);
676
+ await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
401
677
  }
402
678
  catch (second) {
403
679
  if (!(second instanceof ImageCacheCorruptionError)) {
@@ -411,30 +687,50 @@ export class MsbCliBackend {
411
687
  }
412
688
  }
413
689
  /**
414
- * One boot attempt: spawns the attached `msb run` child and polls until the
415
- * sandbox reaches Running. `state.attached` and (for non-keepAlive specs)
416
- * `startedNames` are populated only on success; on any failure the child is reaped here (for
690
+ * One boot attempt, dispatched by shape: an ordinary spec (no
691
+ * `checkpointRef`) drives `run()`'s ATTACHED supervision model
692
+ * (`bootRunOnce`); a checkpoint-restore spec drives `msb restore`'s
693
+ * fundamentally different DETACHED shape (`bootRestoreOnce`) — see each
694
+ * method's own doc. Resets the per-attempt diagnostics tail before either:
695
+ * a retried boot must not blend its tail with the failed attempt's.
696
+ * `restoreLaunchMode` is passed straight through to `bootRestoreOnce`
697
+ * (irrelevant to `bootRunOnce`'s own ordinary `run` path, which never
698
+ * brokers — see `RestoreLaunchMode`'s own doc).
699
+ */
700
+ async bootOnce(msbPath, handle, state, restoreLaunchMode = "direct") {
701
+ state.logTail = [];
702
+ state.attachedExited = false;
703
+ if (handle.spec.checkpointRef !== undefined) {
704
+ await this.bootRestoreOnce(msbPath, handle, state, restoreLaunchMode);
705
+ return;
706
+ }
707
+ await this.bootRunOnce(msbPath, handle, state);
708
+ }
709
+ /**
710
+ * One RUN boot attempt: spawns ATTACHED `msb run`, which stays alive as
711
+ * the sandbox's own supervisor for its whole lifetime, and polls
712
+ * `msb ls --format json` until the name shows Running.
713
+ *
714
+ * `state.attached` and (for non-keepAlive specs) `startedNames` are
715
+ * populated only on success; on any failure the child is reaped here (for
417
716
  * the classified early-exit failures it has already exited; a readiness
418
717
  * timeout leaves it alive and it is hard-killed) so a failed attempt leaves
419
718
  * no live process or registered cleanup state behind — the caller owns
420
- * retry policy, never cleanup (an exit-0 fast-exit success is the one
421
- * exception: it IS success, so `state.attached`/`startedNames` ARE
422
- * populated for it — see below). An early exit is classified from the
423
- * child's combined output: the image-cache-corruption signature throws
424
- * `ImageCacheCorruptionError` (the one failure `start()` heals and
425
- * retries), a host-port bind conflict throws `PortBindConflictError`, and
426
- * otherwise — if the exit code was 0 — `isCompletedFastExit` gets a chance
427
- * to reclassify the exit as a workload that ran to completion before this
428
- * poll loop could ever observe Running (msb 0.6.16's convergent-lifecycle
429
- * rework; see that method's own doc); anything else surfaces the raw
430
- * output as an ordinary boot failure, unchanged.
719
+ * retry policy, never cleanup (an exit-0 success without a live child is
720
+ * the one exception: the fast-exit case below, see `isCompletedFastExit`'s
721
+ * own doc). An early exit is classified from the child's combined output:
722
+ * the image-cache-corruption signature throws `ImageCacheCorruptionError`
723
+ * (the one failure `start()` heals and retries), a host-port bind conflict
724
+ * throws `PortBindConflictError`, and otherwise — if the exit code was 0 —
725
+ * `isCompletedFastExit` gets a chance to reclassify the exit as a workload
726
+ * that ran to completion before this poll loop could ever observe Running
727
+ * (msb 0.6.16's convergent-lifecycle rework; see that method's own doc);
728
+ * anything else surfaces the raw output as an ordinary boot failure,
729
+ * unchanged.
431
730
  */
432
- async bootOnce(msbPath, handle, state) {
433
- // Fresh per-attempt diagnostics: a retried boot must not blend its tail
434
- // with the failed attempt's.
435
- state.logTail = [];
436
- state.attachedExited = false;
437
- const child = spawn(msbPath, MsbCommands.run(handle.spec), { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
731
+ async bootRunOnce(msbPath, handle, state) {
732
+ const argv = MsbCommands.run(handle.spec);
733
+ const child = spawn(msbPath, argv, { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
438
734
  // Merge stdout+stderr into one tail, kept only for the boot diagnostics
439
735
  // below: this pipe is the sole carrier of msb's own output (registry/pull
440
736
  // errors, crash output printed before the sandbox exists). It is not a
@@ -479,8 +775,9 @@ export class MsbCliBackend {
479
775
  }
480
776
  return;
481
777
  }
482
- throw new BackendError(`msb run for sandbox ${handle.id} exited (code ${exited.code ?? "unknown"}) before reaching ` +
483
- `Running — check the image entrypoint and 'msb run' output below:\n${output}`);
778
+ throw new BackendError(`msb run for sandbox ${handle.id} exited (code ${exited.code ?? "unknown"}) ` +
779
+ `before reaching Running — check the image entrypoint and 'msb run' ` +
780
+ `output below:\n${output}`);
484
781
  }
485
782
  if ((await this.runningSandboxNames(msbPath)).has(handle.id)) {
486
783
  state.attached = child;
@@ -501,9 +798,400 @@ export class MsbCliBackend {
501
798
  await sleep(READINESS_POLL_MS);
502
799
  }
503
800
  }
801
+ /**
802
+ * One RESTORE boot attempt. `msb restore` is not `run`'s attached
803
+ * supervision model at all — EMPIRICALLY VERIFIED against msb 0.7.1 (its
804
+ * own `restore.rs` doc: "Restore a snapshot into a new detached
805
+ * sandbox"), the restore CLI process exits — typically within seconds,
806
+ * often with little or no stdout — the INSTANT activation succeeds, while
807
+ * the sandbox keeps booting in the background and only reaches Running
808
+ * some time after that (live-confirmed: `msb ls` shows Running, and exec
809
+ * works, only once the restore process has already exited). A clean exit
810
+ * is therefore not itself a completed boot, and a nonzero exit is msb's
811
+ * own failure signal with the reason on stdout/stderr — the inverse of
812
+ * `run`'s attached child, whose exit before Running always means failure.
813
+ *
814
+ * Two phases follow directly from those two facts:
815
+ * 1. Spawn `msb restore ...` and wait for IT to exit, bounded by
816
+ * `FIRST_RUN_PULL_TIMEOUT_MS` — the same boot budget `bootRunOnce`
817
+ * polls against; a restore's own activation is exposed to the same
818
+ * "this could be a slow cold operation" uncertainty a pull is.
819
+ * Classified from the combined output the same way any boot's early
820
+ * exit is (install-lock, state-db, image-cache-corruption,
821
+ * port-bind-conflict — see `bootClassified`'s own doc on why a
822
+ * restore boot needs the identical classification an ordinary one
823
+ * gets) — a nonzero, unclassified exit throws an ordinary
824
+ * `BackendError` carrying the output, same shape as `bootRunOnce`'s.
825
+ * 2. Once it exits 0, poll `msb ls` for the sandbox to reach Running —
826
+ * the same `READINESS_POLL_MS` cadence and a fresh instance of the
827
+ * identical `FIRST_RUN_PULL_TIMEOUT_MS` budget the attached path's
828
+ * own poll uses (so a restore boot gets the identical readiness
829
+ * patience an ordinary one does, counted from when there is
830
+ * actually something to poll for rather than shaved out of phase
831
+ * 1's own budget). The sandbox settling on exactly `"Stopped"`, or
832
+ * its name disappearing from a `msb ls` listing that itself came
833
+ * back, is a definite failure the moment it is observed — never
834
+ * worth waiting out the rest of the budget for, unlike a status
835
+ * that simply hasn't reached Running yet.
836
+ * 3. The `ls` probe itself failing to run or return in time (spawn
837
+ * error, or the same `LOGS_TIMEOUT_MS` `invoke` enforces on every
838
+ * other probe) is a different thing entirely from the sandbox's name
839
+ * being missing from a listing that DID come back, even though both
840
+ * collapse to the same `undefined` once the probe's own promise is
841
+ * caught — see `isCompletedFastExit`'s own doc, whose sibling
842
+ * contract this mirrors: a probe failure must never be upgraded into
843
+ * a false, specific diagnosis ("disappeared from msb ls entirely" is
844
+ * exactly that kind of false claim when the truth is just that `ls`
845
+ * itself didn't answer in time). A failed probe is therefore treated
846
+ * as "not yet confirmed either way" and retried within budget —
847
+ * install-lock races and an overloaded msb daemon are the same
848
+ * transient shapes this file already documents for `run`/`restore`/
849
+ * `stop`, and one `ls` hiccup during the readiness window must not
850
+ * turn into a spurious restore failure.
851
+ *
852
+ * The `msb restore` CLI process itself is never held onto — it has already
853
+ * exited by the time phase 2 even starts, and msb itself (out-of-process)
854
+ * is the sandbox's own supervisor from here on, the same as it always was.
855
+ * But a restored sandbox reaches Running with ONLY its guest agent inside
856
+ * — the captured workload never re-executes on its own (EMPIRICALLY
857
+ * VERIFIED against msb 0.7.1) — so once Running is confirmed, this method
858
+ * calls `reviveWorkload` to start it itself BEFORE returning: THAT call is
859
+ * what populates `state.attached` (a workload-revival `msb exec` child,
860
+ * not the restore CLI process), and `stop()`'s ordinary attached-child
861
+ * handling (exit-based death detection, the SIGKILL escalation) applies to
862
+ * IT exactly as it always did for `bootRunOnce`'s own attached `msb run`
863
+ * child. See `reviveWorkload`'s own doc for the full revival contract,
864
+ * including the typed error a checkpoint predating workload-cmdline
865
+ * capture throws instead of booting silently idle.
866
+ *
867
+ * Phase 1 (spawning the restore itself and reading back its own exit code
868
+ * + combined output) is dispatched by `launchMode` to either
869
+ * `launchRestoreDirect` (the unconditional spawn this method has always
870
+ * used) or `launchRestoreViaBroker` (POLICY v2's Windows job-object
871
+ * escape hatch — see `RestoreLaunchMode`/`restore-broker.ts`'s own doc).
872
+ * Both return the exact same shape, classified identically right below —
873
+ * a brokered attempt's output goes through the SAME predicates
874
+ * (`isImageCacheCorruption`, `isRestoreAccessDeniedFailure`, ...) a direct
875
+ * attempt's always has, since the underlying `msb restore` invocation and
876
+ * its failure signatures are identical either way; only how its stdout/
877
+ * stderr and exit code got back to this process differs. A broker launch
878
+ * can also come back `"unconfirmed"` (its own `ecFile` never appeared
879
+ * within its bound) — see `RestoreBrokerUnconfirmed`'s own doc — which
880
+ * skips classification entirely and falls straight through to phase 2
881
+ * exactly as a confirmed exit 0 does, since that phase's own `msb ls` poll
882
+ * is activation-gated and settles the question on its own regardless.
883
+ */
884
+ async bootRestoreOnce(msbPath, handle, state, launchMode = "direct") {
885
+ const launch = launchMode === "broker"
886
+ ? await this.launchRestoreViaBroker(msbPath, handle, state)
887
+ : await this.launchRestoreDirect(msbPath, handle, state);
888
+ if (launch.kind === "completed") {
889
+ const { exitCode, output } = launch;
890
+ if (isImageCacheCorruption(output)) {
891
+ throw new ImageCacheCorruptionError(output);
892
+ }
893
+ if (isMsbStateDbError(output)) {
894
+ throw new StateDbError(output);
895
+ }
896
+ if (isMsbInstallLockActive(output)) {
897
+ throw new InstallLockActiveError(output);
898
+ }
899
+ if (isPortBindConflictOutput(output)) {
900
+ throw new PortBindConflictError(`msb restore for sandbox ${handle.id} could not bind a host port: ${output}`);
901
+ }
902
+ if (isRestoreAccessDeniedFailure(output)) {
903
+ throw new RestoreAccessDeniedError(output);
904
+ }
905
+ if (isSandboxAlreadyExistsFailure(output)) {
906
+ throw new SandboxAlreadyExistsError(output);
907
+ }
908
+ if (exitCode !== 0) {
909
+ throw new BackendError(`msb restore for sandbox ${handle.id} exited (code ${exitCode}) — check the snapshot ref and ` +
910
+ `'msb restore' output below:\n${output}`);
911
+ }
912
+ }
913
+ // Exit 0 (direct or brokered), or the broker's own "unconfirmed" outcome
914
+ // — the restore CLI's own task is done, or close enough to trust `msb
915
+ // ls` to settle it (see this method's own doc) — poll for Running the
916
+ // same way bootRunOnce does, under a fresh instance of the identical
917
+ // budget.
918
+ const readyDeadline = Date.now() + FIRST_RUN_PULL_TIMEOUT_MS;
919
+ let lastSeenStatus;
920
+ for (;;) {
921
+ const ls = await invoke(msbPath, MsbCommands.ls(), LOGS_TIMEOUT_MS).catch(() => undefined);
922
+ // `ls === undefined` here means the probe itself never came back
923
+ // (spawn error, or its own LOGS_TIMEOUT_MS timing out) — a DIFFERENT
924
+ // thing from a returned listing that genuinely omits the sandbox's
925
+ // name (see this method's own doc, point 3, and `isCompletedFastExit`'s
926
+ // sibling doc). Only a listing that actually came back gets to update
927
+ // `lastSeenStatus` or trigger the Stopped/disappeared fast-fail below;
928
+ // a failed probe falls straight through to the readiness-budget check
929
+ // and is retried, never asserted as "the sandbox disappeared."
930
+ if (ls !== undefined) {
931
+ lastSeenStatus = statusOf(ls.stdout, handle.id);
932
+ if (lastSeenStatus === "Running") {
933
+ // `msb restore` boots a restored sandbox with only its guest agent
934
+ // inside — the captured workload never re-runs on its own (see
935
+ // this method's own doc and `reviveWorkload`'s) — so this backend
936
+ // starts it itself before registering the boot as complete. A
937
+ // `reviveWorkload` failure propagates unclassified: never register
938
+ // `startedNames` for a sandbox whose workload never actually came
939
+ // up.
940
+ await this.reviveWorkload(msbPath, handle, state);
941
+ if (!handle.spec.keepAlive) {
942
+ this.startedNames.add(handle.id);
943
+ }
944
+ return;
945
+ }
946
+ if (lastSeenStatus === "Stopped" || lastSeenStatus === undefined) {
947
+ throw new BackendError(await this.restoreBootFailureMessage(msbPath, handle.id, lastSeenStatus, false));
948
+ }
949
+ }
950
+ if (Date.now() >= readyDeadline) {
951
+ throw new BackendError(await this.restoreBootFailureMessage(msbPath, handle.id, lastSeenStatus, true));
952
+ }
953
+ await sleep(READINESS_POLL_MS);
954
+ }
955
+ }
956
+ /**
957
+ * `bootRestoreOnce`'s own DIRECT launch — the unconditional child-process
958
+ * spawn every restore attempt used before POLICY v2's broker escalation
959
+ * existed, extracted unchanged from that method's own former body so the
960
+ * two `RestoreLaunchMode`s share one classification point (see
961
+ * `bootRestoreOnce`'s own doc). Bounded by the same `FIRST_RUN_PULL_TIMEOUT_MS`
962
+ * budget `bootRunOnce` polls against, and always resolves `"completed"` —
963
+ * a direct spawn has no analog of the broker's own `"unconfirmed"` outcome,
964
+ * since this process reads the child's real exit code directly.
965
+ */
966
+ async launchRestoreDirect(msbPath, handle, state) {
967
+ const argv = MsbCommands.restore(handle.spec);
968
+ const child = spawn(msbPath, argv, { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
969
+ const stdoutDone = drainTail(child.stdout, state.logTail);
970
+ const stderrDone = drainTail(child.stderr, state.logTail);
971
+ const exitCode = await new Promise((resolveExit, rejectExit) => {
972
+ let settled = false;
973
+ const timer = setTimeout(() => {
974
+ if (settled) {
975
+ return;
976
+ }
977
+ settled = true;
978
+ child.kill("SIGKILL");
979
+ rejectExit(new BackendError(`msb restore for sandbox ${handle.id} did not exit within ${FIRST_RUN_PULL_TIMEOUT_MS / 1000}s — ` +
980
+ `msb itself may be unresponsive; last output:\n${state.logTail.join("\n")}`));
981
+ }, FIRST_RUN_PULL_TIMEOUT_MS);
982
+ child.once("exit", (code) => {
983
+ if (settled) {
984
+ return;
985
+ }
986
+ settled = true;
987
+ clearTimeout(timer);
988
+ resolveExit(code ?? -1);
989
+ });
990
+ });
991
+ await Promise.all([stdoutDone, stderrDone]);
992
+ return { kind: "completed", exitCode, output: state.logTail.join("\n") };
993
+ }
994
+ /**
995
+ * `bootRestoreOnce`'s own BROKERED launch — POLICY v2's Windows job-object
996
+ * escape hatch (see `RestoreLaunchMode`/`restore-broker.ts`'s own module
997
+ * doc on the root cause and the live-validated WMI mitigation). Calls the
998
+ * injected `restoreBroker` seam with the exact same argv a direct attempt
999
+ * would spawn (`MsbCommands.restore`, unmodified) and the same
1000
+ * `FIRST_RUN_PULL_TIMEOUT_MS` budget.
1001
+ *
1002
+ * The broker's own result is either passed straight through
1003
+ * (`"completed"`/`"unconfirmed"`, both appended to `state.logTail` for the
1004
+ * same pre-Running diagnostics purpose `drainTail` serves on the direct
1005
+ * path) or, if the broker call itself REJECTS — broker infrastructure
1006
+ * failing (a missing `powershell.exe`, a script-file write failure, WMI
1007
+ * refusing to even create the process; see `RestoreBrokerLauncher`'s own
1008
+ * doc on what counts as this vs. an ordinary brokered-restore failure) —
1009
+ * caught here and turned into a one-attempt fallback to
1010
+ * `launchRestoreDirect` instead (POLICY v2 point 5: the broker must never
1011
+ * become a new single point of failure). That fallback is scoped to THIS
1012
+ * attempt only; it does not touch the caller's own `RestoreLaunchMode` for
1013
+ * whatever attempt comes after it.
1014
+ */
1015
+ async launchRestoreViaBroker(msbPath, handle, state) {
1016
+ const argv = MsbCommands.restore(handle.spec);
1017
+ let result;
1018
+ try {
1019
+ result = await this.restoreBroker(msbPath, argv, FIRST_RUN_PULL_TIMEOUT_MS);
1020
+ }
1021
+ catch (err) {
1022
+ pushLinesToTail(state.logTail, `[restore broker] broker infrastructure failed for sandbox ${handle.id}, falling back to a direct ` +
1023
+ `attempt: ${err instanceof Error ? err.message : String(err)}`);
1024
+ return this.launchRestoreDirect(msbPath, handle, state);
1025
+ }
1026
+ if (result.kind === "unconfirmed") {
1027
+ pushLinesToTail(state.logTail, `[restore broker] its own exit-code file never appeared within its wait bound for sandbox ` +
1028
+ `${handle.id} — proceeding to the ls-poll phase rather than guessing an outcome.`);
1029
+ return { kind: "unconfirmed" };
1030
+ }
1031
+ pushLinesToTail(state.logTail, result.output);
1032
+ return { kind: "completed", exitCode: result.exitCode, output: result.output };
1033
+ }
1034
+ /**
1035
+ * Renders a restore boot's post-mortem message once its poll for Running
1036
+ * has definitely failed (a settled `"Stopped"`, the name disappearing
1037
+ * from `msb ls` outright, or the readiness budget itself running out).
1038
+ * `msb restore` itself already exited 0 by this point, so the CLI
1039
+ * process's own output (already surfaced for a nonzero exit, above) has
1040
+ * nothing left to add — `msb logs --source system` is the one channel
1041
+ * that can still show what happened to the sandbox's own background boot
1042
+ * (see `MsbCommands.systemLog`'s own doc and `isCompletedFastExit`, its
1043
+ * other caller). Best-effort: a failed probe falls back to a plain "no
1044
+ * output" note rather than letting a diagnostics-gathering failure itself
1045
+ * throw and mask the real one.
1046
+ */
1047
+ async restoreBootFailureMessage(msbPath, id, status, timedOut) {
1048
+ const systemLog = await invoke(msbPath, MsbCommands.systemLog(id), LOGS_TIMEOUT_MS).catch(() => undefined);
1049
+ const diagnostics = systemLog !== undefined && systemLog.stdout.trim() !== ""
1050
+ ? `'msb logs ${id} --source system' output:\n${systemLog.stdout}`
1051
+ : `'msb logs ${id} --source system' produced no output`;
1052
+ // `status` is the last CONFIRMED status a returned `msb ls` listing
1053
+ // actually reported — never a value fabricated from a failed probe (see
1054
+ // bootRestoreOnce's own doc, point 3): `undefined` here on the timedOut
1055
+ // branch means every `ls` probe across the whole readiness budget failed
1056
+ // to even come back, not that a listing came back without the name in
1057
+ // it (that is the non-timedOut, "disappeared" branch below, which is
1058
+ // only ever reached from a listing that DID come back).
1059
+ const reason = timedOut
1060
+ ? `did not reach Running within ${FIRST_RUN_PULL_TIMEOUT_MS / 1000}s of 'msb restore' exiting ` +
1061
+ `(last confirmed status: ${status ?? "none — 'msb ls' never returned a usable listing"})`
1062
+ : status === undefined
1063
+ ? `disappeared from 'msb ls' entirely`
1064
+ : `settled as '${status}' in 'msb ls'`;
1065
+ return `sandbox ${id} was restored ('msb restore' exited 0) but ${reason} before ever reaching Running — ${diagnostics}`;
1066
+ }
1067
+ /**
1068
+ * Starts the workload a restore itself never re-runs. EMPIRICALLY VERIFIED
1069
+ * against msb 0.7.1: a restored sandbox reaches Running with ONLY its
1070
+ * guest agent inside (`guest ps` shows `/init.krun` and kernel threads —
1071
+ * the captured workload command does not re-execute; `msb start`/`msb
1072
+ * logs` on such a sandbox are equally idle/empty). Only ever called from
1073
+ * `bootRestoreOnce`, once it has confirmed Running — see this class's own
1074
+ * doc on `bootRestoreOnce`.
1075
+ *
1076
+ * Spawns a LONG-LIVED, attached `msb exec [-e K=V]... <name> -- <argv>`
1077
+ * session (`MsbCommands.execWithEnv`, env from `handle.spec.env` — a
1078
+ * restore's own `msb restore` has no `-e`/`--env` flag at all, so this exec
1079
+ * is the one place a restored sandbox's guest ever sees it again) — this
1080
+ * becomes the sandbox's own workload from here on, and exec sessions are
1081
+ * exactly what msb's own log capture records (the primary session's
1082
+ * stdout/stderr land in `exec.log`, served by `msb logs`/`-f` — see
1083
+ * `MsbCommands.exec`'s own doc). This exec child slots into the EXACT SAME
1084
+ * `state.attached` role `bootRunOnce`'s own attached `msb run` child fills
1085
+ * for an ordinary boot: child-exit-based death detection, reap-on-stop
1086
+ * (`stop()`'s SIGKILL escalation), and every other attached-child teardown
1087
+ * semantic apply to it unchanged — the detached-restore round left that
1088
+ * slot merely optional, never removed it.
1089
+ *
1090
+ * `handle.spec.command` is ALREADY the fully-resolved workload argv by the
1091
+ * time this runs, in priority order: an explicit command the source
1092
+ * container carried, or — when it had none — the guest cmdline
1093
+ * `createCheckpoint` captured at checkpoint time, merged in by
1094
+ * `createCheckpoint` itself (for its own immediate reboot) or by
1095
+ * `fromCheckpointRegistryEntry` (for a registry-mediated restore, same or
1096
+ * later process — see both functions' own docs). `undefined` here means
1097
+ * NEITHER source exists — an old registry entry predating capture, or one
1098
+ * whose capture attempt itself failed — so this throws
1099
+ * `CheckpointWorkloadCommandMissingError` itself rather than depending on
1100
+ * a caller to have checked first: never boot a restored sandbox silently
1101
+ * idle.
1102
+ *
1103
+ * The exec child gets a brief settle window (`EXEC_REVIVE_SETTLE_MS`) to
1104
+ * prove it isn't an immediate boot failure before this returns success —
1105
+ * mirroring `bootRunOnce`'s own exit-vs-Running race, except a restore's
1106
+ * exec session has no separate "Running" signal of its own to poll for
1107
+ * (the SANDBOX already reports Running regardless of whether this exec
1108
+ * succeeds), so a settle window is what that race collapses to here. An
1109
+ * exit observed within the window is classified exactly like
1110
+ * `bootRunOnce`'s own early exit: exit 0 counts as success only when
1111
+ * `isCompletedFastExit` ALSO confirms it (the sandbox itself settled
1112
+ * Stopped with the boot-completion marker — the repo's existing fast-exit-
1113
+ * completion semantics, reused verbatim); any other exit — nonzero, or
1114
+ * exit 0 without that confirmation — throws a `BackendError` carrying the
1115
+ * exec child's own output, the same failure shape an attached run's early
1116
+ * exit already has.
1117
+ */
1118
+ async reviveWorkload(msbPath, handle, state) {
1119
+ const command = handle.spec.command;
1120
+ if (command === undefined) {
1121
+ throw new CheckpointWorkloadCommandMissingError(handle.spec.checkpointRef ?? handle.id);
1122
+ }
1123
+ state.logTail = [];
1124
+ const argv = MsbCommands.execWithEnv(handle.id, handle.spec.env, command);
1125
+ const child = spawn(msbPath, argv, { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
1126
+ const stdoutDone = drainTail(child.stdout, state.logTail);
1127
+ const stderrDone = drainTail(child.stderr, state.logTail);
1128
+ let exited;
1129
+ child.once("exit", (code) => {
1130
+ exited = { code };
1131
+ state.attachedExited = true;
1132
+ });
1133
+ const settleDeadline = Date.now() + EXEC_REVIVE_SETTLE_MS;
1134
+ while (exited === undefined && Date.now() < settleDeadline) {
1135
+ await sleep(READINESS_POLL_MS);
1136
+ }
1137
+ if (exited === undefined) {
1138
+ // Stayed up through the whole settle window — the ordinary long-lived
1139
+ // case. Slots into state.attached exactly like bootRunOnce's own
1140
+ // attached child.
1141
+ state.attached = child;
1142
+ return;
1143
+ }
1144
+ await Promise.all([stdoutDone, stderrDone]);
1145
+ const output = state.logTail.join("\n");
1146
+ if (exited.code === 0 && (await this.isCompletedFastExit(msbPath, handle.id))) {
1147
+ // Mirrors bootRunOnce's own fast-exit success case: the workload ran to
1148
+ // completion so quickly this settle window caught its natural exit
1149
+ // rather than a crash. No live child left to hold onto — state.attached
1150
+ // stays undefined, same as bootRunOnce's own fast-exit branch.
1151
+ return;
1152
+ }
1153
+ throw new BackendError(`msb exec for sandbox ${handle.id}'s revived workload exited (code ${exited.code ?? "unknown"}) before ` +
1154
+ `staying up — check the workload command and its output below:\n${output}`);
1155
+ }
1156
+ /**
1157
+ * Best-effort: execs `CAPTURE_WORKLOAD_CMDLINE_SCRIPT` in `handle`'s guest
1158
+ * and parses its stdout via `parseCapturedWorkloadCmdline`. Only ever
1159
+ * called from `createCheckpoint`, BEFORE it stops the sandbox (see that
1160
+ * method's own doc), and only when `handle.spec.command` is undefined — an
1161
+ * explicit command needs no capture. Never throws: an exec failure
1162
+ * (nonzero exit, the exec channel itself erroring) and unparseable output
1163
+ * both resolve `undefined` rather than failing the checkpoint — the
1164
+ * captured cmdline is a best-effort fallback, and its absence is fully
1165
+ * handled later, at restore time (`reviveWorkload` throws
1166
+ * `CheckpointWorkloadCommandMissingError` when neither an explicit nor a
1167
+ * captured command exists).
1168
+ */
1169
+ async captureGuestWorkloadCmdline(handle) {
1170
+ try {
1171
+ const result = await this.exec(handle, ["sh", "-c", CAPTURE_WORKLOAD_CMDLINE_SCRIPT]);
1172
+ if (result.exitCode !== 0) {
1173
+ return undefined;
1174
+ }
1175
+ return parseCapturedWorkloadCmdline(result.stdout);
1176
+ }
1177
+ catch {
1178
+ return undefined;
1179
+ }
1180
+ }
1181
+ /**
1182
+ * SPI implementation of `SandboxBackend.capturedWorkloadCommand` — see its
1183
+ * own doc. Reads back whatever `createCheckpoint` most recently stashed on
1184
+ * this sandbox's `HandleState` (keyed by name, so it survives the
1185
+ * stop/snapshot/reboot cycle's own handle-object churn); `undefined` if
1186
+ * this sandbox was never checkpointed, or its checkpoint needed no
1187
+ * capture, or the capture attempt failed.
1188
+ */
1189
+ capturedWorkloadCommand(handle) {
1190
+ return this.handles.get(handle.id)?.capturedCommand;
1191
+ }
504
1192
  /**
505
1193
  * The fast-exit post-mortem classification: only ever consulted from
506
- * `bootOnce` above, and only once the attached `msb run` child has already
1194
+ * `bootRunOnce` above, and only once the attached `msb run` child has already
507
1195
  * exited with code 0 before Running was observed. msb 0.6.16's
508
1196
  * convergent-lifecycle rework means a workload that finishes quickly is
509
1197
  * never observed `"Running"` at all — only `"Starting"`, then the attached
@@ -543,11 +1231,16 @@ export class MsbCliBackend {
543
1231
  }
544
1232
  await invoke(msbPath, MsbCommands.stop(handle.id), STOP_TIMEOUT_MS).catch(() => { });
545
1233
  const attached = state?.attached;
546
- // The attached `msb run` child is msb's own supervisor for this sandbox:
547
- // it stays alive for the sandbox's entire lifetime and only exits once
548
- // the `msb stop` call just above lands, so the common path here is
549
- // "attach a listener, then observe the exit that our own stop just
550
- // caused." state.attachedExited exists for the other case: if the child
1234
+ // The attached child is either an ordinary boot's `msb run` process —
1235
+ // msb's own supervisor for this sandbox, staying alive for its entire
1236
+ // lifetime — or, for a restored sandbox, the `msb exec` session
1237
+ // `reviveWorkload` spawned to revive its captured workload (msb itself,
1238
+ // out-of-process, is the actual supervisor there; this exec child is
1239
+ // just the workload session riding inside it). Either way it stays alive
1240
+ // until the sandbox itself stops, and only exits once the `msb stop`
1241
+ // call just above lands, so the common path here is "attach a listener,
1242
+ // then observe the exit that our own stop just caused." state.attachedExited
1243
+ // exists for the other case: if the child
551
1244
  // had already died before this method ever ran (crashed, or killed by
552
1245
  // something external), start()'s own listener already flipped it, and
553
1246
  // Node never replays a past "exit" event to a listener attached after
@@ -586,30 +1279,340 @@ export class MsbCliBackend {
586
1279
  this.startedNames.delete(handle.id);
587
1280
  this.handles.delete(handle.id);
588
1281
  }
1282
+ /**
1283
+ * `bootClassified`'s own retry for a `RestoreAccessDeniedError` hit on the
1284
+ * ordinary `start()`/`fromCheckpoint().start()` path (its default
1285
+ * `retryAccessDenied: true`) — the sibling of `rebootUnderFreshName`
1286
+ * (`createCheckpoint`'s own reboot retry) for the SAME classified failure,
1287
+ * built from the SAME live-verified policy: `msb restore --name X`
1288
+ * validates the artifact FIRST — an integrity failure exits 1 and leaves
1289
+ * NO sandbox record — but a failure AFTER validation (this access-denied
1290
+ * signature included) leaves `X` behind as a STOPPED SANDBOX RECORD, and
1291
+ * any retry of `restore --name X` then fails outright with msb's own
1292
+ * "already exists" refusal rather than hitting the transient a second
1293
+ * time. A same-name retry (what this method replaces) therefore only ever
1294
+ * proves that collision — never the actual transient — so this mints a
1295
+ * NEW name from `nextSandboxName()` for every retry instead, the same
1296
+ * generator `GenericContainer.start()`'s own boot loop and
1297
+ * `rebootUnderFreshName` both use, tracking it in the reaper ledger BEFORE
1298
+ * that attempt's restore runs and best-effort `msb rm`-ing (then
1299
+ * untracking) a failed attempt's own name before advancing — identical
1300
+ * per-attempt bookkeeping to `rebootUnderFreshName`, just owned one layer
1301
+ * lower.
1302
+ *
1303
+ * The difference from `rebootUnderFreshName` is entirely about WHO owns
1304
+ * the re-keying, not the retry policy itself: `createCheckpoint` calls
1305
+ * `rebootUnderFreshName` with no live sandbox left at all (the source was
1306
+ * already stopped and removed as part of the snapshot step), so EVERY
1307
+ * attempt — including the first — mints a fresh name, and the winning
1308
+ * handle is handed back for `createCheckpoint` itself to re-key. Here,
1309
+ * `handle` is already live under its own name when `first` (the initial
1310
+ * `RestoreAccessDeniedError`) is caught, and `bootClassified`'s caller
1311
+ * (`start()`) has no re-keying step of its own — it just awaits
1312
+ * `bootClassified` and returns — so this method re-keys
1313
+ * `this.handles`/`this.startedNames` AND mutates the caller's own `handle`
1314
+ * object IN PLACE itself on success, the same "one carve-out"
1315
+ * `SandboxHandle`'s own interface doc allows for `createCheckpoint`'s
1316
+ * reboot (see that method's own doc on `handle.id`/`handle.spec`
1317
+ * mutation).
1318
+ *
1319
+ * `handle`'s own original name was already tracked in the reaper ledger by
1320
+ * `GenericContainer.start()`'s own boot loop before `create()`/`start()`
1321
+ * ever ran (mirroring `rebootUnderFreshName`'s treatment of the checkpoint
1322
+ * reboot's own `originalName`) — this method best-effort `msb rm`'s it
1323
+ * once `first` is confirmed retryable, but deliberately leaves its own
1324
+ * ledger entry alone, for the same not-found-tolerant sweep to find,
1325
+ * rather than untracking a name this method never tracked itself.
1326
+ *
1327
+ * Retries up to `RESTORE_ACCESS_DENIED_RETRY_LIMIT` times,
1328
+ * `RESTORE_ACCESS_DENIED_RETRY_DELAY_MS` apart — unchanged from the
1329
+ * same-name retry this replaces. Each attempt goes through
1330
+ * `bootClassified` itself with `retryAccessDenied: false` (so a
1331
+ * `StateDbError`/`InstallLockActiveError`/`ImageCacheCorruptionError` hit
1332
+ * mid-retry still gets its own ordinary classified handling; only
1333
+ * `RestoreAccessDeniedError`'s own retry is owned here) and treats either
1334
+ * `RestoreAccessDeniedError` or `SandboxAlreadyExistsError` as retryable —
1335
+ * a freshly minted name should never collide, but `rebootUnderFreshName`'s
1336
+ * own live-CI evidence says a loaded host can still refuse one, and the
1337
+ * SAME fresh-naming response (rm the failed name, mint another) covers
1338
+ * both signatures identically. Anything else propagates immediately,
1339
+ * unretried.
1340
+ *
1341
+ * POLICY v2's broker escalation applies here too, on the exact same terms
1342
+ * as `rebootUnderFreshName`'s own (see that method's own doc): this
1343
+ * method is only ever entered once the reboot's first, always-direct
1344
+ * attempt has already hit `RestoreAccessDeniedError` (`first`), so on a
1345
+ * Windows host (`isWindowsPlatform()`) every retry THIS method itself
1346
+ * performs is, by construction, an attempt after that trigger — there is
1347
+ * no "first attempt still direct" sub-case to preserve inside this loop
1348
+ * the way `rebootUnderFreshName` has to across its own. `launchMode` is
1349
+ * therefore decided once, before the loop, not re-evaluated per attempt.
1350
+ */
1351
+ async retryRestoreAfterAccessDenied(msbPath, handle, state, first) {
1352
+ const originalName = handle.id;
1353
+ // Best-effort: `first` may have left a stopped sandbox record behind
1354
+ // under `originalName` (see this method's own doc) — cheap cleanup, and
1355
+ // correctness never depends on it, since no retry below ever reuses this
1356
+ // name. Its own ledger entry is deliberately left alone; see this
1357
+ // method's own doc.
1358
+ await invoke(msbPath, MsbCommands.rm(originalName), STOP_TIMEOUT_MS).catch(() => { });
1359
+ const launchMode = this.isWindowsPlatform() ? "broker" : "direct";
1360
+ let last = first;
1361
+ for (let attempt = 1; attempt <= RESTORE_ACCESS_DENIED_RETRY_LIMIT; attempt++) {
1362
+ await sleep(RESTORE_ACCESS_DENIED_RETRY_DELAY_MS);
1363
+ const name = nextSandboxName();
1364
+ if (!handle.spec.keepAlive) {
1365
+ await trackSandbox(name);
1366
+ }
1367
+ const candidate = { id: name, spec: { ...handle.spec, name } };
1368
+ try {
1369
+ await this.bootClassified(msbPath, candidate, state, { retryAccessDenied: false, restoreLaunchMode: launchMode });
1370
+ // Success under `name` — re-key this backend's own runtime
1371
+ // registries (see class doc on `handles`) and publish the new
1372
+ // identity onto the CALLER's own handle object, IN PLACE, exactly
1373
+ // like `createCheckpoint`'s own reboot does (see its own doc).
1374
+ this.handles.delete(originalName);
1375
+ this.handles.set(name, state);
1376
+ this.startedNames.delete(originalName);
1377
+ const mutableHandle = handle;
1378
+ mutableHandle.id = name;
1379
+ mutableHandle.spec = candidate.spec;
1380
+ return;
1381
+ }
1382
+ catch (err) {
1383
+ const classified = err instanceof RestoreAccessDeniedError || err instanceof SandboxAlreadyExistsError;
1384
+ if (!classified) {
1385
+ if (!handle.spec.keepAlive) {
1386
+ await untrackSandbox(name);
1387
+ }
1388
+ throw err;
1389
+ }
1390
+ // Best-effort, result ignored either way — see this method's own
1391
+ // doc on why correctness no longer depends on it.
1392
+ await invoke(msbPath, MsbCommands.rm(name), STOP_TIMEOUT_MS).catch(() => { });
1393
+ if (!handle.spec.keepAlive) {
1394
+ await untrackSandbox(name);
1395
+ }
1396
+ last = err;
1397
+ }
1398
+ }
1399
+ throw new BackendError(`msb restore for sandbox ${originalName} kept hitting msb's Windows access-denied/"sandbox already ` +
1400
+ `exists" refusal across ${RESTORE_ACCESS_DENIED_RETRY_LIMIT} retries, each under a freshly minted ` +
1401
+ `name — this is normally a brief file-handle release lag that clears within one retry, so a failure ` +
1402
+ `held this long looks like a genuinely stuck lock on this host.\n${last.output}`);
1403
+ }
1404
+ /**
1405
+ * `createCheckpoint`'s own reboot step. Mints a NEW sandbox name from
1406
+ * `nextSandboxName()` (the same generator `GenericContainer.start()`'s own
1407
+ * boot loop uses) for EVERY attempt, tracks it in the reaper ledger BEFORE
1408
+ * that attempt's restore runs (mirroring `GenericContainer.start()`'s own
1409
+ * `trackSandbox` call, keepAlive-excluded the same way), and retries two
1410
+ * classified failure signatures on that basis instead of surfacing either
1411
+ * immediately: msb's "sandbox already exists" refusal
1412
+ * (`SandboxAlreadyExistsError`, see `isSandboxAlreadyExistsFailure`'s own
1413
+ * doc) and its Windows access-denied refusal on the just-written snapshot
1414
+ * artifact (`RestoreAccessDeniedError`, see `isRestoreAccessDeniedFailure`'s
1415
+ * own doc — retried here rather than inside `bootClassified` itself, via
1416
+ * that method's own `retryAccessDenied: false`, see its doc).
1417
+ *
1418
+ * LIVE-VERIFIED POLICY this retry is built from: `msb restore --name X`
1419
+ * validates the artifact FIRST — an integrity failure exits 1 and leaves NO
1420
+ * sandbox record — but a failure AFTER validation (a block-device open
1421
+ * PermissionDenied on unix; `RestoreAccessDeniedError`'s own signature on
1422
+ * Windows) leaves `X` behind as a STOPPED SANDBOX RECORD visible in `msb
1423
+ * ls`, and any retry of `restore --name X` then fails with msb's own
1424
+ * "already exists" refusal — so a same-name retry of EITHER classified
1425
+ * failure only ever collides with itself. Reusing a failed attempt's name
1426
+ * is therefore never safe (`msb rm` alone is not reliably enough either: on
1427
+ * Windows it frees only the DB record, not the on-disk directory, which can
1428
+ * outlive it well past this budget — see the fresh-name reboot's own doc on
1429
+ * `createCheckpoint`) — the only universally safe policy is a NEW name per
1430
+ * attempt, same as this backend's ordinary `start()` boot loop already uses
1431
+ * for its own, unrelated port-conflict retries. On a classified failure
1432
+ * this best-effort `msb rm`'s the failed attempt's own name anyway (result
1433
+ * ignored — cheap cleanup, and correctness no longer depends on it now
1434
+ * that the next attempt never reuses that name) and untracks it from the
1435
+ * ledger, before advancing to a freshly minted name for the next attempt.
1436
+ *
1437
+ * `this.checkpointRebootAlreadyExistsRetryBudgetMs`/`_RetryDelayMs` back
1438
+ * this loop's overall deadline/delay rather than the bare
1439
+ * `CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS`/`_DELAY_MS` constants
1440
+ * directly, so a budget-exhaustion test can shrink them to milliseconds
1441
+ * instead of actually blocking for the real ~30s — see those fields' own
1442
+ * doc. Only `SandboxAlreadyExistsError`/`RestoreAccessDeniedError` are
1443
+ * retried here; any other failure `bootClassified` throws (on the first
1444
+ * attempt or a later one) propagates immediately, unretried, WITHOUT
1445
+ * advancing to a new name — `createCheckpoint`'s own caller untracks that
1446
+ * attempt's name on this path (see its own catch block). Not reached by
1447
+ * the ordinary `start()` path: a `GenericContainer.fromCheckpoint(cp)
1448
+ * .start()` restore's own `RestoreAccessDeniedError` is retried by its own
1449
+ * sibling, `retryRestoreAfterAccessDenied` (see that method's own doc) —
1450
+ * same fresh-naming policy, applied one layer lower since that path has no
1451
+ * separate caller left to hand a replacement handle to for re-keying.
1452
+ *
1453
+ * Returns the WINNING attempt's own `SandboxHandle` (a fresh object, never
1454
+ * `undefined`) — `createCheckpoint` re-keys its runtime registries and the
1455
+ * caller's live handle from this, not from a name minted up front.
1456
+ *
1457
+ * POLICY v2's broker escalation (see `RestoreLaunchMode`/`restore-broker.ts`'s
1458
+ * own module doc on the Windows job-object root cause and the live-
1459
+ * validated WMI mitigation) lives here: `launchMode` starts `"direct"` —
1460
+ * the first attempt of ANY reboot is always a direct spawn, unconditionally,
1461
+ * even on Windows — and flips to `"broker"` the moment an attempt hits
1462
+ * `RestoreAccessDeniedError` on a Windows host (`isWindowsPlatform()`);
1463
+ * every attempt from then on, for the rest of THIS reboot, launches
1464
+ * brokered instead, never downgrading back to direct even if a later
1465
+ * attempt's own failure is only `SandboxAlreadyExistsError` (still
1466
+ * retried, but no longer the trigger — only access-denied escalates).
1467
+ * Off Windows, `launchMode` never leaves `"direct"` at all — this reboot's
1468
+ * whole retry loop behaves exactly as it did before this policy existed.
1469
+ */
1470
+ async rebootUnderFreshName(msbPath, sourceSpec, effectiveRef, capturedCommand, state) {
1471
+ const deadline = Date.now() + this.checkpointRebootAlreadyExistsRetryBudgetMs;
1472
+ let attempt = 0;
1473
+ let last;
1474
+ let launchMode = "direct";
1475
+ for (;;) {
1476
+ attempt += 1;
1477
+ const name = nextSandboxName();
1478
+ if (!sourceSpec.keepAlive) {
1479
+ await trackSandbox(name);
1480
+ }
1481
+ const rebootHandle = {
1482
+ id: name,
1483
+ spec: { ...sourceSpec, name, checkpointRef: effectiveRef, command: sourceSpec.command ?? capturedCommand },
1484
+ };
1485
+ try {
1486
+ await this.bootClassified(msbPath, rebootHandle, state, { retryAccessDenied: false, restoreLaunchMode: launchMode });
1487
+ return rebootHandle;
1488
+ }
1489
+ catch (err) {
1490
+ const classified = err instanceof SandboxAlreadyExistsError || err instanceof RestoreAccessDeniedError;
1491
+ if (!classified) {
1492
+ if (!sourceSpec.keepAlive) {
1493
+ await untrackSandbox(name);
1494
+ }
1495
+ throw err;
1496
+ }
1497
+ if (err instanceof RestoreAccessDeniedError && this.isWindowsPlatform()) {
1498
+ // POLICY v2: escalate for every remaining attempt of this reboot —
1499
+ // see this method's own doc on `launchMode`.
1500
+ launchMode = "broker";
1501
+ }
1502
+ // Best-effort: ignore the result either way — a "not found" is as
1503
+ // fine as an actual removal, and this is cleanup, not something the
1504
+ // next attempt (a brand-new name) depends on for correctness.
1505
+ await invoke(msbPath, MsbCommands.rm(name), STOP_TIMEOUT_MS).catch(() => { });
1506
+ if (!sourceSpec.keepAlive) {
1507
+ await untrackSandbox(name);
1508
+ }
1509
+ last = err;
1510
+ if (Date.now() >= deadline) {
1511
+ throw new BackendError(`msb restore kept hitting msb's "sandbox already exists"/access-denied refusal for ` +
1512
+ `${this.checkpointRebootAlreadyExistsRetryBudgetMs / 1000}s across ${attempt} attempt` +
1513
+ `${attempt === 1 ? "" : "s"}, each under a freshly minted name — msb's own on-disk sandbox ` +
1514
+ `directory can lag its database record's own release on a loaded Windows host well past a short ` +
1515
+ `wait, but a refusal held this long looks like a genuinely stuck host rather than a release ` +
1516
+ `race.\nlast attempt ('${name}'):\n${last.output}`);
1517
+ }
1518
+ await sleep(this.checkpointRebootAlreadyExistsRetryDelayMs);
1519
+ }
1520
+ }
1521
+ }
589
1522
  /**
590
1523
  * The stop/snapshot/reboot cycle: `msb stop <name>` (reusing this
591
1524
  * backend's own `stop()`, which also quiesces the attached child and any
592
- * network-link tunnels), `msb snapshot create --from <name> <ref>`, then
593
- * `msb rm <name>` followed by a fresh ATTACHED boot of the SAME name from
594
- * that snapshot — never `msb start`. Upstream's detached-start path
595
- * (`Sandbox::start_detached`) passes `CREATE_BREAKAWAY_FROM_JOB` on
596
- * Windows, which `ERROR_ACCESS_DENIED`s outright whenever the msb CLI runs
597
- * inside a job object that doesn't grant breakaway rights — a Gradle/
598
- * cargo/node test runner on a hosted Windows runner, or any process that
599
- * embeds this library inside its own restrictive job object — and that
600
- * denial is deterministic, not transient, so no retry shape fixes it.
601
- * Attached `msb run` (this backend's normal boot, including `--from-snapshot`
602
- * boots) never hits it, which is why the reboot reuses `bootOnce` instead
603
- * of resuming in place: the stopped sandbox's disk state is already IN the
604
- * snapshot, so `rm`-ing it first and booting a fresh attached sandbox
605
- * under the same name/ports/env/memory (via a spec identical to
606
- * `handle.spec` except `checkpointRef` set to the new `ref`) reproduces
607
- * the exact same observable contract. `bootOnce` swaps `state.attached` to
608
- * the freshly spawned child itself; nothing about `this.handles`/
609
- * `startedNames` or the reaping ledger changes, since the name never
610
- * changed. Its workload restarts from scratch (the VM reboots), which is
611
- * why `capabilities.checkpointRestartsWorkload` is `true` here and the
612
- * generic layer re-runs the wait strategy after this returns.
1525
+ * network-link tunnels), `msb snapshot create --from-sandbox <name> <ref>`
1526
+ * (whose printed artifact path — never `ref` itself — becomes the
1527
+ * EFFECTIVE checkpoint ref this method returns; see
1528
+ * `parseSnapshotCreateArtifactPath`), then `msb rm <name>` followed by
1529
+ * `msb restore <effective-ref> --name <fresh-name>` of a FRESH name from
1530
+ * that snapshot (via `rebootUnderFreshName`/`bootRestoreOnce`, see
1531
+ * `MsbCommands.restore`'s own doc — no `--disk-only`, which a disk-scope
1532
+ * snapshot rejects) — never `msb start`. The fresh name — never the
1533
+ * original — is minted by `nextSandboxName()` (the same generator
1534
+ * `GenericContainer.start()`'s own ordinary boot loop uses; see
1535
+ * `core/sandbox-name.ts`), and a NEW one is minted for EVERY attempt, never
1536
+ * just once for the whole reboot: EMPIRICALLY VERIFIED against a real msb
1537
+ * 0.7.1 binary, a restore that fails PAST msb's own artifact validation
1538
+ * (its Windows access-denied signature, `RestoreAccessDeniedError`) leaves
1539
+ * its `--name` behind as a STOPPED SANDBOX RECORD, and any retry under that
1540
+ * SAME name then collides with msb's own restore-time collision check
1541
+ * (`existing.is_some() || dir_exists`) immediately — CONFIRMED on Windows
1542
+ * CI, where five checkpoint tests collided on their fresh reboot names for
1543
+ * the retry loop's ENTIRE budget once the first attempt hit that
1544
+ * access-denied failure (see `CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS`'s
1545
+ * own doc). A same-name retry therefore only ever retries through a race of
1546
+ * its own making; a NEW name per attempt sidesteps it structurally — a name
1547
+ * nothing else has ever used can never collide with a record or a lingering
1548
+ * directory left by a name nothing will ever restore under again (see
1549
+ * `rebootUnderFreshName`'s own doc for the full retry policy, including its
1550
+ * best-effort `msb rm` of each failed attempt's own name). On success, this method
1551
+ * mutates `handle.id` and `handle.spec` (name only — see below) IN PLACE
1552
+ * on the caller's own `handle`, so every subsequent operation against it
1553
+ * (exec/logs/stop/rm, and — one layer up, via the SAME `SandboxHandle`
1554
+ * reference — `GenericContainer.checkpoint()`'s own post-reboot
1555
+ * `installNetworkLinks`/wait-strategy re-run and `capturedWorkloadCommand()`
1556
+ * call) targets the sandbox actually running now. `SandboxBackend`'s own
1557
+ * interface doc calls `SandboxHandle` "immutable" — this is the one
1558
+ * carve-out, and only this method makes it: a checkpoint reboot is the one
1559
+ * operation that changes what sandbox a handle even refers to. `ports`,
1560
+ * `env`, and `memoryLimitMb` are untouched by the rename — same ports, env,
1561
+ * and memory ceiling as before the reboot, only the name differs.
1562
+ * Upstream's
1563
+ * detached-start path (`Sandbox::start_detached`) passes
1564
+ * `CREATE_BREAKAWAY_FROM_JOB` on Windows, which `ERROR_ACCESS_DENIED`s
1565
+ * outright whenever the msb CLI runs inside a job object that doesn't
1566
+ * grant breakaway rights — a Gradle/cargo/node test runner on a hosted
1567
+ * Windows runner, or any process that embeds this library inside its own
1568
+ * restrictive job object — and that denial is deterministic, not
1569
+ * transient, so no retry shape fixes it. Before 0.7.1, attached `msb run`
1570
+ * (including its `--from-snapshot` boots) never hit this, which is why the
1571
+ * reboot has always gone through the classified boot path (`bootClassified`)
1572
+ * rather than `msb start`. `msb restore` has no attached/detached
1573
+ * distinction of its own — its CLI process always calls `sandbox.detach()`
1574
+ * internally and exits once activation is confirmed (see
1575
+ * `bootRestoreOnce`'s own doc on why its exit is a SUCCESS signal, not
1576
+ * something to reap) — so whether it shares upstream's
1577
+ * `CREATE_BREAKAWAY_FROM_JOB` path on Windows is NOT verified here: this
1578
+ * backend never runs the real msb binary (see this repo's hard "never boot
1579
+ * a sandbox" rule), so this is a real open question for CI, not something
1580
+ * this migration could confirm either way. If it recurs, it will surface
1581
+ * as an ordinary unclassified `BackendError` out of `bootRestoreOnce`
1582
+ * (deterministic exit-code failure, not one of the classified transients
1583
+ * above), the same shape a genuine breakaway denial always took.
1584
+ * `rm`-ing the sandbox first and restoring a fresh one under a fresh
1585
+ * name, same ports/memory (via a spec identical to `handle.spec` except
1586
+ * `name`/`checkpointRef` set to the fresh name and the EFFECTIVE ref this
1587
+ * method discovers, below — env is no longer threaded through at all, see
1588
+ * `MsbCommands.restore`) reproduces the exact same observable contract
1589
+ * MODULO the name itself, which was always an implementation detail, not
1590
+ * part of what a checkpoint promises to preserve. This backend's own
1591
+ * `handles`/`startedNames` registries ARE re-keyed from the original name
1592
+ * to the WINNING attempt's name (see below), and each attempt's own fresh
1593
+ * name is tracked in the reaping ledger before that attempt's restore is
1594
+ * even attempted, exactly like an ordinary `create()` — a failed attempt's
1595
+ * name is untracked again by `rebootUnderFreshName` itself, while the
1596
+ * original name's own ledger entry is deliberately left alone throughout,
1597
+ * for the ledger's existing not-found-tolerant sweep to find (it was
1598
+ * already `msb rm`-ed above). Its workload restarts from scratch (the VM
1599
+ * reboots), which is why `capabilities.checkpointRestartsWorkload` is
1600
+ * `true` here and the generic layer re-runs the wait strategy after this
1601
+ * returns — AFTER `bootRestoreOnce` has already revived the workload
1602
+ * itself via `reviveWorkload` (see that method's own doc): `msb restore`
1603
+ * boots the reboot idle, only the guest agent inside, so `bootRestoreOnce`
1604
+ * no longer leaves `state.attached` untouched the way it did before this
1605
+ * revival step existed — it now carries the workload-revival exec child,
1606
+ * the same attached-child slot `bootRunOnce`'s own `msb run` child fills
1607
+ * for an ordinary boot.
1608
+ *
1609
+ * Before stopping the source sandbox, this also best-effort captures its
1610
+ * guest workload cmdline (`captureGuestWorkloadCmdline`) when
1611
+ * `handle.spec.command` is undefined — the image's own default entrypoint
1612
+ * was running, so there is no explicit command for `reviveWorkload` to
1613
+ * fall back on at either this method's own immediate reboot or a later,
1614
+ * registry-mediated restore. See that method's own doc for why a capture
1615
+ * failure never fails the checkpoint itself.
613
1616
  *
614
1617
  * If the snapshot step fails, the sandbox is left stopped — no
615
1618
  * best-effort restart, since that restart would itself be the broken `msb
@@ -621,6 +1624,22 @@ export class MsbCliBackend {
621
1624
  *
622
1625
  * Refuses outright, before any of the above, when `handle.spec.tmpfsRootMb`
623
1626
  * is set: a tmpfs root has nothing on disk for a snapshot to capture.
1627
+ *
1628
+ * `ref` is the WORKING ref this method is asked to checkpoint under — its
1629
+ * basename becomes the snapshot create call's `<name>` argument (still
1630
+ * meaningful: it lands in msb's own index, see `MsbCommands.snapshotCreate`'s
1631
+ * own doc) and, for a path ref, its dirname becomes `--dest-dir`. It is
1632
+ * NOT necessarily where the artifact ends up: EMPIRICALLY VERIFIED against
1633
+ * a real msb 0.7.1 binary, `snapshot create` always writes under
1634
+ * `<destDir-or-default>/<sandbox>/snap_<32-hex-digest>`, a path `name`
1635
+ * never determines. This method therefore parses that real artifact path
1636
+ * back out of the command's own stdout (last non-empty line, required to
1637
+ * be absolute — see `parseSnapshotCreateArtifactPath`) and returns THAT as
1638
+ * the EFFECTIVE ref — the one used for the reboot below and the one the
1639
+ * caller (`GenericContainer.checkpoint()`) stores in the registry and
1640
+ * hands back on the `Checkpoint` it returns. Malformed or unrecognizable
1641
+ * output (empty, no absolute last line) throws a `BackendError` quoting
1642
+ * the raw, unparsed output verbatim rather than guessing a ref.
624
1643
  */
625
1644
  async createCheckpoint(handle, ref) {
626
1645
  if (handle.spec.tmpfsRootMb !== undefined) {
@@ -630,58 +1649,146 @@ export class MsbCliBackend {
630
1649
  throw new TmpfsRootCheckpointError();
631
1650
  }
632
1651
  const msbPath = await this.msbPath();
1652
+ // Captured up front: `handle.id` itself is mutated in place, below, once
1653
+ // the reboot under the fresh name has actually succeeded — every
1654
+ // reference to the SOURCE sandbox's own name in this method (the
1655
+ // stop/snapshot/rm steps, the pre-mutation error messages) goes through
1656
+ // this local instead, never `handle.id` directly, so it stays correct
1657
+ // regardless of when that mutation happens.
1658
+ const originalName = handle.id;
1659
+ const state = this.handles.get(originalName);
1660
+ if (state === undefined) {
1661
+ throw new BackendError(`no handle state for sandbox '${originalName}' — create() was never called for it`);
1662
+ }
1663
+ // Guest cmdline capture — BEFORE stopping the source sandbox (see
1664
+ // captureGuestWorkloadCmdline's own doc) — only when there is no
1665
+ // explicit command for a restore to fall back on later. Best-effort: a
1666
+ // capture failure never fails the checkpoint itself (captureGuestWorkloadCmdline
1667
+ // already swallows it into `undefined`); its absence only surfaces
1668
+ // later, at restore time, as CheckpointWorkloadCommandMissingError.
1669
+ // Stashed on `state` (kept by reference across this method's own
1670
+ // re-keying below, so it survives the remove+reboot churn) for
1671
+ // `capturedWorkloadCommand()` to hand back to `GenericContainer.checkpoint()`
1672
+ // afterward.
1673
+ const capturedCommand = handle.spec.command === undefined ? await this.captureGuestWorkloadCmdline(handle) : undefined;
1674
+ state.capturedCommand = capturedCommand;
633
1675
  await this.stop(handle);
634
- // A path ref (see checkpoint/ref.ts) stores its artifact under the ref
635
- // itself: mkdirs the parent, then hands msb the ref's own basename as
636
- // the snapshot name and the parent as --dest-dir, so the artifact msb
637
- // writes lands at exactly <parent>/<basename> === ref. A bare-name ref
638
- // (pre-dest-dir checkpoints, still restorable) keeps going through
639
- // msb's own default snapshot store, unchanged.
1676
+ // A path ref (see checkpoint/ref.ts) hands msb the ref's own basename as
1677
+ // the snapshot NAME and the parent as --dest-dir — the parent directory
1678
+ // is honored, but (since msb 0.7.1) the artifact itself lands nested
1679
+ // under <parent>/<sandbox>/snap_<digest>, never literally at `ref`; see
1680
+ // this method's own doc. A bare-name ref (pre-dest-dir checkpoints,
1681
+ // still restorable) keeps going through msb's own default snapshot
1682
+ // store, unchanged.
640
1683
  const isPathRef = path.isAbsolute(ref);
641
1684
  if (isPathRef) {
642
1685
  await fs.mkdir(path.dirname(ref), { recursive: true });
643
1686
  }
644
1687
  const snapshotArgv = isPathRef
645
- ? MsbCommands.snapshotCreate(handle.id, path.basename(ref), path.dirname(ref))
646
- : MsbCommands.snapshotCreate(handle.id, ref);
1688
+ ? MsbCommands.snapshotCreate(originalName, path.basename(ref), path.dirname(ref))
1689
+ : MsbCommands.snapshotCreate(originalName, ref);
647
1690
  const snap = await invoke(msbPath, snapshotArgv, CHECKPOINT_TIMEOUT_MS);
648
1691
  if (snap.exitCode !== 0) {
649
- throw new BackendError(`msb snapshot create --from ${handle.id} ${ref} failed (exit ${snap.exitCode}): ${snap.stderr.trim()} — ` +
650
- `the sandbox is left stopped; run 'msb start ${handle.id}' by hand to bring it back up.`);
1692
+ throw new BackendError(`msb snapshot create --from ${originalName} ${ref} failed (exit ${snap.exitCode}): ${snap.stderr.trim()} — ` +
1693
+ `the sandbox is left stopped; run 'msb start ${originalName}' by hand to bring it back up.`);
651
1694
  }
652
- await invoke(msbPath, MsbCommands.rm(handle.id), STOP_TIMEOUT_MS).catch(() => { });
653
- const state = this.handles.get(handle.id);
654
- if (state === undefined) {
655
- throw new BackendError(`no handle state for sandbox '${handle.id}' — create() was never called for it`);
1695
+ const effectiveRef = parseSnapshotCreateArtifactPath(snap.stdout);
1696
+ if (effectiveRef === undefined) {
1697
+ throw new BackendError(`msb snapshot create --from ${originalName} ${ref} did not print a recognizable artifact path as its ` +
1698
+ `last line — the sandbox is left stopped; run 'msb start ${originalName}' by hand to bring it back up. ` +
1699
+ `Raw output:\n${snap.stdout}${snap.stderr}`);
656
1700
  }
657
- const rebootHandle = { id: handle.id, spec: { ...handle.spec, checkpointRef: ref } };
1701
+ await invoke(msbPath, MsbCommands.rm(originalName), STOP_TIMEOUT_MS).catch(() => { });
1702
+ // `command` resolves the same way `fromCheckpointRegistryEntry` resolves
1703
+ // it for a later, registry-mediated restore: the source's own explicit
1704
+ // command first, the guest cmdline just captured above as the fallback —
1705
+ // so `reviveWorkload`, inside the reboot this triggers next, already has
1706
+ // the fully-resolved workload argv without any restore-time lookup. This
1707
+ // merged `command` (and `checkpointRef`) live ONLY on each attempt's own
1708
+ // handle inside `rebootUnderFreshName`, never on the caller's own
1709
+ // `handle.spec` (mutated below) — that spec must keep reading back
1710
+ // exactly what the SOURCE container's own `spec.command` was (`undefined`
1711
+ // when there was none), since `GenericContainer.checkpoint()` reads
1712
+ // `handle.spec` again right after this returns to build the
1713
+ // named-checkpoint registry entry, which pins that same "explicit
1714
+ // command vs. captured fallback" distinction as two separate fields (see
1715
+ // `CheckpointRegistryEntry`'s own doc).
1716
+ //
1717
+ // `rebootUnderFreshName` owns minting a NEW name per attempt (never
1718
+ // `originalName`, never a prior failed attempt's own name), tracking each
1719
+ // one in the reaping ledger before that attempt's restore runs, and —
1720
+ // per attempt — best-effort `msb rm`-ing and untracking a failed name
1721
+ // before advancing to another fresh one (see that method's own doc for
1722
+ // the full policy). `originalName`'s own ledger entry is deliberately
1723
+ // left alone throughout: it was already `msb rm`-ed above, and the
1724
+ // ledger's sweep is already not-found-tolerant for exactly this shape (a
1725
+ // name the ledger still lists but msb itself has no record of).
1726
+ let rebootHandle;
658
1727
  try {
659
- await this.bootClassified(msbPath, rebootHandle, state);
1728
+ rebootHandle = await this.rebootUnderFreshName(msbPath, handle.spec, effectiveRef, capturedCommand, state);
660
1729
  }
661
1730
  catch (err) {
662
1731
  const detail = err instanceof Error ? err.message : String(err);
663
- throw new BackendError(`sandbox '${handle.id}' was removed after a successful checkpoint snapshot, but booting a fresh ` +
1732
+ throw new BackendError(`sandbox '${originalName}' was removed after a successful checkpoint snapshot, but booting a fresh ` +
664
1733
  `sandbox back up from that snapshot failed: ${detail} — the sandbox's disk state is preserved in ` +
665
- `checkpoint '${ref}', restorable via GenericContainer.fromCheckpoint().`);
1734
+ `checkpoint '${effectiveRef}', restorable via GenericContainer.fromCheckpoint().`);
666
1735
  }
1736
+ const freshName = rebootHandle.id;
1737
+ // The reboot succeeded under `freshName` — the WINNING attempt's own
1738
+ // name, not `originalName` — re-key this backend's own runtime
1739
+ // registries (see class doc on `handles`), then publish the new identity
1740
+ // onto the CALLER's own handle object, IN PLACE, so every subsequent
1741
+ // operation on it targets the sandbox actually running now (see this
1742
+ // method's own doc). Only `id`/`spec.name` change on the live handle —
1743
+ // `spec.command`/`spec.checkpointRef` stay exactly as they were on the
1744
+ // SOURCE spec; see the comment above for why.
1745
+ this.handles.delete(originalName);
1746
+ this.handles.set(freshName, state);
1747
+ this.startedNames.delete(originalName);
1748
+ const mutableHandle = handle;
1749
+ mutableHandle.id = freshName;
1750
+ mutableHandle.spec = { ...handle.spec, name: freshName };
1751
+ return effectiveRef;
667
1752
  }
668
1753
  /**
669
- * Best-effort `msb snapshot rm <basename(ref)>` — "not found" is success,
670
- * the same contract as `removeByName`. msb's own removal deletes both its
671
- * index entry and the dest-dir artifact for a path ref, but afterwards this
672
- * also best-effort recursively deletes the ref path itself: if msb's index
673
- * ever loses track of an artifact without deleting it, the directory would
674
- * otherwise linger under the cache dir forever. That recursive delete is
675
- * gated on `looksLikeCheckpointArtifactDir` first — a `ref` is caller-
676
- * supplied (a corrupt registry entry, a hand-edited env var, …), and a
677
- * `fs.rm(ref, { recursive: true })` on an unverified path would happily
678
- * wipe out an arbitrary directory that merely happens to share its name.
1754
+ * Best-effort `msb snapshot rm <ref> -f` — "not found" is success, the
1755
+ * same contract as `removeByName`. `ref` is passed FULL, never reduced to
1756
+ * `path.basename(ref)`: EMPIRICALLY VERIFIED against a real msb 0.7.1
1757
+ * binary, name-based removal does not resolve at all — the artifact PATH
1758
+ * is the only address that reliably works (see `MsbCommands.snapshotRemove`'s
1759
+ * own doc). Every real caller already hands this the EFFECTIVE ref
1760
+ * `createCheckpoint` returned (an absolute path since 0.7.1), so this is
1761
+ * also simply correct for the common case, not just a defensive choice.
1762
+ *
1763
+ * One failure shape is deliberately NOT swallowed alongside "not found":
1764
+ * msb refuses to remove a snapshot that is still the current HEAD of
1765
+ * older siblings from the same source sandbox (see
1766
+ * `isSnapshotHeadRemovalRefused`) — that refusal propagates as a
1767
+ * `BackendError` naming msb's own remedy, rather than silently doing
1768
+ * nothing, since automatic head rotation is out of scope for this method
1769
+ * (see the checkpoints guide's cleanup section for the documented
1770
+ * limitation).
1771
+ *
1772
+ * msb's own removal deletes both its index entry and the dest-dir
1773
+ * artifact for a path ref, but afterwards this also best-effort
1774
+ * recursively deletes the ref path itself: if msb's index ever loses
1775
+ * track of an artifact without deleting it, the directory would otherwise
1776
+ * linger under the cache dir forever. That recursive delete is gated on
1777
+ * `looksLikeCheckpointArtifactDir` first — a `ref` is caller-supplied (a
1778
+ * corrupt registry entry, a hand-edited env var, …), and a `fs.rm(ref, {
1779
+ * recursive: true })` on an unverified path would happily wipe out an
1780
+ * arbitrary directory that merely happens to share its name.
679
1781
  */
680
1782
  async removeCheckpoint(ref) {
681
1783
  const msbPath = await this.msbPath();
682
1784
  const isPathRef = path.isAbsolute(ref);
683
- const name = isPathRef ? path.basename(ref) : ref;
684
- await invoke(msbPath, MsbCommands.snapshotRemove(name), CHECKPOINT_TIMEOUT_MS).catch(() => { });
1785
+ const result = await invoke(msbPath, MsbCommands.snapshotRemove(ref), CHECKPOINT_TIMEOUT_MS).catch(() => undefined);
1786
+ if (result !== undefined && result.exitCode !== 0 && isSnapshotHeadRemovalRefused(result.stderr)) {
1787
+ throw new BackendError(`msb snapshot rm ${ref} -f was refused (exit ${result.exitCode}): ${result.stderr.trim()} — this ` +
1788
+ `checkpoint is still the newest snapshot of other, older ones from the same source sandbox; ` +
1789
+ `select another snapshot as head first ('msb snapshot head ...', see msb's own message above), ` +
1790
+ `or remove the older siblings first.`);
1791
+ }
685
1792
  if (isPathRef && (await looksLikeCheckpointArtifactDir(ref))) {
686
1793
  await fs.rm(ref, { recursive: true, force: true }).catch(() => { });
687
1794
  }
@@ -726,10 +1833,10 @@ export class MsbCliBackend {
726
1833
  /**
727
1834
  * `msb snapshot save <ref> <destFile>` — writes the `.tar.zst` artifact
728
1835
  * `Checkpoints.exportTo` bundles into its own archive container. Never
729
- * `--with-image` (see the checkpoints guide's own note on why: its import
730
- * fails an integrity check in 0.6.6, so archives never bundle the OCI
731
- * image — the destination machine pulls it on the restored container's
732
- * first boot).
1836
+ * `--with-image` (see the checkpoints guide's own note: this library
1837
+ * doesn't bundle the OCI image into the archive, so the destination
1838
+ * machine pulls it on the restored container's first boot — upstream
1839
+ * supports bundling it, this library just doesn't use that yet).
733
1840
  *
734
1841
  * On Windows, msb 0.6.7 and 0.6.8 fail this call every single time: they
735
1842
  * finish writing the archive to a staging file beside the destination and
@@ -757,51 +1864,66 @@ export class MsbCliBackend {
757
1864
  }
758
1865
  }
759
1866
  /**
760
- * `msb snapshot load <archive>`, then resolves the EFFECTIVE ref: msb
761
- * writes the import under a digest-derived directory name it never lets
762
- * the caller choose (never the archive's own recorded `ref`), parsed from
763
- * the printed artifact path's basename (see `parseImportedDigestDirName`).
1867
+ * `msb snapshot load <archive> --dest <checkpointsDir>`, then resolves the
1868
+ * EFFECTIVE ref: EMPIRICALLY VERIFIED against a real msb 0.7.1 binary, a
1869
+ * successful load prints a `group msb-<hex>: head snap_<digest>
1870
+ * (Initialized)` line, a digest line, and finally the loaded artifact's
1871
+ * own absolute path as its LAST line — `<checkpointsDir>/<generated-
1872
+ * group>/snap_<digest>`, never the archive's own recorded `ref` — parsed
1873
+ * back out by `parseImportedArtifactPath` (the same last-non-empty-line,
1874
+ * must-be-absolute defensive pattern `parseSnapshotCreateArtifactPath`
1875
+ * uses for `snapshot create`). `checkpointsDir` is always this backend's
1876
+ * own checkpoints cache directory (`<cacheDir>/checkpoints`, the same
1877
+ * directory `checkpointRef()` mints created-checkpoint refs under) —
1878
+ * omitting `--dest` would import into msb's own global default snapshot
1879
+ * store instead, outside anywhere this library looks for or cleans up
1880
+ * checkpoints. Created fresh (mkdir -p) before the load, since nothing
1881
+ * guarantees a caller ever created a checkpoint locally first (an
1882
+ * imported-only machine has no reason to have this directory yet).
1883
+ *
1884
+ * This replaces the pre-0.7.1 shape entirely: `snapshot load` used to
1885
+ * write under a digest-derived directory name with no `--dest` of its
1886
+ * own, so this method had to parse out just that bare NAME and then
1887
+ * separately CONFIRM it via `msb snapshot list --format json` before
1888
+ * handing it back (the only way to get a usable ref at all, since the
1889
+ * bare name alone still had to resolve against msb's default store). 0.7.1
1890
+ * prints the full, directly-usable, already-`--dest`-scoped path as part
1891
+ * of `load`'s own output, so that whole list-and-confirm round trip is
1892
+ * gone — dead code that this migration deletes outright, not merely
1893
+ * bypassed.
1894
+ *
764
1895
  * An already-exists failure — msb's own content-addressed dedup — is
765
1896
  * treated as success, since the artifact is already present under that
766
- * digest either way; any OTHER import failure surfaces msb's own stderr.
767
- * The digest-dir basename is then CONFIRMED present via `msb snapshot
768
- * list --format json` (matching it against each entry's
769
- * `name`/`artifact_path`) and returned as-is — never the entry's `digest`
770
- * field. Live-verified against msb 0.6.8: the full `sha256:<64hex>`
771
- * digest does not resolve as a snapshot ref at all (`msb snapshot inspect
772
- * sha256:<full>` fails "snapshot not found"); only the digest-dir name
773
- * resolves for `inspect`/`rm`/`run --from-snapshot`, so it — not the full
774
- * digest — is the ref this must hand back for `hasCheckpoint` and every
775
- * other snapshot-ref call to keep working. `_ref` (the archive's own
776
- * recorded ref) is unused here — msb's importer never takes one, unlike
777
- * docker's, where the effective ref really is the ref passed in.
1897
+ * digest either way, and its stderr is parsed the same way as a success's
1898
+ * stdout; any OTHER import failure surfaces msb's own stderr in a
1899
+ * `BackendError`, and unparseable output (no recognizable absolute last
1900
+ * line — e.g. only the "group ... (Initialized)" line, msb's OWN wording
1901
+ * for a state that is not itself a path) throws a `BackendError` quoting
1902
+ * the raw output verbatim rather than misreading a status line as a ref.
1903
+ * `_ref` (the archive's own recorded ref) is unused here — msb's importer
1904
+ * never takes one, unlike docker's, where the effective ref really is the
1905
+ * ref passed in.
778
1906
  */
779
1907
  async importCheckpoint(srcFile, _ref) {
780
1908
  const msbPath = await this.msbPath();
781
- const imported = await invoke(msbPath, MsbCommands.snapshotImport(srcFile), CHECKPOINT_TIMEOUT_MS);
782
- let digestDirName;
1909
+ const checkpointsDir = path.join(cacheDir(), "checkpoints");
1910
+ await fs.mkdir(checkpointsDir, { recursive: true });
1911
+ const imported = await invoke(msbPath, MsbCommands.snapshotImport(srcFile, checkpointsDir), CHECKPOINT_TIMEOUT_MS);
1912
+ let artifactPath;
783
1913
  if (imported.exitCode === 0) {
784
- digestDirName = parseImportedDigestDirName(imported.stdout);
1914
+ artifactPath = parseImportedArtifactPath(imported.stdout);
785
1915
  }
786
1916
  else if (isSnapshotAlreadyExistsError(imported.stderr)) {
787
- digestDirName = parseImportedDigestDirName(imported.stderr);
1917
+ artifactPath = parseImportedArtifactPath(imported.stderr);
788
1918
  }
789
1919
  else {
790
- throw new BackendError(`msb snapshot load ${srcFile} failed (exit ${imported.exitCode}): ${imported.stderr.trim()}`);
1920
+ throw new BackendError(`msb snapshot load ${srcFile} --dest ${checkpointsDir} failed (exit ${imported.exitCode}): ${imported.stderr.trim()}`);
791
1921
  }
792
- if (digestDirName === undefined) {
793
- throw new BackendError(`msb snapshot load ${srcFile} did not print a recognizable artifact path — output:\n` +
794
- `${imported.stdout}${imported.stderr}`);
1922
+ if (artifactPath === undefined) {
1923
+ throw new BackendError(`msb snapshot load ${srcFile} --dest ${checkpointsDir} did not print a recognizable artifact path as its ` +
1924
+ `last line — output:\n${imported.stdout}${imported.stderr}`);
795
1925
  }
796
- const list = await invoke(msbPath, MsbCommands.snapshotList(), CHECKPOINT_TIMEOUT_MS);
797
- if (list.exitCode !== 0) {
798
- throw new BackendError(`msb snapshot list failed (exit ${list.exitCode}): ${list.stderr.trim()}`);
799
- }
800
- const confirmedRef = confirmDigestDirNamePresent(parseSnapshotList(list.stdout), digestDirName);
801
- if (confirmedRef === undefined) {
802
- throw new BackendError(`imported snapshot '${digestDirName}' from ${srcFile} did not appear in 'msb snapshot list' — could not confirm the import`);
803
- }
804
- return confirmedRef;
1926
+ return artifactPath;
805
1927
  }
806
1928
  async runningSandboxNames(msbPath) {
807
1929
  const result = await invoke(msbPath, MsbCommands.ls(), LOGS_TIMEOUT_MS);
@@ -845,10 +1967,10 @@ export class MsbCliBackend {
845
1967
  return (await invoke(msbPath, MsbCommands.logs(handle.id), LOGS_TIMEOUT_MS)).stdout;
846
1968
  }
847
1969
  /**
848
- * `msb logs -f` never exits once the sandbox stops (confirmed against the
849
- * real msb binary — it blocks on read forever instead of the documented
850
- * clean exit). A watchdog polls `msb ls` in the background; the instant
851
- * the sandbox leaves Running it quiesces the stuck follow child FIRST
1970
+ * `msb logs -f` never exits on its own once the sandbox stops (confirmed
1971
+ * against the real msb 0.7.1 binary — it just never exits, instead of the
1972
+ * documented clean exit). A watchdog polls `msb ls` in the background; the
1973
+ * instant the sandbox leaves Running it quiesces the stuck follow child FIRST
852
1974
  * (kill it, wait for the reader to finish draining whatever was already
853
1975
  * buffered) so `delivered` reflects everything the live stream will ever
854
1976
  * produce, THEN does one authoritative non-follow `msb logs` fetch and
@@ -1057,11 +2179,17 @@ export class MsbCliBackend {
1057
2179
  // Nothing was created for ensureNetwork(); nothing to remove.
1058
2180
  }
1059
2181
  /**
1060
- * Networks are emulated because there is no bridge/subnet the current
1061
- * msb exposes on macOS — the only data path into a running sandbox is the
1062
- * exec channel. Four concerns, each its own guard: reject duplicate guest
1063
- * ports, validate every alias (they get shell-interpolated), probe for
1064
- * `nc`, then install `/etc/hosts` aliases and spawn one tunnel per link.
2182
+ * Networks are emulated because msb gives sandboxes no network shared with
2183
+ * each other, and TCP and UDP links take DIFFERENT routes: a TCP link is
2184
+ * relayed over the exec channel, a UDP link's in-guest forwarder sends
2185
+ * through the sandbox's host gateway to the target's published port.
2186
+ * Validate first (duplicate guest ports, alias charset — both get
2187
+ * shell-interpolated below), then probe for `nc` (every link needs it)
2188
+ * and, only when a UDP link is present, for the forwarder's stricter
2189
+ * dependencies (`-u`/`-e`, `timeout`). `/etc/hosts` aliases install once,
2190
+ * shared by both protocols. From there each link takes its own path: a
2191
+ * TCP link gets an `ExecTunnel`, unchanged; a UDP link gets
2192
+ * `installUdpLink`'s forwarder script.
1065
2193
  */
1066
2194
  async installNetworkLinks(handle, links) {
1067
2195
  if (links.length === 0) {
@@ -1074,18 +2202,60 @@ export class MsbCliBackend {
1074
2202
  if (ncProbe.exitCode !== 0) {
1075
2203
  throw new UnsupportedByBackendError(`network links (no nc/busybox in consumer image '${handle.spec.image}')`, this.name, "run this test with RIGHTSIZE_BACKEND=docker instead");
1076
2204
  }
2205
+ const udpLinks = links.filter((link) => link.protocol === "udp");
2206
+ if (udpLinks.length > 0) {
2207
+ const udpProbe = await this.exec(handle, ["sh", "-c", udpForwarderProbeScript()]);
2208
+ if (udpProbe.exitCode !== 0) {
2209
+ throw new UnsupportedByBackendError(`UDP network links (consumer image '${handle.spec.image}' has no busybox-style nc with -u/-e, or no timeout)`, this.name, "run with the docker backend");
2210
+ }
2211
+ }
1077
2212
  const hostsResult = await this.exec(handle, ["sh", "-c", hostsAliasScript(links)]);
1078
2213
  if (hostsResult.exitCode !== 0) {
1079
2214
  throw new BackendError(`failed to install /etc/hosts aliases in ${handle.id}: ${hostsResult.stderr}`);
1080
2215
  }
1081
2216
  const state = this.handles.get(handle.id);
1082
2217
  for (const link of links) {
2218
+ if (link.protocol === "udp") {
2219
+ await this.installUdpLink(handle, link);
2220
+ continue;
2221
+ }
1083
2222
  const tunnel = new ExecTunnel(msbPath, handle.id, link);
1084
2223
  if (state !== undefined) {
1085
2224
  state.resources.push(tunnel);
1086
2225
  }
1087
2226
  }
1088
2227
  }
2228
+ /**
2229
+ * One UDP link: installs and detached-launches its forwarder script, then
2230
+ * polls `/proc/net/udp{,6}` until the guest port is actually bound — the
2231
+ * launch exec succeeding only proves the script started, not that its
2232
+ * listener has bound yet, and `nc -u -l` itself prints nothing to signal
2233
+ * that. No teardown counterpart, unlike `ExecTunnel`: the forwarder is a
2234
+ * plain guest process with no host-side resource of its own, and guest
2235
+ * processes die with the sandbox. A checkpoint reboot's link replay (see
2236
+ * `GenericContainer.checkpoint`) reruns this exactly like a fresh install
2237
+ * — `/tmp` is tmpfs, so the rebooted guest has no forwarder script left
2238
+ * to find, only a fresh one this rewrites.
2239
+ */
2240
+ async installUdpLink(handle, link) {
2241
+ const install = await this.exec(handle, ["sh", "-c", installUdpForwarderScript(link.guestPort, link.targetHostPort)]);
2242
+ if (install.exitCode !== 0) {
2243
+ throw new BackendError(`failed to install the UDP forwarder for guest port ${link.guestPort} in ${handle.id}: ${install.stderr}`);
2244
+ }
2245
+ const deadline = Date.now() + this.udpLinkReadinessTimeoutMs;
2246
+ for (;;) {
2247
+ const probe = await this.exec(handle, ["sh", "-c", udpReadinessProbeScript(link.guestPort)]);
2248
+ if (probe.exitCode === 0) {
2249
+ return;
2250
+ }
2251
+ if (Date.now() >= deadline) {
2252
+ const log = await this.exec(handle, ["sh", "-c", `tail -c 2000 ${udpForwarderLogPath(link.guestPort)} 2>/dev/null`]);
2253
+ throw new BackendError(`UDP forwarder for guest port ${link.guestPort} in ${handle.id} never bound within ` +
2254
+ `${this.udpLinkReadinessTimeoutMs / 1000}s; forwarder log tail:\n${log.stdout}`);
2255
+ }
2256
+ await sleep(this.udpLinkReadinessPollMs);
2257
+ }
2258
+ }
1089
2259
  /**
1090
2260
  * `msb copy -q <hostPath> <name>:<containerPath>` — the transfer only;
1091
2261
  * `GenericContainer.copyFileToContainer()` already confirmed the sandbox