rightsize 0.7.8 → 0.7.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend-docker/backend.d.ts +4 -1
- package/dist/backend-docker/backend.d.ts.map +1 -1
- package/dist/backend-docker/backend.js +4 -0
- package/dist/backend-docker/backend.js.map +1 -1
- package/dist/backend-msb/backend.d.ts +618 -68
- package/dist/backend-msb/backend.d.ts.map +1 -1
- package/dist/backend-msb/backend.js +1236 -133
- package/dist/backend-msb/backend.js.map +1 -1
- package/dist/backend-msb/commands.d.ts +160 -16
- package/dist/backend-msb/commands.d.ts.map +1 -1
- package/dist/backend-msb/commands.js +197 -31
- package/dist/backend-msb/commands.js.map +1 -1
- package/dist/backend-msb/provisioner.d.ts +1 -1
- package/dist/backend-msb/provisioner.d.ts.map +1 -1
- package/dist/backend-msb/provisioner.js +1 -1
- package/dist/backend-msb/provisioner.js.map +1 -1
- package/dist/backend-msb/restore-access-denied.d.ts +50 -0
- package/dist/backend-msb/restore-access-denied.d.ts.map +1 -0
- package/dist/backend-msb/restore-access-denied.js +52 -0
- package/dist/backend-msb/restore-access-denied.js.map +1 -0
- package/dist/backend-msb/restore-broker.d.ts +181 -0
- package/dist/backend-msb/restore-broker.d.ts.map +1 -0
- package/dist/backend-msb/restore-broker.js +264 -0
- package/dist/backend-msb/restore-broker.js.map +1 -0
- package/dist/backend-msb/sandbox-already-exists.d.ts +32 -0
- package/dist/backend-msb/sandbox-already-exists.d.ts.map +1 -0
- package/dist/backend-msb/sandbox-already-exists.js +35 -0
- package/dist/backend-msb/sandbox-already-exists.js.map +1 -0
- package/dist/backend-msb/snapshot-create.d.ts +22 -0
- package/dist/backend-msb/snapshot-create.d.ts.map +1 -0
- package/dist/backend-msb/snapshot-create.js +33 -0
- package/dist/backend-msb/snapshot-create.js.map +1 -0
- package/dist/backend-msb/snapshot-import.d.ts +30 -14
- package/dist/backend-msb/snapshot-import.d.ts.map +1 -1
- package/dist/backend-msb/snapshot-import.js +35 -17
- package/dist/backend-msb/snapshot-import.js.map +1 -1
- package/dist/backend-msb/snapshot-rm.d.ts +28 -0
- package/dist/backend-msb/snapshot-rm.d.ts.map +1 -0
- package/dist/backend-msb/snapshot-rm.js +30 -0
- package/dist/backend-msb/snapshot-rm.js.map +1 -0
- package/dist/core/backend.d.ts +53 -22
- package/dist/core/backend.d.ts.map +1 -1
- package/dist/core/checkpoint/api.d.ts +2 -2
- package/dist/core/checkpoint/api.js +2 -2
- package/dist/core/checkpoint/registry.d.ts +34 -0
- package/dist/core/checkpoint/registry.d.ts.map +1 -1
- package/dist/core/checkpoint/registry.js +20 -1
- package/dist/core/checkpoint/registry.js.map +1 -1
- package/dist/core/errors.d.ts +49 -0
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +61 -0
- package/dist/core/errors.js.map +1 -1
- package/dist/core/generic-container.d.ts +58 -10
- package/dist/core/generic-container.d.ts.map +1 -1
- package/dist/core/generic-container.js +222 -27
- package/dist/core/generic-container.js.map +1 -1
- package/dist/core/model.d.ts +3 -2
- package/dist/core/model.d.ts.map +1 -1
- package/dist/core/sandbox-name.d.ts +3 -0
- package/dist/core/sandbox-name.d.ts.map +1 -0
- package/dist/core/sandbox-name.js +19 -0
- package/dist/core/sandbox-name.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modules/minio.d.ts +5 -3
- package/dist/modules/minio.d.ts.map +1 -1
- package/dist/modules/minio.js +14 -4
- package/dist/modules/minio.js.map +1 -1
- package/package.json +1 -1
- package/dist/backend-msb/snapshot-list.d.ts +0 -43
- package/dist/backend-msb/snapshot-list.d.ts.map +0 -1
- package/dist/backend-msb/snapshot-list.js +0 -53
- package/dist/backend-msb/snapshot-list.js.map +0 -1
|
@@ -2,7 +2,10 @@ import { spawn, spawnSync } from "node:child_process";
|
|
|
2
2
|
import * as readline from "node:readline";
|
|
3
3
|
import * as fs from "node:fs/promises";
|
|
4
4
|
import * as path from "node:path";
|
|
5
|
-
import { BackendError, PortBindConflictError, UnsupportedByBackendError, TmpfsRootCheckpointError } from "../core/errors.js";
|
|
5
|
+
import { BackendError, PortBindConflictError, UnsupportedByBackendError, TmpfsRootCheckpointError, CheckpointWorkloadCommandMissingError, } from "../core/errors.js";
|
|
6
|
+
import { cacheDir } from "../core/cache-dir.js";
|
|
7
|
+
import { nextSandboxName } from "../core/sandbox-name.js";
|
|
8
|
+
import { trackSandbox, untrackSandbox } from "../core/reaper/init.js";
|
|
6
9
|
import { MsbCommands } from "./commands.js";
|
|
7
10
|
import { runningNames, statusOf } from "./ls-json.js";
|
|
8
11
|
import { hasSandboxStartedMarker } from "./fast-exit.js";
|
|
@@ -12,12 +15,16 @@ import { isImageCacheCorruption } from "./image-cache.js";
|
|
|
12
15
|
import { isMsbStateDbError } from "./state-db.js";
|
|
13
16
|
import { isAgentEndpointNotReady } from "./agent-endpoint.js";
|
|
14
17
|
import { isSnapshotNotFoundError } from "./snapshot-not-found.js";
|
|
15
|
-
import { isSnapshotAlreadyExistsError,
|
|
18
|
+
import { isSnapshotAlreadyExistsError, parseImportedArtifactPath } from "./snapshot-import.js";
|
|
16
19
|
import { isSnapshotSaveAccessDeniedFailure, salvageStagedArchive } from "./snapshot-save-fsync.js";
|
|
17
|
-
import {
|
|
20
|
+
import { parseSnapshotCreateArtifactPath } from "./snapshot-create.js";
|
|
21
|
+
import { isSnapshotHeadRemovalRefused } from "./snapshot-rm.js";
|
|
18
22
|
import { undeliveredLines } from "./follow-replay.js";
|
|
19
23
|
import { requireNoDuplicateGuestPorts, requireAliasesAreValid, hostsAliasScript } from "./network-links.js";
|
|
20
24
|
import { ExecTunnel } from "./exec-tunnel.js";
|
|
25
|
+
import { isRestoreAccessDeniedFailure } from "./restore-access-denied.js";
|
|
26
|
+
import { isSandboxAlreadyExistsFailure } from "./sandbox-already-exists.js";
|
|
27
|
+
import { realRestoreBroker, } from "./restore-broker.js";
|
|
21
28
|
const FIRST_RUN_PULL_TIMEOUT_MS = 600_000; // a cold pull can be slow
|
|
22
29
|
const READINESS_POLL_MS = 300;
|
|
23
30
|
const STOP_TIMEOUT_MS = 60_000;
|
|
@@ -44,6 +51,51 @@ const COPY_TIMEOUT_MS = 120_000;
|
|
|
44
51
|
// already confirmed no longer Running. Never a wait-for-content budget: a
|
|
45
52
|
// stopped sandbox's log cannot grow, so the first successful fetch is final.
|
|
46
53
|
const TERMINAL_FETCH_FAILURE_BUDGET_MS = 10_000;
|
|
54
|
+
// How long `reviveWorkload` gives a freshly-spawned workload-revival exec
|
|
55
|
+
// child to prove it isn't an immediate boot failure before treating it as
|
|
56
|
+
// the ordinary long-lived case. Unlike `bootRunOnce`'s own attached child,
|
|
57
|
+
// there is no separate "reached Running" signal to poll for here — the
|
|
58
|
+
// SANDBOX already reports Running regardless of whether this exec succeeds
|
|
59
|
+
// — so a short settle window is what that same exit-vs-success race
|
|
60
|
+
// collapses to when the only observable signal left is the child's own
|
|
61
|
+
// exit. A few multiples of the ordinary poll cadence is plenty: a failing
|
|
62
|
+
// workload (bad command, missing binary, an immediate usage error) exits
|
|
63
|
+
// within milliseconds of spawn, while a genuine long-lived server never
|
|
64
|
+
// exits on its own at all.
|
|
65
|
+
const EXEC_REVIVE_SETTLE_MS = READINESS_POLL_MS * 3;
|
|
66
|
+
// Bounded retry policy for `msb restore` hitting a Windows access-denied
|
|
67
|
+
// failure on its own just-written snapshot artifact (see
|
|
68
|
+
// `isRestoreAccessDeniedFailure`) — a brief file-handle release lag that
|
|
69
|
+
// normally clears within one retry, so several short-backoff attempts cover
|
|
70
|
+
// the real cases without masking a genuinely stuck lock.
|
|
71
|
+
const RESTORE_ACCESS_DENIED_RETRY_LIMIT = 3;
|
|
72
|
+
const RESTORE_ACCESS_DENIED_RETRY_DELAY_MS = 500;
|
|
73
|
+
/**
|
|
74
|
+
* How long `createCheckpoint`'s own reboot step keeps retrying msb's
|
|
75
|
+
* "sandbox already exists" refusal (see `isSandboxAlreadyExistsFailure`) and
|
|
76
|
+
* its Windows access-denied refusal on the just-written snapshot artifact
|
|
77
|
+
* (see `isRestoreAccessDeniedFailure`), and the pause between attempts — the
|
|
78
|
+
* same install-lock-poll shape `INSTALL_LOCK_RETRY_BUDGET_MS`/
|
|
79
|
+
* `INSTALL_LOCK_RETRY_DELAY_MS` already use. EMPIRICALLY VERIFIED against a
|
|
80
|
+
* real msb 0.7.1 binary (an isolated `MSB_HOME` testbed): `msb restore
|
|
81
|
+
* --name X` validates the artifact FIRST — an integrity failure exits 1 and
|
|
82
|
+
* leaves NO sandbox record — but a failure AFTER validation (matching either
|
|
83
|
+
* classified signature above) leaves `X` behind as a STOPPED SANDBOX RECORD
|
|
84
|
+
* visible in `msb ls`, and msb's own restore-time collision check
|
|
85
|
+
* (`existing.is_some() || dir_exists`, `prepare_create_target` in
|
|
86
|
+
* `sdk/rust/lib/backend/local/sandbox/create.rs`) then refuses any retry
|
|
87
|
+
* under that SAME name — on Windows CI this refusal has been observed
|
|
88
|
+
* holding for the whole span of a same-name retry loop under load. Each
|
|
89
|
+
* retry attempt therefore mints a BRAND NEW name rather than reusing the one
|
|
90
|
+
* that just failed (see `rebootUnderFreshName`), which sidesteps that exact
|
|
91
|
+
* collision structurally — a name nothing else has ever used cannot collide
|
|
92
|
+
* with a record or a lingering on-disk directory belonging to a name nothing
|
|
93
|
+
* will ever restore under again. This budget bounds how long that
|
|
94
|
+
* fresh-name retry loop as a whole keeps advancing before giving up, not a
|
|
95
|
+
* single name's own collision.
|
|
96
|
+
*/
|
|
97
|
+
const CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS = 30_000;
|
|
98
|
+
const CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_DELAY_MS = 2_000;
|
|
47
99
|
/**
|
|
48
100
|
* The boot failure `start()` heals and retries — carries the `msb run`
|
|
49
101
|
* child's combined output for the second-failure diagnostic. Internal to the
|
|
@@ -106,6 +158,45 @@ class InstallLockActiveError extends Error {
|
|
|
106
158
|
this.output = output;
|
|
107
159
|
}
|
|
108
160
|
}
|
|
161
|
+
/**
|
|
162
|
+
* Restore-path classified failure for `isRestoreAccessDeniedFailure` — a
|
|
163
|
+
* Windows-only `msb restore` transient against its own just-written
|
|
164
|
+
* snapshot artifact (see that function's own doc). Internal to the boot
|
|
165
|
+
* path, like its siblings above; `bootClassified` owns the bounded-retry
|
|
166
|
+
* policy. Only ever thrown from `bootRestoreOnce`, since it is a restore
|
|
167
|
+
* invocation's own failure signature — an ordinary `run` boot never touches
|
|
168
|
+
* a snapshot artifact at all.
|
|
169
|
+
*/
|
|
170
|
+
class RestoreAccessDeniedError extends Error {
|
|
171
|
+
output;
|
|
172
|
+
constructor(output) {
|
|
173
|
+
super(`msb restore access-denied on its own snapshot artifact:\n${output}`);
|
|
174
|
+
this.output = output;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Restore-path classified failure for `isSandboxAlreadyExistsFailure` — msb
|
|
179
|
+
* refusing `restore` because a sandbox under this name already exists (its
|
|
180
|
+
* database record, its on-disk directory, or both — see that function's own
|
|
181
|
+
* doc). Internal to the boot path, like its siblings above; only ever
|
|
182
|
+
* thrown from `bootRestoreOnce`. `bootClassified` itself never retries this
|
|
183
|
+
* one (unlike its siblings) — it simply propagates, so the ordinary
|
|
184
|
+
* `start()` path (a `GenericContainer.fromCheckpoint(cp).start()` restore
|
|
185
|
+
* of a name that turns out to still be live) surfaces it immediately, a
|
|
186
|
+
* real error. Only `createCheckpoint`'s own reboot step, via
|
|
187
|
+
* `rebootUnderFreshName`, retries it — under a NEWLY minted name each time,
|
|
188
|
+
* never the one that just collided (see that method's own doc) — a caller
|
|
189
|
+
* reusing a live name is never this backend's own race to hide, but the
|
|
190
|
+
* checkpoint cycle's own historical `rm`-then-restore of the SAME name was
|
|
191
|
+
* exactly that race.
|
|
192
|
+
*/
|
|
193
|
+
class SandboxAlreadyExistsError extends Error {
|
|
194
|
+
output;
|
|
195
|
+
constructor(output) {
|
|
196
|
+
super(`msb restore refused — a sandbox with this name already exists:\n${output}`);
|
|
197
|
+
this.output = output;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
109
200
|
/**
|
|
110
201
|
* How long to wait before retrying a boot that hit msb's state-database
|
|
111
202
|
* error — enough for a winning concurrent invocation's migration transaction
|
|
@@ -192,16 +283,17 @@ function describeHeal(heal) {
|
|
|
192
283
|
}
|
|
193
284
|
/**
|
|
194
285
|
* True when `ref` (an absolute path ref) looks like a checkpoint artifact
|
|
195
|
-
* this backend itself would have written — a directory named `
|
|
196
|
-
* (
|
|
197
|
-
*
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
*
|
|
201
|
-
*
|
|
286
|
+
* this backend itself would have written — a directory named `snap_<hex>`
|
|
287
|
+
* (msb's own snapshot-store naming since 0.7.1 — see
|
|
288
|
+
* `parseSnapshotCreateArtifactPath`) containing a `snapshot.json` file, the
|
|
289
|
+
* same shape `hasCheckpoint` checks for. `removeCheckpoint`'s recursive
|
|
290
|
+
* delete is gated on this: `ref` is caller-supplied (a corrupt or
|
|
291
|
+
* hand-edited registry entry), and skipping the shape check would let an
|
|
292
|
+
* arbitrary path get `fs.rm(..., { recursive: true })`'d just because it
|
|
293
|
+
* happened to be passed in as a "ref".
|
|
202
294
|
*/
|
|
203
295
|
async function looksLikeCheckpointArtifactDir(ref) {
|
|
204
|
-
if (
|
|
296
|
+
if (!/^snap_[0-9a-f]+$/i.test(path.basename(ref))) {
|
|
205
297
|
return false;
|
|
206
298
|
}
|
|
207
299
|
const stat = await fs.stat(ref).catch(() => undefined);
|
|
@@ -234,6 +326,90 @@ function drainTail(stream, tail) {
|
|
|
234
326
|
rl.on("close", () => resolveDrain());
|
|
235
327
|
});
|
|
236
328
|
}
|
|
329
|
+
/**
|
|
330
|
+
* `drainTail`'s sibling for text this backend already has in hand rather
|
|
331
|
+
* than a live stream — a brokered restore's own combined output
|
|
332
|
+
* (`launchRestoreViaBroker`), or a diagnostic note about the broker itself.
|
|
333
|
+
* Same `TAIL_LINES` cap, same shift-oldest behavior, so a brokered attempt's
|
|
334
|
+
* pre-Running diagnostics tail looks exactly like a direct attempt's
|
|
335
|
+
* regardless of which path produced it.
|
|
336
|
+
*/
|
|
337
|
+
function pushLinesToTail(tail, text) {
|
|
338
|
+
for (const line of text.split("\n")) {
|
|
339
|
+
tail.push(line);
|
|
340
|
+
if (tail.length > TAIL_LINES) {
|
|
341
|
+
tail.shift();
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* Busybox-ash-compatible: finds the first non-kernel child of PID 1 in the
|
|
347
|
+
* guest and prints its `/proc/<pid>/cmdline` RAW — NUL-separated with a
|
|
348
|
+
* trailing NUL, the kernel's own on-disk shape, never re-encoded by this
|
|
349
|
+
* script itself — to stdout, exit 0; exits 1 if none is found. Walks
|
|
350
|
+
* every `/proc/<pid>/stat` in NUMERIC pid order (`ls | sort -n`, since a bare
|
|
351
|
+
* shell glob sorts lexically — "10" before "2" — which is wrong for "the
|
|
352
|
+
* first child") and, for each, reads field 4 (`ppid`, 1-indexed per
|
|
353
|
+
* `/proc/pid/stat`'s own `pid (comm) state ppid ...` layout), skipping any
|
|
354
|
+
* whose `ppid` isn't `1`. `comm` is split out via the FIRST `(` and the LAST
|
|
355
|
+
* `)` (`${st#*(}` / `${comm%)*}`, both plain POSIX parameter expansion, no
|
|
356
|
+
* `extglob` needed) rather than a naive single-`(`/`)` split, since a
|
|
357
|
+
* process name may itself contain spaces (or, in principle, parens) — the
|
|
358
|
+
* same robustness a greedy sed capture between the first "(" and the last
|
|
359
|
+
* ")" would get. Two names are excluded even though
|
|
360
|
+
* their `ppid` may show `1`: the guest's own init (`init.krun`) and any
|
|
361
|
+
* bracketed kernel-thread name (`[kworker/0:1]`, `[ksoftirqd/0]`, ...) — the
|
|
362
|
+
* shape every kernel thread's `comm` renders as. Read by `MsbCliBackend`'s
|
|
363
|
+
* own `createCheckpoint`, BEFORE it stops the source sandbox — see that
|
|
364
|
+
* method's own doc — via a plain `exec`, never anything backend-specific
|
|
365
|
+
* beyond that: this is guest-side shell, not msb CLI surface.
|
|
366
|
+
*/
|
|
367
|
+
const CAPTURE_WORKLOAD_CMDLINE_SCRIPT = [
|
|
368
|
+
// A marker comment, not functional shell — lets the msb fixture (and any
|
|
369
|
+
// future test double) recognize this exact exec call by its script
|
|
370
|
+
// content rather than guessing from argv shape alone.
|
|
371
|
+
"# rightsize:capture-workload-cmdline",
|
|
372
|
+
"for pid in $(ls /proc 2>/dev/null | grep -E '^[0-9]+$' | sort -n); do",
|
|
373
|
+
" st=$(cat /proc/$pid/stat 2>/dev/null) || continue",
|
|
374
|
+
" rest=${st##*) }",
|
|
375
|
+
" set -- $rest",
|
|
376
|
+
" ppid=$2",
|
|
377
|
+
" if [ \"$ppid\" != \"1\" ]; then continue; fi",
|
|
378
|
+
" comm=${st#*(}",
|
|
379
|
+
" comm=${comm%)*}",
|
|
380
|
+
" case \"$comm\" in",
|
|
381
|
+
" init.krun|\\[*\\]) continue ;;",
|
|
382
|
+
" esac",
|
|
383
|
+
" cat /proc/$pid/cmdline",
|
|
384
|
+
" exit 0",
|
|
385
|
+
"done",
|
|
386
|
+
"exit 1",
|
|
387
|
+
].join("\n");
|
|
388
|
+
/**
|
|
389
|
+
* Parses `CAPTURE_WORKLOAD_CMDLINE_SCRIPT`'s stdout — the discovered
|
|
390
|
+
* process's raw `/proc/<pid>/cmdline` bytes, NUL-separated with a trailing
|
|
391
|
+
* NUL — into an argv. `captureGuestWorkloadCmdline` fetches this through the
|
|
392
|
+
* ordinary `exec()` (`invoke()`'s line-based reconstruction, not
|
|
393
|
+
* `fetchStdoutExact`'s byte-exact one — see that function's own doc), which
|
|
394
|
+
* appends a trailing `\n` to any non-empty output; since the guest script's
|
|
395
|
+
* own output never contains a real newline of its own (it is exactly the
|
|
396
|
+
* NUL-joined cmdline bytes), that single appended `\n` is always the whole
|
|
397
|
+
* of `invoke()`'s own last "line" and is stripped here before splitting,
|
|
398
|
+
* rather than being read back as a bogus trailing argv element. `undefined`
|
|
399
|
+
* on anything that isn't a clean, non-empty argv after that: empty output
|
|
400
|
+
* (the script found no matching child, exited 1, or the guest's own cmdline
|
|
401
|
+
* was itself empty) or content that splits into zero non-empty tokens. Never
|
|
402
|
+
* throws — see `captureGuestWorkloadCmdline`'s own doc on why a capture
|
|
403
|
+
* failure must never fail the checkpoint that triggered it.
|
|
404
|
+
*/
|
|
405
|
+
function parseCapturedWorkloadCmdline(stdout) {
|
|
406
|
+
if (stdout.length === 0) {
|
|
407
|
+
return undefined;
|
|
408
|
+
}
|
|
409
|
+
const withoutTrailingNewline = stdout.endsWith("\n") ? stdout.slice(0, -1) : stdout;
|
|
410
|
+
const parts = withoutTrailingNewline.split("\0").filter((s) => s.length > 0);
|
|
411
|
+
return parts.length > 0 ? parts : undefined;
|
|
412
|
+
}
|
|
237
413
|
/**
|
|
238
414
|
* The attached-mode CLI driver: every sandbox this backend starts runs as a
|
|
239
415
|
* held child process (`msb run`, no `-d`) because detached mode never
|
|
@@ -265,6 +441,10 @@ export class MsbCliBackend {
|
|
|
265
441
|
/** The stop/snapshot/reboot cycle reboots the microVM — the workload restarts. */
|
|
266
442
|
checkpointRestartsWorkload: true,
|
|
267
443
|
};
|
|
444
|
+
// Both keyed by a sandbox's CURRENT name (== SandboxHandle.id) — normally
|
|
445
|
+
// stable for a handle's whole lifetime, except across createCheckpoint's
|
|
446
|
+
// own reboot, which re-keys both from the source sandbox's name to the
|
|
447
|
+
// fresh one it boots under (see that method's own doc).
|
|
268
448
|
handles = new Map();
|
|
269
449
|
startedNames = new Set();
|
|
270
450
|
// Mirrors msbPathPromise's eventual value as soon as it settles, purely so
|
|
@@ -272,8 +452,27 @@ export class MsbCliBackend {
|
|
|
272
452
|
// anything) has a best-effort synchronous read of it. Never written to
|
|
273
453
|
// except by this one .then() below; never awaited anywhere else.
|
|
274
454
|
resolvedMsbPath;
|
|
275
|
-
|
|
455
|
+
// Test-only override seam for rebootUnderFreshName's own budget/delay
|
|
456
|
+
// — defaults to the real CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS/
|
|
457
|
+
// _DELAY_MS constants so production behavior is unchanged. Without this, a
|
|
458
|
+
// budget-exhaustion red-proof would mean a unit test actually blocking for
|
|
459
|
+
// the real ~30s; a test reaches these the same way it already reaches
|
|
460
|
+
// `handles`/`startedNames` elsewhere in this suite — an unsafe cast — to
|
|
461
|
+
// shrink them to milliseconds instead.
|
|
462
|
+
checkpointRebootAlreadyExistsRetryBudgetMs = CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS;
|
|
463
|
+
checkpointRebootAlreadyExistsRetryDelayMs = CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_DELAY_MS;
|
|
464
|
+
// POLICY v2's broker escalation — see `RestoreLaunchMode`/`restore-broker.ts`.
|
|
465
|
+
restoreBroker;
|
|
466
|
+
// Test-only override for the Windows-only escalation gate
|
|
467
|
+
// (`isWindowsPlatform`) — `undefined` in production, where the real
|
|
468
|
+
// `process.platform` always decides. Reached the same unsafe-cast way this
|
|
469
|
+
// suite already reaches `checkpointRebootAlreadyExistsRetryBudgetMs`
|
|
470
|
+
// above, so both the escalation trigger AND the "non-Windows never
|
|
471
|
+
// brokers" invariant are exercisable without an actual Windows host.
|
|
472
|
+
platformOverrideForTests = undefined;
|
|
473
|
+
constructor(msbPathPromise, options = {}) {
|
|
276
474
|
this.msbPathPromise = msbPathPromise;
|
|
475
|
+
this.restoreBroker = options.restoreBroker ?? realRestoreBroker;
|
|
277
476
|
this.msbPathPromise.then((p) => {
|
|
278
477
|
this.resolvedMsbPath = p;
|
|
279
478
|
}, () => {
|
|
@@ -282,11 +481,24 @@ export class MsbCliBackend {
|
|
|
282
481
|
// when it awaits msbPathPromise itself.
|
|
283
482
|
});
|
|
284
483
|
}
|
|
484
|
+
/**
|
|
485
|
+
* Whether this process should treat itself as running on Windows for
|
|
486
|
+
* POLICY v2's broker-escalation gate — `process.platform === "win32"` in
|
|
487
|
+
* production, or `platformOverrideForTests` when a test has set it. The
|
|
488
|
+
* ONLY thing this gates is whether `rebootUnderFreshName`/
|
|
489
|
+
* `retryRestoreAfterAccessDenied` ever set `RestoreLaunchMode` to
|
|
490
|
+
* `"broker"`; `bootOnce`/`bootRestoreOnce` themselves stay platform-
|
|
491
|
+
* agnostic dispatchers on whatever mode they're handed, so the broker path
|
|
492
|
+
* itself is exercisable on any host once a test picks it explicitly.
|
|
493
|
+
*/
|
|
494
|
+
isWindowsPlatform() {
|
|
495
|
+
return (this.platformOverrideForTests ?? process.platform) === "win32";
|
|
496
|
+
}
|
|
285
497
|
async msbPath() {
|
|
286
498
|
return this.msbPathPromise;
|
|
287
499
|
}
|
|
288
500
|
async create(spec) {
|
|
289
|
-
this.handles.set(spec.name, { attached: undefined, attachedExited: false, resources: [], logTail: [] });
|
|
501
|
+
this.handles.set(spec.name, { attached: undefined, attachedExited: false, resources: [], logTail: [], capturedCommand: undefined });
|
|
290
502
|
return { id: spec.name, spec };
|
|
291
503
|
}
|
|
292
504
|
/**
|
|
@@ -331,14 +543,48 @@ export class MsbCliBackend {
|
|
|
331
543
|
* knows: the install-lock poll, the one-shot state-database retry, and the
|
|
332
544
|
* one-shot image-cache heal. Both the ordinary `start()` path and the
|
|
333
545
|
* checkpoint cycle's post-snapshot reboot come through here — a reboot
|
|
334
|
-
* from a snapshot
|
|
335
|
-
*
|
|
336
|
-
*
|
|
546
|
+
* from a snapshot (now `msb restore ...`, see `bootRestoreOnce`) is
|
|
547
|
+
* as exposed to msb's transients as any other boot, and skipping the
|
|
548
|
+
* classification there turned a passing install-lock poll into an
|
|
549
|
+
* immediate checkpoint failure on a live Windows run. The image-cache heal
|
|
550
|
+
* targets `handle.spec.image`, which for a restore is the checkpoint ref
|
|
551
|
+
* GenericContainer.fromCheckpoint() threaded through as the builder's
|
|
552
|
+
* "image" — never a real OCI reference `msb image remove` can act on
|
|
553
|
+
* meaningfully; this predates the 0.7.1 migration (the same was true of
|
|
554
|
+
* `--from-snapshot` boots) and is unchanged here.
|
|
555
|
+
*
|
|
556
|
+
* `retryAccessDenied` (default `true`) gates ONLY the
|
|
557
|
+
* `RestoreAccessDeniedError` branch below — every other classified retry
|
|
558
|
+
* always runs regardless. The ordinary `start()` path (including a plain
|
|
559
|
+
* `GenericContainer.fromCheckpoint(cp).start()` restore) leaves it at the
|
|
560
|
+
* default, delegating to `retryRestoreAfterAccessDenied` — see that
|
|
561
|
+
* method's own doc for why it, like `createCheckpoint`'s own reboot, must
|
|
562
|
+
* mint a fresh name for every retry rather than reusing the one that just
|
|
563
|
+
* failed: the live-verified dossier this backend's fresh-naming behavior
|
|
564
|
+
* is built from says a restore that fails PAST msb's own artifact
|
|
565
|
+
* validation (this access-denied signature included) leaves the attempted
|
|
566
|
+
* name behind as a stopped sandbox record, so a same-name retry only ever
|
|
567
|
+
* collides with that record instead of retrying the actual transient.
|
|
568
|
+
* `createCheckpoint`'s own reboot (`rebootUnderFreshName`) passes `false`:
|
|
569
|
+
* it owns this error class itself, one layer up, via its own fresh-naming
|
|
570
|
+
* loop, and needs this error to propagate on the first hit rather than
|
|
571
|
+
* being retried here at all.
|
|
572
|
+
*
|
|
573
|
+
* `restoreLaunchMode` (default `"direct"`) is threaded straight down to
|
|
574
|
+
* every `bootOnce` call this method makes, including its own internal
|
|
575
|
+
* install-lock/state-db/image-cache retries — POLICY v2's broker
|
|
576
|
+
* escalation, once triggered by the caller (`rebootUnderFreshName`/
|
|
577
|
+
* `retryRestoreAfterAccessDenied`), applies to EVERY remaining attempt of
|
|
578
|
+
* that reboot, not just the ones those callers themselves loop over, so an
|
|
579
|
+
* install-lock hit mid-brokered-attempt must retry brokered too rather
|
|
580
|
+
* than silently reverting to a direct spawn. See `RestoreLaunchMode`'s own
|
|
581
|
+
* doc.
|
|
337
582
|
*/
|
|
338
|
-
async bootClassified(msbPath, handle, state) {
|
|
583
|
+
async bootClassified(msbPath, handle, state, options = { retryAccessDenied: true }) {
|
|
584
|
+
const restoreLaunchMode = options.restoreLaunchMode ?? "direct";
|
|
339
585
|
let firstOutput;
|
|
340
586
|
try {
|
|
341
|
-
await this.bootOnce(msbPath, handle, state);
|
|
587
|
+
await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
|
|
342
588
|
return;
|
|
343
589
|
}
|
|
344
590
|
catch (first) {
|
|
@@ -355,7 +601,7 @@ export class MsbCliBackend {
|
|
|
355
601
|
while (Date.now() < deadline) {
|
|
356
602
|
await sleep(INSTALL_LOCK_RETRY_DELAY_MS);
|
|
357
603
|
try {
|
|
358
|
-
await this.bootOnce(msbPath, handle, state);
|
|
604
|
+
await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
|
|
359
605
|
return;
|
|
360
606
|
}
|
|
361
607
|
catch (again) {
|
|
@@ -377,7 +623,7 @@ export class MsbCliBackend {
|
|
|
377
623
|
// image-cache heal below.
|
|
378
624
|
await sleep(STATE_DB_RETRY_DELAY_MS);
|
|
379
625
|
try {
|
|
380
|
-
await this.bootOnce(msbPath, handle, state);
|
|
626
|
+
await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
|
|
381
627
|
return;
|
|
382
628
|
}
|
|
383
629
|
catch (second) {
|
|
@@ -390,6 +636,17 @@ export class MsbCliBackend {
|
|
|
390
636
|
`host.\nfirst attempt:\n${first.output}\nafter retry:\n${second.output}`);
|
|
391
637
|
}
|
|
392
638
|
}
|
|
639
|
+
if (first instanceof RestoreAccessDeniedError) {
|
|
640
|
+
if (!options.retryAccessDenied) {
|
|
641
|
+
// The checkpoint reboot's own fresh-naming retry
|
|
642
|
+
// (`rebootUnderFreshName`) owns this error class itself, one layer
|
|
643
|
+
// up — see this method's own doc on `retryAccessDenied`. Propagate
|
|
644
|
+
// immediately rather than retrying it here at all.
|
|
645
|
+
throw first;
|
|
646
|
+
}
|
|
647
|
+
await this.retryRestoreAfterAccessDenied(msbPath, handle, state, first);
|
|
648
|
+
return;
|
|
649
|
+
}
|
|
393
650
|
if (!(first instanceof ImageCacheCorruptionError)) {
|
|
394
651
|
throw first;
|
|
395
652
|
}
|
|
@@ -397,7 +654,7 @@ export class MsbCliBackend {
|
|
|
397
654
|
}
|
|
398
655
|
const heal = await invoke(msbPath, MsbCommands.imageRemove(handle.spec.image), STOP_TIMEOUT_MS).catch((e) => e);
|
|
399
656
|
try {
|
|
400
|
-
await this.bootOnce(msbPath, handle, state);
|
|
657
|
+
await this.bootOnce(msbPath, handle, state, restoreLaunchMode);
|
|
401
658
|
}
|
|
402
659
|
catch (second) {
|
|
403
660
|
if (!(second instanceof ImageCacheCorruptionError)) {
|
|
@@ -411,30 +668,50 @@ export class MsbCliBackend {
|
|
|
411
668
|
}
|
|
412
669
|
}
|
|
413
670
|
/**
|
|
414
|
-
* One boot attempt
|
|
415
|
-
*
|
|
416
|
-
* `
|
|
671
|
+
* One boot attempt, dispatched by shape: an ordinary spec (no
|
|
672
|
+
* `checkpointRef`) drives `run()`'s ATTACHED supervision model
|
|
673
|
+
* (`bootRunOnce`); a checkpoint-restore spec drives `msb restore`'s
|
|
674
|
+
* fundamentally different DETACHED shape (`bootRestoreOnce`) — see each
|
|
675
|
+
* method's own doc. Resets the per-attempt diagnostics tail before either:
|
|
676
|
+
* a retried boot must not blend its tail with the failed attempt's.
|
|
677
|
+
* `restoreLaunchMode` is passed straight through to `bootRestoreOnce`
|
|
678
|
+
* (irrelevant to `bootRunOnce`'s own ordinary `run` path, which never
|
|
679
|
+
* brokers — see `RestoreLaunchMode`'s own doc).
|
|
680
|
+
*/
|
|
681
|
+
async bootOnce(msbPath, handle, state, restoreLaunchMode = "direct") {
|
|
682
|
+
state.logTail = [];
|
|
683
|
+
state.attachedExited = false;
|
|
684
|
+
if (handle.spec.checkpointRef !== undefined) {
|
|
685
|
+
await this.bootRestoreOnce(msbPath, handle, state, restoreLaunchMode);
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
688
|
+
await this.bootRunOnce(msbPath, handle, state);
|
|
689
|
+
}
|
|
690
|
+
/**
|
|
691
|
+
* One RUN boot attempt: spawns ATTACHED `msb run`, which stays alive as
|
|
692
|
+
* the sandbox's own supervisor for its whole lifetime, and polls
|
|
693
|
+
* `msb ls --format json` until the name shows Running.
|
|
694
|
+
*
|
|
695
|
+
* `state.attached` and (for non-keepAlive specs) `startedNames` are
|
|
696
|
+
* populated only on success; on any failure the child is reaped here (for
|
|
417
697
|
* the classified early-exit failures it has already exited; a readiness
|
|
418
698
|
* timeout leaves it alive and it is hard-killed) so a failed attempt leaves
|
|
419
699
|
* no live process or registered cleanup state behind — the caller owns
|
|
420
|
-
* retry policy, never cleanup (an exit-0
|
|
421
|
-
* exception:
|
|
422
|
-
*
|
|
423
|
-
*
|
|
424
|
-
*
|
|
425
|
-
*
|
|
426
|
-
*
|
|
427
|
-
*
|
|
428
|
-
*
|
|
429
|
-
*
|
|
430
|
-
*
|
|
700
|
+
* retry policy, never cleanup (an exit-0 success without a live child is
|
|
701
|
+
* the one exception: the fast-exit case below, see `isCompletedFastExit`'s
|
|
702
|
+
* own doc). An early exit is classified from the child's combined output:
|
|
703
|
+
* the image-cache-corruption signature throws `ImageCacheCorruptionError`
|
|
704
|
+
* (the one failure `start()` heals and retries), a host-port bind conflict
|
|
705
|
+
* throws `PortBindConflictError`, and otherwise — if the exit code was 0 —
|
|
706
|
+
* `isCompletedFastExit` gets a chance to reclassify the exit as a workload
|
|
707
|
+
* that ran to completion before this poll loop could ever observe Running
|
|
708
|
+
* (msb 0.6.16's convergent-lifecycle rework; see that method's own doc);
|
|
709
|
+
* anything else surfaces the raw output as an ordinary boot failure,
|
|
710
|
+
* unchanged.
|
|
431
711
|
*/
|
|
432
|
-
async
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
state.logTail = [];
|
|
436
|
-
state.attachedExited = false;
|
|
437
|
-
const child = spawn(msbPath, MsbCommands.run(handle.spec), { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
|
|
712
|
+
async bootRunOnce(msbPath, handle, state) {
|
|
713
|
+
const argv = MsbCommands.run(handle.spec);
|
|
714
|
+
const child = spawn(msbPath, argv, { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
|
|
438
715
|
// Merge stdout+stderr into one tail, kept only for the boot diagnostics
|
|
439
716
|
// below: this pipe is the sole carrier of msb's own output (registry/pull
|
|
440
717
|
// errors, crash output printed before the sandbox exists). It is not a
|
|
@@ -479,8 +756,9 @@ export class MsbCliBackend {
|
|
|
479
756
|
}
|
|
480
757
|
return;
|
|
481
758
|
}
|
|
482
|
-
throw new BackendError(`msb run for sandbox ${handle.id} exited (code ${exited.code ?? "unknown"})
|
|
483
|
-
`Running — check the image entrypoint and 'msb run'
|
|
759
|
+
throw new BackendError(`msb run for sandbox ${handle.id} exited (code ${exited.code ?? "unknown"}) ` +
|
|
760
|
+
`before reaching Running — check the image entrypoint and 'msb run' ` +
|
|
761
|
+
`output below:\n${output}`);
|
|
484
762
|
}
|
|
485
763
|
if ((await this.runningSandboxNames(msbPath)).has(handle.id)) {
|
|
486
764
|
state.attached = child;
|
|
@@ -501,9 +779,400 @@ export class MsbCliBackend {
|
|
|
501
779
|
await sleep(READINESS_POLL_MS);
|
|
502
780
|
}
|
|
503
781
|
}
|
|
782
|
+
/**
|
|
783
|
+
* One RESTORE boot attempt. `msb restore` is not `run`'s attached
|
|
784
|
+
* supervision model at all — EMPIRICALLY VERIFIED against msb 0.7.1 (its
|
|
785
|
+
* own `restore.rs` doc: "Restore a snapshot into a new detached
|
|
786
|
+
* sandbox"), the restore CLI process exits — typically within seconds,
|
|
787
|
+
* often with little or no stdout — the INSTANT activation succeeds, while
|
|
788
|
+
* the sandbox keeps booting in the background and only reaches Running
|
|
789
|
+
* some time after that (live-confirmed: `msb ls` shows Running, and exec
|
|
790
|
+
* works, only once the restore process has already exited). A clean exit
|
|
791
|
+
* is therefore not itself a completed boot, and a nonzero exit is msb's
|
|
792
|
+
* own failure signal with the reason on stdout/stderr — the inverse of
|
|
793
|
+
* `run`'s attached child, whose exit before Running always means failure.
|
|
794
|
+
*
|
|
795
|
+
* Two phases follow directly from those two facts:
|
|
796
|
+
* 1. Spawn `msb restore ...` and wait for IT to exit, bounded by
|
|
797
|
+
* `FIRST_RUN_PULL_TIMEOUT_MS` — the same boot budget `bootRunOnce`
|
|
798
|
+
* polls against; a restore's own activation is exposed to the same
|
|
799
|
+
* "this could be a slow cold operation" uncertainty a pull is.
|
|
800
|
+
* Classified from the combined output the same way any boot's early
|
|
801
|
+
* exit is (install-lock, state-db, image-cache-corruption,
|
|
802
|
+
* port-bind-conflict — see `bootClassified`'s own doc on why a
|
|
803
|
+
* restore boot needs the identical classification an ordinary one
|
|
804
|
+
* gets) — a nonzero, unclassified exit throws an ordinary
|
|
805
|
+
* `BackendError` carrying the output, same shape as `bootRunOnce`'s.
|
|
806
|
+
* 2. Once it exits 0, poll `msb ls` for the sandbox to reach Running —
|
|
807
|
+
* the same `READINESS_POLL_MS` cadence and a fresh instance of the
|
|
808
|
+
* identical `FIRST_RUN_PULL_TIMEOUT_MS` budget the attached path's
|
|
809
|
+
* own poll uses (so a restore boot gets the identical readiness
|
|
810
|
+
* patience an ordinary one does, counted from when there is
|
|
811
|
+
* actually something to poll for rather than shaved out of phase
|
|
812
|
+
* 1's own budget). The sandbox settling on exactly `"Stopped"`, or
|
|
813
|
+
* its name disappearing from a `msb ls` listing that itself came
|
|
814
|
+
* back, is a definite failure the moment it is observed — never
|
|
815
|
+
* worth waiting out the rest of the budget for, unlike a status
|
|
816
|
+
* that simply hasn't reached Running yet.
|
|
817
|
+
* 3. The `ls` probe itself failing to run or return in time (spawn
|
|
818
|
+
* error, or the same `LOGS_TIMEOUT_MS` `invoke` enforces on every
|
|
819
|
+
* other probe) is a different thing entirely from the sandbox's name
|
|
820
|
+
* being missing from a listing that DID come back, even though both
|
|
821
|
+
* collapse to the same `undefined` once the probe's own promise is
|
|
822
|
+
* caught — see `isCompletedFastExit`'s own doc, whose sibling
|
|
823
|
+
* contract this mirrors: a probe failure must never be upgraded into
|
|
824
|
+
* a false, specific diagnosis ("disappeared from msb ls entirely" is
|
|
825
|
+
* exactly that kind of false claim when the truth is just that `ls`
|
|
826
|
+
* itself didn't answer in time). A failed probe is therefore treated
|
|
827
|
+
* as "not yet confirmed either way" and retried within budget —
|
|
828
|
+
* install-lock races and an overloaded msb daemon are the same
|
|
829
|
+
* transient shapes this file already documents for `run`/`restore`/
|
|
830
|
+
* `stop`, and one `ls` hiccup during the readiness window must not
|
|
831
|
+
* turn into a spurious restore failure.
|
|
832
|
+
*
|
|
833
|
+
* The `msb restore` CLI process itself is never held onto — it has already
|
|
834
|
+
* exited by the time phase 2 even starts, and msb itself (out-of-process)
|
|
835
|
+
* is the sandbox's own supervisor from here on, the same as it always was.
|
|
836
|
+
* But a restored sandbox reaches Running with ONLY its guest agent inside
|
|
837
|
+
* — the captured workload never re-executes on its own (EMPIRICALLY
|
|
838
|
+
* VERIFIED against msb 0.7.1) — so once Running is confirmed, this method
|
|
839
|
+
* calls `reviveWorkload` to start it itself BEFORE returning: THAT call is
|
|
840
|
+
* what populates `state.attached` (a workload-revival `msb exec` child,
|
|
841
|
+
* not the restore CLI process), and `stop()`'s ordinary attached-child
|
|
842
|
+
* handling (exit-based death detection, the SIGKILL escalation) applies to
|
|
843
|
+
* IT exactly as it always did for `bootRunOnce`'s own attached `msb run`
|
|
844
|
+
* child. See `reviveWorkload`'s own doc for the full revival contract,
|
|
845
|
+
* including the typed error a checkpoint predating workload-cmdline
|
|
846
|
+
* capture throws instead of booting silently idle.
|
|
847
|
+
*
|
|
848
|
+
* Phase 1 (spawning the restore itself and reading back its own exit code
|
|
849
|
+
* + combined output) is dispatched by `launchMode` to either
|
|
850
|
+
* `launchRestoreDirect` (the unconditional spawn this method has always
|
|
851
|
+
* used) or `launchRestoreViaBroker` (POLICY v2's Windows job-object
|
|
852
|
+
* escape hatch — see `RestoreLaunchMode`/`restore-broker.ts`'s own doc).
|
|
853
|
+
* Both return the exact same shape, classified identically right below —
|
|
854
|
+
* a brokered attempt's output goes through the SAME predicates
|
|
855
|
+
* (`isImageCacheCorruption`, `isRestoreAccessDeniedFailure`, ...) a direct
|
|
856
|
+
* attempt's always has, since the underlying `msb restore` invocation and
|
|
857
|
+
* its failure signatures are identical either way; only how its stdout/
|
|
858
|
+
* stderr and exit code got back to this process differs. A broker launch
|
|
859
|
+
* can also come back `"unconfirmed"` (its own `ecFile` never appeared
|
|
860
|
+
* within its bound) — see `RestoreBrokerUnconfirmed`'s own doc — which
|
|
861
|
+
* skips classification entirely and falls straight through to phase 2
|
|
862
|
+
* exactly as a confirmed exit 0 does, since that phase's own `msb ls` poll
|
|
863
|
+
* is activation-gated and settles the question on its own regardless.
|
|
864
|
+
*/
|
|
865
|
+
async bootRestoreOnce(msbPath, handle, state, launchMode = "direct") {
|
|
866
|
+
const launch = launchMode === "broker"
|
|
867
|
+
? await this.launchRestoreViaBroker(msbPath, handle, state)
|
|
868
|
+
: await this.launchRestoreDirect(msbPath, handle, state);
|
|
869
|
+
if (launch.kind === "completed") {
|
|
870
|
+
const { exitCode, output } = launch;
|
|
871
|
+
if (isImageCacheCorruption(output)) {
|
|
872
|
+
throw new ImageCacheCorruptionError(output);
|
|
873
|
+
}
|
|
874
|
+
if (isMsbStateDbError(output)) {
|
|
875
|
+
throw new StateDbError(output);
|
|
876
|
+
}
|
|
877
|
+
if (isMsbInstallLockActive(output)) {
|
|
878
|
+
throw new InstallLockActiveError(output);
|
|
879
|
+
}
|
|
880
|
+
if (isPortBindConflictOutput(output)) {
|
|
881
|
+
throw new PortBindConflictError(`msb restore for sandbox ${handle.id} could not bind a host port: ${output}`);
|
|
882
|
+
}
|
|
883
|
+
if (isRestoreAccessDeniedFailure(output)) {
|
|
884
|
+
throw new RestoreAccessDeniedError(output);
|
|
885
|
+
}
|
|
886
|
+
if (isSandboxAlreadyExistsFailure(output)) {
|
|
887
|
+
throw new SandboxAlreadyExistsError(output);
|
|
888
|
+
}
|
|
889
|
+
if (exitCode !== 0) {
|
|
890
|
+
throw new BackendError(`msb restore for sandbox ${handle.id} exited (code ${exitCode}) — check the snapshot ref and ` +
|
|
891
|
+
`'msb restore' output below:\n${output}`);
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
// Exit 0 (direct or brokered), or the broker's own "unconfirmed" outcome
|
|
895
|
+
// — the restore CLI's own task is done, or close enough to trust `msb
|
|
896
|
+
// ls` to settle it (see this method's own doc) — poll for Running the
|
|
897
|
+
// same way bootRunOnce does, under a fresh instance of the identical
|
|
898
|
+
// budget.
|
|
899
|
+
const readyDeadline = Date.now() + FIRST_RUN_PULL_TIMEOUT_MS;
|
|
900
|
+
let lastSeenStatus;
|
|
901
|
+
for (;;) {
|
|
902
|
+
const ls = await invoke(msbPath, MsbCommands.ls(), LOGS_TIMEOUT_MS).catch(() => undefined);
|
|
903
|
+
// `ls === undefined` here means the probe itself never came back
|
|
904
|
+
// (spawn error, or its own LOGS_TIMEOUT_MS timing out) — a DIFFERENT
|
|
905
|
+
// thing from a returned listing that genuinely omits the sandbox's
|
|
906
|
+
// name (see this method's own doc, point 3, and `isCompletedFastExit`'s
|
|
907
|
+
// sibling doc). Only a listing that actually came back gets to update
|
|
908
|
+
// `lastSeenStatus` or trigger the Stopped/disappeared fast-fail below;
|
|
909
|
+
// a failed probe falls straight through to the readiness-budget check
|
|
910
|
+
// and is retried, never asserted as "the sandbox disappeared."
|
|
911
|
+
if (ls !== undefined) {
|
|
912
|
+
lastSeenStatus = statusOf(ls.stdout, handle.id);
|
|
913
|
+
if (lastSeenStatus === "Running") {
|
|
914
|
+
// `msb restore` boots a restored sandbox with only its guest agent
|
|
915
|
+
// inside — the captured workload never re-runs on its own (see
|
|
916
|
+
// this method's own doc and `reviveWorkload`'s) — so this backend
|
|
917
|
+
// starts it itself before registering the boot as complete. A
|
|
918
|
+
// `reviveWorkload` failure propagates unclassified: never register
|
|
919
|
+
// `startedNames` for a sandbox whose workload never actually came
|
|
920
|
+
// up.
|
|
921
|
+
await this.reviveWorkload(msbPath, handle, state);
|
|
922
|
+
if (!handle.spec.keepAlive) {
|
|
923
|
+
this.startedNames.add(handle.id);
|
|
924
|
+
}
|
|
925
|
+
return;
|
|
926
|
+
}
|
|
927
|
+
if (lastSeenStatus === "Stopped" || lastSeenStatus === undefined) {
|
|
928
|
+
throw new BackendError(await this.restoreBootFailureMessage(msbPath, handle.id, lastSeenStatus, false));
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
if (Date.now() >= readyDeadline) {
|
|
932
|
+
throw new BackendError(await this.restoreBootFailureMessage(msbPath, handle.id, lastSeenStatus, true));
|
|
933
|
+
}
|
|
934
|
+
await sleep(READINESS_POLL_MS);
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
/**
|
|
938
|
+
* `bootRestoreOnce`'s own DIRECT launch — the unconditional child-process
|
|
939
|
+
* spawn every restore attempt used before POLICY v2's broker escalation
|
|
940
|
+
* existed, extracted unchanged from that method's own former body so the
|
|
941
|
+
* two `RestoreLaunchMode`s share one classification point (see
|
|
942
|
+
* `bootRestoreOnce`'s own doc). Bounded by the same `FIRST_RUN_PULL_TIMEOUT_MS`
|
|
943
|
+
* budget `bootRunOnce` polls against, and always resolves `"completed"` —
|
|
944
|
+
* a direct spawn has no analog of the broker's own `"unconfirmed"` outcome,
|
|
945
|
+
* since this process reads the child's real exit code directly.
|
|
946
|
+
*/
|
|
947
|
+
async launchRestoreDirect(msbPath, handle, state) {
|
|
948
|
+
const argv = MsbCommands.restore(handle.spec);
|
|
949
|
+
const child = spawn(msbPath, argv, { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
|
|
950
|
+
const stdoutDone = drainTail(child.stdout, state.logTail);
|
|
951
|
+
const stderrDone = drainTail(child.stderr, state.logTail);
|
|
952
|
+
const exitCode = await new Promise((resolveExit, rejectExit) => {
|
|
953
|
+
let settled = false;
|
|
954
|
+
const timer = setTimeout(() => {
|
|
955
|
+
if (settled) {
|
|
956
|
+
return;
|
|
957
|
+
}
|
|
958
|
+
settled = true;
|
|
959
|
+
child.kill("SIGKILL");
|
|
960
|
+
rejectExit(new BackendError(`msb restore for sandbox ${handle.id} did not exit within ${FIRST_RUN_PULL_TIMEOUT_MS / 1000}s — ` +
|
|
961
|
+
`msb itself may be unresponsive; last output:\n${state.logTail.join("\n")}`));
|
|
962
|
+
}, FIRST_RUN_PULL_TIMEOUT_MS);
|
|
963
|
+
child.once("exit", (code) => {
|
|
964
|
+
if (settled) {
|
|
965
|
+
return;
|
|
966
|
+
}
|
|
967
|
+
settled = true;
|
|
968
|
+
clearTimeout(timer);
|
|
969
|
+
resolveExit(code ?? -1);
|
|
970
|
+
});
|
|
971
|
+
});
|
|
972
|
+
await Promise.all([stdoutDone, stderrDone]);
|
|
973
|
+
return { kind: "completed", exitCode, output: state.logTail.join("\n") };
|
|
974
|
+
}
|
|
975
|
+
/**
|
|
976
|
+
* `bootRestoreOnce`'s own BROKERED launch — POLICY v2's Windows job-object
|
|
977
|
+
* escape hatch (see `RestoreLaunchMode`/`restore-broker.ts`'s own module
|
|
978
|
+
* doc on the root cause and the live-validated WMI mitigation). Calls the
|
|
979
|
+
* injected `restoreBroker` seam with the exact same argv a direct attempt
|
|
980
|
+
* would spawn (`MsbCommands.restore`, unmodified) and the same
|
|
981
|
+
* `FIRST_RUN_PULL_TIMEOUT_MS` budget.
|
|
982
|
+
*
|
|
983
|
+
* The broker's own result is either passed straight through
|
|
984
|
+
* (`"completed"`/`"unconfirmed"`, both appended to `state.logTail` for the
|
|
985
|
+
* same pre-Running diagnostics purpose `drainTail` serves on the direct
|
|
986
|
+
* path) or, if the broker call itself REJECTS — broker infrastructure
|
|
987
|
+
* failing (a missing `powershell.exe`, a script-file write failure, WMI
|
|
988
|
+
* refusing to even create the process; see `RestoreBrokerLauncher`'s own
|
|
989
|
+
* doc on what counts as this vs. an ordinary brokered-restore failure) —
|
|
990
|
+
* caught here and turned into a one-attempt fallback to
|
|
991
|
+
* `launchRestoreDirect` instead (POLICY v2 point 5: the broker must never
|
|
992
|
+
* become a new single point of failure). That fallback is scoped to THIS
|
|
993
|
+
* attempt only; it does not touch the caller's own `RestoreLaunchMode` for
|
|
994
|
+
* whatever attempt comes after it.
|
|
995
|
+
*/
|
|
996
|
+
async launchRestoreViaBroker(msbPath, handle, state) {
|
|
997
|
+
const argv = MsbCommands.restore(handle.spec);
|
|
998
|
+
let result;
|
|
999
|
+
try {
|
|
1000
|
+
result = await this.restoreBroker(msbPath, argv, FIRST_RUN_PULL_TIMEOUT_MS);
|
|
1001
|
+
}
|
|
1002
|
+
catch (err) {
|
|
1003
|
+
pushLinesToTail(state.logTail, `[restore broker] broker infrastructure failed for sandbox ${handle.id}, falling back to a direct ` +
|
|
1004
|
+
`attempt: ${err instanceof Error ? err.message : String(err)}`);
|
|
1005
|
+
return this.launchRestoreDirect(msbPath, handle, state);
|
|
1006
|
+
}
|
|
1007
|
+
if (result.kind === "unconfirmed") {
|
|
1008
|
+
pushLinesToTail(state.logTail, `[restore broker] its own exit-code file never appeared within its wait bound for sandbox ` +
|
|
1009
|
+
`${handle.id} — proceeding to the ls-poll phase rather than guessing an outcome.`);
|
|
1010
|
+
return { kind: "unconfirmed" };
|
|
1011
|
+
}
|
|
1012
|
+
pushLinesToTail(state.logTail, result.output);
|
|
1013
|
+
return { kind: "completed", exitCode: result.exitCode, output: result.output };
|
|
1014
|
+
}
|
|
1015
|
+
/**
|
|
1016
|
+
* Renders a restore boot's post-mortem message once its poll for Running
|
|
1017
|
+
* has definitely failed (a settled `"Stopped"`, the name disappearing
|
|
1018
|
+
* from `msb ls` outright, or the readiness budget itself running out).
|
|
1019
|
+
* `msb restore` itself already exited 0 by this point, so the CLI
|
|
1020
|
+
* process's own output (already surfaced for a nonzero exit, above) has
|
|
1021
|
+
* nothing left to add — `msb logs --source system` is the one channel
|
|
1022
|
+
* that can still show what happened to the sandbox's own background boot
|
|
1023
|
+
* (see `MsbCommands.systemLog`'s own doc and `isCompletedFastExit`, its
|
|
1024
|
+
* other caller). Best-effort: a failed probe falls back to a plain "no
|
|
1025
|
+
* output" note rather than letting a diagnostics-gathering failure itself
|
|
1026
|
+
* throw and mask the real one.
|
|
1027
|
+
*/
|
|
1028
|
+
async restoreBootFailureMessage(msbPath, id, status, timedOut) {
|
|
1029
|
+
const systemLog = await invoke(msbPath, MsbCommands.systemLog(id), LOGS_TIMEOUT_MS).catch(() => undefined);
|
|
1030
|
+
const diagnostics = systemLog !== undefined && systemLog.stdout.trim() !== ""
|
|
1031
|
+
? `'msb logs ${id} --source system' output:\n${systemLog.stdout}`
|
|
1032
|
+
: `'msb logs ${id} --source system' produced no output`;
|
|
1033
|
+
// `status` is the last CONFIRMED status a returned `msb ls` listing
|
|
1034
|
+
// actually reported — never a value fabricated from a failed probe (see
|
|
1035
|
+
// bootRestoreOnce's own doc, point 3): `undefined` here on the timedOut
|
|
1036
|
+
// branch means every `ls` probe across the whole readiness budget failed
|
|
1037
|
+
// to even come back, not that a listing came back without the name in
|
|
1038
|
+
// it (that is the non-timedOut, "disappeared" branch below, which is
|
|
1039
|
+
// only ever reached from a listing that DID come back).
|
|
1040
|
+
const reason = timedOut
|
|
1041
|
+
? `did not reach Running within ${FIRST_RUN_PULL_TIMEOUT_MS / 1000}s of 'msb restore' exiting ` +
|
|
1042
|
+
`(last confirmed status: ${status ?? "none — 'msb ls' never returned a usable listing"})`
|
|
1043
|
+
: status === undefined
|
|
1044
|
+
? `disappeared from 'msb ls' entirely`
|
|
1045
|
+
: `settled as '${status}' in 'msb ls'`;
|
|
1046
|
+
return `sandbox ${id} was restored ('msb restore' exited 0) but ${reason} before ever reaching Running — ${diagnostics}`;
|
|
1047
|
+
}
|
|
1048
|
+
/**
|
|
1049
|
+
* Starts the workload a restore itself never re-runs. EMPIRICALLY VERIFIED
|
|
1050
|
+
* against msb 0.7.1: a restored sandbox reaches Running with ONLY its
|
|
1051
|
+
* guest agent inside (`guest ps` shows `/init.krun` and kernel threads —
|
|
1052
|
+
* the captured workload command does not re-execute; `msb start`/`msb
|
|
1053
|
+
* logs` on such a sandbox are equally idle/empty). Only ever called from
|
|
1054
|
+
* `bootRestoreOnce`, once it has confirmed Running — see this class's own
|
|
1055
|
+
* doc on `bootRestoreOnce`.
|
|
1056
|
+
*
|
|
1057
|
+
* Spawns a LONG-LIVED, attached `msb exec [-e K=V]... <name> -- <argv>`
|
|
1058
|
+
* session (`MsbCommands.execWithEnv`, env from `handle.spec.env` — a
|
|
1059
|
+
* restore's own `msb restore` has no `-e`/`--env` flag at all, so this exec
|
|
1060
|
+
* is the one place a restored sandbox's guest ever sees it again) — this
|
|
1061
|
+
* becomes the sandbox's own workload from here on, and exec sessions are
|
|
1062
|
+
* exactly what msb's own log capture records (the primary session's
|
|
1063
|
+
* stdout/stderr land in `exec.log`, served by `msb logs`/`-f` — see
|
|
1064
|
+
* `MsbCommands.exec`'s own doc). This exec child slots into the EXACT SAME
|
|
1065
|
+
* `state.attached` role `bootRunOnce`'s own attached `msb run` child fills
|
|
1066
|
+
* for an ordinary boot: child-exit-based death detection, reap-on-stop
|
|
1067
|
+
* (`stop()`'s SIGKILL escalation), and every other attached-child teardown
|
|
1068
|
+
* semantic apply to it unchanged — the detached-restore round left that
|
|
1069
|
+
* slot merely optional, never removed it.
|
|
1070
|
+
*
|
|
1071
|
+
* `handle.spec.command` is ALREADY the fully-resolved workload argv by the
|
|
1072
|
+
* time this runs, in priority order: an explicit command the source
|
|
1073
|
+
* container carried, or — when it had none — the guest cmdline
|
|
1074
|
+
* `createCheckpoint` captured at checkpoint time, merged in by
|
|
1075
|
+
* `createCheckpoint` itself (for its own immediate reboot) or by
|
|
1076
|
+
* `fromCheckpointRegistryEntry` (for a registry-mediated restore, same or
|
|
1077
|
+
* later process — see both functions' own docs). `undefined` here means
|
|
1078
|
+
* NEITHER source exists — an old registry entry predating capture, or one
|
|
1079
|
+
* whose capture attempt itself failed — so this throws
|
|
1080
|
+
* `CheckpointWorkloadCommandMissingError` itself rather than depending on
|
|
1081
|
+
* a caller to have checked first: never boot a restored sandbox silently
|
|
1082
|
+
* idle.
|
|
1083
|
+
*
|
|
1084
|
+
* The exec child gets a brief settle window (`EXEC_REVIVE_SETTLE_MS`) to
|
|
1085
|
+
* prove it isn't an immediate boot failure before this returns success —
|
|
1086
|
+
* mirroring `bootRunOnce`'s own exit-vs-Running race, except a restore's
|
|
1087
|
+
* exec session has no separate "Running" signal of its own to poll for
|
|
1088
|
+
* (the SANDBOX already reports Running regardless of whether this exec
|
|
1089
|
+
* succeeds), so a settle window is what that race collapses to here. An
|
|
1090
|
+
* exit observed within the window is classified exactly like
|
|
1091
|
+
* `bootRunOnce`'s own early exit: exit 0 counts as success only when
|
|
1092
|
+
* `isCompletedFastExit` ALSO confirms it (the sandbox itself settled
|
|
1093
|
+
* Stopped with the boot-completion marker — the repo's existing fast-exit-
|
|
1094
|
+
* completion semantics, reused verbatim); any other exit — nonzero, or
|
|
1095
|
+
* exit 0 without that confirmation — throws a `BackendError` carrying the
|
|
1096
|
+
* exec child's own output, the same failure shape an attached run's early
|
|
1097
|
+
* exit already has.
|
|
1098
|
+
*/
|
|
1099
|
+
async reviveWorkload(msbPath, handle, state) {
|
|
1100
|
+
const command = handle.spec.command;
|
|
1101
|
+
if (command === undefined) {
|
|
1102
|
+
throw new CheckpointWorkloadCommandMissingError(handle.spec.checkpointRef ?? handle.id);
|
|
1103
|
+
}
|
|
1104
|
+
state.logTail = [];
|
|
1105
|
+
const argv = MsbCommands.execWithEnv(handle.id, handle.spec.env, command);
|
|
1106
|
+
const child = spawn(msbPath, argv, { stdio: [CLOSED_STDIN, "pipe", "pipe"] });
|
|
1107
|
+
const stdoutDone = drainTail(child.stdout, state.logTail);
|
|
1108
|
+
const stderrDone = drainTail(child.stderr, state.logTail);
|
|
1109
|
+
let exited;
|
|
1110
|
+
child.once("exit", (code) => {
|
|
1111
|
+
exited = { code };
|
|
1112
|
+
state.attachedExited = true;
|
|
1113
|
+
});
|
|
1114
|
+
const settleDeadline = Date.now() + EXEC_REVIVE_SETTLE_MS;
|
|
1115
|
+
while (exited === undefined && Date.now() < settleDeadline) {
|
|
1116
|
+
await sleep(READINESS_POLL_MS);
|
|
1117
|
+
}
|
|
1118
|
+
if (exited === undefined) {
|
|
1119
|
+
// Stayed up through the whole settle window — the ordinary long-lived
|
|
1120
|
+
// case. Slots into state.attached exactly like bootRunOnce's own
|
|
1121
|
+
// attached child.
|
|
1122
|
+
state.attached = child;
|
|
1123
|
+
return;
|
|
1124
|
+
}
|
|
1125
|
+
await Promise.all([stdoutDone, stderrDone]);
|
|
1126
|
+
const output = state.logTail.join("\n");
|
|
1127
|
+
if (exited.code === 0 && (await this.isCompletedFastExit(msbPath, handle.id))) {
|
|
1128
|
+
// Mirrors bootRunOnce's own fast-exit success case: the workload ran to
|
|
1129
|
+
// completion so quickly this settle window caught its natural exit
|
|
1130
|
+
// rather than a crash. No live child left to hold onto — state.attached
|
|
1131
|
+
// stays undefined, same as bootRunOnce's own fast-exit branch.
|
|
1132
|
+
return;
|
|
1133
|
+
}
|
|
1134
|
+
throw new BackendError(`msb exec for sandbox ${handle.id}'s revived workload exited (code ${exited.code ?? "unknown"}) before ` +
|
|
1135
|
+
`staying up — check the workload command and its output below:\n${output}`);
|
|
1136
|
+
}
|
|
1137
|
+
/**
|
|
1138
|
+
* Best-effort: execs `CAPTURE_WORKLOAD_CMDLINE_SCRIPT` in `handle`'s guest
|
|
1139
|
+
* and parses its stdout via `parseCapturedWorkloadCmdline`. Only ever
|
|
1140
|
+
* called from `createCheckpoint`, BEFORE it stops the sandbox (see that
|
|
1141
|
+
* method's own doc), and only when `handle.spec.command` is undefined — an
|
|
1142
|
+
* explicit command needs no capture. Never throws: an exec failure
|
|
1143
|
+
* (nonzero exit, the exec channel itself erroring) and unparseable output
|
|
1144
|
+
* both resolve `undefined` rather than failing the checkpoint — the
|
|
1145
|
+
* captured cmdline is a best-effort fallback, and its absence is fully
|
|
1146
|
+
* handled later, at restore time (`reviveWorkload` throws
|
|
1147
|
+
* `CheckpointWorkloadCommandMissingError` when neither an explicit nor a
|
|
1148
|
+
* captured command exists).
|
|
1149
|
+
*/
|
|
1150
|
+
async captureGuestWorkloadCmdline(handle) {
|
|
1151
|
+
try {
|
|
1152
|
+
const result = await this.exec(handle, ["sh", "-c", CAPTURE_WORKLOAD_CMDLINE_SCRIPT]);
|
|
1153
|
+
if (result.exitCode !== 0) {
|
|
1154
|
+
return undefined;
|
|
1155
|
+
}
|
|
1156
|
+
return parseCapturedWorkloadCmdline(result.stdout);
|
|
1157
|
+
}
|
|
1158
|
+
catch {
|
|
1159
|
+
return undefined;
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
/**
|
|
1163
|
+
* SPI implementation of `SandboxBackend.capturedWorkloadCommand` — see its
|
|
1164
|
+
* own doc. Reads back whatever `createCheckpoint` most recently stashed on
|
|
1165
|
+
* this sandbox's `HandleState` (keyed by name, so it survives the
|
|
1166
|
+
* stop/snapshot/reboot cycle's own handle-object churn); `undefined` if
|
|
1167
|
+
* this sandbox was never checkpointed, or its checkpoint needed no
|
|
1168
|
+
* capture, or the capture attempt failed.
|
|
1169
|
+
*/
|
|
1170
|
+
capturedWorkloadCommand(handle) {
|
|
1171
|
+
return this.handles.get(handle.id)?.capturedCommand;
|
|
1172
|
+
}
|
|
504
1173
|
/**
|
|
505
1174
|
* The fast-exit post-mortem classification: only ever consulted from
|
|
506
|
-
* `
|
|
1175
|
+
* `bootRunOnce` above, and only once the attached `msb run` child has already
|
|
507
1176
|
* exited with code 0 before Running was observed. msb 0.6.16's
|
|
508
1177
|
* convergent-lifecycle rework means a workload that finishes quickly is
|
|
509
1178
|
* never observed `"Running"` at all — only `"Starting"`, then the attached
|
|
@@ -543,11 +1212,16 @@ export class MsbCliBackend {
|
|
|
543
1212
|
}
|
|
544
1213
|
await invoke(msbPath, MsbCommands.stop(handle.id), STOP_TIMEOUT_MS).catch(() => { });
|
|
545
1214
|
const attached = state?.attached;
|
|
546
|
-
// The attached
|
|
547
|
-
//
|
|
548
|
-
//
|
|
549
|
-
//
|
|
550
|
-
//
|
|
1215
|
+
// The attached child is either an ordinary boot's `msb run` process —
|
|
1216
|
+
// msb's own supervisor for this sandbox, staying alive for its entire
|
|
1217
|
+
// lifetime — or, for a restored sandbox, the `msb exec` session
|
|
1218
|
+
// `reviveWorkload` spawned to revive its captured workload (msb itself,
|
|
1219
|
+
// out-of-process, is the actual supervisor there; this exec child is
|
|
1220
|
+
// just the workload session riding inside it). Either way it stays alive
|
|
1221
|
+
// until the sandbox itself stops, and only exits once the `msb stop`
|
|
1222
|
+
// call just above lands, so the common path here is "attach a listener,
|
|
1223
|
+
// then observe the exit that our own stop just caused." state.attachedExited
|
|
1224
|
+
// exists for the other case: if the child
|
|
551
1225
|
// had already died before this method ever ran (crashed, or killed by
|
|
552
1226
|
// something external), start()'s own listener already flipped it, and
|
|
553
1227
|
// Node never replays a past "exit" event to a listener attached after
|
|
@@ -586,30 +1260,340 @@ export class MsbCliBackend {
|
|
|
586
1260
|
this.startedNames.delete(handle.id);
|
|
587
1261
|
this.handles.delete(handle.id);
|
|
588
1262
|
}
|
|
1263
|
+
/**
|
|
1264
|
+
* `bootClassified`'s own retry for a `RestoreAccessDeniedError` hit on the
|
|
1265
|
+
* ordinary `start()`/`fromCheckpoint().start()` path (its default
|
|
1266
|
+
* `retryAccessDenied: true`) — the sibling of `rebootUnderFreshName`
|
|
1267
|
+
* (`createCheckpoint`'s own reboot retry) for the SAME classified failure,
|
|
1268
|
+
* built from the SAME live-verified policy: `msb restore --name X`
|
|
1269
|
+
* validates the artifact FIRST — an integrity failure exits 1 and leaves
|
|
1270
|
+
* NO sandbox record — but a failure AFTER validation (this access-denied
|
|
1271
|
+
* signature included) leaves `X` behind as a STOPPED SANDBOX RECORD, and
|
|
1272
|
+
* any retry of `restore --name X` then fails outright with msb's own
|
|
1273
|
+
* "already exists" refusal rather than hitting the transient a second
|
|
1274
|
+
* time. A same-name retry (what this method replaces) therefore only ever
|
|
1275
|
+
* proves that collision — never the actual transient — so this mints a
|
|
1276
|
+
* NEW name from `nextSandboxName()` for every retry instead, the same
|
|
1277
|
+
* generator `GenericContainer.start()`'s own boot loop and
|
|
1278
|
+
* `rebootUnderFreshName` both use, tracking it in the reaper ledger BEFORE
|
|
1279
|
+
* that attempt's restore runs and best-effort `msb rm`-ing (then
|
|
1280
|
+
* untracking) a failed attempt's own name before advancing — identical
|
|
1281
|
+
* per-attempt bookkeeping to `rebootUnderFreshName`, just owned one layer
|
|
1282
|
+
* lower.
|
|
1283
|
+
*
|
|
1284
|
+
* The difference from `rebootUnderFreshName` is entirely about WHO owns
|
|
1285
|
+
* the re-keying, not the retry policy itself: `createCheckpoint` calls
|
|
1286
|
+
* `rebootUnderFreshName` with no live sandbox left at all (the source was
|
|
1287
|
+
* already stopped and removed as part of the snapshot step), so EVERY
|
|
1288
|
+
* attempt — including the first — mints a fresh name, and the winning
|
|
1289
|
+
* handle is handed back for `createCheckpoint` itself to re-key. Here,
|
|
1290
|
+
* `handle` is already live under its own name when `first` (the initial
|
|
1291
|
+
* `RestoreAccessDeniedError`) is caught, and `bootClassified`'s caller
|
|
1292
|
+
* (`start()`) has no re-keying step of its own — it just awaits
|
|
1293
|
+
* `bootClassified` and returns — so this method re-keys
|
|
1294
|
+
* `this.handles`/`this.startedNames` AND mutates the caller's own `handle`
|
|
1295
|
+
* object IN PLACE itself on success, the same "one carve-out"
|
|
1296
|
+
* `SandboxHandle`'s own interface doc allows for `createCheckpoint`'s
|
|
1297
|
+
* reboot (see that method's own doc on `handle.id`/`handle.spec`
|
|
1298
|
+
* mutation).
|
|
1299
|
+
*
|
|
1300
|
+
* `handle`'s own original name was already tracked in the reaper ledger by
|
|
1301
|
+
* `GenericContainer.start()`'s own boot loop before `create()`/`start()`
|
|
1302
|
+
* ever ran (mirroring `rebootUnderFreshName`'s treatment of the checkpoint
|
|
1303
|
+
* reboot's own `originalName`) — this method best-effort `msb rm`'s it
|
|
1304
|
+
* once `first` is confirmed retryable, but deliberately leaves its own
|
|
1305
|
+
* ledger entry alone, for the same not-found-tolerant sweep to find,
|
|
1306
|
+
* rather than untracking a name this method never tracked itself.
|
|
1307
|
+
*
|
|
1308
|
+
* Retries up to `RESTORE_ACCESS_DENIED_RETRY_LIMIT` times,
|
|
1309
|
+
* `RESTORE_ACCESS_DENIED_RETRY_DELAY_MS` apart — unchanged from the
|
|
1310
|
+
* same-name retry this replaces. Each attempt goes through
|
|
1311
|
+
* `bootClassified` itself with `retryAccessDenied: false` (so a
|
|
1312
|
+
* `StateDbError`/`InstallLockActiveError`/`ImageCacheCorruptionError` hit
|
|
1313
|
+
* mid-retry still gets its own ordinary classified handling; only
|
|
1314
|
+
* `RestoreAccessDeniedError`'s own retry is owned here) and treats either
|
|
1315
|
+
* `RestoreAccessDeniedError` or `SandboxAlreadyExistsError` as retryable —
|
|
1316
|
+
* a freshly minted name should never collide, but `rebootUnderFreshName`'s
|
|
1317
|
+
* own live-CI evidence says a loaded host can still refuse one, and the
|
|
1318
|
+
* SAME fresh-naming response (rm the failed name, mint another) covers
|
|
1319
|
+
* both signatures identically. Anything else propagates immediately,
|
|
1320
|
+
* unretried.
|
|
1321
|
+
*
|
|
1322
|
+
* POLICY v2's broker escalation applies here too, on the exact same terms
|
|
1323
|
+
* as `rebootUnderFreshName`'s own (see that method's own doc): this
|
|
1324
|
+
* method is only ever entered once the reboot's first, always-direct
|
|
1325
|
+
* attempt has already hit `RestoreAccessDeniedError` (`first`), so on a
|
|
1326
|
+
* Windows host (`isWindowsPlatform()`) every retry THIS method itself
|
|
1327
|
+
* performs is, by construction, an attempt after that trigger — there is
|
|
1328
|
+
* no "first attempt still direct" sub-case to preserve inside this loop
|
|
1329
|
+
* the way `rebootUnderFreshName` has to across its own. `launchMode` is
|
|
1330
|
+
* therefore decided once, before the loop, not re-evaluated per attempt.
|
|
1331
|
+
*/
|
|
1332
|
+
async retryRestoreAfterAccessDenied(msbPath, handle, state, first) {
|
|
1333
|
+
const originalName = handle.id;
|
|
1334
|
+
// Best-effort: `first` may have left a stopped sandbox record behind
|
|
1335
|
+
// under `originalName` (see this method's own doc) — cheap cleanup, and
|
|
1336
|
+
// correctness never depends on it, since no retry below ever reuses this
|
|
1337
|
+
// name. Its own ledger entry is deliberately left alone; see this
|
|
1338
|
+
// method's own doc.
|
|
1339
|
+
await invoke(msbPath, MsbCommands.rm(originalName), STOP_TIMEOUT_MS).catch(() => { });
|
|
1340
|
+
const launchMode = this.isWindowsPlatform() ? "broker" : "direct";
|
|
1341
|
+
let last = first;
|
|
1342
|
+
for (let attempt = 1; attempt <= RESTORE_ACCESS_DENIED_RETRY_LIMIT; attempt++) {
|
|
1343
|
+
await sleep(RESTORE_ACCESS_DENIED_RETRY_DELAY_MS);
|
|
1344
|
+
const name = nextSandboxName();
|
|
1345
|
+
if (!handle.spec.keepAlive) {
|
|
1346
|
+
await trackSandbox(name);
|
|
1347
|
+
}
|
|
1348
|
+
const candidate = { id: name, spec: { ...handle.spec, name } };
|
|
1349
|
+
try {
|
|
1350
|
+
await this.bootClassified(msbPath, candidate, state, { retryAccessDenied: false, restoreLaunchMode: launchMode });
|
|
1351
|
+
// Success under `name` — re-key this backend's own runtime
|
|
1352
|
+
// registries (see class doc on `handles`) and publish the new
|
|
1353
|
+
// identity onto the CALLER's own handle object, IN PLACE, exactly
|
|
1354
|
+
// like `createCheckpoint`'s own reboot does (see its own doc).
|
|
1355
|
+
this.handles.delete(originalName);
|
|
1356
|
+
this.handles.set(name, state);
|
|
1357
|
+
this.startedNames.delete(originalName);
|
|
1358
|
+
const mutableHandle = handle;
|
|
1359
|
+
mutableHandle.id = name;
|
|
1360
|
+
mutableHandle.spec = candidate.spec;
|
|
1361
|
+
return;
|
|
1362
|
+
}
|
|
1363
|
+
catch (err) {
|
|
1364
|
+
const classified = err instanceof RestoreAccessDeniedError || err instanceof SandboxAlreadyExistsError;
|
|
1365
|
+
if (!classified) {
|
|
1366
|
+
if (!handle.spec.keepAlive) {
|
|
1367
|
+
await untrackSandbox(name);
|
|
1368
|
+
}
|
|
1369
|
+
throw err;
|
|
1370
|
+
}
|
|
1371
|
+
// Best-effort, result ignored either way — see this method's own
|
|
1372
|
+
// doc on why correctness no longer depends on it.
|
|
1373
|
+
await invoke(msbPath, MsbCommands.rm(name), STOP_TIMEOUT_MS).catch(() => { });
|
|
1374
|
+
if (!handle.spec.keepAlive) {
|
|
1375
|
+
await untrackSandbox(name);
|
|
1376
|
+
}
|
|
1377
|
+
last = err;
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
throw new BackendError(`msb restore for sandbox ${originalName} kept hitting msb's Windows access-denied/"sandbox already ` +
|
|
1381
|
+
`exists" refusal across ${RESTORE_ACCESS_DENIED_RETRY_LIMIT} retries, each under a freshly minted ` +
|
|
1382
|
+
`name — this is normally a brief file-handle release lag that clears within one retry, so a failure ` +
|
|
1383
|
+
`held this long looks like a genuinely stuck lock on this host.\n${last.output}`);
|
|
1384
|
+
}
|
|
1385
|
+
/**
|
|
1386
|
+
* `createCheckpoint`'s own reboot step. Mints a NEW sandbox name from
|
|
1387
|
+
* `nextSandboxName()` (the same generator `GenericContainer.start()`'s own
|
|
1388
|
+
* boot loop uses) for EVERY attempt, tracks it in the reaper ledger BEFORE
|
|
1389
|
+
* that attempt's restore runs (mirroring `GenericContainer.start()`'s own
|
|
1390
|
+
* `trackSandbox` call, keepAlive-excluded the same way), and retries two
|
|
1391
|
+
* classified failure signatures on that basis instead of surfacing either
|
|
1392
|
+
* immediately: msb's "sandbox already exists" refusal
|
|
1393
|
+
* (`SandboxAlreadyExistsError`, see `isSandboxAlreadyExistsFailure`'s own
|
|
1394
|
+
* doc) and its Windows access-denied refusal on the just-written snapshot
|
|
1395
|
+
* artifact (`RestoreAccessDeniedError`, see `isRestoreAccessDeniedFailure`'s
|
|
1396
|
+
* own doc — retried here rather than inside `bootClassified` itself, via
|
|
1397
|
+
* that method's own `retryAccessDenied: false`, see its doc).
|
|
1398
|
+
*
|
|
1399
|
+
* LIVE-VERIFIED POLICY this retry is built from: `msb restore --name X`
|
|
1400
|
+
* validates the artifact FIRST — an integrity failure exits 1 and leaves NO
|
|
1401
|
+
* sandbox record — but a failure AFTER validation (a block-device open
|
|
1402
|
+
* PermissionDenied on unix; `RestoreAccessDeniedError`'s own signature on
|
|
1403
|
+
* Windows) leaves `X` behind as a STOPPED SANDBOX RECORD visible in `msb
|
|
1404
|
+
* ls`, and any retry of `restore --name X` then fails with msb's own
|
|
1405
|
+
* "already exists" refusal — so a same-name retry of EITHER classified
|
|
1406
|
+
* failure only ever collides with itself. Reusing a failed attempt's name
|
|
1407
|
+
* is therefore never safe (`msb rm` alone is not reliably enough either: on
|
|
1408
|
+
* Windows it frees only the DB record, not the on-disk directory, which can
|
|
1409
|
+
* outlive it well past this budget — see the fresh-name reboot's own doc on
|
|
1410
|
+
* `createCheckpoint`) — the only universally safe policy is a NEW name per
|
|
1411
|
+
* attempt, same as this backend's ordinary `start()` boot loop already uses
|
|
1412
|
+
* for its own, unrelated port-conflict retries. On a classified failure
|
|
1413
|
+
* this best-effort `msb rm`'s the failed attempt's own name anyway (result
|
|
1414
|
+
* ignored — cheap cleanup, and correctness no longer depends on it now
|
|
1415
|
+
* that the next attempt never reuses that name) and untracks it from the
|
|
1416
|
+
* ledger, before advancing to a freshly minted name for the next attempt.
|
|
1417
|
+
*
|
|
1418
|
+
* `this.checkpointRebootAlreadyExistsRetryBudgetMs`/`_RetryDelayMs` back
|
|
1419
|
+
* this loop's overall deadline/delay rather than the bare
|
|
1420
|
+
* `CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS`/`_DELAY_MS` constants
|
|
1421
|
+
* directly, so a budget-exhaustion test can shrink them to milliseconds
|
|
1422
|
+
* instead of actually blocking for the real ~30s — see those fields' own
|
|
1423
|
+
* doc. Only `SandboxAlreadyExistsError`/`RestoreAccessDeniedError` are
|
|
1424
|
+
* retried here; any other failure `bootClassified` throws (on the first
|
|
1425
|
+
* attempt or a later one) propagates immediately, unretried, WITHOUT
|
|
1426
|
+
* advancing to a new name — `createCheckpoint`'s own caller untracks that
|
|
1427
|
+
* attempt's name on this path (see its own catch block). Not reached by
|
|
1428
|
+
* the ordinary `start()` path: a `GenericContainer.fromCheckpoint(cp)
|
|
1429
|
+
* .start()` restore's own `RestoreAccessDeniedError` is retried by its own
|
|
1430
|
+
* sibling, `retryRestoreAfterAccessDenied` (see that method's own doc) —
|
|
1431
|
+
* same fresh-naming policy, applied one layer lower since that path has no
|
|
1432
|
+
* separate caller left to hand a replacement handle to for re-keying.
|
|
1433
|
+
*
|
|
1434
|
+
* Returns the WINNING attempt's own `SandboxHandle` (a fresh object, never
|
|
1435
|
+
* `undefined`) — `createCheckpoint` re-keys its runtime registries and the
|
|
1436
|
+
* caller's live handle from this, not from a name minted up front.
|
|
1437
|
+
*
|
|
1438
|
+
* POLICY v2's broker escalation (see `RestoreLaunchMode`/`restore-broker.ts`'s
|
|
1439
|
+
* own module doc on the Windows job-object root cause and the live-
|
|
1440
|
+
* validated WMI mitigation) lives here: `launchMode` starts `"direct"` —
|
|
1441
|
+
* the first attempt of ANY reboot is always a direct spawn, unconditionally,
|
|
1442
|
+
* even on Windows — and flips to `"broker"` the moment an attempt hits
|
|
1443
|
+
* `RestoreAccessDeniedError` on a Windows host (`isWindowsPlatform()`);
|
|
1444
|
+
* every attempt from then on, for the rest of THIS reboot, launches
|
|
1445
|
+
* brokered instead, never downgrading back to direct even if a later
|
|
1446
|
+
* attempt's own failure is only `SandboxAlreadyExistsError` (still
|
|
1447
|
+
* retried, but no longer the trigger — only access-denied escalates).
|
|
1448
|
+
* Off Windows, `launchMode` never leaves `"direct"` at all — this reboot's
|
|
1449
|
+
* whole retry loop behaves exactly as it did before this policy existed.
|
|
1450
|
+
*/
|
|
1451
|
+
async rebootUnderFreshName(msbPath, sourceSpec, effectiveRef, capturedCommand, state) {
|
|
1452
|
+
const deadline = Date.now() + this.checkpointRebootAlreadyExistsRetryBudgetMs;
|
|
1453
|
+
let attempt = 0;
|
|
1454
|
+
let last;
|
|
1455
|
+
let launchMode = "direct";
|
|
1456
|
+
for (;;) {
|
|
1457
|
+
attempt += 1;
|
|
1458
|
+
const name = nextSandboxName();
|
|
1459
|
+
if (!sourceSpec.keepAlive) {
|
|
1460
|
+
await trackSandbox(name);
|
|
1461
|
+
}
|
|
1462
|
+
const rebootHandle = {
|
|
1463
|
+
id: name,
|
|
1464
|
+
spec: { ...sourceSpec, name, checkpointRef: effectiveRef, command: sourceSpec.command ?? capturedCommand },
|
|
1465
|
+
};
|
|
1466
|
+
try {
|
|
1467
|
+
await this.bootClassified(msbPath, rebootHandle, state, { retryAccessDenied: false, restoreLaunchMode: launchMode });
|
|
1468
|
+
return rebootHandle;
|
|
1469
|
+
}
|
|
1470
|
+
catch (err) {
|
|
1471
|
+
const classified = err instanceof SandboxAlreadyExistsError || err instanceof RestoreAccessDeniedError;
|
|
1472
|
+
if (!classified) {
|
|
1473
|
+
if (!sourceSpec.keepAlive) {
|
|
1474
|
+
await untrackSandbox(name);
|
|
1475
|
+
}
|
|
1476
|
+
throw err;
|
|
1477
|
+
}
|
|
1478
|
+
if (err instanceof RestoreAccessDeniedError && this.isWindowsPlatform()) {
|
|
1479
|
+
// POLICY v2: escalate for every remaining attempt of this reboot —
|
|
1480
|
+
// see this method's own doc on `launchMode`.
|
|
1481
|
+
launchMode = "broker";
|
|
1482
|
+
}
|
|
1483
|
+
// Best-effort: ignore the result either way — a "not found" is as
|
|
1484
|
+
// fine as an actual removal, and this is cleanup, not something the
|
|
1485
|
+
// next attempt (a brand-new name) depends on for correctness.
|
|
1486
|
+
await invoke(msbPath, MsbCommands.rm(name), STOP_TIMEOUT_MS).catch(() => { });
|
|
1487
|
+
if (!sourceSpec.keepAlive) {
|
|
1488
|
+
await untrackSandbox(name);
|
|
1489
|
+
}
|
|
1490
|
+
last = err;
|
|
1491
|
+
if (Date.now() >= deadline) {
|
|
1492
|
+
throw new BackendError(`msb restore kept hitting msb's "sandbox already exists"/access-denied refusal for ` +
|
|
1493
|
+
`${this.checkpointRebootAlreadyExistsRetryBudgetMs / 1000}s across ${attempt} attempt` +
|
|
1494
|
+
`${attempt === 1 ? "" : "s"}, each under a freshly minted name — msb's own on-disk sandbox ` +
|
|
1495
|
+
`directory can lag its database record's own release on a loaded Windows host well past a short ` +
|
|
1496
|
+
`wait, but a refusal held this long looks like a genuinely stuck host rather than a release ` +
|
|
1497
|
+
`race.\nlast attempt ('${name}'):\n${last.output}`);
|
|
1498
|
+
}
|
|
1499
|
+
await sleep(this.checkpointRebootAlreadyExistsRetryDelayMs);
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
589
1503
|
/**
|
|
590
1504
|
* The stop/snapshot/reboot cycle: `msb stop <name>` (reusing this
|
|
591
1505
|
* backend's own `stop()`, which also quiesces the attached child and any
|
|
592
|
-
* network-link tunnels), `msb snapshot create --from <name> <ref
|
|
593
|
-
*
|
|
594
|
-
*
|
|
595
|
-
*
|
|
596
|
-
*
|
|
597
|
-
*
|
|
598
|
-
*
|
|
599
|
-
*
|
|
600
|
-
*
|
|
601
|
-
*
|
|
602
|
-
*
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
* `
|
|
607
|
-
*
|
|
608
|
-
*
|
|
609
|
-
*
|
|
610
|
-
*
|
|
611
|
-
*
|
|
612
|
-
*
|
|
1506
|
+
* network-link tunnels), `msb snapshot create --from-sandbox <name> <ref>`
|
|
1507
|
+
* (whose printed artifact path — never `ref` itself — becomes the
|
|
1508
|
+
* EFFECTIVE checkpoint ref this method returns; see
|
|
1509
|
+
* `parseSnapshotCreateArtifactPath`), then `msb rm <name>` followed by
|
|
1510
|
+
* `msb restore <effective-ref> --name <fresh-name>` of a FRESH name from
|
|
1511
|
+
* that snapshot (via `rebootUnderFreshName`/`bootRestoreOnce`, see
|
|
1512
|
+
* `MsbCommands.restore`'s own doc — no `--disk-only`, which a disk-scope
|
|
1513
|
+
* snapshot rejects) — never `msb start`. The fresh name — never the
|
|
1514
|
+
* original — is minted by `nextSandboxName()` (the same generator
|
|
1515
|
+
* `GenericContainer.start()`'s own ordinary boot loop uses; see
|
|
1516
|
+
* `core/sandbox-name.ts`), and a NEW one is minted for EVERY attempt, never
|
|
1517
|
+
* just once for the whole reboot: EMPIRICALLY VERIFIED against a real msb
|
|
1518
|
+
* 0.7.1 binary, a restore that fails PAST msb's own artifact validation
|
|
1519
|
+
* (its Windows access-denied signature, `RestoreAccessDeniedError`) leaves
|
|
1520
|
+
* its `--name` behind as a STOPPED SANDBOX RECORD, and any retry under that
|
|
1521
|
+
* SAME name then collides with msb's own restore-time collision check
|
|
1522
|
+
* (`existing.is_some() || dir_exists`) immediately — CONFIRMED on Windows
|
|
1523
|
+
* CI, where five checkpoint tests collided on their fresh reboot names for
|
|
1524
|
+
* the retry loop's ENTIRE budget once the first attempt hit that
|
|
1525
|
+
* access-denied failure (see `CHECKPOINT_REBOOT_ALREADY_EXISTS_RETRY_BUDGET_MS`'s
|
|
1526
|
+
* own doc). A same-name retry therefore only ever retries through a race of
|
|
1527
|
+
* its own making; a NEW name per attempt sidesteps it structurally — a name
|
|
1528
|
+
* nothing else has ever used can never collide with a record or a lingering
|
|
1529
|
+
* directory left by a name nothing will ever restore under again (see
|
|
1530
|
+
* `rebootUnderFreshName`'s own doc for the full retry policy, including its
|
|
1531
|
+
* best-effort `msb rm` of each failed attempt's own name). On success, this method
|
|
1532
|
+
* mutates `handle.id` and `handle.spec` (name only — see below) IN PLACE
|
|
1533
|
+
* on the caller's own `handle`, so every subsequent operation against it
|
|
1534
|
+
* (exec/logs/stop/rm, and — one layer up, via the SAME `SandboxHandle`
|
|
1535
|
+
* reference — `GenericContainer.checkpoint()`'s own post-reboot
|
|
1536
|
+
* `installNetworkLinks`/wait-strategy re-run and `capturedWorkloadCommand()`
|
|
1537
|
+
* call) targets the sandbox actually running now. `SandboxBackend`'s own
|
|
1538
|
+
* interface doc calls `SandboxHandle` "immutable" — this is the one
|
|
1539
|
+
* carve-out, and only this method makes it: a checkpoint reboot is the one
|
|
1540
|
+
* operation that changes what sandbox a handle even refers to. `ports`,
|
|
1541
|
+
* `env`, and `memoryLimitMb` are untouched by the rename — same ports, env,
|
|
1542
|
+
* and memory ceiling as before the reboot, only the name differs.
|
|
1543
|
+
* Upstream's
|
|
1544
|
+
* detached-start path (`Sandbox::start_detached`) passes
|
|
1545
|
+
* `CREATE_BREAKAWAY_FROM_JOB` on Windows, which `ERROR_ACCESS_DENIED`s
|
|
1546
|
+
* outright whenever the msb CLI runs inside a job object that doesn't
|
|
1547
|
+
* grant breakaway rights — a Gradle/cargo/node test runner on a hosted
|
|
1548
|
+
* Windows runner, or any process that embeds this library inside its own
|
|
1549
|
+
* restrictive job object — and that denial is deterministic, not
|
|
1550
|
+
* transient, so no retry shape fixes it. Before 0.7.1, attached `msb run`
|
|
1551
|
+
* (including its `--from-snapshot` boots) never hit this, which is why the
|
|
1552
|
+
* reboot has always gone through the classified boot path (`bootClassified`)
|
|
1553
|
+
* rather than `msb start`. `msb restore` has no attached/detached
|
|
1554
|
+
* distinction of its own — its CLI process always calls `sandbox.detach()`
|
|
1555
|
+
* internally and exits once activation is confirmed (see
|
|
1556
|
+
* `bootRestoreOnce`'s own doc on why its exit is a SUCCESS signal, not
|
|
1557
|
+
* something to reap) — so whether it shares upstream's
|
|
1558
|
+
* `CREATE_BREAKAWAY_FROM_JOB` path on Windows is NOT verified here: this
|
|
1559
|
+
* backend never runs the real msb binary (see this repo's hard "never boot
|
|
1560
|
+
* a sandbox" rule), so this is a real open question for CI, not something
|
|
1561
|
+
* this migration could confirm either way. If it recurs, it will surface
|
|
1562
|
+
* as an ordinary unclassified `BackendError` out of `bootRestoreOnce`
|
|
1563
|
+
* (deterministic exit-code failure, not one of the classified transients
|
|
1564
|
+
* above), the same shape a genuine breakaway denial always took.
|
|
1565
|
+
* `rm`-ing the sandbox first and restoring a fresh one under a fresh
|
|
1566
|
+
* name, same ports/memory (via a spec identical to `handle.spec` except
|
|
1567
|
+
* `name`/`checkpointRef` set to the fresh name and the EFFECTIVE ref this
|
|
1568
|
+
* method discovers, below — env is no longer threaded through at all, see
|
|
1569
|
+
* `MsbCommands.restore`) reproduces the exact same observable contract
|
|
1570
|
+
* MODULO the name itself, which was always an implementation detail, not
|
|
1571
|
+
* part of what a checkpoint promises to preserve. This backend's own
|
|
1572
|
+
* `handles`/`startedNames` registries ARE re-keyed from the original name
|
|
1573
|
+
* to the WINNING attempt's name (see below), and each attempt's own fresh
|
|
1574
|
+
* name is tracked in the reaping ledger before that attempt's restore is
|
|
1575
|
+
* even attempted, exactly like an ordinary `create()` — a failed attempt's
|
|
1576
|
+
* name is untracked again by `rebootUnderFreshName` itself, while the
|
|
1577
|
+
* original name's own ledger entry is deliberately left alone throughout,
|
|
1578
|
+
* for the ledger's existing not-found-tolerant sweep to find (it was
|
|
1579
|
+
* already `msb rm`-ed above). Its workload restarts from scratch (the VM
|
|
1580
|
+
* reboots), which is why `capabilities.checkpointRestartsWorkload` is
|
|
1581
|
+
* `true` here and the generic layer re-runs the wait strategy after this
|
|
1582
|
+
* returns — AFTER `bootRestoreOnce` has already revived the workload
|
|
1583
|
+
* itself via `reviveWorkload` (see that method's own doc): `msb restore`
|
|
1584
|
+
* boots the reboot idle, only the guest agent inside, so `bootRestoreOnce`
|
|
1585
|
+
* no longer leaves `state.attached` untouched the way it did before this
|
|
1586
|
+
* revival step existed — it now carries the workload-revival exec child,
|
|
1587
|
+
* the same attached-child slot `bootRunOnce`'s own `msb run` child fills
|
|
1588
|
+
* for an ordinary boot.
|
|
1589
|
+
*
|
|
1590
|
+
* Before stopping the source sandbox, this also best-effort captures its
|
|
1591
|
+
* guest workload cmdline (`captureGuestWorkloadCmdline`) when
|
|
1592
|
+
* `handle.spec.command` is undefined — the image's own default entrypoint
|
|
1593
|
+
* was running, so there is no explicit command for `reviveWorkload` to
|
|
1594
|
+
* fall back on at either this method's own immediate reboot or a later,
|
|
1595
|
+
* registry-mediated restore. See that method's own doc for why a capture
|
|
1596
|
+
* failure never fails the checkpoint itself.
|
|
613
1597
|
*
|
|
614
1598
|
* If the snapshot step fails, the sandbox is left stopped — no
|
|
615
1599
|
* best-effort restart, since that restart would itself be the broken `msb
|
|
@@ -621,6 +1605,22 @@ export class MsbCliBackend {
|
|
|
621
1605
|
*
|
|
622
1606
|
* Refuses outright, before any of the above, when `handle.spec.tmpfsRootMb`
|
|
623
1607
|
* is set: a tmpfs root has nothing on disk for a snapshot to capture.
|
|
1608
|
+
*
|
|
1609
|
+
* `ref` is the WORKING ref this method is asked to checkpoint under — its
|
|
1610
|
+
* basename becomes the snapshot create call's `<name>` argument (still
|
|
1611
|
+
* meaningful: it lands in msb's own index, see `MsbCommands.snapshotCreate`'s
|
|
1612
|
+
* own doc) and, for a path ref, its dirname becomes `--dest-dir`. It is
|
|
1613
|
+
* NOT necessarily where the artifact ends up: EMPIRICALLY VERIFIED against
|
|
1614
|
+
* a real msb 0.7.1 binary, `snapshot create` always writes under
|
|
1615
|
+
* `<destDir-or-default>/<sandbox>/snap_<32-hex-digest>`, a path `name`
|
|
1616
|
+
* never determines. This method therefore parses that real artifact path
|
|
1617
|
+
* back out of the command's own stdout (last non-empty line, required to
|
|
1618
|
+
* be absolute — see `parseSnapshotCreateArtifactPath`) and returns THAT as
|
|
1619
|
+
* the EFFECTIVE ref — the one used for the reboot below and the one the
|
|
1620
|
+
* caller (`GenericContainer.checkpoint()`) stores in the registry and
|
|
1621
|
+
* hands back on the `Checkpoint` it returns. Malformed or unrecognizable
|
|
1622
|
+
* output (empty, no absolute last line) throws a `BackendError` quoting
|
|
1623
|
+
* the raw, unparsed output verbatim rather than guessing a ref.
|
|
624
1624
|
*/
|
|
625
1625
|
async createCheckpoint(handle, ref) {
|
|
626
1626
|
if (handle.spec.tmpfsRootMb !== undefined) {
|
|
@@ -630,58 +1630,146 @@ export class MsbCliBackend {
|
|
|
630
1630
|
throw new TmpfsRootCheckpointError();
|
|
631
1631
|
}
|
|
632
1632
|
const msbPath = await this.msbPath();
|
|
1633
|
+
// Captured up front: `handle.id` itself is mutated in place, below, once
|
|
1634
|
+
// the reboot under the fresh name has actually succeeded — every
|
|
1635
|
+
// reference to the SOURCE sandbox's own name in this method (the
|
|
1636
|
+
// stop/snapshot/rm steps, the pre-mutation error messages) goes through
|
|
1637
|
+
// this local instead, never `handle.id` directly, so it stays correct
|
|
1638
|
+
// regardless of when that mutation happens.
|
|
1639
|
+
const originalName = handle.id;
|
|
1640
|
+
const state = this.handles.get(originalName);
|
|
1641
|
+
if (state === undefined) {
|
|
1642
|
+
throw new BackendError(`no handle state for sandbox '${originalName}' — create() was never called for it`);
|
|
1643
|
+
}
|
|
1644
|
+
// Guest cmdline capture — BEFORE stopping the source sandbox (see
|
|
1645
|
+
// captureGuestWorkloadCmdline's own doc) — only when there is no
|
|
1646
|
+
// explicit command for a restore to fall back on later. Best-effort: a
|
|
1647
|
+
// capture failure never fails the checkpoint itself (captureGuestWorkloadCmdline
|
|
1648
|
+
// already swallows it into `undefined`); its absence only surfaces
|
|
1649
|
+
// later, at restore time, as CheckpointWorkloadCommandMissingError.
|
|
1650
|
+
// Stashed on `state` (kept by reference across this method's own
|
|
1651
|
+
// re-keying below, so it survives the remove+reboot churn) for
|
|
1652
|
+
// `capturedWorkloadCommand()` to hand back to `GenericContainer.checkpoint()`
|
|
1653
|
+
// afterward.
|
|
1654
|
+
const capturedCommand = handle.spec.command === undefined ? await this.captureGuestWorkloadCmdline(handle) : undefined;
|
|
1655
|
+
state.capturedCommand = capturedCommand;
|
|
633
1656
|
await this.stop(handle);
|
|
634
|
-
// A path ref (see checkpoint/ref.ts)
|
|
635
|
-
//
|
|
636
|
-
//
|
|
637
|
-
//
|
|
638
|
-
// (pre-dest-dir checkpoints,
|
|
639
|
-
// msb's own default snapshot
|
|
1657
|
+
// A path ref (see checkpoint/ref.ts) hands msb the ref's own basename as
|
|
1658
|
+
// the snapshot NAME and the parent as --dest-dir — the parent directory
|
|
1659
|
+
// is honored, but (since msb 0.7.1) the artifact itself lands nested
|
|
1660
|
+
// under <parent>/<sandbox>/snap_<digest>, never literally at `ref`; see
|
|
1661
|
+
// this method's own doc. A bare-name ref (pre-dest-dir checkpoints,
|
|
1662
|
+
// still restorable) keeps going through msb's own default snapshot
|
|
1663
|
+
// store, unchanged.
|
|
640
1664
|
const isPathRef = path.isAbsolute(ref);
|
|
641
1665
|
if (isPathRef) {
|
|
642
1666
|
await fs.mkdir(path.dirname(ref), { recursive: true });
|
|
643
1667
|
}
|
|
644
1668
|
const snapshotArgv = isPathRef
|
|
645
|
-
? MsbCommands.snapshotCreate(
|
|
646
|
-
: MsbCommands.snapshotCreate(
|
|
1669
|
+
? MsbCommands.snapshotCreate(originalName, path.basename(ref), path.dirname(ref))
|
|
1670
|
+
: MsbCommands.snapshotCreate(originalName, ref);
|
|
647
1671
|
const snap = await invoke(msbPath, snapshotArgv, CHECKPOINT_TIMEOUT_MS);
|
|
648
1672
|
if (snap.exitCode !== 0) {
|
|
649
|
-
throw new BackendError(`msb snapshot create --from ${
|
|
650
|
-
`the sandbox is left stopped; run 'msb start ${
|
|
1673
|
+
throw new BackendError(`msb snapshot create --from ${originalName} ${ref} failed (exit ${snap.exitCode}): ${snap.stderr.trim()} — ` +
|
|
1674
|
+
`the sandbox is left stopped; run 'msb start ${originalName}' by hand to bring it back up.`);
|
|
651
1675
|
}
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
1676
|
+
const effectiveRef = parseSnapshotCreateArtifactPath(snap.stdout);
|
|
1677
|
+
if (effectiveRef === undefined) {
|
|
1678
|
+
throw new BackendError(`msb snapshot create --from ${originalName} ${ref} did not print a recognizable artifact path as its ` +
|
|
1679
|
+
`last line — the sandbox is left stopped; run 'msb start ${originalName}' by hand to bring it back up. ` +
|
|
1680
|
+
`Raw output:\n${snap.stdout}${snap.stderr}`);
|
|
656
1681
|
}
|
|
657
|
-
|
|
1682
|
+
await invoke(msbPath, MsbCommands.rm(originalName), STOP_TIMEOUT_MS).catch(() => { });
|
|
1683
|
+
// `command` resolves the same way `fromCheckpointRegistryEntry` resolves
|
|
1684
|
+
// it for a later, registry-mediated restore: the source's own explicit
|
|
1685
|
+
// command first, the guest cmdline just captured above as the fallback —
|
|
1686
|
+
// so `reviveWorkload`, inside the reboot this triggers next, already has
|
|
1687
|
+
// the fully-resolved workload argv without any restore-time lookup. This
|
|
1688
|
+
// merged `command` (and `checkpointRef`) live ONLY on each attempt's own
|
|
1689
|
+
// handle inside `rebootUnderFreshName`, never on the caller's own
|
|
1690
|
+
// `handle.spec` (mutated below) — that spec must keep reading back
|
|
1691
|
+
// exactly what the SOURCE container's own `spec.command` was (`undefined`
|
|
1692
|
+
// when there was none), since `GenericContainer.checkpoint()` reads
|
|
1693
|
+
// `handle.spec` again right after this returns to build the
|
|
1694
|
+
// named-checkpoint registry entry, which pins that same "explicit
|
|
1695
|
+
// command vs. captured fallback" distinction as two separate fields (see
|
|
1696
|
+
// `CheckpointRegistryEntry`'s own doc).
|
|
1697
|
+
//
|
|
1698
|
+
// `rebootUnderFreshName` owns minting a NEW name per attempt (never
|
|
1699
|
+
// `originalName`, never a prior failed attempt's own name), tracking each
|
|
1700
|
+
// one in the reaping ledger before that attempt's restore runs, and —
|
|
1701
|
+
// per attempt — best-effort `msb rm`-ing and untracking a failed name
|
|
1702
|
+
// before advancing to another fresh one (see that method's own doc for
|
|
1703
|
+
// the full policy). `originalName`'s own ledger entry is deliberately
|
|
1704
|
+
// left alone throughout: it was already `msb rm`-ed above, and the
|
|
1705
|
+
// ledger's sweep is already not-found-tolerant for exactly this shape (a
|
|
1706
|
+
// name the ledger still lists but msb itself has no record of).
|
|
1707
|
+
let rebootHandle;
|
|
658
1708
|
try {
|
|
659
|
-
await this.
|
|
1709
|
+
rebootHandle = await this.rebootUnderFreshName(msbPath, handle.spec, effectiveRef, capturedCommand, state);
|
|
660
1710
|
}
|
|
661
1711
|
catch (err) {
|
|
662
1712
|
const detail = err instanceof Error ? err.message : String(err);
|
|
663
|
-
throw new BackendError(`sandbox '${
|
|
1713
|
+
throw new BackendError(`sandbox '${originalName}' was removed after a successful checkpoint snapshot, but booting a fresh ` +
|
|
664
1714
|
`sandbox back up from that snapshot failed: ${detail} — the sandbox's disk state is preserved in ` +
|
|
665
|
-
`checkpoint '${
|
|
1715
|
+
`checkpoint '${effectiveRef}', restorable via GenericContainer.fromCheckpoint().`);
|
|
666
1716
|
}
|
|
1717
|
+
const freshName = rebootHandle.id;
|
|
1718
|
+
// The reboot succeeded under `freshName` — the WINNING attempt's own
|
|
1719
|
+
// name, not `originalName` — re-key this backend's own runtime
|
|
1720
|
+
// registries (see class doc on `handles`), then publish the new identity
|
|
1721
|
+
// onto the CALLER's own handle object, IN PLACE, so every subsequent
|
|
1722
|
+
// operation on it targets the sandbox actually running now (see this
|
|
1723
|
+
// method's own doc). Only `id`/`spec.name` change on the live handle —
|
|
1724
|
+
// `spec.command`/`spec.checkpointRef` stay exactly as they were on the
|
|
1725
|
+
// SOURCE spec; see the comment above for why.
|
|
1726
|
+
this.handles.delete(originalName);
|
|
1727
|
+
this.handles.set(freshName, state);
|
|
1728
|
+
this.startedNames.delete(originalName);
|
|
1729
|
+
const mutableHandle = handle;
|
|
1730
|
+
mutableHandle.id = freshName;
|
|
1731
|
+
mutableHandle.spec = { ...handle.spec, name: freshName };
|
|
1732
|
+
return effectiveRef;
|
|
667
1733
|
}
|
|
668
1734
|
/**
|
|
669
|
-
* Best-effort `msb snapshot rm <
|
|
670
|
-
*
|
|
671
|
-
*
|
|
672
|
-
*
|
|
673
|
-
*
|
|
674
|
-
*
|
|
675
|
-
*
|
|
676
|
-
*
|
|
677
|
-
*
|
|
678
|
-
*
|
|
1735
|
+
* Best-effort `msb snapshot rm <ref> -f` — "not found" is success, the
|
|
1736
|
+
* same contract as `removeByName`. `ref` is passed FULL, never reduced to
|
|
1737
|
+
* `path.basename(ref)`: EMPIRICALLY VERIFIED against a real msb 0.7.1
|
|
1738
|
+
* binary, name-based removal does not resolve at all — the artifact PATH
|
|
1739
|
+
* is the only address that reliably works (see `MsbCommands.snapshotRemove`'s
|
|
1740
|
+
* own doc). Every real caller already hands this the EFFECTIVE ref
|
|
1741
|
+
* `createCheckpoint` returned (an absolute path since 0.7.1), so this is
|
|
1742
|
+
* also simply correct for the common case, not just a defensive choice.
|
|
1743
|
+
*
|
|
1744
|
+
* One failure shape is deliberately NOT swallowed alongside "not found":
|
|
1745
|
+
* msb refuses to remove a snapshot that is still the current HEAD of
|
|
1746
|
+
* older siblings from the same source sandbox (see
|
|
1747
|
+
* `isSnapshotHeadRemovalRefused`) — that refusal propagates as a
|
|
1748
|
+
* `BackendError` naming msb's own remedy, rather than silently doing
|
|
1749
|
+
* nothing, since automatic head rotation is out of scope for this method
|
|
1750
|
+
* (see the checkpoints guide's cleanup section for the documented
|
|
1751
|
+
* limitation).
|
|
1752
|
+
*
|
|
1753
|
+
* msb's own removal deletes both its index entry and the dest-dir
|
|
1754
|
+
* artifact for a path ref, but afterwards this also best-effort
|
|
1755
|
+
* recursively deletes the ref path itself: if msb's index ever loses
|
|
1756
|
+
* track of an artifact without deleting it, the directory would otherwise
|
|
1757
|
+
* linger under the cache dir forever. That recursive delete is gated on
|
|
1758
|
+
* `looksLikeCheckpointArtifactDir` first — a `ref` is caller-supplied (a
|
|
1759
|
+
* corrupt registry entry, a hand-edited env var, …), and a `fs.rm(ref, {
|
|
1760
|
+
* recursive: true })` on an unverified path would happily wipe out an
|
|
1761
|
+
* arbitrary directory that merely happens to share its name.
|
|
679
1762
|
*/
|
|
680
1763
|
async removeCheckpoint(ref) {
|
|
681
1764
|
const msbPath = await this.msbPath();
|
|
682
1765
|
const isPathRef = path.isAbsolute(ref);
|
|
683
|
-
const
|
|
684
|
-
|
|
1766
|
+
const result = await invoke(msbPath, MsbCommands.snapshotRemove(ref), CHECKPOINT_TIMEOUT_MS).catch(() => undefined);
|
|
1767
|
+
if (result !== undefined && result.exitCode !== 0 && isSnapshotHeadRemovalRefused(result.stderr)) {
|
|
1768
|
+
throw new BackendError(`msb snapshot rm ${ref} -f was refused (exit ${result.exitCode}): ${result.stderr.trim()} — this ` +
|
|
1769
|
+
`checkpoint is still the newest snapshot of other, older ones from the same source sandbox; ` +
|
|
1770
|
+
`select another snapshot as head first ('msb snapshot head ...', see msb's own message above), ` +
|
|
1771
|
+
`or remove the older siblings first.`);
|
|
1772
|
+
}
|
|
685
1773
|
if (isPathRef && (await looksLikeCheckpointArtifactDir(ref))) {
|
|
686
1774
|
await fs.rm(ref, { recursive: true, force: true }).catch(() => { });
|
|
687
1775
|
}
|
|
@@ -757,51 +1845,66 @@ export class MsbCliBackend {
|
|
|
757
1845
|
}
|
|
758
1846
|
}
|
|
759
1847
|
/**
|
|
760
|
-
* `msb snapshot load <archive>`, then resolves the
|
|
761
|
-
*
|
|
762
|
-
*
|
|
763
|
-
* the
|
|
1848
|
+
* `msb snapshot load <archive> --dest <checkpointsDir>`, then resolves the
|
|
1849
|
+
* EFFECTIVE ref: EMPIRICALLY VERIFIED against a real msb 0.7.1 binary, a
|
|
1850
|
+
* successful load prints a `group msb-<hex>: head snap_<digest>
|
|
1851
|
+
* (Initialized)` line, a digest line, and finally the loaded artifact's
|
|
1852
|
+
* own absolute path as its LAST line — `<checkpointsDir>/<generated-
|
|
1853
|
+
* group>/snap_<digest>`, never the archive's own recorded `ref` — parsed
|
|
1854
|
+
* back out by `parseImportedArtifactPath` (the same last-non-empty-line,
|
|
1855
|
+
* must-be-absolute defensive pattern `parseSnapshotCreateArtifactPath`
|
|
1856
|
+
* uses for `snapshot create`). `checkpointsDir` is always this backend's
|
|
1857
|
+
* own checkpoints cache directory (`<cacheDir>/checkpoints`, the same
|
|
1858
|
+
* directory `checkpointRef()` mints created-checkpoint refs under) —
|
|
1859
|
+
* omitting `--dest` would import into msb's own global default snapshot
|
|
1860
|
+
* store instead, outside anywhere this library looks for or cleans up
|
|
1861
|
+
* checkpoints. Created fresh (mkdir -p) before the load, since nothing
|
|
1862
|
+
* guarantees a caller ever created a checkpoint locally first (an
|
|
1863
|
+
* imported-only machine has no reason to have this directory yet).
|
|
1864
|
+
*
|
|
1865
|
+
* This replaces the pre-0.7.1 shape entirely: `snapshot load` used to
|
|
1866
|
+
* write under a digest-derived directory name with no `--dest` of its
|
|
1867
|
+
* own, so this method had to parse out just that bare NAME and then
|
|
1868
|
+
* separately CONFIRM it via `msb snapshot list --format json` before
|
|
1869
|
+
* handing it back (the only way to get a usable ref at all, since the
|
|
1870
|
+
* bare name alone still had to resolve against msb's default store). 0.7.1
|
|
1871
|
+
* prints the full, directly-usable, already-`--dest`-scoped path as part
|
|
1872
|
+
* of `load`'s own output, so that whole list-and-confirm round trip is
|
|
1873
|
+
* gone — dead code that this migration deletes outright, not merely
|
|
1874
|
+
* bypassed.
|
|
1875
|
+
*
|
|
764
1876
|
* An already-exists failure — msb's own content-addressed dedup — is
|
|
765
1877
|
* treated as success, since the artifact is already present under that
|
|
766
|
-
* digest either way
|
|
767
|
-
*
|
|
768
|
-
*
|
|
769
|
-
*
|
|
770
|
-
*
|
|
771
|
-
*
|
|
772
|
-
*
|
|
773
|
-
*
|
|
774
|
-
*
|
|
775
|
-
* other snapshot-ref call to keep working. `_ref` (the archive's own
|
|
776
|
-
* recorded ref) is unused here — msb's importer never takes one, unlike
|
|
777
|
-
* docker's, where the effective ref really is the ref passed in.
|
|
1878
|
+
* digest either way, and its stderr is parsed the same way as a success's
|
|
1879
|
+
* stdout; any OTHER import failure surfaces msb's own stderr in a
|
|
1880
|
+
* `BackendError`, and unparseable output (no recognizable absolute last
|
|
1881
|
+
* line — e.g. only the "group ... (Initialized)" line, msb's OWN wording
|
|
1882
|
+
* for a state that is not itself a path) throws a `BackendError` quoting
|
|
1883
|
+
* the raw output verbatim rather than misreading a status line as a ref.
|
|
1884
|
+
* `_ref` (the archive's own recorded ref) is unused here — msb's importer
|
|
1885
|
+
* never takes one, unlike docker's, where the effective ref really is the
|
|
1886
|
+
* ref passed in.
|
|
778
1887
|
*/
|
|
779
1888
|
async importCheckpoint(srcFile, _ref) {
|
|
780
1889
|
const msbPath = await this.msbPath();
|
|
781
|
-
const
|
|
782
|
-
|
|
1890
|
+
const checkpointsDir = path.join(cacheDir(), "checkpoints");
|
|
1891
|
+
await fs.mkdir(checkpointsDir, { recursive: true });
|
|
1892
|
+
const imported = await invoke(msbPath, MsbCommands.snapshotImport(srcFile, checkpointsDir), CHECKPOINT_TIMEOUT_MS);
|
|
1893
|
+
let artifactPath;
|
|
783
1894
|
if (imported.exitCode === 0) {
|
|
784
|
-
|
|
1895
|
+
artifactPath = parseImportedArtifactPath(imported.stdout);
|
|
785
1896
|
}
|
|
786
1897
|
else if (isSnapshotAlreadyExistsError(imported.stderr)) {
|
|
787
|
-
|
|
1898
|
+
artifactPath = parseImportedArtifactPath(imported.stderr);
|
|
788
1899
|
}
|
|
789
1900
|
else {
|
|
790
|
-
throw new BackendError(`msb snapshot load ${srcFile} failed (exit ${imported.exitCode}): ${imported.stderr.trim()}`);
|
|
791
|
-
}
|
|
792
|
-
if (digestDirName === undefined) {
|
|
793
|
-
throw new BackendError(`msb snapshot load ${srcFile} did not print a recognizable artifact path — output:\n` +
|
|
794
|
-
`${imported.stdout}${imported.stderr}`);
|
|
795
|
-
}
|
|
796
|
-
const list = await invoke(msbPath, MsbCommands.snapshotList(), CHECKPOINT_TIMEOUT_MS);
|
|
797
|
-
if (list.exitCode !== 0) {
|
|
798
|
-
throw new BackendError(`msb snapshot list failed (exit ${list.exitCode}): ${list.stderr.trim()}`);
|
|
1901
|
+
throw new BackendError(`msb snapshot load ${srcFile} --dest ${checkpointsDir} failed (exit ${imported.exitCode}): ${imported.stderr.trim()}`);
|
|
799
1902
|
}
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
1903
|
+
if (artifactPath === undefined) {
|
|
1904
|
+
throw new BackendError(`msb snapshot load ${srcFile} --dest ${checkpointsDir} did not print a recognizable artifact path as its ` +
|
|
1905
|
+
`last line — output:\n${imported.stdout}${imported.stderr}`);
|
|
803
1906
|
}
|
|
804
|
-
return
|
|
1907
|
+
return artifactPath;
|
|
805
1908
|
}
|
|
806
1909
|
async runningSandboxNames(msbPath) {
|
|
807
1910
|
const result = await invoke(msbPath, MsbCommands.ls(), LOGS_TIMEOUT_MS);
|