rightsize 0.7.13 → 0.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/backend-msb/agent-relay.d.ts +47 -0
- package/dist/backend-msb/agent-relay.d.ts.map +1 -0
- package/dist/backend-msb/agent-relay.js +50 -0
- package/dist/backend-msb/agent-relay.js.map +1 -0
- package/dist/backend-msb/backend.d.ts +67 -24
- package/dist/backend-msb/backend.d.ts.map +1 -1
- package/dist/backend-msb/backend.js +144 -26
- package/dist/backend-msb/backend.js.map +1 -1
- package/dist/backend-msb/commands.d.ts +37 -20
- package/dist/backend-msb/commands.d.ts.map +1 -1
- package/dist/backend-msb/commands.js +72 -39
- package/dist/backend-msb/commands.js.map +1 -1
- package/dist/backend-msb/exec-tunnel.d.ts +8 -6
- package/dist/backend-msb/exec-tunnel.d.ts.map +1 -1
- package/dist/backend-msb/exec-tunnel.js +18 -11
- package/dist/backend-msb/exec-tunnel.js.map +1 -1
- package/dist/backend-msb/invoke.d.ts +7 -7
- package/dist/backend-msb/invoke.js +7 -7
- package/dist/backend-msb/provisioner.d.ts +1 -1
- package/dist/backend-msb/provisioner.js +1 -1
- package/dist/backend-msb/restore-missing-bindings.d.ts +38 -0
- package/dist/backend-msb/restore-missing-bindings.d.ts.map +1 -0
- package/dist/backend-msb/restore-missing-bindings.js +40 -0
- package/dist/backend-msb/restore-missing-bindings.js.map +1 -0
- package/dist/core/backend.d.ts +6 -0
- package/dist/core/backend.d.ts.map +1 -1
- package/dist/core/checkpoint/copied-files.d.ts +16 -0
- package/dist/core/checkpoint/copied-files.d.ts.map +1 -0
- package/dist/core/checkpoint/copied-files.js +29 -0
- package/dist/core/checkpoint/copied-files.js.map +1 -0
- package/dist/core/errors.d.ts +67 -1
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +82 -2
- package/dist/core/errors.js.map +1 -1
- package/dist/core/generic-container.d.ts +28 -4
- package/dist/core/generic-container.d.ts.map +1 -1
- package/dist/core/generic-container.js +38 -4
- package/dist/core/generic-container.js.map +1 -1
- package/dist/core/model.d.ts +12 -3
- package/dist/core/model.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -213,7 +213,9 @@ backend-specific rather than behavioral divergences:
|
|
|
213
213
|
make it durable and rediscoverable across runs via `Checkpoints.find`/
|
|
214
214
|
`list`/`remove`, registered under the rightsize cache dir. `Checkpoints.exportTo`/
|
|
215
215
|
`importFrom` bundle a checkpoint into a portable archive for moving it
|
|
216
|
-
between machines and CI caches.
|
|
216
|
+
between machines and CI caches. Files added with `withCopyFileToContainer`
|
|
217
|
+
are mounts of host files and are not part of a checkpoint: add them again
|
|
218
|
+
on the `fromCheckpoint()` container. See
|
|
217
219
|
[Checkpoint / restore](https://ngriaznov.github.io/rightsize-node/guide/checkpoints).
|
|
218
220
|
- **Runtime file copy.** `copyFileToContainer` / `copyContentToContainer` /
|
|
219
221
|
`copyFileFromContainer` move files and directories into or out of an
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* True if `output` (an `msb run` or `msb restore` invocation's combined
|
|
3
|
+
* stdout+stderr) is msb giving up on a boot because the sandbox process had
|
|
4
|
+
* already exited cleanly, without leaving a boot-error record, by the time the
|
|
5
|
+
* host went to connect to the guest agent. Captured on a Windows CI runner
|
|
6
|
+
* (a brokered restore, where PowerShell decorated the output and mangled the
|
|
7
|
+
* arrow):
|
|
8
|
+
*
|
|
9
|
+
* ```text
|
|
10
|
+
* msb.exe : error: failed to start "rz-731fdcdc-8"
|
|
11
|
+
* At line:1 char:1
|
|
12
|
+
* ...
|
|
13
|
+
* → other: sandbox process exited (exit code: 0) before agent relay became available
|
|
14
|
+
* → run `msb logs --source system rz-731fdcdc-8` for full diagnostics
|
|
15
|
+
* ```
|
|
16
|
+
*
|
|
17
|
+
* and, undecorated, from an attached `msb run`:
|
|
18
|
+
*
|
|
19
|
+
* ```text
|
|
20
|
+
* error: failed to start "rz-731fdcdc-8"
|
|
21
|
+
* → other: sandbox process exited (exit code: 0) before agent relay became available
|
|
22
|
+
* → run `msb logs --source system rz-731fdcdc-8` for full diagnostics
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* The status in parentheses is Rust's `ExitStatus` Display: `exit code: N` on
|
|
26
|
+
* Windows (hex for high-bit codes, `exit code: 0xc0000409`), `exit status: N`
|
|
27
|
+
* or `signal: N (SIGABRT)` on Unix. Only a clean exit is the transient: it has
|
|
28
|
+
* been seen only under load, on attached run boots and on restores, and the
|
|
29
|
+
* same boot succeeds a moment later. A crash exit with the same message is
|
|
30
|
+
* deterministic (a TAB in a baked env var made msb 0.6.x abort with `signal: 6
|
|
31
|
+
* (SIGABRT)` every time, see the Cassandra module) and fails identically on a
|
|
32
|
+
* retry, so it is left as an ordinary boot failure.
|
|
33
|
+
*
|
|
34
|
+
* Matches two independent substrings, not the whole sentence: the arrow, the
|
|
35
|
+
* `other:` and `error:` prefixes and the line structure all change with the
|
|
36
|
+
* host's console and with PowerShell's capture, and no platform gate is
|
|
37
|
+
* needed. The closing parenthesis in the exit marker keeps `(exit code:
|
|
38
|
+
* 0xc0000409)` and `(exit code: 10)` from matching.
|
|
39
|
+
*
|
|
40
|
+
* What msb leaves behind is the stopped sandbox record under the name, which
|
|
41
|
+
* makes a plain same-name retry unsafe (`msb run` would restart the persisted
|
|
42
|
+
* config and ignore its new flags, `msb restore` refuses with "already
|
|
43
|
+
* exists"), so the caller removes it with `msb rm -f` first and boots again
|
|
44
|
+
* under the same name, exactly once.
|
|
45
|
+
*/
|
|
46
|
+
export declare function isAgentRelayUnavailable(output: string): boolean;
|
|
47
|
+
//# sourceMappingURL=agent-relay.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-relay.d.ts","sourceRoot":"","sources":["../../src/backend-msb/agent-relay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAK/D"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* True if `output` (an `msb run` or `msb restore` invocation's combined
|
|
3
|
+
* stdout+stderr) is msb giving up on a boot because the sandbox process had
|
|
4
|
+
* already exited cleanly, without leaving a boot-error record, by the time the
|
|
5
|
+
* host went to connect to the guest agent. Captured on a Windows CI runner
|
|
6
|
+
* (a brokered restore, where PowerShell decorated the output and mangled the
|
|
7
|
+
* arrow):
|
|
8
|
+
*
|
|
9
|
+
* ```text
|
|
10
|
+
* msb.exe : error: failed to start "rz-731fdcdc-8"
|
|
11
|
+
* At line:1 char:1
|
|
12
|
+
* ...
|
|
13
|
+
* → other: sandbox process exited (exit code: 0) before agent relay became available
|
|
14
|
+
* → run `msb logs --source system rz-731fdcdc-8` for full diagnostics
|
|
15
|
+
* ```
|
|
16
|
+
*
|
|
17
|
+
* and, undecorated, from an attached `msb run`:
|
|
18
|
+
*
|
|
19
|
+
* ```text
|
|
20
|
+
* error: failed to start "rz-731fdcdc-8"
|
|
21
|
+
* → other: sandbox process exited (exit code: 0) before agent relay became available
|
|
22
|
+
* → run `msb logs --source system rz-731fdcdc-8` for full diagnostics
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* The status in parentheses is Rust's `ExitStatus` Display: `exit code: N` on
|
|
26
|
+
* Windows (hex for high-bit codes, `exit code: 0xc0000409`), `exit status: N`
|
|
27
|
+
* or `signal: N (SIGABRT)` on Unix. Only a clean exit is the transient: it has
|
|
28
|
+
* been seen only under load, on attached run boots and on restores, and the
|
|
29
|
+
* same boot succeeds a moment later. A crash exit with the same message is
|
|
30
|
+
* deterministic (a TAB in a baked env var made msb 0.6.x abort with `signal: 6
|
|
31
|
+
* (SIGABRT)` every time, see the Cassandra module) and fails identically on a
|
|
32
|
+
* retry, so it is left as an ordinary boot failure.
|
|
33
|
+
*
|
|
34
|
+
* Matches two independent substrings, not the whole sentence: the arrow, the
|
|
35
|
+
* `other:` and `error:` prefixes and the line structure all change with the
|
|
36
|
+
* host's console and with PowerShell's capture, and no platform gate is
|
|
37
|
+
* needed. The closing parenthesis in the exit marker keeps `(exit code:
|
|
38
|
+
* 0xc0000409)` and `(exit code: 10)` from matching.
|
|
39
|
+
*
|
|
40
|
+
* What msb leaves behind is the stopped sandbox record under the name, which
|
|
41
|
+
* makes a plain same-name retry unsafe (`msb run` would restart the persisted
|
|
42
|
+
* config and ignore its new flags, `msb restore` refuses with "already
|
|
43
|
+
* exists"), so the caller removes it with `msb rm -f` first and boots again
|
|
44
|
+
* under the same name, exactly once.
|
|
45
|
+
*/
|
|
46
|
+
export function isAgentRelayUnavailable(output) {
|
|
47
|
+
return (output.includes("before agent relay became available") &&
|
|
48
|
+
(output.includes("(exit code: 0)") || output.includes("(exit status: 0)")));
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=agent-relay.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-relay.js","sourceRoot":"","sources":["../../src/backend-msb/agent-relay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAc;IACpD,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,qCAAqC,CAAC;QACtD,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAC3E,CAAC;AACJ,CAAC"}
|
|
@@ -79,6 +79,7 @@ export declare class MsbCliBackend implements SandboxBackend {
|
|
|
79
79
|
private platformOverrideForTests;
|
|
80
80
|
private udpLinkReadinessPollMs;
|
|
81
81
|
private udpLinkReadinessTimeoutMs;
|
|
82
|
+
private agentRelayRetryDelayMs;
|
|
82
83
|
constructor(msbPathPromise: Promise<string>, options?: MsbCliBackendOptions);
|
|
83
84
|
/**
|
|
84
85
|
* Whether this process should treat itself as running on Windows for
|
|
@@ -126,17 +127,18 @@ export declare class MsbCliBackend implements SandboxBackend {
|
|
|
126
127
|
/**
|
|
127
128
|
* A boot attempt wrapped in every classified-transient retry this backend
|
|
128
129
|
* knows: the install-lock poll, the one-shot state-database retry, and the
|
|
129
|
-
* one-shot image-cache heal.
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
130
|
+
* one-shot image-cache heal. (The one-shot agent-relay retry sits one level
|
|
131
|
+
* lower, inside `bootOnce`, so each `bootOnce` call below gets it too.)
|
|
132
|
+
* Both the ordinary `start()` path and the checkpoint cycle's post-snapshot
|
|
133
|
+
* reboot come through here — a reboot from a snapshot (now `msb restore
|
|
134
|
+
* ...`, see `bootRestoreOnce`) is as exposed to msb's transients as any
|
|
135
|
+
* other boot, and skipping the classification there turned a passing
|
|
136
|
+
* install-lock poll into an immediate checkpoint failure on a live Windows
|
|
137
|
+
* run. The image-cache heal targets `handle.spec.image`, which for a
|
|
138
|
+
* restore is the checkpoint ref GenericContainer.fromCheckpoint() threaded
|
|
139
|
+
* through as the builder's "image" — never a real OCI reference `msb image
|
|
140
|
+
* remove` can act on meaningfully; this predates the 0.7.1 migration (the
|
|
141
|
+
* same was true of `--from-snapshot` boots) and is unchanged here.
|
|
140
142
|
*
|
|
141
143
|
* `retryAccessDenied` (default `true`) gates ONLY the
|
|
142
144
|
* `RestoreAccessDeniedError` branch below — every other classified retry
|
|
@@ -166,6 +168,27 @@ export declare class MsbCliBackend implements SandboxBackend {
|
|
|
166
168
|
* doc.
|
|
167
169
|
*/
|
|
168
170
|
private bootClassified;
|
|
171
|
+
/**
|
|
172
|
+
* One boot, with its one agent-relay retry. Every caller that boots a
|
|
173
|
+
* sandbox goes through here: `bootClassified`'s first attempt and each of
|
|
174
|
+
* its own retries (install-lock poll, state-database, image-cache heal), and
|
|
175
|
+
* through it the ordinary `start()` path, the reuse boot, direct and
|
|
176
|
+
* brokered restores, and the checkpoint reboot's fresh-name loops
|
|
177
|
+
* (`retryRestoreAfterAccessDenied`, `rebootUnderFreshName`).
|
|
178
|
+
*
|
|
179
|
+
* When an attempt's sandbox process exits cleanly before msb's agent relay
|
|
180
|
+
* comes up (see `isAgentRelayUnavailable`) this removes the failed sandbox
|
|
181
|
+
* (`msb rm -f <name>`, result ignored — msb leaves the record behind as
|
|
182
|
+
* Stopped, and a plain same-name `run` would restart that persisted config
|
|
183
|
+
* while a same-name `restore` is refused as "already exists"), waits
|
|
184
|
+
* `AGENT_RELAY_RETRY_DELAY_MS`, and boots again with the SAME name, spec and
|
|
185
|
+
* `restoreLaunchMode`. Exactly once: a second relay failure throws a
|
|
186
|
+
* `BackendError` carrying that attempt's output, and any other error from
|
|
187
|
+
* either attempt propagates untouched for the caller's own policy. The name
|
|
188
|
+
* never changes, so the reaper ledger, `handles` and `startedNames` need no
|
|
189
|
+
* adjustment — a failed attempt registers nothing in the last two.
|
|
190
|
+
*/
|
|
191
|
+
private bootOnce;
|
|
169
192
|
/**
|
|
170
193
|
* One boot attempt, dispatched by shape: an ordinary spec (no
|
|
171
194
|
* `checkpointRef`) drives `run()`'s ATTACHED supervision model
|
|
@@ -177,7 +200,7 @@ export declare class MsbCliBackend implements SandboxBackend {
|
|
|
177
200
|
* (irrelevant to `bootRunOnce`'s own ordinary `run` path, which never
|
|
178
201
|
* brokers — see `RestoreLaunchMode`'s own doc).
|
|
179
202
|
*/
|
|
180
|
-
private
|
|
203
|
+
private bootAttempt;
|
|
181
204
|
/**
|
|
182
205
|
* One RUN boot attempt: spawns ATTACHED `msb run`, which stays alive as
|
|
183
206
|
* the sandbox's own supervisor for its whole lifetime, and polls
|
|
@@ -193,7 +216,9 @@ export declare class MsbCliBackend implements SandboxBackend {
|
|
|
193
216
|
* own doc). An early exit is classified from the child's combined output:
|
|
194
217
|
* the image-cache-corruption signature throws `ImageCacheCorruptionError`
|
|
195
218
|
* (the one failure `start()` heals and retries), a host-port bind conflict
|
|
196
|
-
* throws `PortBindConflictError`,
|
|
219
|
+
* throws `PortBindConflictError`, a sandbox process that exited before its
|
|
220
|
+
* agent relay came up throws `AgentRelayUnavailableError` (retried once by
|
|
221
|
+
* `bootOnce`), and otherwise — if the exit code was 0 —
|
|
197
222
|
* `isCompletedFastExit` gets a chance to reclassify the exit as a workload
|
|
198
223
|
* that ran to completion before this poll loop could ever observe Running
|
|
199
224
|
* (msb 0.6.16's convergent-lifecycle rework; see that method's own doc);
|
|
@@ -221,9 +246,9 @@ export declare class MsbCliBackend implements SandboxBackend {
|
|
|
221
246
|
* "this could be a slow cold operation" uncertainty a pull is.
|
|
222
247
|
* Classified from the combined output the same way any boot's early
|
|
223
248
|
* exit is (install-lock, state-db, image-cache-corruption,
|
|
224
|
-
* port-bind-conflict — see `bootClassified`'s own doc on
|
|
225
|
-
* restore boot needs the identical classification an ordinary
|
|
226
|
-
* gets) — a nonzero, unclassified exit throws an ordinary
|
|
249
|
+
* port-bind-conflict, agent-relay — see `bootClassified`'s own doc on
|
|
250
|
+
* why a restore boot needs the identical classification an ordinary
|
|
251
|
+
* one gets) — a nonzero, unclassified exit throws an ordinary
|
|
227
252
|
* `BackendError` carrying the output, same shape as `bootRunOnce`'s.
|
|
228
253
|
* 2. Once it exits 0, poll `msb ls` for the sandbox to reach Running —
|
|
229
254
|
* the same `READINESS_POLL_MS` cadence and a fresh instance of the
|
|
@@ -277,12 +302,16 @@ export declare class MsbCliBackend implements SandboxBackend {
|
|
|
277
302
|
* (`isImageCacheCorruption`, `isRestoreAccessDeniedFailure`, ...) a direct
|
|
278
303
|
* attempt's always has, since the underlying `msb restore` invocation and
|
|
279
304
|
* its failure signatures are identical either way; only how its stdout/
|
|
280
|
-
* stderr and exit code got back to this process differs.
|
|
281
|
-
*
|
|
282
|
-
*
|
|
283
|
-
*
|
|
284
|
-
*
|
|
285
|
-
*
|
|
305
|
+
* stderr and exit code got back to this process differs. One of those
|
|
306
|
+
* signatures is not transient: msb 0.7.8 refusing a restore that leaves out
|
|
307
|
+
* a file copied in at start (`isRestoreMissingBindingsFailure`) is checked
|
|
308
|
+
* first and thrown as `CheckpointMissingCopiedFilesError`, which no caller
|
|
309
|
+
* retries. A broker launch can also come back `"unconfirmed"` (its own
|
|
310
|
+
* `ecFile` never appeared within its bound) — see
|
|
311
|
+
* `RestoreBrokerUnconfirmed`'s own doc — which skips classification
|
|
312
|
+
* entirely and falls straight through to phase 2 exactly as a confirmed
|
|
313
|
+
* exit 0 does, since that phase's own `msb ls` poll is activation-gated and
|
|
314
|
+
* settles the question on its own regardless.
|
|
286
315
|
*/
|
|
287
316
|
private bootRestoreOnce;
|
|
288
317
|
/**
|
|
@@ -608,8 +637,9 @@ export declare class MsbCliBackend implements SandboxBackend {
|
|
|
608
637
|
* interface doc calls `SandboxHandle` "immutable" — this is the one
|
|
609
638
|
* carve-out, and only this method makes it: a checkpoint reboot is the one
|
|
610
639
|
* operation that changes what sandbox a handle even refers to. `ports`,
|
|
611
|
-
* `env`, and `memoryLimitMb` are untouched by the rename — same
|
|
612
|
-
* and memory ceiling as before the reboot, only
|
|
640
|
+
* `env`, `mounts`, and `memoryLimitMb` are untouched by the rename — same
|
|
641
|
+
* ports, env, copied files and memory ceiling as before the reboot, only
|
|
642
|
+
* the name differs.
|
|
613
643
|
* Upstream's
|
|
614
644
|
* detached-start path (`Sandbox::start_detached`) passes
|
|
615
645
|
* `CREATE_BREAKAWAY_FROM_JOB` on Windows, which `ERROR_ACCESS_DENIED`s
|
|
@@ -676,6 +706,19 @@ export declare class MsbCliBackend implements SandboxBackend {
|
|
|
676
706
|
* Refuses outright, before any of the above, when `handle.spec.tmpfsRootMb`
|
|
677
707
|
* is set: a tmpfs root has nothing on disk for a snapshot to capture.
|
|
678
708
|
*
|
|
709
|
+
* Files copied in at start (`handle.spec.mounts`) are bind mounts of host
|
|
710
|
+
* files, so the snapshot holds no content for them. The reboot mounts each
|
|
711
|
+
* one again from its host file (`MsbCommands.restore` emits one `--volume`
|
|
712
|
+
* per mount), which is also what msb 0.7.8 requires: it refuses a restore
|
|
713
|
+
* that leaves one out. The host files therefore have to still exist, and
|
|
714
|
+
* this method checks that, before any of the above, with
|
|
715
|
+
* `CheckpointHostFileMissingError` (ahead of the stop, so a missing file
|
|
716
|
+
* never costs the caller its running sandbox). A reuse sandbox adopted
|
|
717
|
+
* through `findRunning` carries the builder's own spec, mounts included
|
|
718
|
+
* (they are part of the reuse identity), so its reboot supplies them like
|
|
719
|
+
* any other. One adopted by a process whose backend never called `create()`
|
|
720
|
+
* for it has no handle state and is refused before anything is stopped.
|
|
721
|
+
*
|
|
679
722
|
* `ref` is the WORKING ref this method is asked to checkpoint under — its
|
|
680
723
|
* basename becomes the snapshot create call's `<name>` argument (still
|
|
681
724
|
* meaningful: it lands in msb's own index, see `MsbCommands.snapshotCreate`'s
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../../src/backend-msb/backend.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../../src/backend-msb/backend.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC3I,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AA6BlE,OAAO,EAGL,KAAK,qBAAqB,EAE3B,MAAM,qBAAqB,CAAC;AAyH7B;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAE9D;AAqUD;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;OAUG;IACH,aAAa,CAAC,EAAE,qBAAqB,CAAC;CACvC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,aAAc,YAAW,cAAc;IA0DhD,OAAO,CAAC,QAAQ,CAAC,cAAc;IAzDjC,kHAAkH;IAClH,QAAQ,CAAC,IAAI,kBAAkB;IAC/B,QAAQ,CAAC,sBAAsB,SAAS;IACxC,+HAA+H;IAC/H,QAAQ,CAAC,YAAY,EAAE,mBAAmB,CAOxC;IAMF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAqB;IAKlD,OAAO,CAAC,eAAe,CAAqB;IAQ5C,OAAO,CAAC,0CAA0C,CAAoD;IACtG,OAAO,CAAC,yCAAyC,CAAmD;IAEpG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IAOtD,OAAO,CAAC,wBAAwB,CAA0C;IAO1E,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,OAAO,CAAC,yBAAyB,CAAiC;IAKlE,OAAO,CAAC,sBAAsB,CAA8B;gBAGzC,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,EAChD,OAAO,GAAE,oBAAyB;IAepC;;;;;;;;;OASG;IACH,OAAO,CAAC,iBAAiB;YAIX,OAAO;IAIf,MAAM,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAKzD;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACG,KAAK,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IASjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;YACW,cAAc;IAmG5B;;;;;;;;;;;;;;;;;;;OAmBG;YACW,QAAQ;IA8BtB;;;;;;;;;;OAUG;YACW,WAAW;IAgBzB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;YACW,WAAW;IAgFzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAsFG;YACW,eAAe;IA8F7B;;;;;;;;;OASG;YACW,mBAAmB;IAuCjC;;;;;;;;;;;;;;;;;;;;OAoBG;YACW,sBAAsB;IA6BpC;;;;;;;;;;;;OAYG;YACW,yBAAyB;IA2BvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkDG;YACW,cAAc;IA8C5B;;;;;;;;;;;;OAYG;YACW,2BAA2B;IAYzC;;;;;;;OAOG;IACH,uBAAuB,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,SAAS;IAIjF;;;;;;;;;;;;;;;;;;;;;;OAsBG;YACW,mBAAmB;IAS3B,IAAI,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAsD1C,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoEG;YACW,6BAA6B;IA8D3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiEG;YACW,oBAAoB;IA4DlC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuIG;IACG,gBAAgB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmI3E;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACG,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBlD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAkBlD;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBpE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACG,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YA0BxD,mBAAmB;IAKjC;;;;;;;;;OASG;IACG,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAalF;;;;;;;;;;OAUG;IACG,IAAI,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAKlD;;;;;;;;;;;;;;;;;;OAkBG;IACG,UAAU,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC;IA+DhG;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,OAAO,CAAC,mBAAmB;IAyE3B;;;;;;;;OAQG;YACW,mBAAmB;IAmC3B,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhD,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAItD;;;;;;;;;;;;OAYG;IACG,mBAAmB,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA+ClG;;;;;;;;;;;OAWG;YACW,cAAc;IAyB5B;;;;;;OAMG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUpG,oEAAoE;IAC9D,iBAAiB,CAAC,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAahG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAU5B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAsB7B;;;;;;;;;OASG;IACG,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAMjC,yBAAyB;IASvC;;;;;;OAMG;IACG,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAS1E,oNAAoN;IAC9M,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,CAAC;CAItD"}
|
|
@@ -2,7 +2,8 @@ import { spawn, spawnSync } from "node:child_process";
|
|
|
2
2
|
import * as readline from "node:readline";
|
|
3
3
|
import * as fs from "node:fs/promises";
|
|
4
4
|
import * as path from "node:path";
|
|
5
|
-
import { BackendError, PortBindConflictError, UnsupportedByBackendError, TmpfsRootCheckpointError, CheckpointWorkloadCommandMissingError, } from "../core/errors.js";
|
|
5
|
+
import { BackendError, PortBindConflictError, UnsupportedByBackendError, TmpfsRootCheckpointError, CheckpointWorkloadCommandMissingError, CheckpointMissingCopiedFilesError, } from "../core/errors.js";
|
|
6
|
+
import { requireCopiedHostFilesExist } from "../core/checkpoint/copied-files.js";
|
|
6
7
|
import { cacheDir } from "../core/cache-dir.js";
|
|
7
8
|
import { nextSandboxName } from "../core/sandbox-name.js";
|
|
8
9
|
import { trackSandbox, untrackSandbox } from "../core/reaper/init.js";
|
|
@@ -13,6 +14,7 @@ import { invoke, CLOSED_STDIN } from "./invoke.js";
|
|
|
13
14
|
import { isPortBindConflictOutput } from "./port-conflict.js";
|
|
14
15
|
import { isImageCacheCorruption } from "./image-cache.js";
|
|
15
16
|
import { isMsbStateDbError } from "./state-db.js";
|
|
17
|
+
import { isAgentRelayUnavailable } from "./agent-relay.js";
|
|
16
18
|
import { isAgentEndpointNotReady } from "./agent-endpoint.js";
|
|
17
19
|
import { isSnapshotNotFoundError } from "./snapshot-not-found.js";
|
|
18
20
|
import { isSnapshotAlreadyExistsError, parseImportedArtifactPath } from "./snapshot-import.js";
|
|
@@ -24,6 +26,7 @@ import { requireNoDuplicateGuestPorts, requireAliasesAreValid, hostsAliasScript,
|
|
|
24
26
|
import { ExecTunnel } from "./exec-tunnel.js";
|
|
25
27
|
import { isRestoreAccessDeniedFailure } from "./restore-access-denied.js";
|
|
26
28
|
import { isSandboxAlreadyExistsFailure } from "./sandbox-already-exists.js";
|
|
29
|
+
import { isRestoreMissingBindingsFailure } from "./restore-missing-bindings.js";
|
|
27
30
|
import { realRestoreBroker, } from "./restore-broker.js";
|
|
28
31
|
const FIRST_RUN_PULL_TIMEOUT_MS = 600_000; // a cold pull can be slow
|
|
29
32
|
const READINESS_POLL_MS = 300;
|
|
@@ -131,6 +134,20 @@ class StateDbError extends Error {
|
|
|
131
134
|
this.output = output;
|
|
132
135
|
}
|
|
133
136
|
}
|
|
137
|
+
/**
|
|
138
|
+
* The boot failure `bootOnce` retries once — the sandbox process exited
|
|
139
|
+
* cleanly before msb's agent relay came up (see `isAgentRelayUnavailable`).
|
|
140
|
+
* Thrown from both `bootRunOnce` and `bootRestoreOnce`, and caught by
|
|
141
|
+
* `bootOnce` itself, so unlike its siblings above it never reaches
|
|
142
|
+
* `bootClassified` or any caller of it.
|
|
143
|
+
*/
|
|
144
|
+
class AgentRelayUnavailableError extends Error {
|
|
145
|
+
output;
|
|
146
|
+
constructor(output) {
|
|
147
|
+
super(`msb sandbox process exited before its agent relay came up:\n${output}`);
|
|
148
|
+
this.output = output;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
134
151
|
/**
|
|
135
152
|
* True if `output` (an `msb run` invocation's combined output) is msb
|
|
136
153
|
* refusing to run anything while its internal install lock is held. Captured
|
|
@@ -211,6 +228,14 @@ class SandboxAlreadyExistsError extends Error {
|
|
|
211
228
|
* to commit; the retry's own `msb run` startup dwarfs this either way.
|
|
212
229
|
*/
|
|
213
230
|
const STATE_DB_RETRY_DELAY_MS = 500;
|
|
231
|
+
/**
|
|
232
|
+
* How long to wait between a boot whose sandbox process exited before its
|
|
233
|
+
* agent relay came up (see `isAgentRelayUnavailable`) and its one retry. The
|
|
234
|
+
* failed sandbox is removed (`msb rm -f`) first; the pause just lets msb and
|
|
235
|
+
* the OS finish tearing the dead process down before the same name boots
|
|
236
|
+
* again.
|
|
237
|
+
*/
|
|
238
|
+
const AGENT_RELAY_RETRY_DELAY_MS = 2_000;
|
|
214
239
|
/**
|
|
215
240
|
* How long the boot path keeps polling while msb's install-operation lock is
|
|
216
241
|
* held, and the pause between attempts (see `isMsbInstallLockActive`) —
|
|
@@ -489,6 +514,11 @@ export class MsbCliBackend {
|
|
|
489
514
|
// red-proof runs in milliseconds rather than the real 5s.
|
|
490
515
|
udpLinkReadinessPollMs = UDP_LINK_READINESS_POLL_MS;
|
|
491
516
|
udpLinkReadinessTimeoutMs = UDP_LINK_READINESS_TIMEOUT_MS;
|
|
517
|
+
// Test-only override seam for bootOnce's pause before its one agent-relay
|
|
518
|
+
// retry — defaults to the real AGENT_RELAY_RETRY_DELAY_MS constant, reached
|
|
519
|
+
// the same unsafe-cast way as the seams above so a retry test does not
|
|
520
|
+
// block for the real 2s.
|
|
521
|
+
agentRelayRetryDelayMs = AGENT_RELAY_RETRY_DELAY_MS;
|
|
492
522
|
constructor(msbPathPromise, options = {}) {
|
|
493
523
|
this.msbPathPromise = msbPathPromise;
|
|
494
524
|
this.restoreBroker = options.restoreBroker ?? realRestoreBroker;
|
|
@@ -560,17 +590,18 @@ export class MsbCliBackend {
|
|
|
560
590
|
/**
|
|
561
591
|
* A boot attempt wrapped in every classified-transient retry this backend
|
|
562
592
|
* knows: the install-lock poll, the one-shot state-database retry, and the
|
|
563
|
-
* one-shot image-cache heal.
|
|
564
|
-
*
|
|
565
|
-
*
|
|
566
|
-
*
|
|
567
|
-
*
|
|
568
|
-
*
|
|
569
|
-
*
|
|
570
|
-
*
|
|
571
|
-
*
|
|
572
|
-
*
|
|
573
|
-
*
|
|
593
|
+
* one-shot image-cache heal. (The one-shot agent-relay retry sits one level
|
|
594
|
+
* lower, inside `bootOnce`, so each `bootOnce` call below gets it too.)
|
|
595
|
+
* Both the ordinary `start()` path and the checkpoint cycle's post-snapshot
|
|
596
|
+
* reboot come through here — a reboot from a snapshot (now `msb restore
|
|
597
|
+
* ...`, see `bootRestoreOnce`) is as exposed to msb's transients as any
|
|
598
|
+
* other boot, and skipping the classification there turned a passing
|
|
599
|
+
* install-lock poll into an immediate checkpoint failure on a live Windows
|
|
600
|
+
* run. The image-cache heal targets `handle.spec.image`, which for a
|
|
601
|
+
* restore is the checkpoint ref GenericContainer.fromCheckpoint() threaded
|
|
602
|
+
* through as the builder's "image" — never a real OCI reference `msb image
|
|
603
|
+
* remove` can act on meaningfully; this predates the 0.7.1 migration (the
|
|
604
|
+
* same was true of `--from-snapshot` boots) and is unchanged here.
|
|
574
605
|
*
|
|
575
606
|
* `retryAccessDenied` (default `true`) gates ONLY the
|
|
576
607
|
* `RestoreAccessDeniedError` branch below — every other classified retry
|
|
@@ -686,6 +717,50 @@ export class MsbCliBackend {
|
|
|
686
717
|
`directory under MSB_HOME).\nfirst attempt:\n${firstOutput}\nafter heal + retry:\n${second.output}`);
|
|
687
718
|
}
|
|
688
719
|
}
|
|
720
|
+
/**
|
|
721
|
+
* One boot, with its one agent-relay retry. Every caller that boots a
|
|
722
|
+
* sandbox goes through here: `bootClassified`'s first attempt and each of
|
|
723
|
+
* its own retries (install-lock poll, state-database, image-cache heal), and
|
|
724
|
+
* through it the ordinary `start()` path, the reuse boot, direct and
|
|
725
|
+
* brokered restores, and the checkpoint reboot's fresh-name loops
|
|
726
|
+
* (`retryRestoreAfterAccessDenied`, `rebootUnderFreshName`).
|
|
727
|
+
*
|
|
728
|
+
* When an attempt's sandbox process exits cleanly before msb's agent relay
|
|
729
|
+
* comes up (see `isAgentRelayUnavailable`) this removes the failed sandbox
|
|
730
|
+
* (`msb rm -f <name>`, result ignored — msb leaves the record behind as
|
|
731
|
+
* Stopped, and a plain same-name `run` would restart that persisted config
|
|
732
|
+
* while a same-name `restore` is refused as "already exists"), waits
|
|
733
|
+
* `AGENT_RELAY_RETRY_DELAY_MS`, and boots again with the SAME name, spec and
|
|
734
|
+
* `restoreLaunchMode`. Exactly once: a second relay failure throws a
|
|
735
|
+
* `BackendError` carrying that attempt's output, and any other error from
|
|
736
|
+
* either attempt propagates untouched for the caller's own policy. The name
|
|
737
|
+
* never changes, so the reaper ledger, `handles` and `startedNames` need no
|
|
738
|
+
* adjustment — a failed attempt registers nothing in the last two.
|
|
739
|
+
*/
|
|
740
|
+
async bootOnce(msbPath, handle, state, restoreLaunchMode = "direct") {
|
|
741
|
+
try {
|
|
742
|
+
await this.bootAttempt(msbPath, handle, state, restoreLaunchMode);
|
|
743
|
+
return;
|
|
744
|
+
}
|
|
745
|
+
catch (first) {
|
|
746
|
+
if (!(first instanceof AgentRelayUnavailableError)) {
|
|
747
|
+
throw first;
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
await invoke(msbPath, MsbCommands.rmForce(handle.id), STOP_TIMEOUT_MS).catch(() => { });
|
|
751
|
+
await sleep(this.agentRelayRetryDelayMs);
|
|
752
|
+
try {
|
|
753
|
+
await this.bootAttempt(msbPath, handle, state, restoreLaunchMode);
|
|
754
|
+
}
|
|
755
|
+
catch (second) {
|
|
756
|
+
if (!(second instanceof AgentRelayUnavailableError)) {
|
|
757
|
+
throw second;
|
|
758
|
+
}
|
|
759
|
+
throw new BackendError(`msb ${handle.spec.checkpointRef !== undefined ? "restore" : "run"} for sandbox ${handle.id}: msb's ` +
|
|
760
|
+
`sandbox process exited before its agent relay came up on both attempts (the retry ran after ` +
|
|
761
|
+
`'msb rm -f' of the failed sandbox and a short pause).\nlast attempt:\n${second.output}`);
|
|
762
|
+
}
|
|
763
|
+
}
|
|
689
764
|
/**
|
|
690
765
|
* One boot attempt, dispatched by shape: an ordinary spec (no
|
|
691
766
|
* `checkpointRef`) drives `run()`'s ATTACHED supervision model
|
|
@@ -697,7 +772,7 @@ export class MsbCliBackend {
|
|
|
697
772
|
* (irrelevant to `bootRunOnce`'s own ordinary `run` path, which never
|
|
698
773
|
* brokers — see `RestoreLaunchMode`'s own doc).
|
|
699
774
|
*/
|
|
700
|
-
async
|
|
775
|
+
async bootAttempt(msbPath, handle, state, restoreLaunchMode) {
|
|
701
776
|
state.logTail = [];
|
|
702
777
|
state.attachedExited = false;
|
|
703
778
|
if (handle.spec.checkpointRef !== undefined) {
|
|
@@ -721,7 +796,9 @@ export class MsbCliBackend {
|
|
|
721
796
|
* own doc). An early exit is classified from the child's combined output:
|
|
722
797
|
* the image-cache-corruption signature throws `ImageCacheCorruptionError`
|
|
723
798
|
* (the one failure `start()` heals and retries), a host-port bind conflict
|
|
724
|
-
* throws `PortBindConflictError`,
|
|
799
|
+
* throws `PortBindConflictError`, a sandbox process that exited before its
|
|
800
|
+
* agent relay came up throws `AgentRelayUnavailableError` (retried once by
|
|
801
|
+
* `bootOnce`), and otherwise — if the exit code was 0 —
|
|
725
802
|
* `isCompletedFastExit` gets a chance to reclassify the exit as a workload
|
|
726
803
|
* that ran to completion before this poll loop could ever observe Running
|
|
727
804
|
* (msb 0.6.16's convergent-lifecycle rework; see that method's own doc);
|
|
@@ -761,6 +838,9 @@ export class MsbCliBackend {
|
|
|
761
838
|
if (isPortBindConflictOutput(output)) {
|
|
762
839
|
throw new PortBindConflictError(`msb run for sandbox ${handle.id} could not bind a host port: ${output}`);
|
|
763
840
|
}
|
|
841
|
+
if (isAgentRelayUnavailable(output)) {
|
|
842
|
+
throw new AgentRelayUnavailableError(output);
|
|
843
|
+
}
|
|
764
844
|
if (exited.code === 0 && (await this.isCompletedFastExit(msbPath, handle.id))) {
|
|
765
845
|
// The workload ran to completion before this poll loop ever
|
|
766
846
|
// observed Running — see isCompletedFastExit's own doc. Started-
|
|
@@ -818,9 +898,9 @@ export class MsbCliBackend {
|
|
|
818
898
|
* "this could be a slow cold operation" uncertainty a pull is.
|
|
819
899
|
* Classified from the combined output the same way any boot's early
|
|
820
900
|
* exit is (install-lock, state-db, image-cache-corruption,
|
|
821
|
-
* port-bind-conflict — see `bootClassified`'s own doc on
|
|
822
|
-
* restore boot needs the identical classification an ordinary
|
|
823
|
-
* gets) — a nonzero, unclassified exit throws an ordinary
|
|
901
|
+
* port-bind-conflict, agent-relay — see `bootClassified`'s own doc on
|
|
902
|
+
* why a restore boot needs the identical classification an ordinary
|
|
903
|
+
* one gets) — a nonzero, unclassified exit throws an ordinary
|
|
824
904
|
* `BackendError` carrying the output, same shape as `bootRunOnce`'s.
|
|
825
905
|
* 2. Once it exits 0, poll `msb ls` for the sandbox to reach Running —
|
|
826
906
|
* the same `READINESS_POLL_MS` cadence and a fresh instance of the
|
|
@@ -874,12 +954,16 @@ export class MsbCliBackend {
|
|
|
874
954
|
* (`isImageCacheCorruption`, `isRestoreAccessDeniedFailure`, ...) a direct
|
|
875
955
|
* attempt's always has, since the underlying `msb restore` invocation and
|
|
876
956
|
* its failure signatures are identical either way; only how its stdout/
|
|
877
|
-
* stderr and exit code got back to this process differs.
|
|
878
|
-
*
|
|
879
|
-
*
|
|
880
|
-
*
|
|
881
|
-
*
|
|
882
|
-
*
|
|
957
|
+
* stderr and exit code got back to this process differs. One of those
|
|
958
|
+
* signatures is not transient: msb 0.7.8 refusing a restore that leaves out
|
|
959
|
+
* a file copied in at start (`isRestoreMissingBindingsFailure`) is checked
|
|
960
|
+
* first and thrown as `CheckpointMissingCopiedFilesError`, which no caller
|
|
961
|
+
* retries. A broker launch can also come back `"unconfirmed"` (its own
|
|
962
|
+
* `ecFile` never appeared within its bound) — see
|
|
963
|
+
* `RestoreBrokerUnconfirmed`'s own doc — which skips classification
|
|
964
|
+
* entirely and falls straight through to phase 2 exactly as a confirmed
|
|
965
|
+
* exit 0 does, since that phase's own `msb ls` poll is activation-gated and
|
|
966
|
+
* settles the question on its own regardless.
|
|
883
967
|
*/
|
|
884
968
|
async bootRestoreOnce(msbPath, handle, state, launchMode = "direct") {
|
|
885
969
|
const launch = launchMode === "broker"
|
|
@@ -887,6 +971,15 @@ export class MsbCliBackend {
|
|
|
887
971
|
: await this.launchRestoreDirect(msbPath, handle, state);
|
|
888
972
|
if (launch.kind === "completed") {
|
|
889
973
|
const { exitCode, output } = launch;
|
|
974
|
+
// First, so none of the transient classifiers below can claim it, and
|
|
975
|
+
// thrown as the public error itself rather than an internal one: nothing
|
|
976
|
+
// retries it (`bootClassified`, `bootOnce` and both fresh-name loops
|
|
977
|
+
// rethrow what they do not recognise), because the same restore is
|
|
978
|
+
// refused the same way every time. Direct and brokered attempts both
|
|
979
|
+
// pass through here, so this is the one place that classifies it.
|
|
980
|
+
if (isRestoreMissingBindingsFailure(output)) {
|
|
981
|
+
throw new CheckpointMissingCopiedFilesError(handle.id, handle.spec.checkpointRef ?? "(unknown)", output);
|
|
982
|
+
}
|
|
890
983
|
if (isImageCacheCorruption(output)) {
|
|
891
984
|
throw new ImageCacheCorruptionError(output);
|
|
892
985
|
}
|
|
@@ -905,6 +998,9 @@ export class MsbCliBackend {
|
|
|
905
998
|
if (isSandboxAlreadyExistsFailure(output)) {
|
|
906
999
|
throw new SandboxAlreadyExistsError(output);
|
|
907
1000
|
}
|
|
1001
|
+
if (isAgentRelayUnavailable(output)) {
|
|
1002
|
+
throw new AgentRelayUnavailableError(output);
|
|
1003
|
+
}
|
|
908
1004
|
if (exitCode !== 0) {
|
|
909
1005
|
throw new BackendError(`msb restore for sandbox ${handle.id} exited (code ${exitCode}) — check the snapshot ref and ` +
|
|
910
1006
|
`'msb restore' output below:\n${output}`);
|
|
@@ -1557,8 +1653,9 @@ export class MsbCliBackend {
|
|
|
1557
1653
|
* interface doc calls `SandboxHandle` "immutable" — this is the one
|
|
1558
1654
|
* carve-out, and only this method makes it: a checkpoint reboot is the one
|
|
1559
1655
|
* operation that changes what sandbox a handle even refers to. `ports`,
|
|
1560
|
-
* `env`, and `memoryLimitMb` are untouched by the rename — same
|
|
1561
|
-
* and memory ceiling as before the reboot, only
|
|
1656
|
+
* `env`, `mounts`, and `memoryLimitMb` are untouched by the rename — same
|
|
1657
|
+
* ports, env, copied files and memory ceiling as before the reboot, only
|
|
1658
|
+
* the name differs.
|
|
1562
1659
|
* Upstream's
|
|
1563
1660
|
* detached-start path (`Sandbox::start_detached`) passes
|
|
1564
1661
|
* `CREATE_BREAKAWAY_FROM_JOB` on Windows, which `ERROR_ACCESS_DENIED`s
|
|
@@ -1625,6 +1722,19 @@ export class MsbCliBackend {
|
|
|
1625
1722
|
* Refuses outright, before any of the above, when `handle.spec.tmpfsRootMb`
|
|
1626
1723
|
* is set: a tmpfs root has nothing on disk for a snapshot to capture.
|
|
1627
1724
|
*
|
|
1725
|
+
* Files copied in at start (`handle.spec.mounts`) are bind mounts of host
|
|
1726
|
+
* files, so the snapshot holds no content for them. The reboot mounts each
|
|
1727
|
+
* one again from its host file (`MsbCommands.restore` emits one `--volume`
|
|
1728
|
+
* per mount), which is also what msb 0.7.8 requires: it refuses a restore
|
|
1729
|
+
* that leaves one out. The host files therefore have to still exist, and
|
|
1730
|
+
* this method checks that, before any of the above, with
|
|
1731
|
+
* `CheckpointHostFileMissingError` (ahead of the stop, so a missing file
|
|
1732
|
+
* never costs the caller its running sandbox). A reuse sandbox adopted
|
|
1733
|
+
* through `findRunning` carries the builder's own spec, mounts included
|
|
1734
|
+
* (they are part of the reuse identity), so its reboot supplies them like
|
|
1735
|
+
* any other. One adopted by a process whose backend never called `create()`
|
|
1736
|
+
* for it has no handle state and is refused before anything is stopped.
|
|
1737
|
+
*
|
|
1628
1738
|
* `ref` is the WORKING ref this method is asked to checkpoint under — its
|
|
1629
1739
|
* basename becomes the snapshot create call's `<name>` argument (still
|
|
1630
1740
|
* meaningful: it lands in msb's own index, see `MsbCommands.snapshotCreate`'s
|
|
@@ -1648,6 +1758,14 @@ export class MsbCliBackend {
|
|
|
1648
1758
|
// away a running sandbox for.
|
|
1649
1759
|
throw new TmpfsRootCheckpointError();
|
|
1650
1760
|
}
|
|
1761
|
+
// Same reasoning for the copied files: the reboot below mounts each one
|
|
1762
|
+
// again from its host file (`msb restore --volume`, see
|
|
1763
|
+
// `MsbCommands.restore`), so a host file that has gone missing would only
|
|
1764
|
+
// be noticed after the source sandbox is stopped, snapshotted and
|
|
1765
|
+
// removed. `GenericContainer.checkpoint()` runs the same check ahead of
|
|
1766
|
+
// replacing a named checkpoint; this one covers a caller that reaches the
|
|
1767
|
+
// backend directly.
|
|
1768
|
+
await requireCopiedHostFilesExist(handle.spec.mounts);
|
|
1651
1769
|
const msbPath = await this.msbPath();
|
|
1652
1770
|
// Captured up front: `handle.id` itself is mutated in place, below, once
|
|
1653
1771
|
// the reboot under the fresh name has actually succeeded — every
|
|
@@ -1731,7 +1849,7 @@ export class MsbCliBackend {
|
|
|
1731
1849
|
const detail = err instanceof Error ? err.message : String(err);
|
|
1732
1850
|
throw new BackendError(`sandbox '${originalName}' was removed after a successful checkpoint snapshot, but booting a fresh ` +
|
|
1733
1851
|
`sandbox back up from that snapshot failed: ${detail} — the sandbox's disk state is preserved in ` +
|
|
1734
|
-
`checkpoint '${effectiveRef}', restorable via GenericContainer.fromCheckpoint()
|
|
1852
|
+
`checkpoint '${effectiveRef}', restorable via GenericContainer.fromCheckpoint().`, err);
|
|
1735
1853
|
}
|
|
1736
1854
|
const freshName = rebootHandle.id;
|
|
1737
1855
|
// The reboot succeeded under `freshName` — the WINNING attempt's own
|