start-command 0.37.9 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +7 -2
- package/bunfig.toml +4 -0
- package/eslint.config.mjs +1 -0
- package/package.json +4 -4
- package/src/bin/cli.js +62 -26
- package/src/lib/args-parser-queries.js +45 -1
- package/src/lib/args-parser.js +19 -0
- package/src/lib/cgroup-memory.js +95 -23
- package/src/lib/command-builder.js +4 -0
- package/src/lib/container-removal.js +4 -5
- package/src/lib/detached-finalize.js +1 -0
- package/src/lib/docker-command-handoff.js +34 -6
- package/src/lib/docker-init-option.js +65 -0
- package/src/lib/docker-inspect-batch.js +176 -0
- package/src/lib/docker-resource-options.js +21 -2
- package/src/lib/docker-runtime-args.js +19 -3
- package/src/lib/execution-attempt.js +19 -9
- package/src/lib/execution-chain.js +182 -0
- package/src/lib/execution-control.js +8 -6
- package/src/lib/execution-env.js +120 -0
- package/src/lib/execution-identity.js +180 -0
- package/src/lib/execution-recovery.js +21 -1
- package/src/lib/execution-resume.js +21 -1
- package/src/lib/execution-store.js +30 -23
- package/src/lib/exit-evidence.js +139 -7
- package/src/lib/isolation.js +21 -3
- package/src/lib/launch-identity-options.js +81 -0
- package/src/lib/launch-identity.js +411 -0
- package/src/lib/log-uploader.js +15 -1
- package/src/lib/output-blocks.js +11 -0
- package/src/lib/query-commands.js +18 -3
- package/src/lib/resume-resources.js +2 -2
- package/src/lib/screen-isolation.js +11 -2
- package/src/lib/spawn-helpers.js +33 -16
- package/src/lib/status-formatter.js +39 -131
- package/src/lib/status-probe.js +149 -0
- package/src/lib/store-decode-cache.js +141 -0
- package/src/lib/usage.js +11 -1
- package/test/args-parser-attach-resume.js +1 -1
- package/test/args-parser-control.js +2 -2
- package/test/cargo-lock-version.mjs +121 -0
- package/test/check-rust-coverage.mjs +71 -0
- package/test/check-web-archive.mjs +154 -0
- package/test/ci-workflow-invariants.js +204 -0
- package/test/command-builder-docker-cleanup.js +24 -0
- package/test/command-exit-codes.mjs +91 -0
- package/test/docker-runtime-options.js +2 -0
- package/test/docker-snapshot-safety.js +39 -0
- package/test/execution-resume.js +2 -0
- package/test/fixtures/fake-npm.sh +64 -0
- package/test/fixtures/lychee-report.md +39 -0
- package/test/javascript-first-ci.js +69 -0
- package/test/links-workflow.mjs +127 -0
- package/test/lint-gate.mjs +56 -0
- package/test/npm-publish.mjs +181 -0
- package/test/recheck-broken-links.mjs +452 -0
- package/test/regression-176.js +2 -0
- package/test/regression-187.js +2 -0
- package/test/regression-191.js +4 -0
- package/test/regression-193.js +2 -0
- package/test/regression-198.js +2 -0
- package/test/regression-207.js +8 -2
- package/test/regression-224.js +2 -0
- package/test/regression-229.js +267 -0
- package/test/regression-230.js +234 -0
- package/test/regression-231.js +298 -0
- package/test/regression-236.js +250 -0
- package/test/regression-238.js +129 -0
- package/test/regression-241.js +254 -0
- package/test/regression-242.js +241 -0
- package/test/regression-243.js +493 -0
- package/test/regression-244.js +471 -0
- package/test/release-push-retry.mjs +172 -0
- package/test/run-js-tests.mjs +15 -3
- package/test/setup/private-snapshot-lock.js +17 -0
- package/test/shared-golden.js +44 -2
- package/test/transient-links.mjs +139 -0
|
@@ -14,7 +14,7 @@ const {
|
|
|
14
14
|
formatTextOptionValue,
|
|
15
15
|
} = require('./status-links-notation');
|
|
16
16
|
const { collectProcessIds } = require('./execution-control');
|
|
17
|
-
const { getDockerCommand
|
|
17
|
+
const { getDockerCommand } = require('./docker-cleanup');
|
|
18
18
|
const {
|
|
19
19
|
describeExitCode,
|
|
20
20
|
CGROUP_OOM_EXIT_REASON,
|
|
@@ -25,136 +25,31 @@ const {
|
|
|
25
25
|
FATAL_MARKER_TAIL_BYTES,
|
|
26
26
|
readLogTail,
|
|
27
27
|
} = require('./isolation-log-utils');
|
|
28
|
-
const { normalizeDockerTimestamp } = require('./docker-post-mortem');
|
|
29
28
|
const { END_TIME_SOURCE } = require('./detached-finalize');
|
|
30
29
|
const { formatCgroupMemory } = require('./cgroup-memory');
|
|
30
|
+
const { addLostWatcherEvidence } = require('./exit-evidence');
|
|
31
31
|
const {
|
|
32
32
|
attachContainerRemoval,
|
|
33
|
-
dockerInspect,
|
|
34
33
|
persistContainerRemoval,
|
|
35
34
|
} = require('./container-removal');
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
* as "stopped"; it means "unknown", so callers can keep the session running
|
|
49
|
-
* instead of fabricating a terminal `-1` result.
|
|
50
|
-
*
|
|
51
|
-
* @param {string} sessionName - Container name
|
|
52
|
-
* @param {Object|null} [probe] - Receives `sessionNotFound` (issue #225)
|
|
53
|
-
* @returns {{running: boolean, exitCode: number|null, oomKilled: boolean|null}|null} State,
|
|
54
|
-
* or null when the container cannot be inspected (not found yet, removed, or docker error)
|
|
55
|
-
*/
|
|
56
|
-
function inspectDockerState(sessionName, probe = null) {
|
|
57
|
-
const { stdout, notFound } = dockerInspect(
|
|
58
|
-
getDockerCommand(),
|
|
59
|
-
dockerProbeSpawnOptions(),
|
|
60
|
-
'{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}} ' +
|
|
61
|
-
'{{.State.StartedAt}} {{.State.FinishedAt}}',
|
|
62
|
-
sessionName
|
|
63
|
-
);
|
|
64
|
-
if (probe) {
|
|
65
|
-
// "No such container" is a fact, unlike a docker that cannot be reached
|
|
66
|
-
// (issue #225); it still reads as unknown liveness here (issue #136).
|
|
67
|
-
probe.sessionNotFound = notFound;
|
|
68
|
-
}
|
|
69
|
-
if (stdout === null) {
|
|
70
|
-
return null;
|
|
71
|
-
}
|
|
72
|
-
// Parsed positionally and defensively: an older docker (or a stubbed one)
|
|
73
|
-
// may answer with fewer fields than the template asks for.
|
|
74
|
-
const [runningRaw, exitRaw, oomKilledRaw, startedRaw, finishedRaw] = stdout
|
|
75
|
-
.trim()
|
|
76
|
-
.split(/\s+/);
|
|
77
|
-
const exitCode = Number.parseInt(exitRaw, 10);
|
|
35
|
+
const { withDockerInspectBatch } = require('./docker-inspect-batch');
|
|
36
|
+
const {
|
|
37
|
+
backendExitCode,
|
|
38
|
+
dockerProbeSpawnOptions,
|
|
39
|
+
inspectDockerState,
|
|
40
|
+
isDetachedDockerRecord,
|
|
41
|
+
readDockerState,
|
|
42
|
+
resolveOomObservation,
|
|
43
|
+
} = require('./status-probe');
|
|
44
|
+
|
|
45
|
+
/** How the status probes run docker. */
|
|
46
|
+
function dockerProbe() {
|
|
78
47
|
return {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
oomKilled:
|
|
82
|
-
oomKilledRaw === 'true' ? true : oomKilledRaw === 'false' ? false : null,
|
|
83
|
-
startedAt: normalizeDockerTimestamp(startedRaw),
|
|
84
|
-
finishedAt: normalizeDockerTimestamp(finishedRaw),
|
|
48
|
+
dockerCommand: getDockerCommand(),
|
|
49
|
+
spawnOptions: dockerProbeSpawnOptions(),
|
|
85
50
|
};
|
|
86
51
|
}
|
|
87
52
|
|
|
88
|
-
function isDetachedDockerRecord(record) {
|
|
89
|
-
const opts = record.options || {};
|
|
90
|
-
return (
|
|
91
|
-
opts.isolated === 'docker' &&
|
|
92
|
-
opts.isolationMode === 'detached' &&
|
|
93
|
-
Boolean(opts.sessionName)
|
|
94
|
-
);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
function dockerProbeSpawnOptions() {
|
|
98
|
-
return getDockerSpawnOptions({
|
|
99
|
-
encoding: 'utf8',
|
|
100
|
-
env: process.env,
|
|
101
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
function readDockerState(record, probe = null) {
|
|
106
|
-
const opts = record.options || {};
|
|
107
|
-
if (opts.isolated !== 'docker' || !opts.sessionName) {
|
|
108
|
-
return null;
|
|
109
|
-
}
|
|
110
|
-
const state = inspectDockerState(opts.sessionName, probe);
|
|
111
|
-
if (
|
|
112
|
-
record.attempt &&
|
|
113
|
-
state?.startedAt &&
|
|
114
|
-
Date.parse(state.startedAt) < Date.parse(record.attempt.startedAt)
|
|
115
|
-
) {
|
|
116
|
-
return null;
|
|
117
|
-
}
|
|
118
|
-
return state;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* Best-effort terminal exit code reported by the isolation backend itself
|
|
123
|
-
* (currently docker via `docker inspect .State.ExitCode`). Returns null when
|
|
124
|
-
* the backend cannot provide a real code, so callers never surface the `-1`
|
|
125
|
-
* sentinel for a session whose real exit code is simply not available yet.
|
|
126
|
-
* A running container has no terminal exit code (docker reports `0` for it),
|
|
127
|
-
* so only a stopped container contributes one.
|
|
128
|
-
* @param {{running: boolean, exitCode: number|null}|null} dockerState - Inspected state
|
|
129
|
-
* @returns {number|null}
|
|
130
|
-
*/
|
|
131
|
-
function backendExitCode(dockerState) {
|
|
132
|
-
return dockerState && !dockerState.running ? dockerState.exitCode : null;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/**
|
|
136
|
-
* Reconcile the OOM observation from the stored record and from `docker inspect`.
|
|
137
|
-
*
|
|
138
|
-
* `State.OOMKilled` is a container-cgroup flag: the kernel sets it when ANY
|
|
139
|
-
* process in the cgroup is OOM-killed and it is never cleared for the life of
|
|
140
|
-
* the container (moby/moby#47618). It is therefore an *observation*, never a
|
|
141
|
-
* verdict about the session (issue #151) — a container that lost one child
|
|
142
|
-
* process keeps running and can still exit `0`. Once observed, the flag stays
|
|
143
|
-
* `true` for the record.
|
|
144
|
-
* @param {Object} record - Execution record
|
|
145
|
-
* @param {{oomKilled: boolean|null}|null} dockerState - Inspected state
|
|
146
|
-
* @returns {boolean|undefined} Observation, or undefined when nothing is known
|
|
147
|
-
*/
|
|
148
|
-
function resolveOomObservation(record, dockerState) {
|
|
149
|
-
if (record.oomKilled === true || dockerState?.oomKilled === true) {
|
|
150
|
-
return true;
|
|
151
|
-
}
|
|
152
|
-
if (record.oomKilled === false || dockerState?.oomKilled === false) {
|
|
153
|
-
return false;
|
|
154
|
-
}
|
|
155
|
-
return undefined;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
53
|
/**
|
|
159
54
|
* Check if a detached isolation session is still running
|
|
160
55
|
* @param {Object} record - Execution record
|
|
@@ -433,6 +328,7 @@ function resolveDetachedStatus(record, logTail, probe = null) {
|
|
|
433
328
|
(oomKilled === true ? 137 : -1);
|
|
434
329
|
}
|
|
435
330
|
applyEndTime(enriched, dockerState, footer.finishedAt);
|
|
331
|
+
addLostWatcherEvidence(enriched, record, dockerState, logTail);
|
|
436
332
|
}
|
|
437
333
|
|
|
438
334
|
return enriched;
|
|
@@ -563,10 +459,7 @@ function enrichDetachedStatus(record) {
|
|
|
563
459
|
const logTail = record
|
|
564
460
|
? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
|
|
565
461
|
: null;
|
|
566
|
-
const probe =
|
|
567
|
-
dockerCommand: getDockerCommand(),
|
|
568
|
-
spawnOptions: dockerProbeSpawnOptions(),
|
|
569
|
-
};
|
|
462
|
+
const probe = dockerProbe();
|
|
570
463
|
const resolved = attachContainerRemoval(
|
|
571
464
|
resolveDetachedStatus(record, logTail, probe),
|
|
572
465
|
probe
|
|
@@ -729,6 +622,7 @@ function formatRecordAsText(record) {
|
|
|
729
622
|
`UUID: ${obj.uuid}`,
|
|
730
623
|
`Status: ${obj.status}`,
|
|
731
624
|
`Command: ${obj.command}`,
|
|
625
|
+
...require('./execution-chain').identityTextLines(obj),
|
|
732
626
|
`Exit Code: ${obj.exitCode !== null ? describeExitCode(obj.exitCode).text : 'N/A'}`,
|
|
733
627
|
...(obj.oomKilled !== undefined
|
|
734
628
|
? [`OOM Killed: ${obj.oomKilled}`]
|
|
@@ -919,12 +813,22 @@ function queryStatus(store, identifier, outputFormat) {
|
|
|
919
813
|
};
|
|
920
814
|
}
|
|
921
815
|
try {
|
|
922
|
-
// Enrich detached execution status with live session check
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
816
|
+
// Enrich detached execution status with live session check; its docker
|
|
817
|
+
// probes are answered by one inspect (issue #243).
|
|
818
|
+
const withProcessIds = withDockerInspectBatch(
|
|
819
|
+
[record],
|
|
820
|
+
dockerProbe(),
|
|
821
|
+
() => {
|
|
822
|
+
const enrichedRecord = enrichDetachedStatus(record);
|
|
823
|
+
persistContainerRemoval(store, record, enrichedRecord);
|
|
824
|
+
return attachProcessIds(enrichedRecord);
|
|
825
|
+
}
|
|
826
|
+
);
|
|
926
827
|
// Attach currentTime so callers can see how long an executing command has been running
|
|
927
|
-
const withCurrentTime =
|
|
828
|
+
const withCurrentTime = require('./execution-chain').attachIdentity(
|
|
829
|
+
attachCurrentTime(withProcessIds),
|
|
830
|
+
store
|
|
831
|
+
);
|
|
928
832
|
return {
|
|
929
833
|
success: true,
|
|
930
834
|
output: formatRecord(withCurrentTime, outputFormat || 'links-notation'),
|
|
@@ -948,8 +852,12 @@ function listExecutions(store, outputFormat, options = {}) {
|
|
|
948
852
|
}
|
|
949
853
|
|
|
950
854
|
try {
|
|
855
|
+
const all = store.getAll();
|
|
856
|
+
// One docker inspect for every listed container (issue #243).
|
|
951
857
|
const enriched = sortRecordsByStartTimeDesc(
|
|
952
|
-
|
|
858
|
+
withDockerInspectBatch(all, dockerProbe(), () =>
|
|
859
|
+
all.map(prepareRecordForList)
|
|
860
|
+
)
|
|
953
861
|
);
|
|
954
862
|
// --running reports the reconciled status, not the stored one: a record
|
|
955
863
|
// whose session already died is listed as executed even if the store still
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live docker state of a detached execution, for `--status` and `--list`.
|
|
3
|
+
*
|
|
4
|
+
* Split out of `status-formatter` (as `status_probe.rs` is in Rust) so the
|
|
5
|
+
* probing stays separate from the rendering and both files stay under the
|
|
6
|
+
* repository line budget (issue #243).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
|
|
10
|
+
const { normalizeDockerTimestamp } = require('./docker-post-mortem');
|
|
11
|
+
const {
|
|
12
|
+
CONTAINER_STATE_REMOVED,
|
|
13
|
+
dockerInspect,
|
|
14
|
+
} = require('./container-removal');
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Inspect the live state of a detached docker container by name.
|
|
18
|
+
*
|
|
19
|
+
* Distinguishes three outcomes that matter for status reporting:
|
|
20
|
+
* - the container exists and is running,
|
|
21
|
+
* - the container exists but has stopped (with a real exit code), and
|
|
22
|
+
* - the container cannot be inspected at all.
|
|
23
|
+
*
|
|
24
|
+
* The last case is crucial on slow Docker-in-Docker hosts (issue #136): right
|
|
25
|
+
* after `docker run -d` returns, `docker inspect <name>` can transiently fail
|
|
26
|
+
* because the container is not visible yet. A failed inspect must NOT be read
|
|
27
|
+
* as "stopped"; it means "unknown", so callers can keep the session running
|
|
28
|
+
* instead of fabricating a terminal `-1` result.
|
|
29
|
+
*
|
|
30
|
+
* @param {string} sessionName - Container name
|
|
31
|
+
* @param {Object|null} [probe] - Receives `sessionNotFound` (issue #225)
|
|
32
|
+
* @returns {{running: boolean, exitCode: number|null, oomKilled: boolean|null}|null} State,
|
|
33
|
+
* or null when the container cannot be inspected (not found yet, removed, or docker error)
|
|
34
|
+
*/
|
|
35
|
+
function inspectDockerState(sessionName, probe = null) {
|
|
36
|
+
const { stdout, notFound } = dockerInspect(
|
|
37
|
+
getDockerCommand(),
|
|
38
|
+
dockerProbeSpawnOptions(),
|
|
39
|
+
'{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}} ' +
|
|
40
|
+
'{{.State.StartedAt}} {{.State.FinishedAt}}',
|
|
41
|
+
sessionName
|
|
42
|
+
);
|
|
43
|
+
if (probe) {
|
|
44
|
+
// "No such container" is a fact, unlike a docker that cannot be reached
|
|
45
|
+
// (issue #225); it still reads as unknown liveness here (issue #136).
|
|
46
|
+
probe.sessionNotFound = notFound;
|
|
47
|
+
}
|
|
48
|
+
if (stdout === null) {
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
// Parsed positionally and defensively: an older docker (or a stubbed one)
|
|
52
|
+
// may answer with fewer fields than the template asks for.
|
|
53
|
+
const [runningRaw, exitRaw, oomKilledRaw, startedRaw, finishedRaw] = stdout
|
|
54
|
+
.trim()
|
|
55
|
+
.split(/\s+/);
|
|
56
|
+
const exitCode = Number.parseInt(exitRaw, 10);
|
|
57
|
+
return {
|
|
58
|
+
running: runningRaw === 'true',
|
|
59
|
+
exitCode: Number.isFinite(exitCode) ? exitCode : null,
|
|
60
|
+
oomKilled:
|
|
61
|
+
oomKilledRaw === 'true' ? true : oomKilledRaw === 'false' ? false : null,
|
|
62
|
+
startedAt: normalizeDockerTimestamp(startedRaw),
|
|
63
|
+
finishedAt: normalizeDockerTimestamp(finishedRaw),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function isDetachedDockerRecord(record) {
|
|
68
|
+
const opts = record.options || {};
|
|
69
|
+
return (
|
|
70
|
+
opts.isolated === 'docker' &&
|
|
71
|
+
opts.isolationMode === 'detached' &&
|
|
72
|
+
Boolean(opts.sessionName)
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function dockerProbeSpawnOptions() {
|
|
77
|
+
return getDockerSpawnOptions({
|
|
78
|
+
encoding: 'utf8',
|
|
79
|
+
env: process.env,
|
|
80
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function readDockerState(record, probe = null) {
|
|
85
|
+
const opts = record.options || {};
|
|
86
|
+
if (opts.isolated !== 'docker' || !opts.sessionName) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
if (record.containerState === CONTAINER_STATE_REMOVED) {
|
|
90
|
+
// A removed container stays removed: a relaunch clears the marker, so
|
|
91
|
+
// there is nothing left to ask Docker (issue #243).
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
const state = inspectDockerState(opts.sessionName, probe);
|
|
95
|
+
if (
|
|
96
|
+
record.attempt &&
|
|
97
|
+
state?.startedAt &&
|
|
98
|
+
Date.parse(state.startedAt) < Date.parse(record.attempt.startedAt)
|
|
99
|
+
) {
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
return state;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Best-effort terminal exit code reported by the isolation backend itself
|
|
107
|
+
* (currently docker via `docker inspect .State.ExitCode`). Returns null when
|
|
108
|
+
* the backend cannot provide a real code, so callers never surface the `-1`
|
|
109
|
+
* sentinel for a session whose real exit code is simply not available yet.
|
|
110
|
+
* A running container has no terminal exit code (docker reports `0` for it),
|
|
111
|
+
* so only a stopped container contributes one.
|
|
112
|
+
* @param {{running: boolean, exitCode: number|null}|null} dockerState - Inspected state
|
|
113
|
+
* @returns {number|null}
|
|
114
|
+
*/
|
|
115
|
+
function backendExitCode(dockerState) {
|
|
116
|
+
return dockerState && !dockerState.running ? dockerState.exitCode : null;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Reconcile the OOM observation from the stored record and from `docker inspect`.
|
|
121
|
+
*
|
|
122
|
+
* `State.OOMKilled` is a container-cgroup flag: the kernel sets it when ANY
|
|
123
|
+
* process in the cgroup is OOM-killed and it is never cleared for the life of
|
|
124
|
+
* the container (moby/moby#47618). It is therefore an *observation*, never a
|
|
125
|
+
* verdict about the session (issue #151) — a container that lost one child
|
|
126
|
+
* process keeps running and can still exit `0`. Once observed, the flag stays
|
|
127
|
+
* `true` for the record.
|
|
128
|
+
* @param {Object} record - Execution record
|
|
129
|
+
* @param {{oomKilled: boolean|null}|null} dockerState - Inspected state
|
|
130
|
+
* @returns {boolean|undefined} Observation, or undefined when nothing is known
|
|
131
|
+
*/
|
|
132
|
+
function resolveOomObservation(record, dockerState) {
|
|
133
|
+
if (record.oomKilled === true || dockerState?.oomKilled === true) {
|
|
134
|
+
return true;
|
|
135
|
+
}
|
|
136
|
+
if (record.oomKilled === false || dockerState?.oomKilled === false) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
module.exports = {
|
|
143
|
+
backendExitCode,
|
|
144
|
+
dockerProbeSpawnOptions,
|
|
145
|
+
inspectDockerState,
|
|
146
|
+
isDetachedDockerRecord,
|
|
147
|
+
readDockerState,
|
|
148
|
+
resolveOomObservation,
|
|
149
|
+
};
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decoded execution store, shared within one process and between processes
|
|
3
|
+
* (issue #243).
|
|
4
|
+
*
|
|
5
|
+
* Decoding `executions.lino` costs about 1-4 ms per record, and one
|
|
6
|
+
* `--status` used to decode it up to three times (`get`, then `getAll` twice
|
|
7
|
+
* for the execution chain). The file is still read on every call, so a write
|
|
8
|
+
* by any process is seen at once, but it is decoded again only when its
|
|
9
|
+
* content differs from the last decoded copy. The content itself is the key:
|
|
10
|
+
* unlike an mtime, it cannot miss two same-size writes within one clock tick.
|
|
11
|
+
*
|
|
12
|
+
* A new process (every `$ --status` poll) still had to decode once. The
|
|
13
|
+
* decoded records are therefore also kept as JSON next to the store, tagged
|
|
14
|
+
* with the SHA-256 of the content they were decoded from; `JSON.parse` of
|
|
15
|
+
* that copy is about 250 times faster than the lino decode. A copy whose
|
|
16
|
+
* digest does not match the current content is ignored and replaced.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const fs = require('fs');
|
|
20
|
+
const crypto = require('crypto');
|
|
21
|
+
const { isDeepStrictEqual } = require('util');
|
|
22
|
+
const { atomicWrite } = require('./atomic-write');
|
|
23
|
+
|
|
24
|
+
/** Format tag of the JSON copy; a different tag is ignored. */
|
|
25
|
+
const DECODED_COPY_FORMAT = 'start-command-decoded-store-js-1';
|
|
26
|
+
|
|
27
|
+
/** @type {Map<string, {content: string, data: *, decodes: number}>} */
|
|
28
|
+
const decodedStores = new Map();
|
|
29
|
+
|
|
30
|
+
function digestOf(content) {
|
|
31
|
+
return crypto.createHash('sha256').update(content).digest('hex');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Records of the JSON copy at `copyPath`, when it was decoded from content
|
|
36
|
+
* with this digest; otherwise undefined.
|
|
37
|
+
*/
|
|
38
|
+
function readDecodedCopy(copyPath, digest) {
|
|
39
|
+
try {
|
|
40
|
+
const copy = JSON.parse(fs.readFileSync(copyPath, 'utf8'));
|
|
41
|
+
if (copy && copy.format === DECODED_COPY_FORMAT && copy.sha256 === digest) {
|
|
42
|
+
return copy.records;
|
|
43
|
+
}
|
|
44
|
+
} catch {
|
|
45
|
+
/* Missing or unreadable: decode instead. */
|
|
46
|
+
}
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Keep `data` as JSON for the next process, unless JSON cannot represent it
|
|
52
|
+
* exactly. Best effort: a read-only folder only costs the next decode.
|
|
53
|
+
*/
|
|
54
|
+
function writeDecodedCopy(copyPath, digest, data) {
|
|
55
|
+
try {
|
|
56
|
+
const json = JSON.stringify({
|
|
57
|
+
format: DECODED_COPY_FORMAT,
|
|
58
|
+
sha256: digest,
|
|
59
|
+
records: data,
|
|
60
|
+
});
|
|
61
|
+
if (!isDeepStrictEqual(JSON.parse(json).records, data)) {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
atomicWrite(copyPath, json);
|
|
65
|
+
} catch {
|
|
66
|
+
/* Not kept; the next process decodes again. */
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Decode `content` of the store at `filePath`, or reuse an earlier decode of
|
|
72
|
+
* the same content. Every call returns its own deep copy, so callers may
|
|
73
|
+
* mutate the records they get.
|
|
74
|
+
* @param {string} filePath - Store file the content was read from
|
|
75
|
+
* @param {string} content - Current file content
|
|
76
|
+
* @param {function(string): *} decode - Decoder for a changed content
|
|
77
|
+
* @param {{copyPath?: string}} [options] - JSON copy shared between processes
|
|
78
|
+
* @returns {{data: *, reused: boolean, source: 'memory'|'copy'|'decode'}}
|
|
79
|
+
*/
|
|
80
|
+
function decodeStoreContent(filePath, content, decode, options = {}) {
|
|
81
|
+
const cached = decodedStores.get(filePath);
|
|
82
|
+
if (cached && cached.content === content) {
|
|
83
|
+
return {
|
|
84
|
+
data: structuredClone(cached.data),
|
|
85
|
+
reused: true,
|
|
86
|
+
source: 'memory',
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
const decodes = cached ? cached.decodes : 0;
|
|
90
|
+
const { copyPath } = options;
|
|
91
|
+
const digest = copyPath ? digestOf(content) : null;
|
|
92
|
+
const copied = copyPath ? readDecodedCopy(copyPath, digest) : undefined;
|
|
93
|
+
if (copied !== undefined) {
|
|
94
|
+
decodedStores.set(filePath, {
|
|
95
|
+
content,
|
|
96
|
+
data: structuredClone(copied),
|
|
97
|
+
decodes,
|
|
98
|
+
});
|
|
99
|
+
return { data: copied, reused: true, source: 'copy' };
|
|
100
|
+
}
|
|
101
|
+
const data = decode(content);
|
|
102
|
+
decodedStores.set(filePath, {
|
|
103
|
+
content,
|
|
104
|
+
data: structuredClone(data),
|
|
105
|
+
decodes: decodes + 1,
|
|
106
|
+
});
|
|
107
|
+
if (copyPath) {
|
|
108
|
+
writeDecodedCopy(copyPath, digest, data);
|
|
109
|
+
}
|
|
110
|
+
return { data, reused: false, source: 'decode' };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* How many times this process decoded the store at `filePath` since it was
|
|
115
|
+
* last forgotten.
|
|
116
|
+
* @param {string} filePath - Store file
|
|
117
|
+
* @returns {number}
|
|
118
|
+
*/
|
|
119
|
+
function decodeCount(filePath) {
|
|
120
|
+
const cached = decodedStores.get(filePath);
|
|
121
|
+
return cached ? cached.decodes : 0;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Drop the decoded copy of a store held by this process.
|
|
126
|
+
* @param {string} [filePath] - Store file; all stores when omitted
|
|
127
|
+
*/
|
|
128
|
+
function forgetDecodedStore(filePath) {
|
|
129
|
+
if (filePath === undefined) {
|
|
130
|
+
decodedStores.clear();
|
|
131
|
+
} else {
|
|
132
|
+
decodedStores.delete(filePath);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
module.exports = {
|
|
137
|
+
DECODED_COPY_FORMAT,
|
|
138
|
+
decodeCount,
|
|
139
|
+
decodeStoreContent,
|
|
140
|
+
forgetDecodedStore,
|
|
141
|
+
};
|
package/src/lib/usage.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** Print usage information */
|
|
2
2
|
function printUsage() {
|
|
3
|
-
console.log(`Usage: $ [options] [--] <command> | $ --status <uuid> [--output-format <fmt>] | $ --list [--running] [--output-format <fmt>] | $ --upload-log <id> | $ --stop <id> | $ --terminate <id> | $ --attach <id> [--read-only] | $ --resume <id> [-- <command>] | $ --resume-all
|
|
3
|
+
console.log(`Usage: $ [options] [--] <command> | $ --status <uuid> [--output-format <fmt>] | $ --list [--running] [--output-format <fmt>] | $ --upload-log <id> | $ --stop <id> | $ --terminate <id> | $ --attach <id> [--read-only] | $ --resume <id> [-- <command>] | $ --resume-all | $ --alias <id> <name> | $ --rename <id> <name>
|
|
4
4
|
|
|
5
5
|
Options:
|
|
6
6
|
--help, -h Show this usage and exit successfully
|
|
@@ -10,18 +10,22 @@ Options:
|
|
|
10
10
|
--session, -s <name> Session name for isolation
|
|
11
11
|
--session-id <uuid> Session UUID for tracking (auto-generated if not provided)
|
|
12
12
|
--session-name <uuid> Alias for --session-id
|
|
13
|
+
--append-attempt Run as the next attempt of the existing --session-id execution
|
|
14
|
+
--continues <id> Record that this execution continues an earlier one
|
|
13
15
|
--image <image> Docker image (optional, defaults to OS-matched image)
|
|
14
16
|
--volume, -v <spec> Docker bind mount/volume host:container[:mode] (repeatable, docker only)
|
|
15
17
|
--mount <spec> Docker --mount spec (repeatable, docker only)
|
|
16
18
|
--env, -e <KEY=VALUE> Environment variable for docker container (repeatable, docker only)
|
|
17
19
|
--label <KEY=VALUE> Container attribution (repeatable, docker only; start-command.* reserved)
|
|
18
20
|
--privileged Run docker container in privileged mode (docker only)
|
|
21
|
+
--init, --no-init Run tini as PID 1 to reap orphaned processes (docker only; default START_COMMAND_DOCKER_INIT=1)
|
|
19
22
|
--network <name> Connect to a named network (repeatable, docker only)
|
|
20
23
|
--network-alias <alias> Add alias to the first network (repeatable, docker only)
|
|
21
24
|
--on-kill-resume <N> Resume a killed (exit 137 / OOM) detached docker session up to N times
|
|
22
25
|
--recovery-command <cmd> Command to run in the same container on such a resume
|
|
23
26
|
--on-kill-resume-delay <min[-max]> Wait a random number of seconds before each such resume (default 0)
|
|
24
27
|
--memory <size|N%|MIN%-MAX%> Docker RAM cap; percentages use daemon capacity
|
|
28
|
+
--memory-reservation <spec> Soft memory limit (Docker size or N%, below --memory)
|
|
25
29
|
--memory-swap <spec> Combined memory+swap cap (defaults to memory, no extra swap)
|
|
26
30
|
--cpus <n|N%|MIN%-MAX%> Docker CPU cap; ranges draw once per creation
|
|
27
31
|
--on-kill-resume-memory <spec> New memory cap before a qualifying OOM recovery
|
|
@@ -64,6 +68,8 @@ Options:
|
|
|
64
68
|
--cleanup-dry-run Show stale records that would be cleaned up (without cleaning)
|
|
65
69
|
--cleanup <id> Remove a Docker execution's containers and snapshot images (never forced)
|
|
66
70
|
--cleanup-dry-run <id> Show the containers and images --cleanup <id> would remove
|
|
71
|
+
--alias <id> <name> Add a name the execution can be addressed by
|
|
72
|
+
--rename <id> <name> Rename the execution, keeping its previous name as an alias
|
|
67
73
|
--version, -v Show version information
|
|
68
74
|
|
|
69
75
|
Examples:
|
|
@@ -75,6 +81,7 @@ Examples:
|
|
|
75
81
|
$ --isolated docker --image ghcr.io/link-foundation/box-js:latest -- bun --version
|
|
76
82
|
$ -i docker -v ~/.config/gh:/root/.config/gh -e TOKEN=abc -- gh repo list
|
|
77
83
|
$ -i docker --image konard/hive-mind-dind:latest --privileged -- solve ...
|
|
84
|
+
$ -i docker --init -d -- solve ... # tini reaps zombies of long-running tasks
|
|
78
85
|
$ --isolated ssh --endpoint user@remote.server -- ls -la
|
|
79
86
|
$ --isolated-user -- npm test # Create isolated user
|
|
80
87
|
$ -u myuser -- npm start # Custom username
|
|
@@ -91,6 +98,9 @@ Examples:
|
|
|
91
98
|
$ --resume my-docker-session # Restart the stored command in the same container
|
|
92
99
|
$ --resume my-docker-session -- bash # Run a different command in the same container
|
|
93
100
|
$ --resume-all # Re-attach every still-running execution
|
|
101
|
+
$ --session-id <uuid> --append-attempt -- npm test # Next attempt, same log
|
|
102
|
+
$ --continues <uuid> -- npm test # Fresh execution chained to <uuid>
|
|
103
|
+
$ --rename <uuid> nightly-build # Address it as nightly-build from now on
|
|
94
104
|
$ -i docker -d --on-kill-resume 3 --recovery-command 'solve --resume' -- solve
|
|
95
105
|
$ -i docker -d --on-kill-resume 3 --on-kill-resume-delay 30-90 -- cargo test
|
|
96
106
|
$ --use-command-stream echo "Hello" # Use command-stream library`);
|
|
@@ -142,7 +142,7 @@ describe('query mode exclusivity', () => {
|
|
|
142
142
|
test(`rejects ${args.join(' ')}`, () => {
|
|
143
143
|
assert.throws(() => {
|
|
144
144
|
parse(args);
|
|
145
|
-
}, /Cannot combine --status, --list, --upload-log, --stop, --terminate, --attach, --resume, --resume-all, or --
|
|
145
|
+
}, /Cannot combine --status, --list, --upload-log, --stop, --terminate, --attach, --resume, --resume-all, --cleanup, --alias, or --rename/);
|
|
146
146
|
});
|
|
147
147
|
}
|
|
148
148
|
});
|
|
@@ -54,13 +54,13 @@ describe('control options', () => {
|
|
|
54
54
|
it('should reject combining query and control modes', () => {
|
|
55
55
|
assert.throws(() => {
|
|
56
56
|
parseArgs(['--status', 'uuid-here', '--stop', 'my-session']);
|
|
57
|
-
}, /Cannot combine --status, --list, --upload-log, --stop, --terminate, --attach, --resume, --resume-all, or --
|
|
57
|
+
}, /Cannot combine --status, --list, --upload-log, --stop, --terminate, --attach, --resume, --resume-all, --cleanup, --alias, or --rename/);
|
|
58
58
|
});
|
|
59
59
|
|
|
60
60
|
it('should reject combining upload-log with control modes', () => {
|
|
61
61
|
assert.throws(() => {
|
|
62
62
|
parseArgs(['--upload-log', 'uuid-here', '--terminate', 'my-session']);
|
|
63
|
-
}, /Cannot combine --status, --list, --upload-log, --stop, --terminate, --attach, --resume, --resume-all, or --
|
|
63
|
+
}, /Cannot combine --status, --list, --upload-log, --stop, --terminate, --attach, --resume, --resume-all, --cleanup, --alias, or --rename/);
|
|
64
64
|
});
|
|
65
65
|
|
|
66
66
|
it('should reject output-format with control modes', () => {
|