start-command 0.39.2 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/eslint.config.mjs +1 -0
- package/package.json +9 -10
- package/src/bin/cli.js +7 -5
- package/src/lib/command-stream.js +19 -5
- package/src/lib/execution-chain.js +2 -2
- package/src/lib/execution-store.js +62 -44
- package/src/lib/log-sanitizer.js +16 -137
- package/test/ci-workflow-invariants.js +18 -0
- package/test/command-stream-log.js +85 -0
- package/test/dependency-freshness.mjs +552 -0
- package/test/execution-store-clink.js +152 -0
- package/test/execution-store.js +49 -5
- package/test/fixtures/executions-lino-objects-codec-0.4.0.lino +1 -0
- package/test/fixtures/executions-lino-objects-codec-0.9.3.lino +26 -0
- package/test/javascript-first-ci.js +29 -0
- package/test/log-sanitizer.js +116 -47
- package/test/regression-187.js +4 -0
- package/test/regression-224.js +4 -2
- package/test/regression-231.js +20 -0
- package/test/release-git.mjs +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# start-command
|
|
2
2
|
|
|
3
|
+
## 0.40.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 47a9dd8: Upgrade every dependency to its latest release, and let CI reject pull requests that fall behind (#253).
|
|
8
|
+
|
|
9
|
+
- **Upgrades:** `@link-foundation/sensitive-data-sanitizer` 0.12.2, `command-stream` 2.6.0, `lino-objects-codec` 0.9.3, the lint tools, the GitHub Actions, the runners (`ubuntu-26.04`, `macos-26`) and the .NET 10 SDK that clink 4.x needs.
|
|
10
|
+
- **Store format:** the execution store is now written in lino-objects-codec 0.9's readable format. Stores written by older releases are still read, but older releases of start cannot read a store written by this one.
|
|
11
|
+
- **Links database:** the store had stayed empty, because clink 4.x rejected every query it sent. It now writes one record-scoped link per property and verifies the writes. The existing `recordToLinksNotation()` helper remains available with the corrected format.
|
|
12
|
+
- **`--use-command-stream`:** command output is appended to the log again. command-stream 2.x returns it as snapshots.
|
|
13
|
+
- **Sanitizer:** start's own `Authorization` masking pass is gone, because the sanitizer now masks only the scheme and token itself. A NUL byte in a log no longer blocks the upload (`--allow-binary`).
|
|
14
|
+
- **Dependency fixes:** preserve Authorization placeholders, bracketed tokens and following control bytes; avoid quadratic scanning of dense credentials. Load command-stream's lightweight `core` entry for command execution.
|
|
15
|
+
- **CI:** `scripts/check-dependency-freshness.mjs` fails a pull request when any npm package, crate, action, CI tool, runtime, runner or translator pin is not its latest release. The one exception is a version that links an open blocking issue.
|
|
16
|
+
|
|
3
17
|
## 0.39.2
|
|
4
18
|
|
|
5
19
|
### Patch Changes
|
package/eslint.config.mjs
CHANGED
|
@@ -80,6 +80,7 @@ export default [
|
|
|
80
80
|
// Web-standard globals Node exposes; used by the repository-level
|
|
81
81
|
// scripts/ directory, which the root config lints (issue #168).
|
|
82
82
|
AbortController: 'readonly',
|
|
83
|
+
AbortSignal: 'readonly',
|
|
83
84
|
URL: 'readonly',
|
|
84
85
|
setTimeout: 'readonly',
|
|
85
86
|
clearTimeout: 'readonly',
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "start-command",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.40.0",
|
|
4
4
|
"description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
|
|
5
5
|
"main": "src/bin/cli.js",
|
|
6
6
|
"exports": {
|
|
@@ -49,12 +49,12 @@
|
|
|
49
49
|
"devDependencies": {
|
|
50
50
|
"@changesets/cli": "^3.0.3",
|
|
51
51
|
"@eslint/js": "^10.0.1",
|
|
52
|
-
"eslint": "^10.
|
|
52
|
+
"eslint": "^10.13.0",
|
|
53
53
|
"eslint-config-prettier": "^10.1.8",
|
|
54
|
-
"eslint-plugin-prettier": "^5.5.
|
|
54
|
+
"eslint-plugin-prettier": "^5.5.7",
|
|
55
55
|
"husky": "^9.1.7",
|
|
56
|
-
"lint-staged": "^17.0
|
|
57
|
-
"prettier": "^3.
|
|
56
|
+
"lint-staged": "^17.6.0",
|
|
57
|
+
"prettier": "^3.9.10"
|
|
58
58
|
},
|
|
59
59
|
"lint-staged": {
|
|
60
60
|
"*.{js,mjs,cjs}": [
|
|
@@ -68,13 +68,12 @@
|
|
|
68
68
|
]
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@link-foundation/sensitive-data-sanitizer": "^0.12.
|
|
72
|
-
"command-stream": "^
|
|
73
|
-
"lino-objects-codec": "^0.
|
|
71
|
+
"@link-foundation/sensitive-data-sanitizer": "^0.12.2",
|
|
72
|
+
"command-stream": "^2.6.0",
|
|
73
|
+
"lino-objects-codec": "^0.9.3"
|
|
74
74
|
},
|
|
75
75
|
"overrides": {
|
|
76
76
|
"brace-expansion": "^5.0.12",
|
|
77
|
-
"flatted": "^3.4.
|
|
78
|
-
"js-yaml": "^4.3.2"
|
|
77
|
+
"flatted": "^3.4.4"
|
|
79
78
|
}
|
|
80
79
|
}
|
package/src/bin/cli.js
CHANGED
|
@@ -814,7 +814,7 @@ async function runDirectWithCommandStream(
|
|
|
814
814
|
sessionId
|
|
815
815
|
) {
|
|
816
816
|
// Lazy load command-stream
|
|
817
|
-
const { getCommandStream } = require('../lib/command-stream');
|
|
817
|
+
const { getCommandStream, outputText } = require('../lib/command-stream');
|
|
818
818
|
const { $, raw } = await getCommandStream();
|
|
819
819
|
|
|
820
820
|
// Get the command name (first word of the actual command to execute)
|
|
@@ -921,11 +921,13 @@ async function runDirectWithCommandStream(
|
|
|
921
921
|
}
|
|
922
922
|
|
|
923
923
|
// Collect output for log
|
|
924
|
-
|
|
925
|
-
|
|
924
|
+
const stdout = outputText(result.stdout);
|
|
925
|
+
const stderr = outputText(result.stderr);
|
|
926
|
+
if (stdout) {
|
|
927
|
+
appendLogFile(logFilePath, stdout);
|
|
926
928
|
}
|
|
927
|
-
if (
|
|
928
|
-
appendLogFile(logFilePath,
|
|
929
|
+
if (stderr) {
|
|
930
|
+
appendLogFile(logFilePath, stderr);
|
|
929
931
|
}
|
|
930
932
|
} catch (err) {
|
|
931
933
|
exitCode = err.code || 1;
|
|
@@ -26,11 +26,24 @@ let commandStream = null;
|
|
|
26
26
|
*/
|
|
27
27
|
async function getCommandStream() {
|
|
28
28
|
if (!commandStream) {
|
|
29
|
-
|
|
29
|
+
// Execution needs no compatibility layers, terminal renderers or REPL.
|
|
30
|
+
commandStream = await import('command-stream/core');
|
|
30
31
|
}
|
|
31
32
|
return commandStream;
|
|
32
33
|
}
|
|
33
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Text of a completed command's stdout or stderr. Since command-stream 2.x
|
|
37
|
+
* these are readable stream snapshots rather than strings, and an empty one
|
|
38
|
+
* is still truthy (#253).
|
|
39
|
+
*
|
|
40
|
+
* @param {unknown} output - result.stdout or result.stderr
|
|
41
|
+
* @returns {string}
|
|
42
|
+
*/
|
|
43
|
+
function outputText(output) {
|
|
44
|
+
return output === undefined || output === null ? '' : String(output);
|
|
45
|
+
}
|
|
46
|
+
|
|
34
47
|
/**
|
|
35
48
|
* Execute a shell command synchronously and return the result
|
|
36
49
|
* Uses command-stream's $ function with .sync() for blocking execution.
|
|
@@ -54,8 +67,8 @@ async function execCommand(command, options = {}) {
|
|
|
54
67
|
const result = $cmd`${command}`.sync();
|
|
55
68
|
|
|
56
69
|
return {
|
|
57
|
-
stdout: (result.stdout
|
|
58
|
-
stderr: (result.stderr
|
|
70
|
+
stdout: outputText(result.stdout).trim(),
|
|
71
|
+
stderr: outputText(result.stderr).trim(),
|
|
59
72
|
code: resolveChildExitCode(result.code),
|
|
60
73
|
};
|
|
61
74
|
} catch (err) {
|
|
@@ -91,8 +104,8 @@ async function execCommandAsync(command, options = {}) {
|
|
|
91
104
|
const result = await $cmd`${command}`;
|
|
92
105
|
|
|
93
106
|
return {
|
|
94
|
-
stdout: (result.stdout
|
|
95
|
-
stderr: (result.stderr
|
|
107
|
+
stdout: outputText(result.stdout).trim(),
|
|
108
|
+
stderr: outputText(result.stderr).trim(),
|
|
96
109
|
code: resolveChildExitCode(result.code),
|
|
97
110
|
};
|
|
98
111
|
} catch (err) {
|
|
@@ -250,6 +263,7 @@ async function runWithHandlers(command, handlers = {}, options = {}) {
|
|
|
250
263
|
|
|
251
264
|
module.exports = {
|
|
252
265
|
getCommandStream,
|
|
266
|
+
outputText,
|
|
253
267
|
execCommand,
|
|
254
268
|
execCommandAsync,
|
|
255
269
|
commandExists,
|
|
@@ -21,7 +21,7 @@ const { executionAliases, hasUserNames } = require('./execution-identity');
|
|
|
21
21
|
function buildChain(records, record) {
|
|
22
22
|
const seen = new Set([record.uuid]);
|
|
23
23
|
const chain = [record];
|
|
24
|
-
for (let cur = record; cur.continues;
|
|
24
|
+
for (let cur = record; cur.continues;) {
|
|
25
25
|
const prev = records.find((r) => r.uuid === cur.continues);
|
|
26
26
|
if (!prev || seen.has(prev.uuid)) {
|
|
27
27
|
break;
|
|
@@ -30,7 +30,7 @@ function buildChain(records, record) {
|
|
|
30
30
|
chain.unshift(prev);
|
|
31
31
|
cur = prev;
|
|
32
32
|
}
|
|
33
|
-
for (let cur = record; ;
|
|
33
|
+
for (let cur = record; ;) {
|
|
34
34
|
const next = records
|
|
35
35
|
.filter((r) => r.continues === cur.uuid && !seen.has(r.uuid))
|
|
36
36
|
.sort((a, b) => Date.parse(b.startTime) - Date.parse(a.startTime))[0];
|
|
@@ -245,6 +245,16 @@ class ExecutionRecord {
|
|
|
245
245
|
}
|
|
246
246
|
}
|
|
247
247
|
|
|
248
|
+
/**
|
|
249
|
+
* Quote text as a clink reference: `"..."` with inner quotes doubled.
|
|
250
|
+
* clink 4.x rejects backslash escapes inside references.
|
|
251
|
+
* @param {string} text
|
|
252
|
+
* @returns {string}
|
|
253
|
+
*/
|
|
254
|
+
function clinkReference(text) {
|
|
255
|
+
return `"${text.replaceAll('"', '""')}"`;
|
|
256
|
+
}
|
|
257
|
+
|
|
248
258
|
/**
|
|
249
259
|
* Check if clink is installed
|
|
250
260
|
* @returns {boolean}
|
|
@@ -343,70 +353,69 @@ class ExecutionStore {
|
|
|
343
353
|
}
|
|
344
354
|
|
|
345
355
|
/**
|
|
346
|
-
* Convert
|
|
347
|
-
*
|
|
356
|
+
* Convert a record's properties to clink links notation.
|
|
357
|
+
*
|
|
358
|
+
* The record is one link `(uuid: ExecutionRecord uuid)` plus one link per
|
|
359
|
+
* property, `(uuid.key: uuid "key=value")`. Property links have the record
|
|
360
|
+
* as their source, so two records that share a value (both `executing`) are
|
|
361
|
+
* still distinct links: doublets are deduplicated by (source, target), and a
|
|
362
|
+
* shared `(status executing)` link would carry only one record's name. The
|
|
363
|
+
* `key=` prefix keeps the target from being a bare number, which clink 4.x
|
|
364
|
+
* reads as a link address even when quoted (issue #253).
|
|
348
365
|
* @param {ExecutionRecord} record
|
|
349
366
|
* @returns {string}
|
|
350
367
|
*/
|
|
351
368
|
recordToLinksNotation(record) {
|
|
352
|
-
// Using clink's string alias feature for readable identifiers
|
|
353
|
-
// Format: (uuid: uuid-value) (pid: pid-value) etc.
|
|
354
369
|
const obj = record.toObject();
|
|
355
|
-
const
|
|
370
|
+
const links = [];
|
|
356
371
|
|
|
357
372
|
for (const [key, value] of Object.entries(obj)) {
|
|
358
373
|
if (value !== null && value !== undefined) {
|
|
359
|
-
|
|
360
|
-
const escapedValue =
|
|
374
|
+
const text =
|
|
361
375
|
typeof value === 'object' ? JSON.stringify(value) : String(value);
|
|
362
|
-
|
|
376
|
+
links.push(
|
|
377
|
+
`(${record.uuid}.${key}: ${record.uuid} ${clinkReference(`${key}=${text}`)})`
|
|
378
|
+
);
|
|
363
379
|
}
|
|
364
380
|
}
|
|
365
381
|
|
|
366
|
-
return
|
|
382
|
+
return links.join(' ');
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/** Build the create query using the same property notation as the public helper. */
|
|
386
|
+
buildClinkCreateQuery(record) {
|
|
387
|
+
return `() ((${record.uuid}: ExecutionRecord ${record.uuid}) ${this.recordToLinksNotation(record)})`;
|
|
367
388
|
}
|
|
368
389
|
|
|
369
390
|
/**
|
|
370
|
-
* Build clink query
|
|
371
|
-
* @param {
|
|
391
|
+
* Build clink query deleting every property link of a record
|
|
392
|
+
* @param {string} uuid
|
|
372
393
|
* @returns {string}
|
|
373
394
|
*/
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
const links = [];
|
|
377
|
-
|
|
378
|
-
// Create main record link
|
|
379
|
-
links.push(`(${record.uuid}: ExecutionRecord ${record.uuid})`);
|
|
380
|
-
|
|
381
|
-
// Create property links
|
|
382
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
383
|
-
if (value !== null && value !== undefined) {
|
|
384
|
-
const escapedValue =
|
|
385
|
-
typeof value === 'object' ? JSON.stringify(value) : String(value);
|
|
386
|
-
// Using format: (uuid.property: property "value")
|
|
387
|
-
links.push(`(${record.uuid}.${key}: ${key} "${escapedValue}")`);
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
// Format: () ((links)) - creates new links
|
|
392
|
-
return `() ((${links.join(') (')}))`;
|
|
395
|
+
buildClinkDeletePropertiesQuery(uuid) {
|
|
396
|
+
return `(($id: ${uuid} $value)) ()`;
|
|
393
397
|
}
|
|
394
398
|
|
|
395
399
|
/**
|
|
396
400
|
* Execute clink command
|
|
397
401
|
* @param {string} query
|
|
402
|
+
* @param {string[]} [extraArgs] - flags appended after `--db <path>`
|
|
398
403
|
* @returns {{success: boolean, output: string}}
|
|
399
404
|
*/
|
|
400
|
-
execClink(query) {
|
|
405
|
+
execClink(query, extraArgs = []) {
|
|
401
406
|
try {
|
|
402
407
|
// The query embeds record values verbatim; a shell string would let a
|
|
403
408
|
// single quote in any of them break out of the argument
|
|
404
409
|
// (CodeQL js/shell-command-constructed-from-input, issue #168).
|
|
405
|
-
const result = execFileSync(
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
+
const result = execFileSync(
|
|
411
|
+
'clink',
|
|
412
|
+
[query, '--db', this.linksDbPath, ...extraArgs],
|
|
413
|
+
{
|
|
414
|
+
encoding: 'utf8',
|
|
415
|
+
timeout: 10000,
|
|
416
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
417
|
+
}
|
|
418
|
+
);
|
|
410
419
|
return { success: true, output: result };
|
|
411
420
|
} catch (err) {
|
|
412
421
|
this.log(`Clink error: ${err.message}`);
|
|
@@ -417,15 +426,22 @@ class ExecutionStore {
|
|
|
417
426
|
/**
|
|
418
427
|
* Write a record to the links database using clink
|
|
419
428
|
* @param {ExecutionRecord} record
|
|
429
|
+
* @param {{isNew?: boolean}} [options] - isNew skips dropping old links
|
|
420
430
|
* @returns {boolean}
|
|
421
431
|
*/
|
|
422
|
-
writeLinksRecord(record) {
|
|
432
|
+
writeLinksRecord(record, { isNew = false } = {}) {
|
|
423
433
|
if (!this.useLinks) {
|
|
424
434
|
return false;
|
|
425
435
|
}
|
|
426
436
|
|
|
427
|
-
|
|
428
|
-
|
|
437
|
+
// Replace, not merge: drop the previous property links first. clink
|
|
438
|
+
// reports an error when the record has no links yet, which is harmless.
|
|
439
|
+
if (!isNew) {
|
|
440
|
+
this.execClink(this.buildClinkDeletePropertiesQuery(record.uuid));
|
|
441
|
+
}
|
|
442
|
+
const result = this.execClink(this.buildClinkCreateQuery(record), [
|
|
443
|
+
'--auto-create-missing-references',
|
|
444
|
+
]);
|
|
429
445
|
|
|
430
446
|
if (result.success) {
|
|
431
447
|
this.log(`Wrote record ${record.uuid} to links database`);
|
|
@@ -444,9 +460,8 @@ class ExecutionStore {
|
|
|
444
460
|
return false;
|
|
445
461
|
}
|
|
446
462
|
|
|
447
|
-
|
|
448
|
-
const
|
|
449
|
-
const result = this.execClink(query);
|
|
463
|
+
this.execClink(this.buildClinkDeletePropertiesQuery(uuid));
|
|
464
|
+
const result = this.execClink(`((${uuid}: ExecutionRecord ${uuid})) ()`);
|
|
450
465
|
return result.success;
|
|
451
466
|
}
|
|
452
467
|
|
|
@@ -492,7 +507,7 @@ class ExecutionStore {
|
|
|
492
507
|
|
|
493
508
|
// Also write to links database if available
|
|
494
509
|
if (this.useLinks) {
|
|
495
|
-
this.writeLinksRecord(record);
|
|
510
|
+
this.writeLinksRecord(record, { isNew: existingIndex < 0 });
|
|
496
511
|
}
|
|
497
512
|
|
|
498
513
|
return true;
|
|
@@ -810,8 +825,10 @@ class ExecutionStore {
|
|
|
810
825
|
|
|
811
826
|
// Query links database for all ExecutionRecord links
|
|
812
827
|
try {
|
|
828
|
+
// Without --changes clink prints nothing for a read-only match
|
|
813
829
|
const queryResult = this.execClink(
|
|
814
|
-
`((($id: ExecutionRecord $uuid)) (($id: ExecutionRecord $uuid)))
|
|
830
|
+
`((($id: ExecutionRecord $uuid)) (($id: ExecutionRecord $uuid)))`,
|
|
831
|
+
['--changes']
|
|
815
832
|
);
|
|
816
833
|
|
|
817
834
|
if (queryResult.success) {
|
|
@@ -928,6 +945,7 @@ module.exports = {
|
|
|
928
945
|
ExecutionStatus,
|
|
929
946
|
LockManager,
|
|
930
947
|
isClinkInstalled,
|
|
948
|
+
clinkReference,
|
|
931
949
|
DEFAULT_APP_FOLDER,
|
|
932
950
|
LINO_DB_FILE,
|
|
933
951
|
LINKS_DB_FILE,
|
package/src/lib/log-sanitizer.js
CHANGED
|
@@ -8,11 +8,10 @@
|
|
|
8
8
|
* local gh hosts.yml (--gh-config). It streams the log from stdin, keeps
|
|
9
9
|
* invalid UTF-8 bytes and writes a new private file atomically (--output).
|
|
10
10
|
*
|
|
11
|
-
*
|
|
12
|
-
* `Authorization: <scheme> <value>`
|
|
13
|
-
* rest of
|
|
14
|
-
*
|
|
15
|
-
* start with [REDACTED]. Remove it with sensitive-data-sanitizer#45.
|
|
11
|
+
* Since 0.12.1 the library masks only the scheme and token of an
|
|
12
|
+
* `Authorization: <scheme> <value>` header, so a logged one-line JSON prompt
|
|
13
|
+
* keeps the rest of its line (issue #249, sensitive-data-sanitizer#45); the
|
|
14
|
+
* pre-pass that did this here is gone (issue #253).
|
|
16
15
|
*/
|
|
17
16
|
|
|
18
17
|
const fs = require('fs');
|
|
@@ -27,89 +26,6 @@ const COMMAND = 'sensitive-data-sanitizer';
|
|
|
27
26
|
const OVERRIDE = 'START_SENSITIVE_DATA_SANITIZER';
|
|
28
27
|
const OPTIONS = ['--credentials-only', '--env-secrets', '--gh-config'];
|
|
29
28
|
|
|
30
|
-
// The longest header and scheme prefix is 122 bytes.
|
|
31
|
-
const HOLDBACK = 160;
|
|
32
|
-
const HEADER =
|
|
33
|
-
/((?:Proxy-)?Authorization[ \t]{0,32}:[ \t]{0,32})(?:Bearer|Basic|Digest|token)[ \t]{1,32}/gi;
|
|
34
|
-
// Shell expressions, placeholders and masked values are not credentials.
|
|
35
|
-
const NOT_A_VALUE = '$<[{';
|
|
36
|
-
|
|
37
|
-
function isValueByte(code) {
|
|
38
|
-
return (
|
|
39
|
-
code > 0x20 &&
|
|
40
|
-
code !== 0x22 &&
|
|
41
|
-
code !== 0x27 &&
|
|
42
|
-
code !== 0x2c &&
|
|
43
|
-
code !== 0x5c
|
|
44
|
-
);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
function valueEnd(text, start) {
|
|
48
|
-
let end = start;
|
|
49
|
-
while (end < text.length && isValueByte(text.charCodeAt(end))) {
|
|
50
|
-
end++;
|
|
51
|
-
}
|
|
52
|
-
return end;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
/**
|
|
56
|
-
* Masks `Authorization: <scheme> <token>` as `Authorization: [REDACTED]`.
|
|
57
|
-
* It works on bytes (latin1 maps each byte to one character), so invalid
|
|
58
|
-
* UTF-8 passes through unchanged, and holds at most HOLDBACK pending bytes.
|
|
59
|
-
*/
|
|
60
|
-
class AuthorizationHeaderMask {
|
|
61
|
-
constructor() {
|
|
62
|
-
this.pending = '';
|
|
63
|
-
this.inValue = false;
|
|
64
|
-
this.redactions = 0;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
push(block, final = false) {
|
|
68
|
-
const text = this.pending + block.toString('latin1');
|
|
69
|
-
this.pending = '';
|
|
70
|
-
let output = '';
|
|
71
|
-
let offset = 0;
|
|
72
|
-
if (this.inValue) {
|
|
73
|
-
offset = valueEnd(text, 0);
|
|
74
|
-
if (offset === text.length && !final) {
|
|
75
|
-
return Buffer.alloc(0);
|
|
76
|
-
}
|
|
77
|
-
this.inValue = false;
|
|
78
|
-
}
|
|
79
|
-
HEADER.lastIndex = offset;
|
|
80
|
-
let match;
|
|
81
|
-
while ((match = HEADER.exec(text))) {
|
|
82
|
-
const valueStart = match.index + match[0].length;
|
|
83
|
-
if (valueStart === text.length && !final) {
|
|
84
|
-
output += text.slice(offset, match.index);
|
|
85
|
-
this.pending = text.slice(match.index);
|
|
86
|
-
return Buffer.from(output, 'latin1');
|
|
87
|
-
}
|
|
88
|
-
if (
|
|
89
|
-
valueStart === text.length ||
|
|
90
|
-
!isValueByte(text.charCodeAt(valueStart)) ||
|
|
91
|
-
NOT_A_VALUE.includes(text[valueStart])
|
|
92
|
-
) {
|
|
93
|
-
continue;
|
|
94
|
-
}
|
|
95
|
-
output += text.slice(offset, match.index + match[1].length) + MARKER;
|
|
96
|
-
this.redactions++;
|
|
97
|
-
offset = valueEnd(text, valueStart);
|
|
98
|
-
if (offset === text.length && !final) {
|
|
99
|
-
this.inValue = true;
|
|
100
|
-
return Buffer.from(output, 'latin1');
|
|
101
|
-
}
|
|
102
|
-
HEADER.lastIndex = offset;
|
|
103
|
-
}
|
|
104
|
-
const safeEnd = final
|
|
105
|
-
? text.length
|
|
106
|
-
: Math.max(offset, text.length - HOLDBACK);
|
|
107
|
-
output += text.slice(offset, safeEnd);
|
|
108
|
-
this.pending = text.slice(safeEnd);
|
|
109
|
-
return Buffer.from(output, 'latin1');
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
|
|
113
29
|
/** Value-free failure: `code` is a fixed ERR_* name, never log content. */
|
|
114
30
|
class SanitizationError extends Error {
|
|
115
31
|
constructor(code, subject = 'Log') {
|
|
@@ -195,10 +111,9 @@ function runSanitizer(args, options, subject) {
|
|
|
195
111
|
* @returns {string}
|
|
196
112
|
*/
|
|
197
113
|
function sanitizeText(text, options = {}) {
|
|
198
|
-
const input = new AuthorizationHeaderMask().push(Buffer.from(text), true);
|
|
199
114
|
const result = runSanitizer(
|
|
200
115
|
['redact', '-', ...OPTIONS],
|
|
201
|
-
{ ...options, input },
|
|
116
|
+
{ ...options, input: text },
|
|
202
117
|
'Text'
|
|
203
118
|
);
|
|
204
119
|
return result.stdout.toString();
|
|
@@ -230,46 +145,6 @@ function countMarkers(filePath) {
|
|
|
230
145
|
}
|
|
231
146
|
}
|
|
232
147
|
|
|
233
|
-
function createPrivateFile(filePath) {
|
|
234
|
-
const descriptor = fs.openSync(filePath, 'wx', 0o600);
|
|
235
|
-
if (process.platform !== 'win32') {
|
|
236
|
-
fs.fchmodSync(descriptor, 0o600);
|
|
237
|
-
}
|
|
238
|
-
return descriptor;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
function maskAuthorizationHeaders(sourcePath, stagePath) {
|
|
242
|
-
const source = fs.openSync(sourcePath, 'r');
|
|
243
|
-
let stage;
|
|
244
|
-
try {
|
|
245
|
-
if (!fs.fstatSync(source).isFile()) {
|
|
246
|
-
throw new SanitizationError('ERR_SOURCE');
|
|
247
|
-
}
|
|
248
|
-
stage = createPrivateFile(stagePath);
|
|
249
|
-
const mask = new AuthorizationHeaderMask();
|
|
250
|
-
const buffer = Buffer.alloc(BLOCK_SIZE);
|
|
251
|
-
let bytes = 0;
|
|
252
|
-
for (;;) {
|
|
253
|
-
const length = fs.readSync(source, buffer, 0, BLOCK_SIZE, null);
|
|
254
|
-
const output = mask.push(buffer.subarray(0, length), length === 0);
|
|
255
|
-
if (output.length) {
|
|
256
|
-
fs.writeSync(stage, output);
|
|
257
|
-
}
|
|
258
|
-
if (length === 0) {
|
|
259
|
-
break;
|
|
260
|
-
}
|
|
261
|
-
bytes += length;
|
|
262
|
-
}
|
|
263
|
-
fs.fsyncSync(stage);
|
|
264
|
-
return bytes;
|
|
265
|
-
} finally {
|
|
266
|
-
fs.closeSync(source);
|
|
267
|
-
if (stage !== undefined) {
|
|
268
|
-
fs.closeSync(stage);
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
|
|
273
148
|
/**
|
|
274
149
|
* Create a private sanitized copy of a log. The source is never modified;
|
|
275
150
|
* any failure removes the copy and throws a value-free SanitizationError.
|
|
@@ -288,19 +163,25 @@ function sanitizeLogToTemp(sourcePath, options = {}) {
|
|
|
288
163
|
if (process.platform !== 'win32') {
|
|
289
164
|
fs.chmodSync(directory, 0o700);
|
|
290
165
|
}
|
|
291
|
-
const stagePath = path.join(directory, 'authorization-masked.log');
|
|
292
166
|
const target = path.join(directory, 'execution.log');
|
|
293
167
|
code = 'ERR_SOURCE';
|
|
294
|
-
const
|
|
295
|
-
|
|
296
|
-
const input = fs.openSync(stagePath, 'r');
|
|
168
|
+
const input = fs.openSync(sourcePath, 'r');
|
|
169
|
+
let bytes;
|
|
297
170
|
try {
|
|
298
|
-
const
|
|
171
|
+
const source = fs.fstatSync(input);
|
|
172
|
+
if (!source.isFile()) {
|
|
173
|
+
throw new SanitizationError('ERR_SOURCE');
|
|
174
|
+
}
|
|
175
|
+
bytes = source.size;
|
|
176
|
+
const size = Math.max(1, bytes);
|
|
177
|
+
code = 'ERR_PRIVATE_FILE';
|
|
299
178
|
runSanitizer(
|
|
300
179
|
[
|
|
301
180
|
'redact',
|
|
302
181
|
'-',
|
|
303
182
|
'--stream',
|
|
183
|
+
// A NUL byte in a terminal transcript must not block the upload.
|
|
184
|
+
'--allow-binary',
|
|
304
185
|
...OPTIONS,
|
|
305
186
|
'--max-bytes',
|
|
306
187
|
String(size),
|
|
@@ -326,7 +207,6 @@ function sanitizeLogToTemp(sourcePath, options = {}) {
|
|
|
326
207
|
`Prepared private sanitized log: ${bytes} bytes scanned, ${Math.max(0, redactions)} redactions.`
|
|
327
208
|
);
|
|
328
209
|
}
|
|
329
|
-
fs.rmSync(stagePath, { force: true });
|
|
330
210
|
return { path: target, cleanup };
|
|
331
211
|
} catch (error) {
|
|
332
212
|
cleanup();
|
|
@@ -341,7 +221,6 @@ function sanitizeLogToTemp(sourcePath, options = {}) {
|
|
|
341
221
|
}
|
|
342
222
|
|
|
343
223
|
module.exports = {
|
|
344
|
-
AuthorizationHeaderMask,
|
|
345
224
|
BLOCK_SIZE,
|
|
346
225
|
SanitizationError,
|
|
347
226
|
resolveSanitizerCommand,
|
|
@@ -504,6 +504,24 @@ describe('CI workflow invariants', () => {
|
|
|
504
504
|
assert.match(lint.body, /cargo metadata --locked/);
|
|
505
505
|
});
|
|
506
506
|
|
|
507
|
+
it('gates pull requests on every dependency being its latest release', () => {
|
|
508
|
+
// Obsolete dependencies, sensitive-data-sanitizer among them, stayed in
|
|
509
|
+
// use unnoticed (issue #253).
|
|
510
|
+
const jsJobs = parseJobs(readWorkflow('js.yml'));
|
|
511
|
+
const freshness = jsJobs.find((job) => job.name === 'dependency-freshness');
|
|
512
|
+
assert.ok(freshness, 'js.yml has no dependency-freshness job');
|
|
513
|
+
assert.match(
|
|
514
|
+
freshness.body,
|
|
515
|
+
/scripts\/check-dependency-freshness\.mjs --lockfiles\n/
|
|
516
|
+
);
|
|
517
|
+
const status = jsJobs.find((job) => job.name === 'pipeline-status');
|
|
518
|
+
assert.ok(parseNeeds(status.body).includes('dependency-freshness'));
|
|
519
|
+
const lint = parseJobs(readWorkflow('rust.yml')).find(
|
|
520
|
+
(job) => job.name === 'lint'
|
|
521
|
+
);
|
|
522
|
+
assert.match(lint.body, /cargo update --dry-run/);
|
|
523
|
+
});
|
|
524
|
+
|
|
507
525
|
it('reports a failed Rust test as a failed test, not low coverage', () => {
|
|
508
526
|
// Without pipefail a failing test reached the threshold check as an empty
|
|
509
527
|
// value: "Coverage % is below the 50% threshold" (issue #234).
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Issue #253: command-stream 2.x returns completed `stdout` and `stderr` as
|
|
3
|
+
* readable stream snapshots instead of strings. `--use-command-stream` must
|
|
4
|
+
* still write the command's output to its log file.
|
|
5
|
+
*/
|
|
6
|
+
const { describe, it, expect } = require('bun:test');
|
|
7
|
+
const { spawnSync } = require('child_process');
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const os = require('os');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const { pathToFileURL } = require('url');
|
|
12
|
+
|
|
13
|
+
const CLI = path.join(__dirname, '../src/bin/cli.js');
|
|
14
|
+
|
|
15
|
+
describe('--use-command-stream log (#253)', () => {
|
|
16
|
+
it('loads execution without compatibility and terminal modules', () => {
|
|
17
|
+
const wrapper = pathToFileURL(
|
|
18
|
+
path.join(__dirname, '../src/lib/command-stream.js')
|
|
19
|
+
).href;
|
|
20
|
+
const result = spawnSync(
|
|
21
|
+
'node',
|
|
22
|
+
[
|
|
23
|
+
'--input-type=module',
|
|
24
|
+
'-e',
|
|
25
|
+
`import { registerHooks } from 'node:module';
|
|
26
|
+
import assert from 'node:assert/strict';
|
|
27
|
+
const extras = [];
|
|
28
|
+
registerHooks({ load(url, context, next) {
|
|
29
|
+
if (/command-stream\\/src\\/(shelljs|terminal|rust-repl)/.test(url)) extras.push(url);
|
|
30
|
+
return next(url, context);
|
|
31
|
+
}});
|
|
32
|
+
const { default: wrapper } = await import(${JSON.stringify(wrapper)});
|
|
33
|
+
const core = await wrapper.getCommandStream();
|
|
34
|
+
assert.equal(typeof core.$, 'function');
|
|
35
|
+
assert.equal(typeof core.ProcessRunner, 'function');
|
|
36
|
+
assert.deepEqual(extras, []);`,
|
|
37
|
+
],
|
|
38
|
+
{ encoding: 'utf8', timeout: 30000 }
|
|
39
|
+
);
|
|
40
|
+
expect(result.error).toBeUndefined();
|
|
41
|
+
expect(result.stderr).toBe('');
|
|
42
|
+
expect(result.status).toBe(0);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it.skipIf(process.platform === 'win32')(
|
|
46
|
+
'writes stdout and stderr to the log file',
|
|
47
|
+
() => {
|
|
48
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'start-253-'));
|
|
49
|
+
try {
|
|
50
|
+
const result = spawnSync(
|
|
51
|
+
process.execPath,
|
|
52
|
+
[
|
|
53
|
+
CLI,
|
|
54
|
+
'--use-command-stream',
|
|
55
|
+
'--',
|
|
56
|
+
'sh',
|
|
57
|
+
'-c',
|
|
58
|
+
'echo out-253; echo err-253 >&2',
|
|
59
|
+
],
|
|
60
|
+
{
|
|
61
|
+
env: {
|
|
62
|
+
...process.env,
|
|
63
|
+
START_APP_FOLDER: path.join(root, 'app'),
|
|
64
|
+
START_LOG_DIR: path.join(root, 'logs'),
|
|
65
|
+
START_DISABLE_AUTO_ISSUE: '1',
|
|
66
|
+
START_DISABLE_LOG_UPLOAD: '1',
|
|
67
|
+
},
|
|
68
|
+
encoding: 'utf8',
|
|
69
|
+
timeout: 30000,
|
|
70
|
+
}
|
|
71
|
+
);
|
|
72
|
+
expect(result.status).toBe(0);
|
|
73
|
+
expect(result.stderr).not.toContain('Could not append log file');
|
|
74
|
+
const logDir = path.join(root, 'logs', 'direct');
|
|
75
|
+
const [logName] = fs.readdirSync(logDir);
|
|
76
|
+
const log = fs.readFileSync(path.join(logDir, logName), 'utf8');
|
|
77
|
+
expect(log).toContain('out-253\n');
|
|
78
|
+
expect(log).toContain('err-253\n');
|
|
79
|
+
} finally {
|
|
80
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
40000
|
|
84
|
+
);
|
|
85
|
+
});
|