start-command 0.39.1 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +2 -1
- package/eslint.config.mjs +1 -0
- package/package.json +9 -9
- package/src/bin/cli.js +7 -5
- package/src/lib/command-stream.js +19 -5
- package/src/lib/execution-chain.js +2 -2
- package/src/lib/execution-store.js +62 -44
- package/src/lib/failure-handler.js +12 -6
- package/src/lib/log-sanitizer.js +191 -229
- package/src/lib/log-uploader.js +6 -3
- package/test/ci-workflow-invariants.js +18 -0
- package/test/command-stream-log.js +85 -0
- package/test/dependency-freshness.mjs +552 -0
- package/test/execution-store-clink.js +152 -0
- package/test/execution-store.js +49 -5
- package/test/fixtures/executions-lino-objects-codec-0.4.0.lino +1 -0
- package/test/fixtures/executions-lino-objects-codec-0.9.3.lino +26 -0
- package/test/javascript-first-ci.js +29 -0
- package/test/log-sanitizer.js +387 -151
- package/test/regression-187.js +4 -0
- package/test/regression-224.js +4 -2
- package/test/regression-231.js +20 -0
- package/test/release-git.mjs +2 -1
package/test/log-sanitizer.js
CHANGED
|
@@ -5,157 +5,310 @@ const fs = require('fs');
|
|
|
5
5
|
const os = require('os');
|
|
6
6
|
const path = require('path');
|
|
7
7
|
const { execFileSync } = require('child_process');
|
|
8
|
+
const { createHash } = require('node:crypto');
|
|
8
9
|
const {
|
|
10
|
+
SanitizationError,
|
|
11
|
+
resolveSanitizerCommand,
|
|
9
12
|
sanitizeLogToTemp,
|
|
10
|
-
|
|
13
|
+
sanitizeText,
|
|
11
14
|
} = require('../src/lib/log-sanitizer');
|
|
12
15
|
|
|
16
|
+
const GH_CONFIG_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'sanitize-gh-'));
|
|
17
|
+
|
|
18
|
+
// Only what the runtime needs, so the host's own credential-named variables
|
|
19
|
+
// and gh login cannot change the expected output.
|
|
20
|
+
function testEnv(extra = {}) {
|
|
21
|
+
const env = { GH_CONFIG_DIR };
|
|
22
|
+
for (const name of [
|
|
23
|
+
'PATH',
|
|
24
|
+
'Path',
|
|
25
|
+
'SystemRoot',
|
|
26
|
+
'SYSTEMROOT',
|
|
27
|
+
'TEMP',
|
|
28
|
+
'TMP',
|
|
29
|
+
'TMPDIR',
|
|
30
|
+
'HOME',
|
|
31
|
+
'USERPROFILE',
|
|
32
|
+
]) {
|
|
33
|
+
if (process.env[name] !== undefined) {
|
|
34
|
+
env[name] = process.env[name];
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return { ...env, ...extra };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function withLog(content, callback) {
|
|
41
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sanitize-test-'));
|
|
42
|
+
const source = path.join(dir, 'source.log');
|
|
43
|
+
fs.writeFileSync(source, content);
|
|
44
|
+
try {
|
|
45
|
+
return callback(source, dir);
|
|
46
|
+
} finally {
|
|
47
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function sanitizeFile(content, env = testEnv()) {
|
|
52
|
+
return withLog(content, (source) => {
|
|
53
|
+
const copy = sanitizeLogToTemp(source, { env });
|
|
54
|
+
try {
|
|
55
|
+
return fs.readFileSync(copy.path);
|
|
56
|
+
} finally {
|
|
57
|
+
copy.cleanup();
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// The three lines from issue #249, which the old rules truncated.
|
|
63
|
+
const ISSUE_249_LINES = [
|
|
64
|
+
"Command: gate='/tmp/hive-mind-disk-baseline-0ce9e170-e6d3-494d-91e1-0d820245d34b'",
|
|
65
|
+
'🧠 Formal AI: endpoint http://172.18.0.2:8080/api/openai/v1',
|
|
66
|
+
'{"bodyPreview":"{\\"messages\\":[{\\"content\\":\\"use: curl -L -H \\\\\\"Authorization: token $(gh auth token)\\\\\\" -o <filename> <url>\\\\n...the rest of a 100 KB prompt...\\"}]}"}',
|
|
67
|
+
'proxy http://myproxy:3128 and http://127.0.0.1:34271/session',
|
|
68
|
+
'Authorization: [REDACTED] stays as it is',
|
|
69
|
+
'account konard@example.com https://github.com/konard',
|
|
70
|
+
];
|
|
71
|
+
|
|
13
72
|
function credentials() {
|
|
14
73
|
return [
|
|
15
74
|
...'pousr'.split('').map((kind) => `gh${kind}_${'a'.repeat(36)}`),
|
|
16
75
|
`github_` + `pat_${'b'.repeat(82)}`,
|
|
17
76
|
`sk-${'c'.repeat(48)}`,
|
|
18
77
|
`sk-` + `proj-${'d'.repeat(90)}`,
|
|
19
|
-
`sk-` + `ant-api03-${'e'.repeat(90)}`,
|
|
20
78
|
`AK` + `IA${'F'.repeat(16)}`,
|
|
21
79
|
`AS` + `IA${'G'.repeat(16)}`,
|
|
22
80
|
'ey' + 'JhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.signature',
|
|
23
|
-
'
|
|
24
|
-
'
|
|
25
|
-
'
|
|
26
|
-
'
|
|
81
|
+
'arbitrary-credential',
|
|
82
|
+
'encoded-credential',
|
|
83
|
+
'password',
|
|
84
|
+
'arbitrary-aws-secret',
|
|
27
85
|
'custom-secret-value',
|
|
28
|
-
`${Buffer.from(' \n{"alg":"HS256"}').toString('base64url')}.${Buffer.from('{"sub":"x"}').toString('base64url')}.signature`,
|
|
29
86
|
];
|
|
30
87
|
}
|
|
31
88
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
89
|
+
describe('log sanitization', () => {
|
|
90
|
+
it('requires the sanitizer release the Rust installer asks for', () => {
|
|
91
|
+
// An older CLI rejects --allow-binary (ERR_ARGUMENT), so the bundled
|
|
92
|
+
// dependency and the Rust installer and version check agree.
|
|
93
|
+
const range =
|
|
94
|
+
require('../package.json').dependencies[
|
|
95
|
+
'@link-foundation/sensitive-data-sanitizer'
|
|
96
|
+
];
|
|
97
|
+
const rust = fs.readFileSync(
|
|
98
|
+
path.join(__dirname, '../../rust/src/lib/log_sanitizer.rs'),
|
|
99
|
+
'utf8'
|
|
100
|
+
);
|
|
101
|
+
const [, major, minor, patch] = range.match(/^\^(\d+)\.(\d+)\.(\d+)$/);
|
|
102
|
+
assert.ok(
|
|
103
|
+
rust.includes(
|
|
104
|
+
`PACKAGE: &str = "@link-foundation/sensitive-data-sanitizer@${range}";`
|
|
105
|
+
)
|
|
106
|
+
);
|
|
107
|
+
assert.ok(
|
|
108
|
+
rust.includes(
|
|
109
|
+
`MINIMUM_VERSION: (u64, u64, u64) = (${major}, ${minor}, ${patch});`
|
|
110
|
+
)
|
|
111
|
+
);
|
|
35
112
|
});
|
|
36
|
-
const blocks = [];
|
|
37
|
-
for (let offset = 0; offset < input.length; offset += blockSize) {
|
|
38
|
-
blocks.push(sanitizer.push(input.subarray(offset, offset + blockSize)));
|
|
39
|
-
}
|
|
40
|
-
blocks.push(sanitizer.push(Buffer.alloc(0), true));
|
|
41
|
-
return Buffer.concat(blocks);
|
|
42
|
-
}
|
|
43
113
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
for (const size of [1, 7, 31, 256, 1024, 65536]) {
|
|
48
|
-
const input = Buffer.from(
|
|
49
|
-
`ordinary utf8: café 😀\n${' '.repeat(Math.max(0, size - 43))}${secrets.join('\n')}\n${'padding\n'.repeat(400)}finished\n`
|
|
50
|
-
);
|
|
51
|
-
const output = sanitize(input, size).toString();
|
|
52
|
-
for (const secret of secrets) {
|
|
53
|
-
assert.ok(!output.includes(secret));
|
|
54
|
-
}
|
|
55
|
-
for (const value of [
|
|
56
|
-
'arbitrary-credential',
|
|
57
|
-
'encoded-credential',
|
|
58
|
-
'password',
|
|
59
|
-
'arbitrary-aws-secret',
|
|
60
|
-
]) {
|
|
61
|
-
assert.ok(!output.includes(value));
|
|
62
|
-
}
|
|
63
|
-
assert.ok(output.startsWith('ordinary utf8: café 😀\n'));
|
|
64
|
-
assert.ok(output.endsWith('\nfinished\n'));
|
|
65
|
-
}
|
|
114
|
+
it('keeps every line from issue #249 and public account details', () => {
|
|
115
|
+
const input = `${ISSUE_249_LINES.join('\n')}\n`;
|
|
116
|
+
assert.strictEqual(sanitizeFile(input).toString(), input);
|
|
66
117
|
});
|
|
67
118
|
|
|
68
|
-
it('
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
119
|
+
it('redacts credentials and credential-named environment values', () => {
|
|
120
|
+
const input = [
|
|
121
|
+
...credentials().slice(0, 9),
|
|
122
|
+
'Authorization: Bearer arbitrary-credential',
|
|
123
|
+
'Proxy-Authorization: Basic encoded-credential',
|
|
124
|
+
'clone https://user:password@example.test/repo.git',
|
|
125
|
+
'AWS_SECRET_ACCESS_KEY=arbitrary-aws-secret',
|
|
126
|
+
'value custom-secret-value end',
|
|
127
|
+
'ordinary utf8: café 😀',
|
|
128
|
+
'',
|
|
129
|
+
].join('\n');
|
|
130
|
+
const output = sanitizeFile(
|
|
131
|
+
input,
|
|
132
|
+
testEnv({ START_TEST_SECRET: 'custom-secret-value' })
|
|
133
|
+
).toString();
|
|
134
|
+
for (const [index, secret] of credentials().entries()) {
|
|
135
|
+
assert.ok(!output.includes(secret), `leaked secret #${index}`);
|
|
136
|
+
}
|
|
137
|
+
for (const kept of [
|
|
138
|
+
'Authorization: [REDACTED]\n',
|
|
139
|
+
'Proxy-Authorization: [REDACTED]\n',
|
|
140
|
+
'clone https://[REDACTED]@example.test/repo.git\n',
|
|
141
|
+
'value [REDACTED] end\n',
|
|
142
|
+
'ordinary utf8: café 😀\n',
|
|
143
|
+
]) {
|
|
144
|
+
assert.ok(output.includes(kept), `missing ${kept}`);
|
|
84
145
|
}
|
|
85
146
|
});
|
|
86
147
|
|
|
87
|
-
it('
|
|
88
|
-
const
|
|
148
|
+
it('masks only the Authorization scheme and token, keeping the rest of the line (#249)', () => {
|
|
149
|
+
const cases = [
|
|
150
|
+
[
|
|
151
|
+
`curl -H 'Authorization: Basic dXNlcjpwYXNz' -o out.txt url`,
|
|
152
|
+
`curl -H 'Authorization: [REDACTED]' -o out.txt url`,
|
|
153
|
+
],
|
|
154
|
+
[
|
|
155
|
+
'Authorization: Bearer abc rest of line',
|
|
156
|
+
'Authorization: [REDACTED] rest of line',
|
|
157
|
+
],
|
|
158
|
+
[
|
|
159
|
+
'Authorization: token abc, other: keep',
|
|
160
|
+
'Authorization: [REDACTED], other: keep',
|
|
161
|
+
],
|
|
162
|
+
[
|
|
163
|
+
'{"h":"Authorization: Bearer abc\\" tail","next":"keep"}',
|
|
164
|
+
'{"h":"Authorization: [REDACTED]\\" tail","next":"keep"}',
|
|
165
|
+
],
|
|
166
|
+
[
|
|
167
|
+
'Authorization: token $(gh auth token) and ${GITHUB_TOKEN}',
|
|
168
|
+
'Authorization: token $(gh auth token) and ${GITHUB_TOKEN}',
|
|
169
|
+
],
|
|
170
|
+
// Since 0.12.2, placeholders and their scheme remain unchanged.
|
|
171
|
+
['Authorization: Bearer <token>', 'Authorization: Bearer <token>'],
|
|
172
|
+
];
|
|
173
|
+
// sensitive-data-sanitizer 0.12.1 does this itself (#45), without the
|
|
174
|
+
// pre-pass start used to run (#253).
|
|
175
|
+
const input = cases.map(([line]) => line).join('\n');
|
|
176
|
+
const expected = cases.map(([, line]) => line).join('\n');
|
|
177
|
+
assert.strictEqual(sanitizeFile(input).toString(), expected);
|
|
178
|
+
assert.strictEqual(sanitizeText(input, { env: testEnv() }), expected);
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
it('masks long Authorization tokens across stream blocks', () => {
|
|
89
182
|
const input = Buffer.from(
|
|
90
|
-
`Authorization: Bearer
|
|
183
|
+
`Authorization: Bearer ${'a'.repeat(200000)} tail\nordinary\n`
|
|
91
184
|
);
|
|
92
|
-
const output = [];
|
|
93
|
-
for (let offset = 0; offset < input.length; offset += 65536) {
|
|
94
|
-
output.push(sanitizer.push(input.subarray(offset, offset + 65536)));
|
|
95
|
-
}
|
|
96
|
-
output.push(sanitizer.push(Buffer.alloc(0), true));
|
|
97
185
|
assert.strictEqual(
|
|
98
|
-
|
|
99
|
-
'[REDACTED]\nordinary\n'
|
|
186
|
+
sanitizeFile(input).toString(),
|
|
187
|
+
'Authorization: [REDACTED] tail\nordinary\n'
|
|
100
188
|
);
|
|
101
189
|
});
|
|
102
190
|
|
|
103
|
-
it('
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
191
|
+
it('preserves Authorization placeholders and control bytes when sanitizing twice (#253)', () => {
|
|
192
|
+
const input = [
|
|
193
|
+
'Authorization: Bearer [REDACTED] already',
|
|
194
|
+
'Authorization: Bearer {{token}} x',
|
|
195
|
+
'Authorization: Bearer <token> x',
|
|
196
|
+
'Authorization: Bearer $GITHUB_TOKEN x',
|
|
197
|
+
'Authorization: Bearer abc\0 tail',
|
|
198
|
+
'Authorization: Bearer abc\x1b[0m tail',
|
|
199
|
+
'{"header":"Authorization: Bearer ab[c]d","next":"keep"}',
|
|
200
|
+
].join('\n');
|
|
201
|
+
const expected = [
|
|
202
|
+
'Authorization: Bearer [REDACTED] already',
|
|
203
|
+
// Arbitrary bracketed values are credentials; known references above
|
|
204
|
+
// stay intact, but every bracket of this value must be removed (#50).
|
|
205
|
+
'Authorization: [REDACTED] x',
|
|
206
|
+
'Authorization: Bearer <token> x',
|
|
207
|
+
'Authorization: Bearer $GITHUB_TOKEN x',
|
|
208
|
+
'Authorization: [REDACTED]\0 tail',
|
|
209
|
+
'Authorization: [REDACTED]\x1b[0m tail',
|
|
210
|
+
'{"header":"Authorization: [REDACTED]","next":"keep"}',
|
|
211
|
+
].join('\n');
|
|
212
|
+
const output = sanitizeFile(input);
|
|
213
|
+
assert.strictEqual(output.toString(), expected);
|
|
214
|
+
assert.deepStrictEqual(sanitizeFile(output), output);
|
|
115
215
|
});
|
|
116
216
|
|
|
117
|
-
it(
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
217
|
+
it(
|
|
218
|
+
'sanitizes dense credentials within a bounded process (#253, sanitizer #51)',
|
|
219
|
+
{ timeout: 25000 },
|
|
220
|
+
() => {
|
|
221
|
+
const count = 5700;
|
|
222
|
+
const input = Array.from({ length: count }, (_, index) => {
|
|
223
|
+
const token = createHash('sha256').update(String(index)).digest('hex');
|
|
224
|
+
return `gh${'p_'}${token.slice(0, 36)} keep\n`;
|
|
225
|
+
}).join('');
|
|
226
|
+
const { command, args } = resolveSanitizerCommand({});
|
|
227
|
+
const output = execFileSync(
|
|
228
|
+
command,
|
|
229
|
+
[
|
|
230
|
+
'--max-old-space-size=384',
|
|
231
|
+
...args,
|
|
232
|
+
'redact',
|
|
233
|
+
'-',
|
|
234
|
+
'--stream',
|
|
235
|
+
'--credentials-only',
|
|
236
|
+
'--allow-binary',
|
|
237
|
+
],
|
|
238
|
+
{
|
|
239
|
+
input,
|
|
240
|
+
env: testEnv(),
|
|
241
|
+
timeout: 20000,
|
|
242
|
+
killSignal: 'SIGKILL',
|
|
243
|
+
maxBuffer: 1024 * 1024,
|
|
244
|
+
}
|
|
245
|
+
);
|
|
246
|
+
assert.strictEqual(output.toString(), '[REDACTED] keep\n'.repeat(count));
|
|
247
|
+
}
|
|
248
|
+
);
|
|
249
|
+
|
|
250
|
+
it('preserves invalid UTF-8 bytes, NUL bytes and line endings', () => {
|
|
251
|
+
const input = Buffer.concat([
|
|
252
|
+
Buffer.from('bad '),
|
|
253
|
+
Buffer.from([0xff, 0xfe, 0xc3]),
|
|
254
|
+
Buffer.from(` gh${'p_'}${'x'.repeat(36)}\r\nnext\rend`),
|
|
255
|
+
]);
|
|
256
|
+
const expected = Buffer.concat([
|
|
257
|
+
Buffer.from('bad '),
|
|
258
|
+
Buffer.from([0xff, 0xfe, 0xc3]),
|
|
259
|
+
Buffer.from(' [REDACTED]\r\nnext\rend'),
|
|
260
|
+
]);
|
|
261
|
+
assert.deepStrictEqual(sanitizeFile(input), expected);
|
|
262
|
+
assert.deepStrictEqual(sanitizeFile(Buffer.alloc(0)), Buffer.alloc(0));
|
|
263
|
+
// --allow-binary: a NUL byte in a terminal transcript no longer blocks
|
|
264
|
+
// the upload with ERR_BINARY (#253), and credentials next to it are masked.
|
|
265
|
+
assert.strictEqual(
|
|
266
|
+
sanitizeFile(`line\0one gh${'p_'}${'x'.repeat(36)}\0two\n`).toString(),
|
|
267
|
+
'line\0one [REDACTED]\0two\n'
|
|
268
|
+
);
|
|
126
269
|
});
|
|
127
270
|
|
|
128
|
-
it('
|
|
129
|
-
const
|
|
130
|
-
GH_TOKEN: 'abcde',
|
|
131
|
-
TEST_PASSWORD: 'cdefg',
|
|
132
|
-
GITHUB_PAT: 'opaque-known-pat',
|
|
133
|
-
});
|
|
271
|
+
it('sanitizes issue text with the same rules', () => {
|
|
272
|
+
const token = `gh${'p_'}${'a'.repeat(36)}`;
|
|
134
273
|
assert.strictEqual(
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
)
|
|
140
|
-
.toString(),
|
|
141
|
-
'[REDACTED] [REDACTED] example.test.invalid'
|
|
274
|
+
sanitizeText(`echo ${token} 😀 'Authorization: token abc' -o x`, {
|
|
275
|
+
env: testEnv(),
|
|
276
|
+
}),
|
|
277
|
+
`echo [REDACTED] 😀 'Authorization: [REDACTED]' -o x`
|
|
142
278
|
);
|
|
143
|
-
const short = new StreamingSanitizer({ GH_TOKEN: 'a' });
|
|
144
279
|
assert.strictEqual(
|
|
145
|
-
|
|
146
|
-
'
|
|
280
|
+
sanitizeText('echo "quoted" $(id) `hostname`', { env: testEnv() }),
|
|
281
|
+
'echo "quoted" $(id) `hostname`'
|
|
282
|
+
);
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
it('resolves the override first and the bundled library otherwise', () => {
|
|
286
|
+
assert.deepStrictEqual(
|
|
287
|
+
resolveSanitizerCommand({ START_SENSITIVE_DATA_SANITIZER: '/x/sds' }),
|
|
288
|
+
{ command: '/x/sds', args: [] }
|
|
289
|
+
);
|
|
290
|
+
const bundled = resolveSanitizerCommand({});
|
|
291
|
+
assert.strictEqual(bundled.command, process.execPath);
|
|
292
|
+
assert.ok(
|
|
293
|
+
bundled.args[0].endsWith(
|
|
294
|
+
path.join(
|
|
295
|
+
'sensitive-data-sanitizer',
|
|
296
|
+
'bin',
|
|
297
|
+
'sensitive-data-sanitizer.js'
|
|
298
|
+
)
|
|
299
|
+
)
|
|
147
300
|
);
|
|
148
301
|
});
|
|
149
302
|
|
|
150
303
|
it('creates a private sanitized copy and removes it without changing the source', () => {
|
|
151
|
-
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sanitize-test-'));
|
|
152
|
-
const source = path.join(dir, 'source.log');
|
|
153
304
|
const input = `gh` + `p_${'x'.repeat(36)}`;
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
const copy = sanitizeLogToTemp(source, { env: {} });
|
|
305
|
+
withLog(input, (source) => {
|
|
306
|
+
const copy = sanitizeLogToTemp(source, { env: testEnv() });
|
|
157
307
|
assert.notStrictEqual(copy.path, source);
|
|
158
308
|
assert.strictEqual(fs.readFileSync(copy.path, 'utf8'), '[REDACTED]');
|
|
309
|
+
assert.deepStrictEqual(fs.readdirSync(path.dirname(copy.path)), [
|
|
310
|
+
'execution.log',
|
|
311
|
+
]);
|
|
159
312
|
if (process.platform !== 'win32') {
|
|
160
313
|
assert.strictEqual(fs.statSync(copy.path).mode & 0o777, 0o600);
|
|
161
314
|
assert.strictEqual(
|
|
@@ -165,76 +318,149 @@ describe('log sanitization', () => {
|
|
|
165
318
|
}
|
|
166
319
|
assert.strictEqual(fs.readFileSync(source, 'utf8'), input);
|
|
167
320
|
copy.cleanup();
|
|
168
|
-
assert.ok(!fs.existsSync(copy.path));
|
|
169
|
-
}
|
|
170
|
-
fs.rmSync(dir, { recursive: true, force: true });
|
|
171
|
-
}
|
|
321
|
+
assert.ok(!fs.existsSync(path.dirname(copy.path)));
|
|
322
|
+
});
|
|
172
323
|
});
|
|
173
324
|
|
|
174
|
-
it('
|
|
325
|
+
it('keeps private permissions even with a permissive umask', () => {
|
|
175
326
|
if (process.platform === 'win32') {
|
|
176
327
|
return;
|
|
177
328
|
}
|
|
178
|
-
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sanitize-creation-'));
|
|
179
|
-
const source = path.join(dir, 'source.log');
|
|
180
|
-
fs.writeFileSync(source, 'ordinary output');
|
|
181
|
-
const originalOpen = fs.openSync;
|
|
182
329
|
const originalUmask = process.umask(0);
|
|
183
|
-
let creationChecked = false;
|
|
184
|
-
let copy;
|
|
185
|
-
fs.openSync = (...args) => {
|
|
186
|
-
if (args[1] !== 'wx') {
|
|
187
|
-
return originalOpen(...args);
|
|
188
|
-
}
|
|
189
|
-
assert.strictEqual(
|
|
190
|
-
fs.statSync(path.dirname(args[0])).mode & 0o777,
|
|
191
|
-
0o700
|
|
192
|
-
);
|
|
193
|
-
const descriptor = originalOpen(...args);
|
|
194
|
-
assert.strictEqual(fs.fstatSync(descriptor).mode & 0o777, 0o600);
|
|
195
|
-
creationChecked = true;
|
|
196
|
-
return descriptor;
|
|
197
|
-
};
|
|
198
330
|
try {
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
331
|
+
withLog('ordinary output', (source) => {
|
|
332
|
+
const copy = sanitizeLogToTemp(source, { env: testEnv() });
|
|
333
|
+
try {
|
|
334
|
+
assert.strictEqual(
|
|
335
|
+
fs.statSync(path.dirname(copy.path)).mode & 0o777,
|
|
336
|
+
0o700
|
|
337
|
+
);
|
|
338
|
+
assert.strictEqual(fs.statSync(copy.path).mode & 0o777, 0o600);
|
|
339
|
+
assert.strictEqual(
|
|
340
|
+
fs.readFileSync(copy.path, 'utf8'),
|
|
341
|
+
'ordinary output'
|
|
342
|
+
);
|
|
343
|
+
} finally {
|
|
344
|
+
copy.cleanup();
|
|
345
|
+
}
|
|
346
|
+
});
|
|
202
347
|
} finally {
|
|
203
|
-
fs.openSync = originalOpen;
|
|
204
348
|
process.umask(originalUmask);
|
|
205
|
-
copy?.cleanup();
|
|
206
|
-
fs.rmSync(dir, { recursive: true, force: true });
|
|
207
349
|
}
|
|
208
350
|
});
|
|
209
351
|
|
|
210
|
-
it('
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
352
|
+
it('reports scanned bytes and added redactions in verbose mode only', () => {
|
|
353
|
+
const messages = [];
|
|
354
|
+
const originalLog = console.log;
|
|
355
|
+
console.log = (message) => messages.push(message);
|
|
356
|
+
try {
|
|
357
|
+
withLog(`[REDACTED] gh${'p_'}${'x'.repeat(36)}\n`, (source) => {
|
|
358
|
+
sanitizeLogToTemp(source, { env: testEnv() }).cleanup();
|
|
359
|
+
sanitizeLogToTemp(source, { env: testEnv(), verbose: true }).cleanup();
|
|
360
|
+
});
|
|
361
|
+
} finally {
|
|
362
|
+
console.log = originalLog;
|
|
363
|
+
}
|
|
364
|
+
assert.deepStrictEqual(messages, [
|
|
365
|
+
'Prepared private sanitized log: 52 bytes scanned, 1 redactions.',
|
|
366
|
+
]);
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
it('fails closed with value-free codes and leaves no copy behind', () => {
|
|
370
|
+
const before = fs
|
|
371
|
+
.readdirSync(os.tmpdir())
|
|
372
|
+
.filter((name) => name.startsWith('start-sanitized-')).length;
|
|
373
|
+
const attempt = (source, env = testEnv()) => {
|
|
374
|
+
try {
|
|
375
|
+
sanitizeLogToTemp(source, { env }).cleanup();
|
|
376
|
+
} catch (error) {
|
|
377
|
+
assert.ok(error instanceof SanitizationError);
|
|
378
|
+
return error;
|
|
379
|
+
}
|
|
380
|
+
assert.fail('sanitization must fail');
|
|
381
|
+
};
|
|
382
|
+
assert.match(attempt('/does-not-exist').message, /\(ERR_SOURCE\)/);
|
|
383
|
+
assert.match(attempt(os.tmpdir()).message, /\(ERR_SOURCE\)/);
|
|
384
|
+
withLog('ordinary log', (source, dir) => {
|
|
385
|
+
assert.strictEqual(
|
|
386
|
+
attempt(
|
|
387
|
+
source,
|
|
388
|
+
testEnv({
|
|
389
|
+
START_SENSITIVE_DATA_SANITIZER: path.join(dir, 'missing'),
|
|
390
|
+
})
|
|
391
|
+
).code,
|
|
392
|
+
'ERR_UNAVAILABLE'
|
|
393
|
+
);
|
|
394
|
+
if (process.platform !== 'win32') {
|
|
395
|
+
const failing = path.join(dir, 'failing-sanitizer');
|
|
396
|
+
fs.writeFileSync(
|
|
397
|
+
failing,
|
|
398
|
+
'#!/bin/sh\necho "secret-from-stderr (ERR_WORKER)" >&2\nexit 2\n',
|
|
399
|
+
{ mode: 0o755 }
|
|
400
|
+
);
|
|
401
|
+
const error = attempt(
|
|
402
|
+
source,
|
|
403
|
+
testEnv({ START_SENSITIVE_DATA_SANITIZER: failing })
|
|
404
|
+
);
|
|
405
|
+
assert.strictEqual(
|
|
406
|
+
error.message,
|
|
407
|
+
'Log sanitization failed (ERR_WORKER); upload blocked.'
|
|
408
|
+
);
|
|
409
|
+
// A sanitizer that succeeds without writing the output blocks too.
|
|
410
|
+
fs.writeFileSync(failing, '#!/bin/sh\nexit 0\n', { mode: 0o755 });
|
|
411
|
+
assert.strictEqual(
|
|
412
|
+
attempt(source, testEnv({ START_SENSITIVE_DATA_SANITIZER: failing }))
|
|
413
|
+
.code,
|
|
414
|
+
'ERR_OUTPUT'
|
|
415
|
+
);
|
|
416
|
+
}
|
|
417
|
+
});
|
|
215
418
|
assert.throws(
|
|
216
|
-
() =>
|
|
217
|
-
|
|
419
|
+
() =>
|
|
420
|
+
sanitizeText('x', {
|
|
421
|
+
env: testEnv({ START_SENSITIVE_DATA_SANITIZER: '/does-not-exist' }),
|
|
422
|
+
}),
|
|
423
|
+
/^Error: Text sanitization failed \(ERR_UNAVAILABLE\); publication blocked\.$/
|
|
218
424
|
);
|
|
425
|
+
const after = fs
|
|
426
|
+
.readdirSync(os.tmpdir())
|
|
427
|
+
.filter((name) => name.startsWith('start-sanitized-')).length;
|
|
428
|
+
assert.ok(after <= before);
|
|
219
429
|
});
|
|
220
430
|
|
|
221
431
|
it('blocks both upload paths on read or write failure and does not expose exception text', () => {
|
|
222
432
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sanitize-errors-'));
|
|
223
433
|
const source = path.join(dir, 'source.log');
|
|
224
434
|
fs.writeFileSync(source, 'ordinary log');
|
|
435
|
+
// A fake uploader, so a regression can never publish the log for real.
|
|
436
|
+
const uploaded = path.join(dir, 'uploaded');
|
|
437
|
+
fs.writeFileSync(
|
|
438
|
+
path.join(dir, 'gh-upload-log'),
|
|
439
|
+
`#!/bin/sh\ntouch "${uploaded}"\necho https://gist.github.com/test/id\n`,
|
|
440
|
+
{ mode: 0o755 }
|
|
441
|
+
);
|
|
442
|
+
const originalPath = process.env.PATH;
|
|
443
|
+
process.env.PATH = `${dir}${path.delimiter}${originalPath}`;
|
|
225
444
|
const { uploadLogPath } = require('../src/lib/log-uploader');
|
|
226
445
|
const { uploadLog } = require('../src/lib/failure-handler');
|
|
227
446
|
const messages = [];
|
|
228
447
|
const originalLog = console.log;
|
|
229
448
|
console.log = (message) => messages.push(message);
|
|
230
449
|
try {
|
|
231
|
-
|
|
450
|
+
// The sanitizer child reads and writes the files itself; start only
|
|
451
|
+
// opens and checks the source.
|
|
452
|
+
for (const method of ['openSync', 'fstatSync']) {
|
|
232
453
|
const original = fs[method];
|
|
233
454
|
fs[method] = () => {
|
|
234
455
|
throw new Error('sensitive error details');
|
|
235
456
|
};
|
|
236
457
|
try {
|
|
237
|
-
|
|
458
|
+
const result = uploadLogPath(source);
|
|
459
|
+
assert.strictEqual(result.success, false);
|
|
460
|
+
assert.strictEqual(
|
|
461
|
+
result.error,
|
|
462
|
+
'Log sanitization failed (ERR_SOURCE); upload blocked.'
|
|
463
|
+
);
|
|
238
464
|
assert.strictEqual(uploadLog(source), null);
|
|
239
465
|
} finally {
|
|
240
466
|
fs[method] = original;
|
|
@@ -245,8 +471,15 @@ describe('log sanitization', () => {
|
|
|
245
471
|
(message) => !message.includes('sensitive error details')
|
|
246
472
|
)
|
|
247
473
|
);
|
|
474
|
+
assert.ok(
|
|
475
|
+
messages.includes(
|
|
476
|
+
'Warning: Log sanitization failed (ERR_SOURCE); upload blocked.'
|
|
477
|
+
)
|
|
478
|
+
);
|
|
479
|
+
assert.ok(!fs.existsSync(uploaded));
|
|
248
480
|
} finally {
|
|
249
481
|
console.log = originalLog;
|
|
482
|
+
process.env.PATH = originalPath;
|
|
250
483
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
251
484
|
}
|
|
252
485
|
});
|
|
@@ -257,7 +490,9 @@ describe('log sanitization', () => {
|
|
|
257
490
|
}
|
|
258
491
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'upload-sanitize-test-'));
|
|
259
492
|
const log = path.join(dir, 'source.log');
|
|
260
|
-
|
|
493
|
+
const line =
|
|
494
|
+
"Command: gate='/tmp/hive-mind-disk-baseline-0ce9e170' http://172.18.0.2:8080/v1";
|
|
495
|
+
fs.writeFileSync(log, `${line} gh` + `p_${'x'.repeat(36)}`);
|
|
261
496
|
fs.writeFileSync(
|
|
262
497
|
path.join(dir, 'gh-upload-log'),
|
|
263
498
|
'#!/bin/sh\nprintf "%s\\n" "$@" > "$CAPTURE_ARGS"\ncat "$1" > "$CAPTURE_LOG"\nls -ld "$1" | cut -c1-10 > "$CAPTURE_MODE"\necho https://gist.github.com/test/id\n',
|
|
@@ -270,15 +505,14 @@ describe('log sanitization', () => {
|
|
|
270
505
|
? `require(${JSON.stringify(require.resolve('../src/lib/failure-handler'))}).uploadLog(process.env.SOURCE_LOG)`
|
|
271
506
|
: `require(${JSON.stringify(require.resolve('../src/lib/log-uploader'))}).uploadLogPath(process.env.SOURCE_LOG, { noSanitize: ${kind === 'optout'} })`;
|
|
272
507
|
execFileSync(process.execPath, ['-e', script], {
|
|
273
|
-
env: {
|
|
274
|
-
...process.env,
|
|
508
|
+
env: testEnv({
|
|
275
509
|
PATH: `${dir}${path.delimiter}${process.env.PATH}`,
|
|
276
510
|
SOURCE_LOG: log,
|
|
277
511
|
CAPTURE_ARGS: path.join(dir, 'args'),
|
|
278
512
|
CAPTURE_LOG: path.join(dir, 'uploaded'),
|
|
279
513
|
CAPTURE_MODE: path.join(dir, 'mode'),
|
|
280
514
|
GH_UPLOAD_PUBLIC: 'true',
|
|
281
|
-
},
|
|
515
|
+
}),
|
|
282
516
|
stdio: 'pipe',
|
|
283
517
|
});
|
|
284
518
|
const args = fs
|
|
@@ -288,7 +522,9 @@ describe('log sanitization', () => {
|
|
|
288
522
|
assert.strictEqual(args[1], '--private');
|
|
289
523
|
assert.strictEqual(
|
|
290
524
|
fs.readFileSync(path.join(dir, 'uploaded'), 'utf8'),
|
|
291
|
-
kind === 'optout'
|
|
525
|
+
kind === 'optout'
|
|
526
|
+
? fs.readFileSync(log, 'utf8')
|
|
527
|
+
: `${line} [REDACTED]`
|
|
292
528
|
);
|
|
293
529
|
if (kind !== 'optout') {
|
|
294
530
|
assert.notStrictEqual(args[0], log);
|