pi-cursor-sdk 0.1.28 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +39 -36
- package/docs/crabbox-platform-testing-lessons.md +508 -0
- package/docs/cursor-dogfood-checklist.md +4 -3
- package/docs/cursor-live-smoke-checklist.md +22 -20
- package/docs/cursor-model-ux-spec.md +13 -13
- package/docs/cursor-native-tool-replay.md +11 -11
- package/docs/cursor-native-tool-visual-audit.md +9 -7
- package/docs/cursor-testing-lessons.md +20 -15
- package/docs/cursor-tool-surfaces.md +5 -5
- package/docs/platform-smoke.md +994 -0
- package/package.json +32 -3
- package/platform-smoke.config.mjs +21 -0
- package/scripts/debug-provider-events.mjs +10 -3
- package/scripts/debug-sdk-events.mjs +10 -2
- package/scripts/isolated-cursor-smoke.sh +4 -4
- package/scripts/lib/cursor-visual-render.mjs +1 -0
- package/scripts/platform-smoke/artifacts.mjs +124 -0
- package/scripts/platform-smoke/assertions.mjs +101 -0
- package/scripts/platform-smoke/card-detect.mjs +96 -0
- package/scripts/platform-smoke/crabbox-runner.mjs +215 -0
- package/scripts/platform-smoke/doctor.mjs +446 -0
- package/scripts/platform-smoke/jsonl-text.mjs +31 -0
- package/scripts/platform-smoke/live-suite-runner.mjs +677 -0
- package/scripts/platform-smoke/platform-build-windows.ps1 +187 -0
- package/scripts/platform-smoke/pty-capture.mjs +131 -0
- package/scripts/platform-smoke/render-ansi.mjs +65 -0
- package/scripts/platform-smoke/scenarios.mjs +186 -0
- package/scripts/platform-smoke/targets.mjs +900 -0
- package/scripts/platform-smoke/visual-evidence.mjs +139 -0
- package/scripts/platform-smoke.mjs +193 -0
- package/scripts/probe-mcp-coldstart.mjs +8 -1
- package/scripts/steering-rpc-smoke.mjs +1 -1
- package/scripts/tmux-live-smoke.sh +3 -3
- package/scripts/visual-tui-smoke.mjs +1 -1
- package/src/context.ts +2 -4
- package/src/cursor-pi-tool-bridge-abort.ts +1 -0
- package/src/cursor-pi-tool-bridge-diagnostics.ts +12 -1
- package/src/cursor-pi-tool-bridge.ts +46 -1
- package/src/cursor-provider-turn-lifecycle-emitter.ts +65 -8
- package/src/cursor-provider-turn-tool-ledger.ts +2 -3
- package/src/cursor-run-final-text.ts +11 -1
- package/src/cursor-skill-tool.ts +273 -0
- package/src/cursor-state.ts +38 -19
- package/src/cursor-tool-lifecycle.ts +1 -1
- package/src/cursor-tool-manifest.ts +1 -1
- package/src/cursor-transcript-utils.ts +7 -3
- package/src/index.ts +3 -0
|
@@ -0,0 +1,900 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Target runner — real suite execution, artifact writing, and fail-through.
|
|
3
|
+
*
|
|
4
|
+
* Each target session: warmup → run suites → artifacts → stop.
|
|
5
|
+
* Live suites execute real Cursor-backed PTY/ConPTY runs and fail through with artifacts.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
9
|
+
import { dirname, isAbsolute, relative, resolve } from "node:path";
|
|
10
|
+
|
|
11
|
+
import { createSuiteDir, writeManifest, writeSummary, writeCommand, writeExitCode, scanArtifacts, scanForSecrets, redactSecrets } from "./artifacts.mjs";
|
|
12
|
+
import { runAssertions } from "./assertions.mjs";
|
|
13
|
+
import { getScenario } from "./scenarios.mjs";
|
|
14
|
+
import { warmupLease, runOnLease, stopLease } from "./crabbox-runner.mjs";
|
|
15
|
+
import { renderAll } from "./render-ansi.mjs";
|
|
16
|
+
import { assertRequiredCards, detectCards, writeCardArtifacts } from "./card-detect.mjs";
|
|
17
|
+
import { collectVisualEvidence } from "./visual-evidence.mjs";
|
|
18
|
+
import { extractContentText, extractFinalTextContent } from "./jsonl-text.mjs";
|
|
19
|
+
|
|
20
|
+
export function platformFor(targetName) {
|
|
21
|
+
return targetName === "windows-native" ? "powershell" : "posix";
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function makeRunId() {
|
|
25
|
+
return `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function finalizeSuiteArtifacts(suiteDir, checks, summaryData, expectedFiles) {
|
|
29
|
+
const assertions = runAssertions(suiteDir, checks);
|
|
30
|
+
writeSummary(suiteDir, { ...summaryData, ok: assertions.ok });
|
|
31
|
+
const expected = assertions.ok ? expectedFiles : [...expectedFiles, "failures.md"];
|
|
32
|
+
const manifest = writeManifest(suiteDir, expected);
|
|
33
|
+
if (manifest.missing.length === 0) return { assertions, manifest };
|
|
34
|
+
|
|
35
|
+
const finalAssertions = runAssertions(suiteDir, [
|
|
36
|
+
...checks,
|
|
37
|
+
{
|
|
38
|
+
id: "artifact-manifest-complete",
|
|
39
|
+
fn: () => false,
|
|
40
|
+
error: `missing required artifact(s): ${manifest.missing.join(", ")}`,
|
|
41
|
+
},
|
|
42
|
+
]);
|
|
43
|
+
writeSummary(suiteDir, { ...summaryData, ok: false });
|
|
44
|
+
const finalManifest = writeManifest(suiteDir, [...expectedFiles, "failures.md"]);
|
|
45
|
+
return { assertions: finalAssertions, manifest: finalManifest };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function writeRedactedFile(path, content) {
|
|
49
|
+
writeFileSync(path, redactSecrets(content ?? ""));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function writeStopLeaseArtifacts(suiteDir, stopResult) {
|
|
53
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.stop.stdout.txt"), stopResult.stdout ?? "");
|
|
54
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.stop.stderr.txt"), stopResult.stderr ?? "");
|
|
55
|
+
writeFileSync(resolve(suiteDir, "crabbox.stop.exit-code.txt"), `code=${stopResult.code}\nsignal=${stopResult.signal ?? "none"}\n`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function stopLeaseCheck(stopResult) {
|
|
59
|
+
return {
|
|
60
|
+
id: "lease-stop",
|
|
61
|
+
fn: () => stopResult?.code === 0,
|
|
62
|
+
error: `Crabbox stop failed (exit ${stopResult?.code ?? "unknown"}); check crabbox.stop.stderr.txt`,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function createLeaseCleanupResult(config, targetName, leaseId, stopResult, runId = makeRunId()) {
|
|
67
|
+
const suiteName = "lease-cleanup";
|
|
68
|
+
const suiteDir = createSuiteDir(config.artifactRoot, runId, targetName, suiteName);
|
|
69
|
+
writeFileSync(resolve(suiteDir, "target.json"), JSON.stringify({
|
|
70
|
+
targetName,
|
|
71
|
+
platform: platformFor(targetName),
|
|
72
|
+
slug: `${config.packageName ?? "pi-cursor-sdk"}-${targetName}`,
|
|
73
|
+
runId,
|
|
74
|
+
writtenAt: new Date().toISOString(),
|
|
75
|
+
}, null, 2));
|
|
76
|
+
writeFileSync(resolve(suiteDir, "suite.json"), JSON.stringify({
|
|
77
|
+
suiteName,
|
|
78
|
+
leaseId,
|
|
79
|
+
writtenAt: new Date().toISOString(),
|
|
80
|
+
}, null, 2));
|
|
81
|
+
writeCommand(suiteDir, `crabbox stop ${targetName} --id ${leaseId}`);
|
|
82
|
+
writeExitCode(suiteDir, stopResult.code, stopResult.signal);
|
|
83
|
+
writeStopLeaseArtifacts(suiteDir, stopResult);
|
|
84
|
+
const { assertions } = finalizeSuiteArtifacts(
|
|
85
|
+
suiteDir,
|
|
86
|
+
[stopLeaseCheck(stopResult)],
|
|
87
|
+
{ target: targetName, suite: suiteName, exitCode: stopResult.code, signal: stopResult.signal, elapsedMs: 0 },
|
|
88
|
+
[
|
|
89
|
+
"summary.json", "target.json", "suite.json", "command.txt", "exit-code.txt",
|
|
90
|
+
"crabbox.stop.stdout.txt", "crabbox.stop.stderr.txt", "crabbox.stop.exit-code.txt", "assertions.json",
|
|
91
|
+
],
|
|
92
|
+
);
|
|
93
|
+
return { ok: assertions.ok, suiteDir, assertions };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function createLeaseCleanupFailureResult(config, targetName, leaseId, stopResult) {
|
|
97
|
+
return { ...createLeaseCleanupResult(config, targetName, leaseId, stopResult), ok: false };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Execute a single suite on a target.
|
|
102
|
+
* Returns { ok, suiteDir, assertions }.
|
|
103
|
+
* On failure, writes fail-through artifacts but does not throw.
|
|
104
|
+
*/
|
|
105
|
+
export async function runTargetSuite(config, targetName, suiteName, leaseSession) {
|
|
106
|
+
const scenario = getScenario(suiteName);
|
|
107
|
+
const runId = leaseSession?.runId ?? makeRunId();
|
|
108
|
+
const suiteDir = createSuiteDir(config.artifactRoot, runId, targetName, suiteName);
|
|
109
|
+
const platform = platformFor(targetName);
|
|
110
|
+
const slug = `${config.packageName ?? "pi-cursor-sdk"}-${targetName}`;
|
|
111
|
+
|
|
112
|
+
console.log(`\n── [${targetName}] ${suiteName} ──`);
|
|
113
|
+
console.log(` runId: ${runId}`);
|
|
114
|
+
console.log(` suiteDir: ${suiteDir}`);
|
|
115
|
+
|
|
116
|
+
// Write metadata
|
|
117
|
+
writeFileSync(resolve(suiteDir, "target.json"), JSON.stringify({
|
|
118
|
+
targetName, platform, slug, runId,
|
|
119
|
+
writtenAt: new Date().toISOString(),
|
|
120
|
+
}, null, 2));
|
|
121
|
+
|
|
122
|
+
writeFileSync(resolve(suiteDir, "suite.json"), JSON.stringify({
|
|
123
|
+
suiteName,
|
|
124
|
+
cursorCalls: scenario?.cursorCalls ?? 0,
|
|
125
|
+
writtenAt: new Date().toISOString(),
|
|
126
|
+
}, null, 2));
|
|
127
|
+
|
|
128
|
+
if (!scenario) {
|
|
129
|
+
const result = failSuite(suiteDir, targetName, suiteName, `unknown suite: ${suiteName}`);
|
|
130
|
+
result.ok = false;
|
|
131
|
+
return result;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// Route to suite-specific executor
|
|
135
|
+
switch (suiteName) {
|
|
136
|
+
case "platform-build":
|
|
137
|
+
return await executePlatformBuild(config, targetName, suiteDir, slug, platform, leaseSession);
|
|
138
|
+
case "cursor-native-visual-matrix":
|
|
139
|
+
case "cursor-bridge-visual-matrix":
|
|
140
|
+
case "cursor-abort-cleanup":
|
|
141
|
+
return await executeLiveSuite(config, targetName, suiteName, suiteDir, slug, leaseSession);
|
|
142
|
+
default:
|
|
143
|
+
return failSuite(suiteDir, targetName, suiteName, `unknown suite: ${suiteName}`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Execute a target session: warm once, sync once, run suites fail-fast, stop once.
|
|
149
|
+
* This is the release-gate path; per-suite runs remain available for diagnosis.
|
|
150
|
+
*/
|
|
151
|
+
export async function runTargetSuites(config, targetName, suiteNames) {
|
|
152
|
+
const slug = `${config.packageName ?? "pi-cursor-sdk"}-${targetName}`;
|
|
153
|
+
const runId = makeRunId();
|
|
154
|
+
console.log(` targetRunId: ${runId}`);
|
|
155
|
+
console.log(` warmup ${targetName}...`);
|
|
156
|
+
const warmup = await warmupLease(targetName, slug, config);
|
|
157
|
+
if (!warmup.ok) {
|
|
158
|
+
const suiteName = suiteNames[0] ?? "platform-build";
|
|
159
|
+
const suiteDir = createSuiteDir(config.artifactRoot, runId, targetName, suiteName);
|
|
160
|
+
writeFileSync(resolve(suiteDir, "target.json"), JSON.stringify({
|
|
161
|
+
targetName, platform: platformFor(targetName), slug, runId,
|
|
162
|
+
writtenAt: new Date().toISOString(),
|
|
163
|
+
}, null, 2));
|
|
164
|
+
writeFileSync(resolve(suiteDir, "suite.json"), JSON.stringify({
|
|
165
|
+
suiteName,
|
|
166
|
+
writtenAt: new Date().toISOString(),
|
|
167
|
+
}, null, 2));
|
|
168
|
+
writeExitCode(suiteDir, warmup.code, warmup.signal);
|
|
169
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.warmup.stdout.txt"), warmup.stdout);
|
|
170
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.warmup.stderr.txt"), warmup.stderr);
|
|
171
|
+
const failed = failSuite(suiteDir, targetName, suiteName, `Crabbox warmup failed (exit ${warmup.code}): ${warmup.stderr.slice(-500)}`);
|
|
172
|
+
return { ok: false, results: [failed] };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const results = [];
|
|
176
|
+
let sync = true;
|
|
177
|
+
const livePrepDir = `.platform-smoke-runs/live-prep-${Date.now()}-${targetName}`;
|
|
178
|
+
let stopResult;
|
|
179
|
+
try {
|
|
180
|
+
for (const suiteName of suiteNames) {
|
|
181
|
+
console.log(` Suite: ${suiteName}`);
|
|
182
|
+
const result = await runTargetSuite(config, targetName, suiteName, { ...warmup, sync, livePrepDir, runId });
|
|
183
|
+
results.push(result);
|
|
184
|
+
sync = false;
|
|
185
|
+
if (!result.ok) break;
|
|
186
|
+
}
|
|
187
|
+
} finally {
|
|
188
|
+
console.log(` stopping lease ${warmup.leaseId}...`);
|
|
189
|
+
stopResult = await stopLease(targetName, warmup.leaseId, config);
|
|
190
|
+
}
|
|
191
|
+
if (stopResult) {
|
|
192
|
+
results.push(createLeaseCleanupResult(config, targetName, warmup.leaseId, stopResult, runId));
|
|
193
|
+
}
|
|
194
|
+
return { ok: results.every((result) => result.ok), results };
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Execute the platform-build suite on a target.
|
|
199
|
+
*
|
|
200
|
+
* Steps:
|
|
201
|
+
* 1. Warmup lease (syncs checkout)
|
|
202
|
+
* 2. Run combined build shell: npm ci, test, typecheck, pack
|
|
203
|
+
* 3. Run separate asserts on output
|
|
204
|
+
* 4. Stop lease
|
|
205
|
+
* 5. Write failure artifacts on any failure
|
|
206
|
+
*/
|
|
207
|
+
async function executePlatformBuild(config, targetName, suiteDir, slug, platform, leaseSession) {
|
|
208
|
+
const startedAt = Date.now();
|
|
209
|
+
const packageName = config.packageName ?? "pi-cursor-sdk";
|
|
210
|
+
const command = buildPlatformBuildCommand(targetName, packageName, config.nodeValidationMajor ?? 24);
|
|
211
|
+
writeCommand(suiteDir, command);
|
|
212
|
+
let warmup = leaseSession;
|
|
213
|
+
const ownsLease = !warmup;
|
|
214
|
+
|
|
215
|
+
if (!warmup) {
|
|
216
|
+
console.log(` warmup ${targetName}...`);
|
|
217
|
+
warmup = await warmupLease(targetName, slug, config);
|
|
218
|
+
if (!warmup.ok) {
|
|
219
|
+
writeExitCode(suiteDir, warmup.code, warmup.signal);
|
|
220
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.warmup.stdout.txt"), warmup.stdout);
|
|
221
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.warmup.stderr.txt"), warmup.stderr);
|
|
222
|
+
return failSuite(suiteDir, targetName, "platform-build",
|
|
223
|
+
`Crabbox warmup failed (exit ${warmup.code}): ${warmup.stderr.slice(-500)}`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
console.log(` executing build shell on ${targetName}...`);
|
|
228
|
+
const result = await runOnLease(targetName, warmup.leaseId, command, {
|
|
229
|
+
shell: true,
|
|
230
|
+
timeout: 600_000,
|
|
231
|
+
sync: leaseSession?.sync,
|
|
232
|
+
config,
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
const elapsed = Date.now() - startedAt;
|
|
236
|
+
|
|
237
|
+
// Write artifact files
|
|
238
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.stdout.txt"), result.stdout);
|
|
239
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.stderr.txt"), result.stderr);
|
|
240
|
+
writeFileSync(resolve(suiteDir, "crabbox.timing.json"), JSON.stringify({
|
|
241
|
+
startedAt: new Date(startedAt).toISOString(),
|
|
242
|
+
elapsedMs: elapsed,
|
|
243
|
+
code: result.code,
|
|
244
|
+
signal: result.signal,
|
|
245
|
+
}, null, 2));
|
|
246
|
+
writeCommand(suiteDir, command);
|
|
247
|
+
writeExitCode(suiteDir, result.code, result.signal);
|
|
248
|
+
|
|
249
|
+
let stopResult;
|
|
250
|
+
if (ownsLease) {
|
|
251
|
+
console.log(` stopping lease ${warmup.leaseId}...`);
|
|
252
|
+
stopResult = await stopLease(targetName, warmup.leaseId, config);
|
|
253
|
+
writeStopLeaseArtifacts(suiteDir, stopResult);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
writePlatformBuildExtracts(suiteDir, result.stdout);
|
|
257
|
+
|
|
258
|
+
// Run redaction scan
|
|
259
|
+
const violations = scanForSecrets(result.stdout + result.stderr);
|
|
260
|
+
if (violations.length > 0) {
|
|
261
|
+
writeFileSync(resolve(suiteDir, "redaction-violations.json"), JSON.stringify(violations, null, 2));
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// Build assertions
|
|
265
|
+
const stdout = result.stdout;
|
|
266
|
+
const exitOk = result.code === 0;
|
|
267
|
+
const markerOk = stdout.includes("PLATFORM_BUILD_OK");
|
|
268
|
+
const nodeMajor = Number(stdout.match(/PLATFORM_NODE_VERSION=v?(\d+)\./)?.[1] ?? 0);
|
|
269
|
+
const nodeVersionOk = nodeMajor >= (config.nodeValidationMajor ?? 24);
|
|
270
|
+
const npmCiOk = /PLATFORM_NPM_CI_EXIT=0/.test(stdout);
|
|
271
|
+
const checkPlatformSmokeOk = /PLATFORM_CHECK_PLATFORM_SMOKE_EXIT=0/.test(stdout);
|
|
272
|
+
const npmTestOk = /PLATFORM_NPM_TEST_EXIT=0/.test(stdout);
|
|
273
|
+
const typecheckOk = /PLATFORM_TYPECHECK_EXIT=0/.test(stdout);
|
|
274
|
+
const npmPackOk = /PLATFORM_NPM_PACK_EXIT=0/.test(stdout) && /PLATFORM_PACKED_TARBALL=\S+/.test(stdout);
|
|
275
|
+
const fixtureOk = /PLATFORM_FIXTURE_EXIT=0/.test(stdout);
|
|
276
|
+
const packedNodeInstallOk = /PLATFORM_PACKED_NODE_INSTALL_EXIT=0/.test(stdout);
|
|
277
|
+
const installOk = /PLATFORM_PI_INSTALL_EXIT=0/.test(stdout);
|
|
278
|
+
const listOutput = section(stdout, "PI_LIST_STDOUT");
|
|
279
|
+
const packageInstallSegment = `node_modules${platform === "powershell" ? "\\" : "/"}${packageName}`;
|
|
280
|
+
const listOk = /PLATFORM_PI_LIST_EXIT=0/.test(stdout) && listOutput.includes(packageName) && listOutput.includes(packageInstallSegment);
|
|
281
|
+
const noPiEDot = !/\bpi\s+-e\s+\./.test(stdout) && !/\bpi\s+--extension\s+\./.test(stdout);
|
|
282
|
+
const noSecrets = violations.length === 0;
|
|
283
|
+
|
|
284
|
+
const checks = [
|
|
285
|
+
{ id: "build-exit-zero", fn: () => exitOk },
|
|
286
|
+
{ id: "build-marker", fn: () => markerOk },
|
|
287
|
+
{ id: "node-version", fn: () => nodeVersionOk },
|
|
288
|
+
{ id: "npm-ci", fn: () => npmCiOk },
|
|
289
|
+
{ id: "check-platform-smoke", fn: () => checkPlatformSmokeOk },
|
|
290
|
+
{ id: "npm-test", fn: () => npmTestOk },
|
|
291
|
+
{ id: "typecheck", fn: () => typecheckOk },
|
|
292
|
+
{ id: "npm-pack", fn: () => npmPackOk },
|
|
293
|
+
{ id: "fixture-workspace", fn: () => fixtureOk },
|
|
294
|
+
{ id: "packed-node-install", fn: () => packedNodeInstallOk },
|
|
295
|
+
{ id: "packed-install", fn: () => installOk },
|
|
296
|
+
{ id: "pi-list", fn: () => listOk },
|
|
297
|
+
{ id: "no-pi-e-dot", fn: () => noPiEDot },
|
|
298
|
+
{ id: "no-secrets", fn: () => noSecrets },
|
|
299
|
+
];
|
|
300
|
+
|
|
301
|
+
if (result.code !== 0 && !markerOk) {
|
|
302
|
+
checks.push({ id: "build-stderr", fn: () => false, error: `exit ${result.code}, check crabbox.stderr.txt` });
|
|
303
|
+
}
|
|
304
|
+
if (stopResult) checks.push(stopLeaseCheck(stopResult));
|
|
305
|
+
|
|
306
|
+
const expectedFiles = [
|
|
307
|
+
"summary.json", "target.json", "suite.json",
|
|
308
|
+
"command.txt", "exit-code.txt",
|
|
309
|
+
"crabbox.stdout.txt", "crabbox.stderr.txt", "crabbox.timing.json",
|
|
310
|
+
"node-version.txt", "npm-version.txt",
|
|
311
|
+
"npm-ci.stdout.txt", "npm-ci.stderr.txt",
|
|
312
|
+
"check-platform-smoke.stdout.txt", "check-platform-smoke.stderr.txt",
|
|
313
|
+
"npm-test.stdout.txt", "npm-test.stderr.txt",
|
|
314
|
+
"typecheck.stdout.txt", "typecheck.stderr.txt",
|
|
315
|
+
"npm-pack.stdout.txt", "npm-pack.stderr.txt",
|
|
316
|
+
"packed-tarball.txt", "packed-node-install.stdout.txt", "packed-node-install.stderr.txt",
|
|
317
|
+
"pi-install.stdout.txt", "pi-install.stderr.txt",
|
|
318
|
+
"pi-list.stdout.txt", "pi-list.stderr.txt",
|
|
319
|
+
"assertions.json",
|
|
320
|
+
];
|
|
321
|
+
if (stopResult) expectedFiles.push("crabbox.stop.stdout.txt", "crabbox.stop.stderr.txt", "crabbox.stop.exit-code.txt");
|
|
322
|
+
const { assertions } = finalizeSuiteArtifacts(suiteDir, checks, {
|
|
323
|
+
target: targetName,
|
|
324
|
+
suite: "platform-build",
|
|
325
|
+
exitCode: result.code,
|
|
326
|
+
signal: result.signal,
|
|
327
|
+
elapsedMs: elapsed,
|
|
328
|
+
}, expectedFiles);
|
|
329
|
+
|
|
330
|
+
console.log(` ${assertions.ok ? "PASS" : "FAIL"} platform-build on ${targetName} (${elapsed}ms)`);
|
|
331
|
+
|
|
332
|
+
return { ok: assertions.ok, suiteDir, assertions };
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* Build a POSIX shell command that runs the full platform-build pipeline
|
|
337
|
+
* and prints a success/failure marker.
|
|
338
|
+
*/
|
|
339
|
+
function section(text, name) {
|
|
340
|
+
const start = `--- ${name} START ---`;
|
|
341
|
+
const end = `--- ${name} END ---`;
|
|
342
|
+
const startIndex = text.indexOf(start);
|
|
343
|
+
if (startIndex === -1) return "";
|
|
344
|
+
const contentStart = startIndex + start.length;
|
|
345
|
+
const endIndex = text.indexOf(end, contentStart);
|
|
346
|
+
const raw = endIndex === -1 ? text.slice(contentStart) : text.slice(contentStart, endIndex);
|
|
347
|
+
return raw.replace(/^\r?\n/, "").replace(/\r?\n$/, "");
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function markerValue(text, name) {
|
|
351
|
+
const match = text.match(new RegExp(`^${name}=(.*)$`, "m"));
|
|
352
|
+
return match?.[1]?.trim() ?? "";
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function writePlatformBuildExtracts(suiteDir, stdout) {
|
|
356
|
+
writeRedactedFile(resolve(suiteDir, "node-version.txt"), `${markerValue(stdout, "PLATFORM_NODE_VERSION")}\n`);
|
|
357
|
+
writeRedactedFile(resolve(suiteDir, "npm-version.txt"), `${markerValue(stdout, "PLATFORM_NPM_VERSION")}\n`);
|
|
358
|
+
writeRedactedFile(resolve(suiteDir, "npm-ci.stdout.txt"), section(stdout, "NPM_CI_STDOUT"));
|
|
359
|
+
writeRedactedFile(resolve(suiteDir, "npm-ci.stderr.txt"), section(stdout, "NPM_CI_STDERR"));
|
|
360
|
+
writeRedactedFile(resolve(suiteDir, "check-platform-smoke.stdout.txt"), section(stdout, "CHECK_PLATFORM_SMOKE_STDOUT"));
|
|
361
|
+
writeRedactedFile(resolve(suiteDir, "check-platform-smoke.stderr.txt"), section(stdout, "CHECK_PLATFORM_SMOKE_STDERR"));
|
|
362
|
+
writeRedactedFile(resolve(suiteDir, "npm-test.stdout.txt"), section(stdout, "NPM_TEST_STDOUT"));
|
|
363
|
+
writeRedactedFile(resolve(suiteDir, "npm-test.stderr.txt"), section(stdout, "NPM_TEST_STDERR"));
|
|
364
|
+
writeRedactedFile(resolve(suiteDir, "typecheck.stdout.txt"), section(stdout, "TYPECHECK_STDOUT"));
|
|
365
|
+
writeRedactedFile(resolve(suiteDir, "typecheck.stderr.txt"), section(stdout, "TYPECHECK_STDERR"));
|
|
366
|
+
writeRedactedFile(resolve(suiteDir, "npm-pack.stdout.txt"), section(stdout, "NPM_PACK_STDOUT"));
|
|
367
|
+
writeRedactedFile(resolve(suiteDir, "npm-pack.stderr.txt"), section(stdout, "NPM_PACK_STDERR"));
|
|
368
|
+
writeRedactedFile(resolve(suiteDir, "packed-tarball.txt"), `${markerValue(stdout, "PLATFORM_PACKED_TARBALL")}\n`);
|
|
369
|
+
writeRedactedFile(resolve(suiteDir, "packed-node-install.stdout.txt"), section(stdout, "PACKED_NODE_INSTALL_STDOUT"));
|
|
370
|
+
writeRedactedFile(resolve(suiteDir, "packed-node-install.stderr.txt"), section(stdout, "PACKED_NODE_INSTALL_STDERR"));
|
|
371
|
+
writeRedactedFile(resolve(suiteDir, "pi-install.stdout.txt"), section(stdout, "PI_INSTALL_STDOUT"));
|
|
372
|
+
writeRedactedFile(resolve(suiteDir, "pi-install.stderr.txt"), section(stdout, "PI_INSTALL_STDERR"));
|
|
373
|
+
writeRedactedFile(resolve(suiteDir, "pi-list.stdout.txt"), section(stdout, "PI_LIST_STDOUT"));
|
|
374
|
+
writeRedactedFile(resolve(suiteDir, "pi-list.stderr.txt"), section(stdout, "PI_LIST_STDERR"));
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
function posixSection(name, command) {
|
|
378
|
+
return [
|
|
379
|
+
`echo "--- ${name} START ---"`,
|
|
380
|
+
command,
|
|
381
|
+
`echo "--- ${name} END ---"`,
|
|
382
|
+
];
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Build a shell command that runs the full platform-build pipeline and packed-install contract.
|
|
387
|
+
*/
|
|
388
|
+
export function buildPlatformBuildCommand(targetName, packageName = "pi-cursor-sdk", nodeValidationMajor = 24) {
|
|
389
|
+
const platform = platformFor(targetName);
|
|
390
|
+
const lines = [];
|
|
391
|
+
if (platform === "posix") {
|
|
392
|
+
lines.push("set -o pipefail");
|
|
393
|
+
lines.push('echo "Starting platform-build in $(pwd) at $(date -u +%Y-%m-%dT%H:%M:%SZ)"');
|
|
394
|
+
lines.push('RUN_ROOT=".platform-smoke-runs/platform-build-$(date -u +%Y%m%dT%H%M%SZ)-$$"');
|
|
395
|
+
lines.push('SOURCE_ROOT="$(pwd)"');
|
|
396
|
+
lines.push('PACK_DIR="$SOURCE_ROOT/$RUN_ROOT/pack"');
|
|
397
|
+
lines.push('TEST_WORKSPACE="$SOURCE_ROOT/$RUN_ROOT/test-workspace"');
|
|
398
|
+
lines.push('PI_PROJECT="$SOURCE_ROOT/$RUN_ROOT/pi-project"');
|
|
399
|
+
lines.push('mkdir -p "$PACK_DIR" "$TEST_WORKSPACE" "$PI_PROJECT"');
|
|
400
|
+
lines.push('echo "PLATFORM_RUN_ROOT=$RUN_ROOT"');
|
|
401
|
+
lines.push('echo "PLATFORM_TEST_WORKSPACE=$TEST_WORKSPACE"');
|
|
402
|
+
lines.push('echo "PLATFORM_PI_PROJECT=$PI_PROJECT"');
|
|
403
|
+
lines.push("");
|
|
404
|
+
lines.push('NODE_VERSION=$(node --version)');
|
|
405
|
+
lines.push('NPM_VERSION=$(npm --version)');
|
|
406
|
+
lines.push('NODE_MAJOR=${NODE_VERSION#v}');
|
|
407
|
+
lines.push('NODE_MAJOR=${NODE_MAJOR%%.*}');
|
|
408
|
+
lines.push('printf "%s\\n" "$NODE_VERSION" > "$PACK_DIR/node-version.txt"');
|
|
409
|
+
lines.push('printf "%s\\n" "$NPM_VERSION" > "$PACK_DIR/npm-version.txt"');
|
|
410
|
+
lines.push('echo "PLATFORM_NODE_VERSION=$NODE_VERSION"');
|
|
411
|
+
lines.push('echo "PLATFORM_NPM_VERSION=$NPM_VERSION"');
|
|
412
|
+
lines.push(`if [ "$NODE_MAJOR" -ge ${nodeValidationMajor} ]; then NODE_VERSION_EXIT=0; else NODE_VERSION_EXIT=1; fi`);
|
|
413
|
+
lines.push('echo "PLATFORM_NODE_VERSION_EXIT=$NODE_VERSION_EXIT"');
|
|
414
|
+
lines.push(...posixSection("NODE_VERSION_STDOUT", 'cat "$PACK_DIR/node-version.txt"'));
|
|
415
|
+
lines.push(...posixSection("NPM_VERSION_STDOUT", 'cat "$PACK_DIR/npm-version.txt"'));
|
|
416
|
+
lines.push('');
|
|
417
|
+
lines.push('echo "=== npm ci ==="');
|
|
418
|
+
lines.push('npm ci >"$PACK_DIR/npm-ci.stdout.txt" 2>"$PACK_DIR/npm-ci.stderr.txt"');
|
|
419
|
+
lines.push("CI_EXIT=$?");
|
|
420
|
+
lines.push('echo "PLATFORM_NPM_CI_EXIT=$CI_EXIT"');
|
|
421
|
+
lines.push(...posixSection("NPM_CI_STDOUT", 'cat "$PACK_DIR/npm-ci.stdout.txt" 2>/dev/null || true'));
|
|
422
|
+
lines.push(...posixSection("NPM_CI_STDERR", 'cat "$PACK_DIR/npm-ci.stderr.txt" 2>/dev/null || true'));
|
|
423
|
+
lines.push("");
|
|
424
|
+
lines.push('echo "=== check:platform-smoke ==="');
|
|
425
|
+
lines.push('PI_CURSOR_SKIP_RELEASE_VERSION_GUARD=1 npm run check:platform-smoke >"$PACK_DIR/check-platform-smoke.stdout.txt" 2>"$PACK_DIR/check-platform-smoke.stderr.txt"');
|
|
426
|
+
lines.push("CHECK_PLATFORM_SMOKE_EXIT=$?");
|
|
427
|
+
lines.push('echo "PLATFORM_CHECK_PLATFORM_SMOKE_EXIT=$CHECK_PLATFORM_SMOKE_EXIT"');
|
|
428
|
+
lines.push(...posixSection("CHECK_PLATFORM_SMOKE_STDOUT", 'cat "$PACK_DIR/check-platform-smoke.stdout.txt" 2>/dev/null || true'));
|
|
429
|
+
lines.push(...posixSection("CHECK_PLATFORM_SMOKE_STDERR", 'cat "$PACK_DIR/check-platform-smoke.stderr.txt" 2>/dev/null || true'));
|
|
430
|
+
lines.push("");
|
|
431
|
+
lines.push('echo "=== npm test ==="');
|
|
432
|
+
lines.push('PI_CURSOR_SKIP_RELEASE_VERSION_GUARD=1 npm test >"$PACK_DIR/npm-test.stdout.txt" 2>"$PACK_DIR/npm-test.stderr.txt"');
|
|
433
|
+
lines.push("TEST_EXIT=$?");
|
|
434
|
+
lines.push('echo "PLATFORM_NPM_TEST_EXIT=$TEST_EXIT"');
|
|
435
|
+
lines.push(...posixSection("NPM_TEST_STDOUT", 'cat "$PACK_DIR/npm-test.stdout.txt" 2>/dev/null || true'));
|
|
436
|
+
lines.push(...posixSection("NPM_TEST_STDERR", 'cat "$PACK_DIR/npm-test.stderr.txt" 2>/dev/null || true'));
|
|
437
|
+
lines.push("");
|
|
438
|
+
lines.push('echo "=== typecheck ==="');
|
|
439
|
+
lines.push('npm run typecheck >"$PACK_DIR/typecheck.stdout.txt" 2>"$PACK_DIR/typecheck.stderr.txt"');
|
|
440
|
+
lines.push("TC_EXIT=$?");
|
|
441
|
+
lines.push('echo "PLATFORM_TYPECHECK_EXIT=$TC_EXIT"');
|
|
442
|
+
lines.push(...posixSection("TYPECHECK_STDOUT", 'cat "$PACK_DIR/typecheck.stdout.txt" 2>/dev/null || true'));
|
|
443
|
+
lines.push(...posixSection("TYPECHECK_STDERR", 'cat "$PACK_DIR/typecheck.stderr.txt" 2>/dev/null || true'));
|
|
444
|
+
lines.push("");
|
|
445
|
+
lines.push('echo "=== npm pack ==="');
|
|
446
|
+
lines.push('PACK_TARBALL=$(npm pack --silent >"$PACK_DIR/npm-pack.stdout.txt" 2>"$PACK_DIR/npm-pack.stderr.txt" && cat "$PACK_DIR/npm-pack.stdout.txt")');
|
|
447
|
+
lines.push("PACK_EXIT=$?");
|
|
448
|
+
lines.push('echo "PLATFORM_NPM_PACK_EXIT=$PACK_EXIT"');
|
|
449
|
+
lines.push(...posixSection("NPM_PACK_STDOUT", 'cat "$PACK_DIR/npm-pack.stdout.txt" 2>/dev/null || true'));
|
|
450
|
+
lines.push(...posixSection("NPM_PACK_STDERR", 'cat "$PACK_DIR/npm-pack.stderr.txt" 2>/dev/null || true'));
|
|
451
|
+
lines.push('if [ -n "$PACK_TARBALL" ] && [ -f "$PACK_TARBALL" ]; then mv "$PACK_TARBALL" "$PACK_DIR/$PACK_TARBALL"; fi');
|
|
452
|
+
lines.push('echo "PLATFORM_PACKED_TARBALL=$PACK_TARBALL"');
|
|
453
|
+
lines.push('printf "%s\\n" "$PACK_TARBALL" > "$PACK_DIR/packed-tarball.txt"');
|
|
454
|
+
lines.push("");
|
|
455
|
+
lines.push('echo "=== fixture workspace ==="');
|
|
456
|
+
lines.push('cp package.json README.md "$TEST_WORKSPACE"/ 2>"$PACK_DIR/fixture.stderr.txt"');
|
|
457
|
+
lines.push('FIXTURE_COPY_EXIT=$?');
|
|
458
|
+
lines.push('cp -R src "$TEST_WORKSPACE"/ 2>>"$PACK_DIR/fixture.stderr.txt"');
|
|
459
|
+
lines.push('SRC_COPY_EXIT=$?');
|
|
460
|
+
lines.push('if [ "$FIXTURE_COPY_EXIT" -eq 0 ] && [ "$SRC_COPY_EXIT" -eq 0 ]; then FIXTURE_EXIT=0; else FIXTURE_EXIT=1; fi');
|
|
461
|
+
lines.push('cat "$PACK_DIR/fixture.stderr.txt"');
|
|
462
|
+
lines.push('echo "PLATFORM_FIXTURE_EXIT=$FIXTURE_EXIT"');
|
|
463
|
+
lines.push("");
|
|
464
|
+
lines.push('echo "=== pi install packed tarball ==="');
|
|
465
|
+
lines.push('PI_CLI="$(pwd)/node_modules/.bin/pi"');
|
|
466
|
+
lines.push('if [ ! -x "$PI_CLI" ]; then PI_CLI="$(command -v pi || true)"; fi');
|
|
467
|
+
lines.push('echo "PLATFORM_PI_CLI=$PI_CLI"');
|
|
468
|
+
lines.push('if [ -n "$PACK_TARBALL" ] && [ -n "$PI_CLI" ] && [ -f "$PACK_DIR/$PACK_TARBALL" ]; then (cd "$PI_PROJECT" && npm init -y >"$PACK_DIR/packed-node-install.stdout.txt" 2>"$PACK_DIR/packed-node-install.stderr.txt" && npm install --no-save "$PACK_DIR/$PACK_TARBALL" >>"$PACK_DIR/packed-node-install.stdout.txt" 2>>"$PACK_DIR/packed-node-install.stderr.txt"); PACKED_NODE_INSTALL_EXIT=$?; else echo "missing pi cli or tarball" >"$PACK_DIR/packed-node-install.stderr.txt"; PACKED_NODE_INSTALL_EXIT=1; fi');
|
|
469
|
+
lines.push('echo "PLATFORM_PACKED_NODE_INSTALL_EXIT=$PACKED_NODE_INSTALL_EXIT"');
|
|
470
|
+
lines.push(...posixSection("PACKED_NODE_INSTALL_STDOUT", 'cat "$PACK_DIR/packed-node-install.stdout.txt" 2>/dev/null || true'));
|
|
471
|
+
lines.push(...posixSection("PACKED_NODE_INSTALL_STDERR", 'cat "$PACK_DIR/packed-node-install.stderr.txt" 2>/dev/null || true'));
|
|
472
|
+
lines.push(`if [ "$PACKED_NODE_INSTALL_EXIT" -eq 0 ] && [ -n "$PI_CLI" ]; then (cd "$PI_PROJECT" && PI_OFFLINE=1 "$PI_CLI" install -l ./node_modules/${packageName} >"$PACK_DIR/pi-install.stdout.txt" 2>"$PACK_DIR/pi-install.stderr.txt"); PI_INSTALL_EXIT=$?; else echo "packed npm install failed or missing pi cli" >"$PACK_DIR/pi-install.stderr.txt"; PI_INSTALL_EXIT=1; fi`);
|
|
473
|
+
lines.push('echo "PLATFORM_PI_INSTALL_EXIT=$PI_INSTALL_EXIT"');
|
|
474
|
+
lines.push(...posixSection("PI_INSTALL_STDOUT", 'cat "$PACK_DIR/pi-install.stdout.txt" 2>/dev/null || true'));
|
|
475
|
+
lines.push(...posixSection("PI_INSTALL_STDERR", 'cat "$PACK_DIR/pi-install.stderr.txt" 2>/dev/null || true'));
|
|
476
|
+
lines.push("");
|
|
477
|
+
lines.push('echo "=== pi list ==="');
|
|
478
|
+
lines.push('if [ -n "$PI_CLI" ]; then (cd "$PI_PROJECT" && PI_OFFLINE=1 "$PI_CLI" list >"$PACK_DIR/pi-list.stdout.txt" 2>"$PACK_DIR/pi-list.stderr.txt"); PI_LIST_EXIT=$?; else echo "missing pi cli" >"$PACK_DIR/pi-list.stderr.txt"; PI_LIST_EXIT=1; fi');
|
|
479
|
+
lines.push('echo "PLATFORM_PI_LIST_EXIT=$PI_LIST_EXIT"');
|
|
480
|
+
lines.push(...posixSection("PI_LIST_STDOUT", 'cat "$PACK_DIR/pi-list.stdout.txt" 2>/dev/null || true'));
|
|
481
|
+
lines.push(...posixSection("PI_LIST_STDERR", 'cat "$PACK_DIR/pi-list.stderr.txt" 2>/dev/null || true'));
|
|
482
|
+
lines.push("");
|
|
483
|
+
lines.push('echo "node=$NODE_VERSION_EXIT ci=$CI_EXIT checkPlatformSmoke=$CHECK_PLATFORM_SMOKE_EXIT test=$TEST_EXIT typecheck=$TC_EXIT pack=$PACK_EXIT fixture=$FIXTURE_EXIT packedNodeInstall=$PACKED_NODE_INSTALL_EXIT install=$PI_INSTALL_EXIT list=$PI_LIST_EXIT"');
|
|
484
|
+
lines.push('if [ "$NODE_VERSION_EXIT" -ne 0 ] || [ "$CI_EXIT" -ne 0 ] || [ "$CHECK_PLATFORM_SMOKE_EXIT" -ne 0 ] || [ "$TEST_EXIT" -ne 0 ] || [ "$TC_EXIT" -ne 0 ] || [ "$PACK_EXIT" -ne 0 ] || [ "$FIXTURE_EXIT" -ne 0 ] || [ "$PACKED_NODE_INSTALL_EXIT" -ne 0 ] || [ "$PI_INSTALL_EXIT" -ne 0 ] || [ "$PI_LIST_EXIT" -ne 0 ]; then');
|
|
485
|
+
lines.push(' echo "PLATFORM_BUILD_FAILED: node=$NODE_VERSION_EXIT ci=$CI_EXIT checkPlatformSmoke=$CHECK_PLATFORM_SMOKE_EXIT test=$TEST_EXIT typecheck=$TC_EXIT pack=$PACK_EXIT fixture=$FIXTURE_EXIT packedNodeInstall=$PACKED_NODE_INSTALL_EXIT install=$PI_INSTALL_EXIT list=$PI_LIST_EXIT"');
|
|
486
|
+
lines.push(" exit 1");
|
|
487
|
+
lines.push("fi");
|
|
488
|
+
lines.push('echo "PLATFORM_BUILD_OK"');
|
|
489
|
+
} else {
|
|
490
|
+
lines.push(`powershell.exe -NoLogo -NoProfile -ExecutionPolicy Bypass -File .\\scripts\\platform-smoke\\platform-build-windows.ps1 -PackageName ${packageName} -NodeValidationMajor ${nodeValidationMajor}`);
|
|
491
|
+
}
|
|
492
|
+
return lines.join("\n");
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
async function executeLiveSuite(config, targetName, suiteName, suiteDir, slug, leaseSession) {
|
|
496
|
+
const scenario = getScenario(suiteName);
|
|
497
|
+
const startedAt = Date.now();
|
|
498
|
+
const command = buildLiveSuiteCommand(config, targetName, suiteName, leaseSession?.livePrepDir);
|
|
499
|
+
writeCommand(suiteDir, command);
|
|
500
|
+
let warmup = leaseSession;
|
|
501
|
+
const ownsLease = !warmup;
|
|
502
|
+
|
|
503
|
+
if (!warmup) {
|
|
504
|
+
console.log(` warmup ${targetName}...`);
|
|
505
|
+
warmup = await warmupLease(targetName, slug, config);
|
|
506
|
+
if (!warmup.ok) {
|
|
507
|
+
writeExitCode(suiteDir, warmup.code, warmup.signal);
|
|
508
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.warmup.stdout.txt"), warmup.stdout);
|
|
509
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.warmup.stderr.txt"), warmup.stderr);
|
|
510
|
+
return failSuite(suiteDir, targetName, suiteName, `Crabbox warmup failed (exit ${warmup.code}): ${warmup.stderr.slice(-500)}`);
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
console.log(` executing live suite on ${targetName}...`);
|
|
515
|
+
const result = await runOnLeaseWithTransientRetry(suiteDir, targetName, warmup.leaseId, command, {
|
|
516
|
+
shell: true,
|
|
517
|
+
timeout: 900_000,
|
|
518
|
+
allowEnv: ["CURSOR_API_KEY"],
|
|
519
|
+
sync: leaseSession?.sync,
|
|
520
|
+
config,
|
|
521
|
+
});
|
|
522
|
+
const elapsed = Date.now() - startedAt;
|
|
523
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.stdout.txt"), result.stdout);
|
|
524
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.stderr.txt"), result.stderr);
|
|
525
|
+
writeFileSync(resolve(suiteDir, "crabbox.timing.json"), JSON.stringify({
|
|
526
|
+
startedAt: new Date(startedAt).toISOString(),
|
|
527
|
+
elapsedMs: elapsed,
|
|
528
|
+
code: result.code,
|
|
529
|
+
signal: result.signal,
|
|
530
|
+
}, null, 2));
|
|
531
|
+
writeExitCode(suiteDir, result.code, result.signal);
|
|
532
|
+
|
|
533
|
+
let stopResult;
|
|
534
|
+
if (ownsLease) {
|
|
535
|
+
console.log(` stopping lease ${warmup.leaseId}...`);
|
|
536
|
+
stopResult = await stopLease(targetName, warmup.leaseId, config);
|
|
537
|
+
writeStopLeaseArtifacts(suiteDir, stopResult);
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
const bundle = extractLiveBundle(suiteDir, result.stdout);
|
|
541
|
+
const liveArtifactDir = resolve(suiteDir, "artifacts");
|
|
542
|
+
mkdirSync(liveArtifactDir, { recursive: true });
|
|
543
|
+
const terminalAnsi = resolve(liveArtifactDir, "terminal.ansi");
|
|
544
|
+
const terminalTxt = resolve(liveArtifactDir, "terminal.txt");
|
|
545
|
+
let renderResult = { pngOk: false };
|
|
546
|
+
let cards = [];
|
|
547
|
+
if (existsSync(terminalAnsi)) {
|
|
548
|
+
renderResult = await renderAll(terminalAnsi, liveArtifactDir, {
|
|
549
|
+
label: `${targetName}-${suiteName}`,
|
|
550
|
+
model: config.cursorModel,
|
|
551
|
+
mode: "agent",
|
|
552
|
+
sessionId: `${targetName}-${suiteName}`,
|
|
553
|
+
});
|
|
554
|
+
}
|
|
555
|
+
if (existsSync(terminalTxt)) {
|
|
556
|
+
cards = detectCards(readFileSync(terminalTxt, "utf8"));
|
|
557
|
+
writeCardArtifacts(liveArtifactDir, cards);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
const statusPath = resolve(liveArtifactDir, "live-status.json");
|
|
561
|
+
const status = readJson(statusPath);
|
|
562
|
+
const terminalText = existsSync(terminalTxt) ? readFileSync(terminalTxt, "utf8") : "";
|
|
563
|
+
const jsonlPath = resolve(liveArtifactDir, "session.jsonl");
|
|
564
|
+
const jsonlRaw = existsSync(jsonlPath) ? readFileSync(jsonlPath, "utf8") : "";
|
|
565
|
+
const cardChecks = assertRequiredCards(liveArtifactDir, cards, scenario?.requiredCards ?? []);
|
|
566
|
+
const jsonlToolNames = collectJsonlToolNames(jsonlRaw);
|
|
567
|
+
const jsonlResults = collectJsonlToolResults(jsonlRaw);
|
|
568
|
+
const usageChecks = collectUsageChecks(jsonlRaw);
|
|
569
|
+
writeFileSync(resolve(liveArtifactDir, "jsonl-tool-names.json"), JSON.stringify([...jsonlToolNames].sort(), null, 2));
|
|
570
|
+
writeFileSync(resolve(liveArtifactDir, "jsonl-tool-results.json"), JSON.stringify(jsonlResults, null, 2));
|
|
571
|
+
const jsonlToolChecks = (scenario?.requiredJSONLTools ?? []).map(({ name }) => ({
|
|
572
|
+
id: `jsonl-tool-${name}`,
|
|
573
|
+
fn: () => jsonlToolNames.has(name),
|
|
574
|
+
}));
|
|
575
|
+
const jsonlResultChecks = (scenario?.requiredJSONLResults ?? []).map((requirement) => ({
|
|
576
|
+
id: `jsonl-result-${requirement.id}`,
|
|
577
|
+
fn: () => jsonlResults.some((result) => matchesJsonlResult(result, requirement)),
|
|
578
|
+
}));
|
|
579
|
+
const bridgeDiagnostics = [
|
|
580
|
+
...collectBridgeDiagnostics(terminalText),
|
|
581
|
+
...collectBridgeDiagnosticFile(resolve(liveArtifactDir, "bridge-diagnostics.jsonl")),
|
|
582
|
+
];
|
|
583
|
+
writeFileSync(resolve(liveArtifactDir, "bridge-diagnostics.json"), JSON.stringify(bridgeDiagnostics, null, 2));
|
|
584
|
+
const bridgeDiagnosticChecks = scenario?.requiredBridgeDiagnostics === "abort" ? [
|
|
585
|
+
{ id: "bridge-diagnostic-run-created", fn: () => bridgeDiagnostics.some((event) => event.event === "run_created") },
|
|
586
|
+
{ id: "bridge-diagnostic-tools-exposed", fn: () => bridgeDiagnostics.some((event) => event.event === "tools_exposed") },
|
|
587
|
+
{ id: "bridge-diagnostic-request-queued", fn: () => bridgeDiagnostics.some((event) => event.event === "request_queued" && event.piToolName === "bash") },
|
|
588
|
+
{ id: "bridge-diagnostic-run-cancelled", fn: () => bridgeDiagnostics.some((event) => event.event === "run_cancelled") },
|
|
589
|
+
{ id: "bridge-diagnostic-request-rejected", fn: () => bridgeDiagnostics.some((event) => event.event === "request_rejected" && event.piToolName === "bash" && event.rejectionKind === "cancelled") },
|
|
590
|
+
] : scenario?.requiredBridgeDiagnostics ? [
|
|
591
|
+
{ id: "bridge-diagnostic-run-created", fn: () => bridgeDiagnostics.some((event) => event.event === "run_created") },
|
|
592
|
+
{ id: "bridge-diagnostic-tools-exposed", fn: () => bridgeDiagnostics.some((event) => event.event === "tools_exposed") },
|
|
593
|
+
{ id: "bridge-diagnostic-request-resolved", fn: () => bridgeDiagnostics.some((event) => event.event === "request_resolved") },
|
|
594
|
+
] : [];
|
|
595
|
+
const visualEvidenceSpecs = scenario?.visualEvidence ?? [];
|
|
596
|
+
const visualEvidence = existsSync(resolve(liveArtifactDir, "terminal.html"))
|
|
597
|
+
? await collectVisualEvidence({
|
|
598
|
+
htmlPath: resolve(liveArtifactDir, "terminal.html"),
|
|
599
|
+
pngPath: resolve(liveArtifactDir, "terminal.full.png"),
|
|
600
|
+
outDir: liveArtifactDir,
|
|
601
|
+
specs: visualEvidenceSpecs,
|
|
602
|
+
})
|
|
603
|
+
: { ok: false, checks: [{ id: "visual-html-present", ok: false, error: "terminal.html missing" }] };
|
|
604
|
+
const visualEvidenceResultChecks = visualEvidenceSpecs
|
|
605
|
+
.filter((spec) => spec.jsonlResultId)
|
|
606
|
+
.map((spec) => ({
|
|
607
|
+
id: `visual-jsonl-state-${spec.id}`,
|
|
608
|
+
fn: () => {
|
|
609
|
+
const visualItem = visualEvidence.items?.find((item) => item.id === spec.id);
|
|
610
|
+
const resultRequirement = scenario?.requiredJSONLResults?.find((requirement) => requirement.id === spec.jsonlResultId);
|
|
611
|
+
return visualItem?.ok === true && Boolean(resultRequirement && jsonlResults.some((result) => matchesJsonlResult(result, resultRequirement)));
|
|
612
|
+
},
|
|
613
|
+
}));
|
|
614
|
+
const violations = [
|
|
615
|
+
...scanForSecrets(result.stdout + result.stderr + terminalText + jsonlRaw).map((violation) => ({ file: "process-output", violation })),
|
|
616
|
+
...bundle.violations,
|
|
617
|
+
...scanArtifacts(suiteDir),
|
|
618
|
+
];
|
|
619
|
+
if (violations.length > 0) writeFileSync(resolve(suiteDir, "redaction-violations.json"), JSON.stringify(violations, null, 2));
|
|
620
|
+
const providerDebugFiles = findFiles(resolve(suiteDir, "cursor-sdk-events"));
|
|
621
|
+
|
|
622
|
+
const checks = [
|
|
623
|
+
{ id: "live-exit-zero", fn: () => result.code === 0 },
|
|
624
|
+
{ id: "bundle-extracted", fn: () => bundle.ok },
|
|
625
|
+
{ id: "live-status-ok", fn: () => status?.ok === true },
|
|
626
|
+
{ id: "cursor-no-fast", fn: () => readJson(resolve(liveArtifactDir, "pi-command.json"))?.args?.includes("--cursor-no-fast") === true },
|
|
627
|
+
{ id: "cursor-model", fn: () => readJson(resolve(liveArtifactDir, "pi-command.json"))?.args?.includes(config.cursorModel) === true },
|
|
628
|
+
{ id: "terminal-ansi", fn: () => existsSync(terminalAnsi) && readFileSync(terminalAnsi).length > 0 },
|
|
629
|
+
{ id: "terminal-text", fn: () => terminalText.length > 0 },
|
|
630
|
+
{ id: "terminal-html", fn: () => existsSync(resolve(liveArtifactDir, "terminal.html")) },
|
|
631
|
+
{ id: "terminal-png", fn: () => renderResult.pngOk && existsSync(resolve(liveArtifactDir, "terminal.final-viewport.png")) },
|
|
632
|
+
{ id: "session-jsonl", fn: () => jsonlRaw.length > 0 },
|
|
633
|
+
{ id: "provider-debug-artifacts", fn: () => providerDebugFiles.some((file) => file.endsWith("session.json")) && providerDebugFiles.length > 1 },
|
|
634
|
+
...(suiteName !== "cursor-abort-cleanup" ? [
|
|
635
|
+
{ id: "jsonl-usage-non-negative", fn: () => usageChecks.seen && usageChecks.nonNegative },
|
|
636
|
+
{ id: "jsonl-cache-zero", fn: () => usageChecks.seen && usageChecks.cacheZero },
|
|
637
|
+
] : []),
|
|
638
|
+
{ id: "final-marker", fn: () => scenario?.finalMarker ? status?.finalMarkerObserved === true : status?.ok === true },
|
|
639
|
+
...(suiteName === "cursor-abort-cleanup" ? [{ id: "abort-no-successful-answer", fn: () => !hasAbortSuccessClaim(jsonlRaw) }] : []),
|
|
640
|
+
{ id: "no-secrets", fn: () => violations.length === 0 },
|
|
641
|
+
...cardChecks.map((check) => ({ id: check.id, fn: () => check.ok })),
|
|
642
|
+
...jsonlToolChecks,
|
|
643
|
+
...jsonlResultChecks,
|
|
644
|
+
...bridgeDiagnosticChecks,
|
|
645
|
+
...(visualEvidence.checks ?? []).map((check) => ({ id: check.id, fn: () => check.ok === true, error: check.error })),
|
|
646
|
+
...visualEvidenceResultChecks,
|
|
647
|
+
];
|
|
648
|
+
if (stopResult) checks.push(stopLeaseCheck(stopResult));
|
|
649
|
+
const expectedFiles = [
|
|
650
|
+
"summary.json", "target.json", "suite.json", "command.txt", "exit-code.txt",
|
|
651
|
+
"crabbox.stdout.txt", "crabbox.stderr.txt", "crabbox.timing.json", "assertions.json",
|
|
652
|
+
"artifacts/terminal.ansi", "artifacts/terminal.txt", "artifacts/terminal.html",
|
|
653
|
+
"artifacts/terminal.full.png", "artifacts/terminal.final-viewport.png", "artifacts/session.jsonl",
|
|
654
|
+
"artifacts/live-status.json", "artifacts/cards/cards.json", "artifacts/cards/index.html",
|
|
655
|
+
"artifacts/visual-evidence.json", "artifacts/jsonl-tool-results.json", "artifacts/bridge-diagnostics.json", "artifacts/bridge-diagnostics.jsonl",
|
|
656
|
+
];
|
|
657
|
+
if (suiteName === "cursor-abort-cleanup") {
|
|
658
|
+
expectedFiles.push("artifacts/abort-started.txt", "logs/process-before.stdout.txt", "logs/process-after.stdout.txt", "logs/leftover-process-check.stdout.txt");
|
|
659
|
+
}
|
|
660
|
+
if (stopResult) expectedFiles.push("crabbox.stop.stdout.txt", "crabbox.stop.stderr.txt", "crabbox.stop.exit-code.txt");
|
|
661
|
+
const { assertions } = finalizeSuiteArtifacts(suiteDir, checks, {
|
|
662
|
+
target: targetName,
|
|
663
|
+
suite: suiteName,
|
|
664
|
+
exitCode: result.code,
|
|
665
|
+
signal: result.signal,
|
|
666
|
+
elapsedMs: elapsed,
|
|
667
|
+
}, expectedFiles);
|
|
668
|
+
console.log(` ${assertions.ok ? "PASS" : "FAIL"} ${suiteName} on ${targetName} (${elapsed}ms)`);
|
|
669
|
+
return { ok: assertions.ok, suiteDir, assertions };
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
async function runOnLeaseWithTransientRetry(suiteDir, targetName, leaseId, command, options) {
|
|
673
|
+
const first = await runOnLease(targetName, leaseId, command, options);
|
|
674
|
+
if (!isTransientCrabboxSshFailure(first)) return first;
|
|
675
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.retry1.stdout.txt"), first.stdout);
|
|
676
|
+
writeRedactedFile(resolve(suiteDir, "crabbox.retry1.stderr.txt"), first.stderr);
|
|
677
|
+
await new Promise((resolveRetry) => setTimeout(resolveRetry, 10_000));
|
|
678
|
+
return await runOnLease(targetName, leaseId, command, { ...options, sync: false });
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
function isTransientCrabboxSshFailure(result) {
|
|
682
|
+
const text = `${result.stdout}\n${result.stderr}`;
|
|
683
|
+
return result.code === 255 && /ssh: connect to host .*\b(Operation timed out|Connection timed out)\b/i.test(text);
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
function buildLiveSuiteCommand(config, targetName, suiteName, prepDir) {
|
|
687
|
+
const model = config.cursorModel ?? "cursor/composer-2-5";
|
|
688
|
+
const packageName = config.packageName ?? "pi-cursor-sdk";
|
|
689
|
+
const prepArgs = prepDir ? ` --prep-dir ${platformFor(targetName) === "powershell" ? prepDir : shellQuote(prepDir)}` : "";
|
|
690
|
+
if (platformFor(targetName) === "powershell") {
|
|
691
|
+
return `powershell.exe -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "node scripts/platform-smoke/live-suite-runner.mjs --suite ${suiteName} --target ${targetName} --model ${model} --package-name ${packageName}${prepArgs}"`;
|
|
692
|
+
}
|
|
693
|
+
return `node scripts/platform-smoke/live-suite-runner.mjs --suite ${shellQuote(suiteName)} --target ${shellQuote(targetName)} --model ${shellQuote(model)} --package-name ${shellQuote(packageName)}${prepArgs}`;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
function extractLiveBundle(suiteDir, stdout) {
|
|
697
|
+
const start = stdout.indexOf("PLATFORM_LIVE_BUNDLE_JSON_START");
|
|
698
|
+
const end = stdout.indexOf("PLATFORM_LIVE_BUNDLE_JSON_END", start);
|
|
699
|
+
if (start === -1 || end === -1) return { ok: false, violations: [] };
|
|
700
|
+
const jsonText = stdout.slice(start + "PLATFORM_LIVE_BUNDLE_JSON_START".length, end).trim();
|
|
701
|
+
let bundle;
|
|
702
|
+
try { bundle = JSON.parse(jsonText); } catch { return { ok: false, violations: [] }; }
|
|
703
|
+
if (!Array.isArray(bundle.files)) return { ok: false, violations: [] };
|
|
704
|
+
const violations = [];
|
|
705
|
+
for (const file of bundle.files) {
|
|
706
|
+
if (!file?.path || typeof file.contentBase64 !== "string") continue;
|
|
707
|
+
if (!isSafeBundlePath(suiteDir, file.path)) return { ok: false, violations };
|
|
708
|
+
const outPath = resolve(suiteDir, file.path);
|
|
709
|
+
mkdirSync(dirname(outPath), { recursive: true });
|
|
710
|
+
const content = Buffer.from(file.contentBase64, "base64");
|
|
711
|
+
if (isTextArtifactPath(file.path)) {
|
|
712
|
+
const text = content.toString("utf8");
|
|
713
|
+
violations.push(...scanForSecrets(text).map((violation) => ({ file: file.path, violation })));
|
|
714
|
+
if (file.path.endsWith("redaction-violations.json")) {
|
|
715
|
+
violations.push(...readRedactionViolationList(text, file.path));
|
|
716
|
+
}
|
|
717
|
+
writeFileSync(outPath, redactSecrets(text));
|
|
718
|
+
} else {
|
|
719
|
+
writeFileSync(outPath, content);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
return { ok: true, violations };
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
function readRedactionViolationList(text, fallbackFile) {
|
|
726
|
+
try {
|
|
727
|
+
const parsed = JSON.parse(text);
|
|
728
|
+
if (!Array.isArray(parsed)) return [];
|
|
729
|
+
return parsed
|
|
730
|
+
.filter((item) => typeof item?.violation === "string")
|
|
731
|
+
.map((item) => ({ file: typeof item.file === "string" ? item.file : fallbackFile, violation: item.violation }));
|
|
732
|
+
} catch {
|
|
733
|
+
return [];
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
function isTextArtifactPath(path) {
|
|
738
|
+
const ext = path.split(".").pop()?.toLowerCase() ?? "";
|
|
739
|
+
return ["txt", "json", "jsonl", "md", "log", "ansi", "html", "yml", "yaml", "js", "mjs", "ts"].includes(ext);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
export function isSafeBundlePath(suiteDir, bundlePath) {
|
|
743
|
+
if (typeof bundlePath !== "string" || bundlePath.length === 0) return false;
|
|
744
|
+
if (isAbsolute(bundlePath) || /^[A-Za-z]:[\\/]/.test(bundlePath)) return false;
|
|
745
|
+
const outPath = resolve(suiteDir, bundlePath);
|
|
746
|
+
const rel = relative(suiteDir, outPath);
|
|
747
|
+
return rel.length > 0 && !rel.startsWith("..") && !isAbsolute(rel);
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
function readJson(path) {
|
|
751
|
+
try { return JSON.parse(readFileSync(path, "utf8")); } catch { return undefined; }
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
function findFiles(root) {
|
|
755
|
+
const files = [];
|
|
756
|
+
function visit(dir) {
|
|
757
|
+
let entries;
|
|
758
|
+
try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
759
|
+
for (const entry of entries) {
|
|
760
|
+
const path = resolve(dir, entry.name);
|
|
761
|
+
if (entry.isDirectory()) visit(path);
|
|
762
|
+
else if (entry.isFile()) files.push(path);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
visit(root);
|
|
766
|
+
return files;
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
function collectBridgeDiagnosticFile(path) {
|
|
770
|
+
let raw;
|
|
771
|
+
try { raw = readFileSync(path, "utf8"); } catch { return []; }
|
|
772
|
+
const events = [];
|
|
773
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
774
|
+
if (!line.trim()) continue;
|
|
775
|
+
try { events.push(JSON.parse(line)); } catch {}
|
|
776
|
+
}
|
|
777
|
+
return events;
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
function collectBridgeDiagnostics(terminalText) {
|
|
781
|
+
const prefix = "[pi-cursor-sdk:bridge] ";
|
|
782
|
+
const events = [];
|
|
783
|
+
for (const line of terminalText.split(/\r?\n/)) {
|
|
784
|
+
const index = line.indexOf(prefix);
|
|
785
|
+
if (index === -1) continue;
|
|
786
|
+
const jsonText = line.slice(index + prefix.length).trim();
|
|
787
|
+
try { events.push(JSON.parse(jsonText)); } catch {}
|
|
788
|
+
}
|
|
789
|
+
return events;
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
function collectUsageChecks(jsonlRaw) {
|
|
793
|
+
let seen = false;
|
|
794
|
+
let nonNegative = true;
|
|
795
|
+
let cacheZero = true;
|
|
796
|
+
for (const line of jsonlRaw.split(/\r?\n/)) {
|
|
797
|
+
if (!line.trim()) continue;
|
|
798
|
+
let event;
|
|
799
|
+
try { event = JSON.parse(line); } catch { continue; }
|
|
800
|
+
const usage = event?.message?.usage;
|
|
801
|
+
if (!usage || typeof usage !== "object") continue;
|
|
802
|
+
seen = true;
|
|
803
|
+
for (const value of Object.values(usage)) {
|
|
804
|
+
if (typeof value === "number" && value < 0) nonNegative = false;
|
|
805
|
+
}
|
|
806
|
+
if (typeof usage.cacheRead === "number" && usage.cacheRead !== 0) cacheZero = false;
|
|
807
|
+
if (typeof usage.cacheWrite === "number" && usage.cacheWrite !== 0) cacheZero = false;
|
|
808
|
+
}
|
|
809
|
+
return { seen, nonNegative, cacheZero };
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
export function hasAbortSuccessClaim(jsonlRaw) {
|
|
813
|
+
for (const line of jsonlRaw.split(/\r?\n/)) {
|
|
814
|
+
if (!line.trim()) continue;
|
|
815
|
+
let event;
|
|
816
|
+
try { event = JSON.parse(line); } catch { continue; }
|
|
817
|
+
const message = event?.message;
|
|
818
|
+
if (message?.role !== "assistant") continue;
|
|
819
|
+
const text = extractFinalTextContent(message.content);
|
|
820
|
+
if (/\b(?:done|complete|completed|success|succeeded|finished)\b/i.test(text)) return true;
|
|
821
|
+
}
|
|
822
|
+
return false;
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
function collectJsonlToolNames(jsonlRaw) {
|
|
826
|
+
const names = new Set();
|
|
827
|
+
for (const line of jsonlRaw.split(/\r?\n/)) {
|
|
828
|
+
if (!line.trim()) continue;
|
|
829
|
+
let event;
|
|
830
|
+
try { event = JSON.parse(line); } catch { continue; }
|
|
831
|
+
const message = event?.message;
|
|
832
|
+
if (typeof message?.toolName === "string") names.add(message.toolName);
|
|
833
|
+
for (const block of message?.content ?? []) {
|
|
834
|
+
if (typeof block?.name === "string") names.add(block.name);
|
|
835
|
+
if (typeof block?.details?.sourceToolName === "string") names.add(block.details.sourceToolName);
|
|
836
|
+
}
|
|
837
|
+
if (typeof message?.details?.sourceToolName === "string") names.add(message.details.sourceToolName);
|
|
838
|
+
}
|
|
839
|
+
return names;
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
function collectJsonlToolResults(jsonlRaw) {
|
|
843
|
+
const results = [];
|
|
844
|
+
for (const line of jsonlRaw.split(/\r?\n/)) {
|
|
845
|
+
if (!line.trim()) continue;
|
|
846
|
+
let event;
|
|
847
|
+
try { event = JSON.parse(line); } catch { continue; }
|
|
848
|
+
const message = event?.message;
|
|
849
|
+
if (message?.role !== "toolResult" || typeof message.toolName !== "string") continue;
|
|
850
|
+
const contentText = extractContentText(message.content);
|
|
851
|
+
results.push({
|
|
852
|
+
toolName: message.toolName,
|
|
853
|
+
isError: message.isError === true,
|
|
854
|
+
sourceToolName: message.details?.sourceToolName,
|
|
855
|
+
path: message.details?.path,
|
|
856
|
+
contentText,
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
return results;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
function matchesJsonlResult(result, requirement) {
|
|
863
|
+
if (requirement.toolName && result.toolName !== requirement.toolName) return false;
|
|
864
|
+
if (requirement.sourceToolName && result.sourceToolName !== requirement.sourceToolName) return false;
|
|
865
|
+
if (typeof requirement.isError === "boolean" && result.isError !== requirement.isError) return false;
|
|
866
|
+
const haystack = `${result.contentText}\n${result.path ?? ""}`;
|
|
867
|
+
if (requirement.contains && !haystack.includes(requirement.contains)) return false;
|
|
868
|
+
if (requirement.pattern && !(new RegExp(requirement.pattern, requirement.flags ?? "i")).test(haystack)) return false;
|
|
869
|
+
return true;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
function shellQuote(value) {
|
|
873
|
+
return `'${String(value).replace(/'/g, `'\\''`)}'`;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* Write a failure suite result. Used for live suite hard failures during
|
|
878
|
+
* warmup/execution and for unknown suites.
|
|
879
|
+
*/
|
|
880
|
+
function failSuite(suiteDir, targetName, suiteName, message) {
|
|
881
|
+
const safeMessage = redactSecrets(message);
|
|
882
|
+
console.log(` FAIL ${suiteName} on ${targetName}: ${safeMessage}`);
|
|
883
|
+
|
|
884
|
+
writeCommand(suiteDir, `# ${suiteName} — ${safeMessage}`);
|
|
885
|
+
writeExitCode(suiteDir, 1, null);
|
|
886
|
+
|
|
887
|
+
const checks = [{ id: "execution", fn: () => false, error: safeMessage }];
|
|
888
|
+
const { assertions } = finalizeSuiteArtifacts(
|
|
889
|
+
suiteDir,
|
|
890
|
+
checks,
|
|
891
|
+
{ target: targetName, suite: suiteName, exitCode: 1, error: safeMessage },
|
|
892
|
+
[
|
|
893
|
+
"summary.json", "target.json", "suite.json",
|
|
894
|
+
"command.txt", "exit-code.txt",
|
|
895
|
+
"assertions.json",
|
|
896
|
+
],
|
|
897
|
+
);
|
|
898
|
+
|
|
899
|
+
return { ok: false, suiteDir, assertions };
|
|
900
|
+
}
|