verikun 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -7
- package/dist/agent/remote.js +5 -1
- package/dist/args.js +1 -0
- package/dist/cli.js +172 -16
- package/dist/drivers/adb.js +27 -6
- package/dist/drivers/ios.js +5 -0
- package/dist/report.js +102 -9
- package/dist/run.js +234 -14
- package/dist/server.js +44 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -122,7 +122,7 @@ vk screenshot # -> ./.verikun/screen.png
|
|
|
122
122
|
|---|---|
|
|
123
123
|
| `run start [name] [--force]` | Begin a named run. One auto-starts on the first action if you don't. |
|
|
124
124
|
| `run status` | Show the active run and its recorded steps. |
|
|
125
|
-
| `run archive [name]` | Write JUnit + HTML report to `./.verikun/runs/<id>/`; exits non-zero if any step failed. |
|
|
125
|
+
| `run archive [name] [--no-logs]` | Write JUnit + HTML report to `./.verikun/runs/<id>/`; exits non-zero if any step failed. Captures `artifacts/logcat.txt` by default. |
|
|
126
126
|
| `run clear` | Discard the active run without a report. |
|
|
127
127
|
|
|
128
128
|
## Test runs & reports
|
|
@@ -130,10 +130,11 @@ vk screenshot # -> ./.verikun/screen.png
|
|
|
130
130
|
Actions are recorded into a **test run** — one auto-starts on the first action
|
|
131
131
|
(set `VERIKUN_NO_RUN=1` to disable). Every command becomes a step with its
|
|
132
132
|
timing, the selector + identifier it resolved through, and pass/fail; a failing
|
|
133
|
-
step also captures a screenshot **and** the UI hierarchy of the page.
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
the report
|
|
133
|
+
step also captures a screenshot **and** the UI hierarchy of the page. Device
|
|
134
|
+
logs are captured automatically at archive time into `artifacts/logcat.txt`
|
|
135
|
+
(session-scoped when possible). You can still run `vk log <package>` mid-run to
|
|
136
|
+
pull a snapshot into a step for the report; `--no-logs` / `VERIKUN_NO_LOGS`
|
|
137
|
+
skips the archive dump on green runs (failures still capture).
|
|
137
138
|
|
|
138
139
|
`vk run archive` finalizes the run into `./.verikun/runs/<id>/`:
|
|
139
140
|
|
|
@@ -141,10 +142,19 @@ the report alongside the failure.
|
|
|
141
142
|
for failed assertions, `<error>` for environment errors, and the resolved
|
|
142
143
|
identifier in `<system-out>`. Drops straight into CI.
|
|
143
144
|
- **`report.html`** — a self-contained report: every step, the identifiers used,
|
|
144
|
-
any screenshots taken, the screenshot + hierarchy of any failed page,
|
|
145
|
-
device
|
|
145
|
+
any screenshots taken, the screenshot + hierarchy of any failed page, a link to
|
|
146
|
+
the full device log, an app-scoped log accordion when the run launched an app,
|
|
147
|
+
and any per-step logs from `vk log`. A run that did not pass says so in a banner
|
|
148
|
+
at the top.
|
|
149
|
+
- **`artifacts/logcat.txt`** — full device log for the run window (default).
|
|
150
|
+
- **`artifacts/logcat-app.txt`** — app-scoped log (when a package/bundle was launched).
|
|
146
151
|
- **`run.json`** — the raw recording.
|
|
147
152
|
|
|
153
|
+
A `vk ai` run can also fail where no single command did — a `repeat` that never
|
|
154
|
+
sees its target, a cost/timeout abort. That verdict is recorded too (as
|
|
155
|
+
`run.json`'s `failure`, plus a failed step carrying the reason and a screenshot),
|
|
156
|
+
so a failed test is never reported as `failures="0"` in the JUnit that CI reads.
|
|
157
|
+
|
|
148
158
|
`vk run archive` exits non-zero when the run contained failures, so the same
|
|
149
159
|
command both produces the report and gates CI.
|
|
150
160
|
|
package/dist/agent/remote.js
CHANGED
|
@@ -112,7 +112,7 @@ function createRemoteBackend(opts, health) {
|
|
|
112
112
|
const execRaw = async (req, record) => {
|
|
113
113
|
const res = await t.postJson('/v1/exec', req, EXEC_TIMEOUT_MS);
|
|
114
114
|
if (record && res.step)
|
|
115
|
-
opts.onStep?.(res.step, decodeArtifacts(res.artifacts));
|
|
115
|
+
opts.onStep?.(res.step, decodeArtifacts(res.artifacts), res.logStart);
|
|
116
116
|
return { code: res.code, error: res.error ? (0, rpc_1.rebuildError)(res.error) : undefined };
|
|
117
117
|
};
|
|
118
118
|
return {
|
|
@@ -121,6 +121,10 @@ function createRemoteBackend(opts, health) {
|
|
|
121
121
|
const res = await t.postJson('/v1/elements', {}, ELEMENTS_TIMEOUT_MS);
|
|
122
122
|
return res.elements;
|
|
123
123
|
},
|
|
124
|
+
async getLogs(logOpts = {}) {
|
|
125
|
+
const res = await t.postJson('/v1/logs', logOpts, ELEMENTS_TIMEOUT_MS);
|
|
126
|
+
return res.logs ?? '';
|
|
127
|
+
},
|
|
124
128
|
async install(appPath) {
|
|
125
129
|
// v1 remote installs are single-file uploads; the extension is the only thing
|
|
126
130
|
// the client tells the server about the artifact (never a path).
|
package/dist/args.js
CHANGED
package/dist/cli.js
CHANGED
|
@@ -50,6 +50,7 @@ exports.chooseLogOpts = chooseLogOpts;
|
|
|
50
50
|
exports.evalAssert = evalAssert;
|
|
51
51
|
exports.tokenizeLine = tokenizeLine;
|
|
52
52
|
exports.withBatchGlobals = withBatchGlobals;
|
|
53
|
+
exports.terminalFailure = terminalFailure;
|
|
53
54
|
exports.executeForServer = executeForServer;
|
|
54
55
|
exports.run = run;
|
|
55
56
|
const node_fs_1 = require("node:fs");
|
|
@@ -975,16 +976,47 @@ function cmdRun(positionals, flags, platform, device) {
|
|
|
975
976
|
case 'archive':
|
|
976
977
|
case 'finish':
|
|
977
978
|
case 'save': {
|
|
978
|
-
const
|
|
979
|
+
const noLogs = (0, args_1.flagBool)(flags, 'no-logs');
|
|
980
|
+
// Best-effort: archive-time log capture needs a device. Prefer the run's
|
|
981
|
+
// bound serial/platform so a multi-device host hits the right one. A
|
|
982
|
+
// missing/broken toolchain must not prevent sealing the report.
|
|
983
|
+
let fetchLogs;
|
|
984
|
+
const active = run_1.Recorder.status();
|
|
985
|
+
const hasFailures = !!active?.steps.some((s) => s.status !== 'passed');
|
|
986
|
+
if ((0, run_1.wantsArchiveLogs)(hasFailures, noLogs)) {
|
|
987
|
+
try {
|
|
988
|
+
const plat = active?.platform === 'ios' || active?.platform === 'android'
|
|
989
|
+
? active.platform
|
|
990
|
+
: platform;
|
|
991
|
+
const driver = (0, drivers_1.getDriver)(plat, active?.device || device);
|
|
992
|
+
fetchLogs = (opts) => driver.getLogs(opts);
|
|
993
|
+
}
|
|
994
|
+
catch (e) {
|
|
995
|
+
(0, output_1.err)(`[verikun] archive log capture unavailable (${e.message})`);
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
const { dir, xmlPath, htmlPath, state } = run_1.Recorder.archive(positionals[1], { noLogs, fetchLogs });
|
|
979
999
|
const { passed, failed } = tally(state.steps);
|
|
980
1000
|
if (asJson) {
|
|
981
|
-
(0, output_1.json)({
|
|
1001
|
+
(0, output_1.json)({
|
|
1002
|
+
archived: dir,
|
|
1003
|
+
report: htmlPath,
|
|
1004
|
+
junit: xmlPath,
|
|
1005
|
+
steps: state.steps.length,
|
|
1006
|
+
passed,
|
|
1007
|
+
failed,
|
|
1008
|
+
...(state.logFile ? { logFile: state.logFile } : {}),
|
|
1009
|
+
});
|
|
982
1010
|
}
|
|
983
1011
|
else {
|
|
984
1012
|
(0, output_1.out)(dir); // primary result: the archived run directory
|
|
985
1013
|
(0, output_1.err)(`archived '${state.name}': ${state.steps.length} step(s), ${passed} passed, ${failed} failed/error`);
|
|
986
1014
|
(0, output_1.err)(` JUnit: ${xmlPath}`);
|
|
987
1015
|
(0, output_1.err)(` HTML: ${htmlPath}`);
|
|
1016
|
+
if (state.logFile)
|
|
1017
|
+
(0, output_1.err)(` Logs: ${(0, node_path_1.join)(dir, state.logFile)}`);
|
|
1018
|
+
if (state.appLogFile)
|
|
1019
|
+
(0, output_1.err)(` App: ${(0, node_path_1.join)(dir, state.appLogFile)}`);
|
|
988
1020
|
}
|
|
989
1021
|
// Exit non-zero when the run contained failures, so CI can gate on it.
|
|
990
1022
|
return failed > 0 ? 1 : 0;
|
|
@@ -1280,6 +1312,7 @@ async function resolveBackend(platform, device, flags) {
|
|
|
1280
1312
|
backend: {
|
|
1281
1313
|
exec: (command, positionals, f) => executeOutcome(command, positionals, f, driver),
|
|
1282
1314
|
getElements: () => driver.getElements(),
|
|
1315
|
+
getLogs: (opts) => driver.getLogs(opts),
|
|
1283
1316
|
install: (appPath) => driver.install(appPath),
|
|
1284
1317
|
reset: (appId) => {
|
|
1285
1318
|
assertSafeAppId(appId);
|
|
@@ -1290,6 +1323,24 @@ async function resolveBackend(platform, device, flags) {
|
|
|
1290
1323
|
driver.clearApp(appId);
|
|
1291
1324
|
},
|
|
1292
1325
|
preflight: () => driver.preflight(),
|
|
1326
|
+
captureFailure: async () => {
|
|
1327
|
+
// Two independent tries: a screencap can succeed where a dump doesn't (and
|
|
1328
|
+
// vice versa), and neither is allowed to derail recording the failure.
|
|
1329
|
+
const out = {};
|
|
1330
|
+
try {
|
|
1331
|
+
out.png = driver.screenshot();
|
|
1332
|
+
}
|
|
1333
|
+
catch {
|
|
1334
|
+
/* device may be gone — that may be why we failed */
|
|
1335
|
+
}
|
|
1336
|
+
try {
|
|
1337
|
+
out.hierarchy = driver.getElements({ all: false });
|
|
1338
|
+
}
|
|
1339
|
+
catch {
|
|
1340
|
+
/* ditto */
|
|
1341
|
+
}
|
|
1342
|
+
return out;
|
|
1343
|
+
},
|
|
1293
1344
|
},
|
|
1294
1345
|
platform,
|
|
1295
1346
|
device,
|
|
@@ -1300,8 +1351,9 @@ async function resolveBackend(platform, device, flags) {
|
|
|
1300
1351
|
url: server,
|
|
1301
1352
|
authKey: (0, args_1.flagStr)(flags, 'auth-key') || process.env.VERIKUN_SERVER_AUTH_KEY || undefined,
|
|
1302
1353
|
// Each remote step is spliced into the local active run so the archived report
|
|
1303
|
-
// is identical to a local run's.
|
|
1304
|
-
|
|
1354
|
+
// is identical to a local run's. logStart travels from the server's device clock
|
|
1355
|
+
// so archive-time / vk log scoping works without a local driver.
|
|
1356
|
+
onStep: (step, artifacts, logStart) => run_1.Recorder.appendForeignStep(step, artifacts, { ...runCtx, logStart }),
|
|
1305
1357
|
};
|
|
1306
1358
|
const health = await (0, remote_1.pingServer)(opts); // fails fast (exit 3) on a bad URL or key
|
|
1307
1359
|
runCtx = { platform: health.platform, device: health.serial };
|
|
@@ -1319,12 +1371,91 @@ async function resolveBackend(platform, device, flags) {
|
|
|
1319
1371
|
await (0, remote_1.pingServer)(opts);
|
|
1320
1372
|
await remote.getElements();
|
|
1321
1373
|
},
|
|
1374
|
+
// Hierarchy only: the server exposes no screenshot route, so a remote run's
|
|
1375
|
+
// engine failure archives without a picture. Honest degrade over a protocol
|
|
1376
|
+
// change here — tracked in #48.
|
|
1377
|
+
captureFailure: async () => {
|
|
1378
|
+
try {
|
|
1379
|
+
return { hierarchy: await remote.getElements() };
|
|
1380
|
+
}
|
|
1381
|
+
catch {
|
|
1382
|
+
return {};
|
|
1383
|
+
}
|
|
1384
|
+
},
|
|
1322
1385
|
},
|
|
1323
1386
|
platform: health.platform,
|
|
1324
1387
|
device: health.serial,
|
|
1325
1388
|
remote: { url: server, version: health.version },
|
|
1326
1389
|
};
|
|
1327
1390
|
}
|
|
1391
|
+
/**
|
|
1392
|
+
* The one terminal-failure record for a non-ok engine result — `null` when the run
|
|
1393
|
+
* passed. Exported for the unit suite.
|
|
1394
|
+
*
|
|
1395
|
+
* Budget and timeout aborts come back from the engine as a bare flag with NO `failure`
|
|
1396
|
+
* object, so their reason is composed here; `where` is `run` because the abort is not
|
|
1397
|
+
* attributable to one node. Both the recorded failure and the `[ai] …` status line are
|
|
1398
|
+
* built from this, so the archive and the console cannot disagree about why a run died.
|
|
1399
|
+
*/
|
|
1400
|
+
function terminalFailure(r, opts) {
|
|
1401
|
+
if (r.ok)
|
|
1402
|
+
return null;
|
|
1403
|
+
if (r.abortedForEnv) {
|
|
1404
|
+
return { where: r.failure?.where ?? 'run', reason: r.failure?.reason ?? 'device unavailable', kind: 'env' };
|
|
1405
|
+
}
|
|
1406
|
+
if (r.abortedForBudget) {
|
|
1407
|
+
return { where: r.failure?.where ?? 'run', reason: `cost ceiling $${opts.maxCostUsd} reached`, kind: 'budget' };
|
|
1408
|
+
}
|
|
1409
|
+
if (r.abortedForTimeout) {
|
|
1410
|
+
return { where: 'run', reason: `run timeout (${Math.round(opts.timeoutMs / 1000)}s) reached`, kind: 'timeout' };
|
|
1411
|
+
}
|
|
1412
|
+
return { where: r.failure?.where ?? 'run', reason: r.failure?.reason ?? 'failed', kind: 'fail' };
|
|
1413
|
+
}
|
|
1414
|
+
/** The `[ai] …` console verdict for a terminal failure, phrased as it always was. */
|
|
1415
|
+
function terminalStatusLine(t) {
|
|
1416
|
+
if (t.kind === 'fail')
|
|
1417
|
+
return `FAIL at ${t.where}: ${t.reason}`;
|
|
1418
|
+
return `ABORTED — ${t.kind === 'env' ? `environment: ${t.reason}` : t.reason}`;
|
|
1419
|
+
}
|
|
1420
|
+
/**
|
|
1421
|
+
* Best-effort archive-time device-log capture via an ExecBackend (local driver or
|
|
1422
|
+
* remote `/v1/logs`). Writes `artifacts/logcat.txt` onto the active run so a later
|
|
1423
|
+
* `Recorder.archive()` finds `logFile` already set. Never throws — a gone device
|
|
1424
|
+
* must not prevent sealing the report.
|
|
1425
|
+
*/
|
|
1426
|
+
async function prefetchArchiveLogs(backend, noLogs = false) {
|
|
1427
|
+
const state = run_1.Recorder.status();
|
|
1428
|
+
if (!state)
|
|
1429
|
+
return;
|
|
1430
|
+
const hasFailures = state.steps.some((s) => s.status !== 'passed');
|
|
1431
|
+
if (!(0, run_1.wantsArchiveLogs)(hasFailures, noLogs))
|
|
1432
|
+
return;
|
|
1433
|
+
if (!backend.getLogs)
|
|
1434
|
+
return;
|
|
1435
|
+
// Skip only when both artifacts are already attached (e.g. a prior prefetch).
|
|
1436
|
+
if (state.logFile && state.appLogFile)
|
|
1437
|
+
return;
|
|
1438
|
+
const window = (0, run_1.archiveLogWindow)(state);
|
|
1439
|
+
try {
|
|
1440
|
+
const full = state.logFile ? undefined : await backend.getLogs(window);
|
|
1441
|
+
const appId = (0, run_1.inferRunAppId)(state);
|
|
1442
|
+
let app;
|
|
1443
|
+
if (appId && !state.appLogFile) {
|
|
1444
|
+
try {
|
|
1445
|
+
app = await backend.getLogs({ ...window, appId, scopedOnly: true });
|
|
1446
|
+
}
|
|
1447
|
+
catch (e) {
|
|
1448
|
+
(0, output_1.err)(`[verikun] could not capture archive app logs (${e.message})`);
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
if (full !== undefined || (app !== undefined && app !== '')) {
|
|
1452
|
+
run_1.Recorder.attachArchiveLogs(full, app);
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
catch (e) {
|
|
1456
|
+
(0, output_1.err)(`[verikun] could not capture archive device logs (${e.message})`);
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1328
1459
|
/**
|
|
1329
1460
|
* Run one natural-language test through a backend and return DATA — no stdout
|
|
1330
1461
|
* writes (stdout stays the caller's one result; progress streams to stderr).
|
|
@@ -1365,10 +1496,14 @@ async function runAiTest(file, opts, backend, platform, device) {
|
|
|
1365
1496
|
if (existing && existing.steps.length > 0) {
|
|
1366
1497
|
// Seal the pre-existing run into the archive instead of letting start(force=true)
|
|
1367
1498
|
// discard it — a manual in-progress run should never be silently lost.
|
|
1499
|
+
await prefetchArchiveLogs(backend);
|
|
1368
1500
|
const sealed = run_1.Recorder.archive();
|
|
1369
1501
|
(0, output_1.err)(`[ai] archived the active run ('${existing.name}', ${existing.steps.length} step(s)) → ${sealed.dir}`);
|
|
1370
1502
|
}
|
|
1371
1503
|
const started = run_1.Recorder.start(`ai: ${(0, node_path_1.basename)(file)}`, platform, device, true);
|
|
1504
|
+
// Prefer --package when the prose never launches by id (rare but possible).
|
|
1505
|
+
if (opts.pkg)
|
|
1506
|
+
run_1.Recorder.annotateRun({ appId: opts.pkg });
|
|
1372
1507
|
// Suppress per-step `out()` so stdout stays the one final result; progress -> stderr.
|
|
1373
1508
|
const prevQuiet = (0, output_1.setOutputQuiet)(true);
|
|
1374
1509
|
let result;
|
|
@@ -1394,7 +1529,15 @@ async function runAiTest(file, opts, backend, platform, device) {
|
|
|
1394
1529
|
// seal the run so it is not left dangling in .verikun/run/ for the next command
|
|
1395
1530
|
// to roll over. Then let the error map to an exit code as usual.
|
|
1396
1531
|
run_1.Recorder.annotateRun({ ai: { ok: false, cost: cost.summaryLine(), modelRepairs: 0, improvements: [] } });
|
|
1532
|
+
// No evidence capture here: a throw at this level usually IS the device dying, so
|
|
1533
|
+
// the capture would fail the same way and only add noise to the error path.
|
|
1534
|
+
run_1.Recorder.recordTerminalFailure({
|
|
1535
|
+
where: 'run',
|
|
1536
|
+
reason: e.message,
|
|
1537
|
+
kind: (0, errors_1.isEnvError)(e) ? 'env' : 'fail',
|
|
1538
|
+
});
|
|
1397
1539
|
try {
|
|
1540
|
+
await prefetchArchiveLogs(backend);
|
|
1398
1541
|
run_1.Recorder.archive();
|
|
1399
1542
|
}
|
|
1400
1543
|
catch (sealErr) {
|
|
@@ -1419,20 +1562,18 @@ async function runAiTest(file, opts, backend, platform, device) {
|
|
|
1419
1562
|
(0, output_1.err)(`[ai] could not cache plan: ${e.message}`);
|
|
1420
1563
|
}
|
|
1421
1564
|
}
|
|
1565
|
+
// A failure the engine produced (a control node giving up, a budget/timeout abort)
|
|
1566
|
+
// never ran through a command, so nothing recorded it — without this the archive
|
|
1567
|
+
// declares the failed test green. Must come BEFORE the archive that renders it.
|
|
1568
|
+
const terminal = terminalFailure(result, opts);
|
|
1569
|
+
if (terminal)
|
|
1570
|
+
run_1.Recorder.recordTerminalFailure(terminal, await backend.captureFailure?.());
|
|
1422
1571
|
run_1.Recorder.annotateRun({
|
|
1423
1572
|
ai: { ok: result.ok, cost: costLine, modelRepairs: result.modelRepairs, improvements: result.improvements },
|
|
1424
1573
|
});
|
|
1574
|
+
await prefetchArchiveLogs(backend);
|
|
1425
1575
|
const { dir, xmlPath, htmlPath, state } = run_1.Recorder.archive();
|
|
1426
|
-
|
|
1427
|
-
? 'PASS'
|
|
1428
|
-
: result.abortedForBudget
|
|
1429
|
-
? `ABORTED — cost ceiling $${opts.maxCostUsd} reached`
|
|
1430
|
-
: result.abortedForTimeout
|
|
1431
|
-
? `ABORTED — run timeout (${Math.round(opts.timeoutMs / 1000)}s) reached`
|
|
1432
|
-
: result.abortedForEnv
|
|
1433
|
-
? `ABORTED — environment: ${result.failure?.reason}`
|
|
1434
|
-
: `FAIL at ${result.failure?.where}: ${result.failure?.reason}`;
|
|
1435
|
-
(0, output_1.err)(`[ai] ${status} · ${costLine}`);
|
|
1576
|
+
(0, output_1.err)(`[ai] ${terminal ? terminalStatusLine(terminal) : 'PASS'} · ${costLine}`);
|
|
1436
1577
|
(0, output_1.err)(`[ai] report: ${htmlPath}`);
|
|
1437
1578
|
if (result.improvements.length) {
|
|
1438
1579
|
(0, output_1.err)(`[ai] ${result.improvements.length} suggested improvement(s) (also in the report):`);
|
|
@@ -1708,11 +1849,23 @@ async function executeForServer(command, positionals, flags, driver, platform) {
|
|
|
1708
1849
|
catch {
|
|
1709
1850
|
/* surfaced by the command handler below */
|
|
1710
1851
|
}
|
|
1852
|
+
// Sample the device clock up front so the caller's run can set logStart (the
|
|
1853
|
+
// ephemeral recorder never persists RunState). Best-effort — empty/unavailable
|
|
1854
|
+
// just means archive / vk log fall back to last-N.
|
|
1855
|
+
let logStart;
|
|
1856
|
+
try {
|
|
1857
|
+
const t = driver.deviceTime();
|
|
1858
|
+
if (t)
|
|
1859
|
+
logStart = t;
|
|
1860
|
+
}
|
|
1861
|
+
catch {
|
|
1862
|
+
/* device clock unavailable */
|
|
1863
|
+
}
|
|
1711
1864
|
const recorder = run_1.Recorder.beginEphemeralStep(command, positionals, flags, platform, serial);
|
|
1712
1865
|
const ctx = { driver, platform, device: serial, positionals, flags, record: recorder };
|
|
1713
1866
|
const outcome = await runRecorded(command, ctx, recorder, driver);
|
|
1714
1867
|
const { step, artifacts } = recorder.takeEphemeral();
|
|
1715
|
-
return { ...outcome, step, artifacts };
|
|
1868
|
+
return { ...outcome, step, artifacts, ...(logStart ? { logStart } : {}) };
|
|
1716
1869
|
}
|
|
1717
1870
|
/**
|
|
1718
1871
|
* Run one already-parsed command for the CLI / `batch`: dispatch meta-commands,
|
|
@@ -1884,7 +2037,10 @@ ENVIRONMENT
|
|
|
1884
2037
|
TEST RUNS (actions are recorded; a run auto-starts on first action)
|
|
1885
2038
|
run start [name] [--force] Begin a named run (else one starts implicitly)
|
|
1886
2039
|
run status Show the active run, its device/session, and steps
|
|
1887
|
-
run archive [name]
|
|
2040
|
+
run archive [name] [--no-logs] Write JUnit + HTML report, move to ./.verikun/runs/<id>/
|
|
2041
|
+
Captures artifacts/logcat.txt by default (session-scoped);
|
|
2042
|
+
--no-logs / VERIKUN_NO_LOGS skips on green runs (failures
|
|
2043
|
+
still capture). Capture is best-effort and never blocks archive.
|
|
1888
2044
|
run clear Discard the active run with no report
|
|
1889
2045
|
An implicit run auto-closes (archives) and rolls over on a device change, a
|
|
1890
2046
|
VERIKUN_SESSION change, or VERIKUN_RUN_IDLE_MIN minutes idle (default 30; 0 off).
|
package/dist/drivers/adb.js
CHANGED
|
@@ -356,15 +356,36 @@ class AdbDriver {
|
|
|
356
356
|
args.push('-t', String(n));
|
|
357
357
|
}
|
|
358
358
|
if (opts.appId) {
|
|
359
|
-
//
|
|
360
|
-
//
|
|
361
|
-
//
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
359
|
+
// Prefer --uid: it survives process death/restart (crash traces stay under the
|
|
360
|
+
// package's uid). Fall back to --pid for a live process when uid isn't known.
|
|
361
|
+
// When neither works: vk log falls through to system-wide so a FATAL EXCEPTION
|
|
362
|
+
// isn't missed; archive accordion passes scopedOnly to keep the dump empty.
|
|
363
|
+
const uid = this.packageUid(opts.appId);
|
|
364
|
+
if (uid) {
|
|
365
|
+
args.push(`--uid=${uid}`);
|
|
366
|
+
}
|
|
367
|
+
else {
|
|
368
|
+
const pid = this.shell(['pidof', opts.appId]).trim().split(/\s+/)[0];
|
|
369
|
+
if (pid)
|
|
370
|
+
args.push(`--pid=${pid}`);
|
|
371
|
+
else if (opts.scopedOnly)
|
|
372
|
+
return '';
|
|
373
|
+
}
|
|
365
374
|
}
|
|
366
375
|
return this.shell(args, 15000);
|
|
367
376
|
}
|
|
377
|
+
/** Android userId for an installed package, or '' if unknown. Used to scope
|
|
378
|
+
* logcat across process restarts (unlike pidof, which only sees a live process). */
|
|
379
|
+
packageUid(appId) {
|
|
380
|
+
try {
|
|
381
|
+
const out = this.shell(['dumpsys', 'package', appId], 10000);
|
|
382
|
+
const m = /\buserId=(\d+)\b/.exec(out);
|
|
383
|
+
return m?.[1] ?? '';
|
|
384
|
+
}
|
|
385
|
+
catch {
|
|
386
|
+
return '';
|
|
387
|
+
}
|
|
388
|
+
}
|
|
368
389
|
deviceTime() {
|
|
369
390
|
// logcat's default timestamp is MM-DD HH:MM:SS.mmm in the device's LOCAL time.
|
|
370
391
|
// Sample it from the device clock with a space-free format (so no device-shell
|
package/dist/drivers/ios.js
CHANGED
|
@@ -433,6 +433,11 @@ class IdbDriver {
|
|
|
433
433
|
const proc = opts.appId.split('.').pop() || opts.appId;
|
|
434
434
|
args.push('--predicate', `process CONTAINS "${proc}"`);
|
|
435
435
|
}
|
|
436
|
+
else if (opts.scopedOnly) {
|
|
437
|
+
// scopedOnly without an appId can't mean anything on iOS — empty rather than
|
|
438
|
+
// a full dump (mirrors Android's "couldn't scope" behaviour).
|
|
439
|
+
return '';
|
|
440
|
+
}
|
|
436
441
|
const out = (0, exec_1.runText)(XCRUN, args, { timeout: 20000 }).stdout;
|
|
437
442
|
if (opts.since)
|
|
438
443
|
return out; // the whole session window
|
package/dist/report.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// recorder (run.ts) / suite runner (suite.ts) own all the I/O. The RunState data
|
|
6
6
|
// model lives in run.ts; we import the types only.
|
|
7
7
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.runFailure = runFailure;
|
|
8
9
|
exports.toJUnitXml = toJUnitXml;
|
|
9
10
|
exports.suiteTotals = suiteTotals;
|
|
10
11
|
exports.toSuiteIndexJson = toSuiteIndexJson;
|
|
@@ -41,12 +42,40 @@ function resolvedLabel(s) {
|
|
|
41
42
|
function fmtDuration(ms) {
|
|
42
43
|
return ms < 1000 ? `${ms} ms` : `${(ms / 1000).toFixed(2)} s`;
|
|
43
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* The run's verdict, taken from the ENGINE rather than inferred from step statuses.
|
|
47
|
+
*
|
|
48
|
+
* A `vk ai` run can fail outside any command — a `repeat` that never sees its target,
|
|
49
|
+
* a budget/timeout abort — and those record no step, so a tally-only report declared
|
|
50
|
+
* the run green (issue #41). `Recorder.recordTerminalFailure` now writes `failure`, and
|
|
51
|
+
* `ai.ok` is the older, coarser signal we still fall back to; between them an
|
|
52
|
+
* unrecorded failure can no longer read as success.
|
|
53
|
+
*/
|
|
54
|
+
function runFailure(run) {
|
|
55
|
+
if (run.failure)
|
|
56
|
+
return run.failure;
|
|
57
|
+
if (run.ai && !run.ai.ok)
|
|
58
|
+
return { where: 'run', reason: 'the run did not pass (no step recorded the failure)' };
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
44
61
|
function counts(run) {
|
|
45
62
|
const passed = run.steps.filter((s) => s.status === 'passed').length;
|
|
46
63
|
const failures = run.steps.filter((s) => s.status === 'failed').length;
|
|
47
64
|
const errors = run.steps.filter((s) => s.status === 'error').length;
|
|
48
65
|
const timeMs = run.steps.reduce((a, s) => a + s.durationMs, 0);
|
|
49
|
-
|
|
66
|
+
// Belt and braces. Normally the terminal failure IS a step by the time we render, so
|
|
67
|
+
// this stays false; it fires only if the failure never reached the recorder, and then
|
|
68
|
+
// both renderers emit one extra entry — hence `tests` grows too, so the tally never
|
|
69
|
+
// disagrees with the testcase list it is supposed to describe.
|
|
70
|
+
const unrecorded = failures + errors === 0 && runFailure(run) !== null;
|
|
71
|
+
return {
|
|
72
|
+
tests: run.steps.length + (unrecorded ? 1 : 0),
|
|
73
|
+
passed,
|
|
74
|
+
failures: failures + (unrecorded ? 1 : 0),
|
|
75
|
+
errors,
|
|
76
|
+
timeMs,
|
|
77
|
+
unrecorded,
|
|
78
|
+
};
|
|
50
79
|
}
|
|
51
80
|
// --- JUnit ----------------------------------------------------------------
|
|
52
81
|
function toJUnitXml(run) {
|
|
@@ -93,16 +122,28 @@ function toJUnitXml(run) {
|
|
|
93
122
|
return ` <testcase ${attrs}>${body}\n </testcase>`;
|
|
94
123
|
})
|
|
95
124
|
.join('\n');
|
|
125
|
+
const f = runFailure(run);
|
|
126
|
+
const unrecordedCase = c.unrecorded && f
|
|
127
|
+
? ` <testcase name="${xmlAttr(`run did not pass (${f.where})`)}" classname="verikun.run" time="0.000">` +
|
|
128
|
+
`\n <failure message="${xmlAttr(f.reason)}" type="AssertionFailure">${xmlText(`${f.where}: ${f.reason}`)}</failure>\n </testcase>`
|
|
129
|
+
: '';
|
|
130
|
+
const allCases = [cases, unrecordedCase].filter(Boolean).join('\n');
|
|
96
131
|
const suiteAttrs = `name="${xmlAttr(run.name)}" tests="${c.tests}" failures="${c.failures}" ` +
|
|
97
132
|
`errors="${c.errors}" time="${suiteTime}" timestamp="${xmlAttr(run.startedAt)}"`;
|
|
133
|
+
const suiteExtras = [];
|
|
134
|
+
if (run.logFile)
|
|
135
|
+
suiteExtras.push(`device log: ${run.logFile}`);
|
|
136
|
+
if (run.ai) {
|
|
137
|
+
suiteExtras.push('vk ai: ' +
|
|
138
|
+
run.ai.cost +
|
|
139
|
+
(run.ai.improvements.length ? '\nSuggested improvements:\n' + run.ai.improvements.join('\n') : ''));
|
|
140
|
+
}
|
|
98
141
|
return (`<?xml version="1.0" encoding="UTF-8"?>\n` +
|
|
99
142
|
`<testsuites name="verikun" tests="${c.tests}" failures="${c.failures}" errors="${c.errors}" time="${suiteTime}">\n` +
|
|
100
143
|
`<testsuite ${suiteAttrs}>\n` +
|
|
101
|
-
`${
|
|
102
|
-
(
|
|
103
|
-
? ` <system-out>${xmlText('
|
|
104
|
-
run.ai.cost +
|
|
105
|
-
(run.ai.improvements.length ? '\nSuggested improvements:\n' + run.ai.improvements.join('\n') : ''))}</system-out>\n`
|
|
144
|
+
`${allCases}\n` +
|
|
145
|
+
(suiteExtras.length
|
|
146
|
+
? ` <system-out>${xmlText(suiteExtras.join('\n'))}</system-out>\n`
|
|
106
147
|
: '') +
|
|
107
148
|
`</testsuite>\n</testsuites>\n`);
|
|
108
149
|
}
|
|
@@ -136,11 +177,18 @@ const STYLE = `
|
|
|
136
177
|
.msg.fail { color:var(--fail); }
|
|
137
178
|
img.shot { display:block; margin-top:10px; max-width:300px; max-height:520px; border:1px solid var(--line); border-radius:6px; }
|
|
138
179
|
details { margin-top:8px; }
|
|
180
|
+
details.run-log { margin-top:20px; background:#fff; border:1px solid var(--line); border-radius:8px; padding:10px 14px; }
|
|
181
|
+
details.run-log > summary { font-weight:600; color:#1f2328; }
|
|
182
|
+
details.run-log > summary a { font-weight:400; }
|
|
183
|
+
details.run-log pre { max-height:480px; }
|
|
139
184
|
summary { cursor:pointer; color:var(--muted); font-size:13px; }
|
|
140
185
|
pre { background:#0d1117; color:#e6edf3; padding:12px; border-radius:6px; overflow:auto; font-size:12px; line-height:1.45; max-height:360px; }
|
|
141
186
|
.aibox { background:#fff; border:1px solid var(--line); border-radius:8px; padding:12px 14px; margin-bottom:20px; font-size:13px; }
|
|
142
187
|
.aibox .cost { font-family:ui-monospace,SFMono-Regular,Menlo,monospace; color:var(--muted); margin-top:4px; }
|
|
143
188
|
.aibox ul { margin:8px 0 0; padding-left:18px; }
|
|
189
|
+
.failbox { background:#fff; border:1px solid var(--fail); border-left-width:4px; border-radius:8px; padding:12px 14px; margin-bottom:20px; font-size:13px; }
|
|
190
|
+
.failbox .where { font-family:ui-monospace,SFMono-Regular,Menlo,monospace; color:var(--muted); }
|
|
191
|
+
.failbox .why { color:var(--fail); margin-top:4px; }
|
|
144
192
|
`;
|
|
145
193
|
function aiPanelHtml(ai) {
|
|
146
194
|
const improvements = ai.improvements.length
|
|
@@ -154,6 +202,14 @@ function aiPanelHtml(ai) {
|
|
|
154
202
|
${improvements}
|
|
155
203
|
</div>`;
|
|
156
204
|
}
|
|
205
|
+
/** The run-level failure, stated once at the top. This page is where a human looks
|
|
206
|
+
* first, so the verdict has to be visible without reading 26 green rows. */
|
|
207
|
+
function failBoxHtml(f) {
|
|
208
|
+
return `<div class="failbox">
|
|
209
|
+
<div><strong>This run did not pass.</strong> <span class="where">${htmlEsc(f.where)}</span></div>
|
|
210
|
+
<div class="why">${htmlEsc(f.reason)}</div>
|
|
211
|
+
</div>`;
|
|
212
|
+
}
|
|
157
213
|
function stepHtml(s) {
|
|
158
214
|
const detail = [];
|
|
159
215
|
if (selectorLabel(s))
|
|
@@ -316,8 +372,14 @@ ${suite.tests.map((x) => suiteTestRow(x, linkBase)).join('\n')}
|
|
|
316
372
|
</html>
|
|
317
373
|
`;
|
|
318
374
|
}
|
|
319
|
-
|
|
375
|
+
/**
|
|
376
|
+
* @param opts.appLog app-scoped logcat body for the bottom accordion (kept out
|
|
377
|
+
* of RunState / run.json — pass the file contents when writing
|
|
378
|
+
* report.html). The full device dump stays a meta-row file link.
|
|
379
|
+
*/
|
|
380
|
+
function toHtml(run, opts = {}) {
|
|
320
381
|
const c = counts(run);
|
|
382
|
+
const failure = runFailure(run);
|
|
321
383
|
const chips = [
|
|
322
384
|
`<span class="chip pass">${c.passed} passed</span>`,
|
|
323
385
|
c.failures ? `<span class="chip fail">${c.failures} failed</span>` : '',
|
|
@@ -332,7 +394,20 @@ function toHtml(run) {
|
|
|
332
394
|
`started ${htmlEsc(run.startedAt)}`,
|
|
333
395
|
run.finishedAt ? `finished ${htmlEsc(run.finishedAt)}` : '',
|
|
334
396
|
run.implicit ? 'implicit run' : '',
|
|
397
|
+
run.logFile ? `<a href="${htmlEsc(run.logFile)}">device log</a>` : '',
|
|
335
398
|
].filter(Boolean);
|
|
399
|
+
// Bottom accordion: app-scoped dump only. The noisy full device log stays a
|
|
400
|
+
// download via the meta link — same shape on every framework (package/uid scope).
|
|
401
|
+
const appLabel = run.appId ? ` for ${htmlEsc(run.appId)}` : '';
|
|
402
|
+
const appFileLink = run.appLogFile
|
|
403
|
+
? ` (<a href="${htmlEsc(run.appLogFile)}">${htmlEsc(run.appLogFile)}</a>)`
|
|
404
|
+
: '';
|
|
405
|
+
const appLogPanel = opts.appLog !== undefined && opts.appLog !== ''
|
|
406
|
+
? `\n <details class="run-log">
|
|
407
|
+
<summary>App log${appLabel}${appFileLink}</summary>
|
|
408
|
+
<pre>${htmlEsc(opts.appLog)}</pre>
|
|
409
|
+
</details>`
|
|
410
|
+
: '';
|
|
336
411
|
return `<!doctype html>
|
|
337
412
|
<html lang="en">
|
|
338
413
|
<head>
|
|
@@ -348,10 +423,28 @@ function toHtml(run) {
|
|
|
348
423
|
<div class="summary">
|
|
349
424
|
${chips}
|
|
350
425
|
</div>
|
|
426
|
+
${failure ? failBoxHtml(failure) : ''}
|
|
351
427
|
${run.ai ? aiPanelHtml(run.ai) : ''}
|
|
352
428
|
<ol class="steps">
|
|
353
|
-
${
|
|
354
|
-
|
|
429
|
+
${[
|
|
430
|
+
...run.steps.map(stepHtml),
|
|
431
|
+
// Only when the failure reached no step — otherwise it is already a red row.
|
|
432
|
+
...(c.unrecorded && failure
|
|
433
|
+
? [
|
|
434
|
+
stepHtml({
|
|
435
|
+
index: run.steps.length,
|
|
436
|
+
command: 'ai',
|
|
437
|
+
name: `run did not pass (${failure.where})`,
|
|
438
|
+
startedAt: run.startedAt,
|
|
439
|
+
durationMs: 0,
|
|
440
|
+
status: 'failed',
|
|
441
|
+
exitCode: 1,
|
|
442
|
+
message: failure.reason,
|
|
443
|
+
}),
|
|
444
|
+
]
|
|
445
|
+
: []),
|
|
446
|
+
].join('\n ')}
|
|
447
|
+
</ol>${appLogPanel}
|
|
355
448
|
</div>
|
|
356
449
|
</body>
|
|
357
450
|
</html>
|
package/dist/run.js
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Recorder = void 0;
|
|
3
|
+
exports.Recorder = exports.RUN_APP_LOG_ARTIFACT = exports.RUN_LOG_ARTIFACT = void 0;
|
|
4
4
|
exports.isRecordable = isRecordable;
|
|
5
|
+
exports.wantsArchiveLogs = wantsArchiveLogs;
|
|
6
|
+
exports.archiveLogWindow = archiveLogWindow;
|
|
7
|
+
exports.inferRunAppId = inferRunAppId;
|
|
5
8
|
exports.runId = runId;
|
|
6
9
|
exports.uniqueDir = uniqueDir;
|
|
7
10
|
exports.rolloverReason = rolloverReason;
|
|
11
|
+
exports.rolloverLogsSameDevice = rolloverLogsSameDevice;
|
|
8
12
|
exports.stepName = stepName;
|
|
9
13
|
const node_fs_1 = require("node:fs");
|
|
10
14
|
const node_path_1 = require("node:path");
|
|
@@ -32,6 +36,58 @@ function isRecordable(command) {
|
|
|
32
36
|
}
|
|
33
37
|
const HIERARCHY_CAP = 24000; // chars of failure hierarchy kept inline in run.json
|
|
34
38
|
const LOG_CAP = 50000; // chars of device logs kept inline in run.json (tail-kept)
|
|
39
|
+
/** Max chars written to the archive-time `artifacts/logcat.txt` (tail-kept). Larger
|
|
40
|
+
* than the per-step LOG_CAP because the file is not inlined into run.json. */
|
|
41
|
+
const LOG_FILE_CAP = 512_000;
|
|
42
|
+
/** When no session `logStart` marker exists, archive captures this many trailing lines. */
|
|
43
|
+
const ARCHIVE_LOG_LINES = 5000;
|
|
44
|
+
exports.RUN_LOG_ARTIFACT = 'artifacts/logcat.txt';
|
|
45
|
+
exports.RUN_APP_LOG_ARTIFACT = 'artifacts/logcat-app.txt';
|
|
46
|
+
const APP_LIFECYCLE = new Set(['launch', 'open', 'stop', 'clear']);
|
|
47
|
+
/** Whether archive-time device-log capture should run. Default on; `--no-logs` /
|
|
48
|
+
* `VERIKUN_NO_LOGS` opts out of *green* runs only — a failed run always captures
|
|
49
|
+
* (best-effort), because that is when the log is load-bearing. Exported for tests. */
|
|
50
|
+
function wantsArchiveLogs(hasFailures, noLogsFlag = false) {
|
|
51
|
+
if (hasFailures)
|
|
52
|
+
return true;
|
|
53
|
+
if (noLogsFlag)
|
|
54
|
+
return false;
|
|
55
|
+
if (process.env.VERIKUN_NO_LOGS)
|
|
56
|
+
return false;
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
/** Window passed to `getLogs` at archive time: session-scoped when `logStart` is
|
|
60
|
+
* known, otherwise a bounded trailing dump. Exported for tests. */
|
|
61
|
+
function archiveLogWindow(state) {
|
|
62
|
+
if (state.logStart)
|
|
63
|
+
return { since: state.logStart };
|
|
64
|
+
return { lines: ARCHIVE_LOG_LINES };
|
|
65
|
+
}
|
|
66
|
+
/** Package / bundle id the run exercised, if we can tell. Prefer an explicit
|
|
67
|
+
* `state.appId` (set on launch/open/stop/clear); otherwise recover from those
|
|
68
|
+
* steps' names. Exported for tests. */
|
|
69
|
+
function inferRunAppId(state) {
|
|
70
|
+
if (state.appId && /^[A-Za-z0-9._-]+$/.test(state.appId))
|
|
71
|
+
return state.appId;
|
|
72
|
+
for (let i = state.steps.length - 1; i >= 0; i--) {
|
|
73
|
+
const s = state.steps[i];
|
|
74
|
+
if (!APP_LIFECYCLE.has(s.command))
|
|
75
|
+
continue;
|
|
76
|
+
// stepName is `launch com.foo` / `clear com.foo` (no flags in the name).
|
|
77
|
+
const m = /^(?:launch|open|stop|clear)\s+([A-Za-z0-9._-]+)\s*$/.exec(s.name);
|
|
78
|
+
if (m)
|
|
79
|
+
return m[1];
|
|
80
|
+
}
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
function truncateLogFile(text) {
|
|
84
|
+
if (text.length <= LOG_FILE_CAP)
|
|
85
|
+
return text;
|
|
86
|
+
return '…(truncated)\n' + text.slice(-LOG_FILE_CAP);
|
|
87
|
+
}
|
|
88
|
+
function runHasFailures(state) {
|
|
89
|
+
return state.steps.some((s) => s.status !== 'passed');
|
|
90
|
+
}
|
|
35
91
|
// --- paths & persistence --------------------------------------------------
|
|
36
92
|
const activeDir = () => (0, node_path_1.join)((0, output_1.artifactDir)(), 'run');
|
|
37
93
|
const archiveBase = () => (0, node_path_1.join)((0, output_1.artifactDir)(), 'runs');
|
|
@@ -49,6 +105,19 @@ function loadState(dir) {
|
|
|
49
105
|
return null;
|
|
50
106
|
}
|
|
51
107
|
}
|
|
108
|
+
/** Relative path of a step's failure screenshot. One place, because a synthetic
|
|
109
|
+
* terminal-failure step has to land on the same convention as a recorded one. */
|
|
110
|
+
const failImagePath = (index) => `artifacts/step-${index}-fail.png`;
|
|
111
|
+
function writeArtifactTo(dir, rel, buf) {
|
|
112
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.join)(dir, 'artifacts'), { recursive: true });
|
|
113
|
+
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(dir, rel), buf);
|
|
114
|
+
}
|
|
115
|
+
/** Compact hierarchy text, HEAD-capped — a failure dump is read top-down, so the
|
|
116
|
+
* first screenful is the useful part (unlike logs, which are kept tail-first). */
|
|
117
|
+
function capHierarchy(els) {
|
|
118
|
+
const text = (0, format_1.formatCompact)(els);
|
|
119
|
+
return text.length > HIERARCHY_CAP ? text.slice(0, HIERARCHY_CAP) + '\n…(truncated)' : text;
|
|
120
|
+
}
|
|
52
121
|
function saveState(dir, state) {
|
|
53
122
|
(0, node_fs_1.mkdirSync)(dir, { recursive: true });
|
|
54
123
|
(0, node_fs_1.writeFileSync)(statePath(dir), JSON.stringify(state, null, 2));
|
|
@@ -123,6 +192,14 @@ function rolloverReason(state, serial, session) {
|
|
|
123
192
|
return `idle for ${fmtAge(ageMs(state))} (>${idle}m)`;
|
|
124
193
|
return null;
|
|
125
194
|
}
|
|
195
|
+
/** Whether the incoming step's driver is safe to use for archive-time logs of the
|
|
196
|
+
* run being sealed. On a device-change rollover the driver is already pointed at
|
|
197
|
+
* the *new* serial — pulling logcat from it would attribute the wrong device's
|
|
198
|
+
* output to the old run. Idle/session rollover on the same device is fine.
|
|
199
|
+
* Exported for tests. */
|
|
200
|
+
function rolloverLogsSameDevice(runDevice, newSerial) {
|
|
201
|
+
return !runDevice || !newSerial || runDevice === newSerial;
|
|
202
|
+
}
|
|
126
203
|
function stepName(command, positionals, flags) {
|
|
127
204
|
const p = positionals;
|
|
128
205
|
const at = typeof flags['at'] === 'string' ? flags['at'] : undefined;
|
|
@@ -187,7 +264,13 @@ class Recorder {
|
|
|
187
264
|
const reason = rolloverReason(state, serial, session);
|
|
188
265
|
if (reason) {
|
|
189
266
|
try {
|
|
190
|
-
|
|
267
|
+
// Only reuse this step's driver when it still targets the run's device.
|
|
268
|
+
// A device-change rollover must not write the new device's logcat into
|
|
269
|
+
// the old run's archive (vk run archive binds a driver to state.device).
|
|
270
|
+
const fetchLogs = driver && rolloverLogsSameDevice(state.device, serial)
|
|
271
|
+
? (opts) => driver.getLogs(opts)
|
|
272
|
+
: undefined;
|
|
273
|
+
const dest = Recorder.seal(state, dir, { fetchLogs });
|
|
191
274
|
(0, output_1.err)(`[verikun] previous run '${state.name}' (${state.steps.length} step(s)) auto-closed → ${dest} (${reason}); starting a fresh run`);
|
|
192
275
|
state = null;
|
|
193
276
|
rolledOver = true;
|
|
@@ -220,6 +303,11 @@ class Recorder {
|
|
|
220
303
|
if (!state.session && session)
|
|
221
304
|
state.session = session;
|
|
222
305
|
}
|
|
306
|
+
// Remember the app under test from lifecycle commands so archive can scope
|
|
307
|
+
// the accordion log without requiring a separate flag.
|
|
308
|
+
if (APP_LIFECYCLE.has(command) && positionals[0] && /^[A-Za-z0-9._-]+$/.test(positionals[0])) {
|
|
309
|
+
state.appId = positionals[0];
|
|
310
|
+
}
|
|
223
311
|
// Anchor the log window at the session's first step (covers both implicit
|
|
224
312
|
// creation and an explicit `vk run start`, which records no marker itself).
|
|
225
313
|
// Best-effort — a missing marker just means `vk log` falls back to last-N.
|
|
@@ -352,8 +440,8 @@ class Recorder {
|
|
|
352
440
|
if (!driver)
|
|
353
441
|
return;
|
|
354
442
|
try {
|
|
355
|
-
this.writeArtifact(
|
|
356
|
-
this.step.failImage =
|
|
443
|
+
this.writeArtifact(failImagePath(this.step.index), driver.screenshot());
|
|
444
|
+
this.step.failImage = failImagePath(this.step.index);
|
|
357
445
|
}
|
|
358
446
|
catch (e) {
|
|
359
447
|
// Best-effort evidence: the device may be gone (often why the step failed). Surface
|
|
@@ -362,8 +450,7 @@ class Recorder {
|
|
|
362
450
|
(0, output_1.err)(`[verikun] could not capture failure screenshot (${e.message})`);
|
|
363
451
|
}
|
|
364
452
|
try {
|
|
365
|
-
|
|
366
|
-
this.step.failHierarchy = text.length > HIERARCHY_CAP ? text.slice(0, HIERARCHY_CAP) + '\n…(truncated)' : text;
|
|
453
|
+
this.step.failHierarchy = capHierarchy(driver.getElements({ all: false }));
|
|
367
454
|
}
|
|
368
455
|
catch (e) {
|
|
369
456
|
if (!quiet)
|
|
@@ -375,8 +462,7 @@ class Recorder {
|
|
|
375
462
|
this.sink[rel] = buf;
|
|
376
463
|
return;
|
|
377
464
|
}
|
|
378
|
-
(
|
|
379
|
-
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(this.dir, rel), buf);
|
|
465
|
+
writeArtifactTo(this.dir, rel, buf);
|
|
380
466
|
}
|
|
381
467
|
commit() {
|
|
382
468
|
this.step.durationMs = Date.now() - this.startMs;
|
|
@@ -386,18 +472,87 @@ class Recorder {
|
|
|
386
472
|
this.state.updatedAt = nowIso();
|
|
387
473
|
saveState(this.dir, this.state);
|
|
388
474
|
}
|
|
389
|
-
/** Finalize a run: write reports next to it, then
|
|
390
|
-
|
|
475
|
+
/** Finalize a run: optionally capture device logs, write reports next to it, then
|
|
476
|
+
* move it into ./.verikun/runs/<id>/. Log capture is best-effort — a device that
|
|
477
|
+
* is gone (often why the run failed) must never prevent sealing the report. */
|
|
478
|
+
static seal(state, dir, opts = {}) {
|
|
479
|
+
Recorder.maybeCaptureArchiveLogs(state, dir, opts);
|
|
391
480
|
state.finishedAt = nowIso();
|
|
392
481
|
state.updatedAt = nowIso();
|
|
393
482
|
saveState(dir, state);
|
|
394
483
|
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(dir, 'report.xml'), (0, report_1.toJUnitXml)(state));
|
|
395
|
-
|
|
484
|
+
// Accordion embeds the *app*-scoped dump (readable); the full device dump stays
|
|
485
|
+
// a file link in the meta row. Missing file is fine — capture is best-effort.
|
|
486
|
+
let appLog;
|
|
487
|
+
if (state.appLogFile) {
|
|
488
|
+
try {
|
|
489
|
+
appLog = (0, node_fs_1.readFileSync)((0, node_path_1.join)(dir, state.appLogFile), 'utf8');
|
|
490
|
+
}
|
|
491
|
+
catch {
|
|
492
|
+
/* report still writes */
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(dir, 'report.html'), (0, report_1.toHtml)(state, { appLog }));
|
|
396
496
|
(0, node_fs_1.mkdirSync)(archiveBase(), { recursive: true });
|
|
397
497
|
const dest = uniqueDir((0, node_path_1.join)(archiveBase(), state.id));
|
|
398
498
|
(0, node_fs_1.renameSync)(dir, dest);
|
|
399
499
|
return dest;
|
|
400
500
|
}
|
|
501
|
+
/** Write archive-time log artifacts on the ACTIVE run without sealing. Used by
|
|
502
|
+
* remote (`--server`) callers that fetch asynchronously before `archive()`.
|
|
503
|
+
* Pass `full` / `app` only for the pieces you have — omitted sides are left alone. */
|
|
504
|
+
static attachArchiveLogs(full, app) {
|
|
505
|
+
const dir = activeDir();
|
|
506
|
+
const state = loadState(dir);
|
|
507
|
+
if (!state)
|
|
508
|
+
return;
|
|
509
|
+
if (full !== undefined) {
|
|
510
|
+
Recorder.writeArchiveLog(state, dir, full, exports.RUN_LOG_ARTIFACT, 'logFile');
|
|
511
|
+
}
|
|
512
|
+
if (app !== undefined && app !== '') {
|
|
513
|
+
if (!state.appId)
|
|
514
|
+
state.appId = inferRunAppId(state);
|
|
515
|
+
Recorder.writeArchiveLog(state, dir, app, exports.RUN_APP_LOG_ARTIFACT, 'appLogFile');
|
|
516
|
+
}
|
|
517
|
+
saveState(dir, state);
|
|
518
|
+
}
|
|
519
|
+
static writeArchiveLog(state, dir, text, rel, field) {
|
|
520
|
+
const body = truncateLogFile(text);
|
|
521
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.join)(dir, 'artifacts'), { recursive: true });
|
|
522
|
+
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(dir, rel), body);
|
|
523
|
+
state[field] = rel;
|
|
524
|
+
}
|
|
525
|
+
static maybeCaptureArchiveLogs(state, dir, opts) {
|
|
526
|
+
if (!wantsArchiveLogs(runHasFailures(state), opts.noLogs))
|
|
527
|
+
return;
|
|
528
|
+
if (!opts.fetchLogs)
|
|
529
|
+
return;
|
|
530
|
+
const window = archiveLogWindow(state);
|
|
531
|
+
if (!state.logFile) {
|
|
532
|
+
try {
|
|
533
|
+
const text = opts.fetchLogs(window);
|
|
534
|
+
if (text !== undefined && text !== null) {
|
|
535
|
+
Recorder.writeArchiveLog(state, dir, text, exports.RUN_LOG_ARTIFACT, 'logFile');
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
catch (e) {
|
|
539
|
+
(0, output_1.err)(`[verikun] could not capture archive device logs (${e.message})`);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
const appId = inferRunAppId(state);
|
|
543
|
+
if (appId)
|
|
544
|
+
state.appId = appId;
|
|
545
|
+
if (appId && !state.appLogFile) {
|
|
546
|
+
try {
|
|
547
|
+
const text = opts.fetchLogs({ ...window, appId, scopedOnly: true });
|
|
548
|
+
if (text)
|
|
549
|
+
Recorder.writeArchiveLog(state, dir, text, exports.RUN_APP_LOG_ARTIFACT, 'appLogFile');
|
|
550
|
+
}
|
|
551
|
+
catch (e) {
|
|
552
|
+
(0, output_1.err)(`[verikun] could not capture archive app logs (${e.message})`);
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
}
|
|
401
556
|
/** One-line context summary for `vk run status`. */
|
|
402
557
|
static contextLine(state) {
|
|
403
558
|
const bits = [];
|
|
@@ -428,6 +583,8 @@ class Recorder {
|
|
|
428
583
|
* a local one. Re-indexes the step, writes its artifact buffers under the new
|
|
429
584
|
* index, and rewrites the step's artifact references to match. Creates an
|
|
430
585
|
* implicit run first if none is active (parity with beginStep's auto-start).
|
|
586
|
+
* `ctx.logStart` (device-clock marker from the server) is recorded on the first
|
|
587
|
+
* splice so archive-time / `vk log` scoping works the same as a local run.
|
|
431
588
|
*/
|
|
432
589
|
static appendForeignStep(step, artifacts = {}, ctx = {}) {
|
|
433
590
|
if (process.env.VERIKUN_NO_RUN)
|
|
@@ -448,6 +605,17 @@ class Recorder {
|
|
|
448
605
|
};
|
|
449
606
|
(0, output_1.err)('[verikun] recording test run (implicit) — archive: `vk run archive` · discard: `vk run clear`');
|
|
450
607
|
}
|
|
608
|
+
// Anchor the log window from the server's device clock (beginEphemeralStep
|
|
609
|
+
// never persists state, so the marker has to travel on the wire).
|
|
610
|
+
if (!state.logStart && ctx.logStart)
|
|
611
|
+
state.logStart = ctx.logStart;
|
|
612
|
+
// Mirror local beginStep: remember the app from lifecycle steps so archive
|
|
613
|
+
// can scope the accordion without a local launch having set state.appId.
|
|
614
|
+
if (APP_LIFECYCLE.has(step.command)) {
|
|
615
|
+
const m = /^(?:launch|open|stop|clear)\s+([A-Za-z0-9._-]+)\s*$/.exec(step.name);
|
|
616
|
+
if (m)
|
|
617
|
+
state.appId = m[1];
|
|
618
|
+
}
|
|
451
619
|
const index = state.steps.length;
|
|
452
620
|
const spliced = { ...step, index };
|
|
453
621
|
for (const [rel, buf] of Object.entries(artifacts)) {
|
|
@@ -490,6 +658,56 @@ class Recorder {
|
|
|
490
658
|
last.message = message;
|
|
491
659
|
saveState(dir, state);
|
|
492
660
|
}
|
|
661
|
+
/**
|
|
662
|
+
* Record a terminal failure the `vk ai` ENGINE produced rather than a command — a
|
|
663
|
+
* control node that gave up (`repeat` exhausted, `when` matched no branch), a
|
|
664
|
+
* budget/timeout abort, an engine-internal throw. None of those go through
|
|
665
|
+
* beginStep, so before this existed nothing marked the run red and the archived
|
|
666
|
+
* report declared a failed test fully green (issue #41).
|
|
667
|
+
*
|
|
668
|
+
* Always records the run-level verdict. Appends a synthetic failed step ONLY when
|
|
669
|
+
* no step is already red — a leaf failure carries its own step and evidence, and
|
|
670
|
+
* counting it twice would be its own kind of lie.
|
|
671
|
+
*/
|
|
672
|
+
static recordTerminalFailure(failure, evidence) {
|
|
673
|
+
if (process.env.VERIKUN_NO_RUN)
|
|
674
|
+
return;
|
|
675
|
+
const dir = activeDir();
|
|
676
|
+
const state = loadState(dir);
|
|
677
|
+
if (!state)
|
|
678
|
+
return;
|
|
679
|
+
state.failure = { where: failure.where, reason: failure.reason };
|
|
680
|
+
if (state.steps.every((s) => s.status === 'passed')) {
|
|
681
|
+
const index = state.steps.length;
|
|
682
|
+
// An environment abort is exit 3 (the box is broken, not the app), matching
|
|
683
|
+
// what `vk ai` itself returns; everything else is an exit-1 assertion failure.
|
|
684
|
+
const env = failure.kind === 'env';
|
|
685
|
+
const step = {
|
|
686
|
+
index,
|
|
687
|
+
command: 'ai',
|
|
688
|
+
name: `ai ${env ? 'aborted' : 'failed'} at ${failure.where}`,
|
|
689
|
+
startedAt: nowIso(),
|
|
690
|
+
durationMs: 0,
|
|
691
|
+
status: env ? 'error' : 'failed',
|
|
692
|
+
exitCode: env ? 3 : 1,
|
|
693
|
+
message: failure.reason,
|
|
694
|
+
};
|
|
695
|
+
if (evidence?.png) {
|
|
696
|
+
try {
|
|
697
|
+
writeArtifactTo(dir, failImagePath(index), evidence.png);
|
|
698
|
+
step.failImage = failImagePath(index);
|
|
699
|
+
}
|
|
700
|
+
catch (e) {
|
|
701
|
+
(0, output_1.err)(`[verikun] could not write the failure screenshot (${e.message})`);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
if (evidence?.hierarchy)
|
|
705
|
+
step.failHierarchy = capHierarchy(evidence.hierarchy);
|
|
706
|
+
state.steps.push(step);
|
|
707
|
+
}
|
|
708
|
+
state.updatedAt = nowIso();
|
|
709
|
+
saveState(dir, state);
|
|
710
|
+
}
|
|
493
711
|
static start(name, platform, device, force) {
|
|
494
712
|
const dir = activeDir();
|
|
495
713
|
const existing = loadState(dir);
|
|
@@ -521,8 +739,10 @@ class Recorder {
|
|
|
521
739
|
(0, node_fs_1.rmSync)(dir, { recursive: true, force: true });
|
|
522
740
|
return existing;
|
|
523
741
|
}
|
|
524
|
-
/** Write JUnit + HTML reports and move the run into ./.verikun/runs/<id>/.
|
|
525
|
-
|
|
742
|
+
/** Write JUnit + HTML reports and move the run into ./.verikun/runs/<id>/.
|
|
743
|
+
* By default captures a bounded device-log tail into `artifacts/logcat.txt`
|
|
744
|
+
* (see wantsArchiveLogs / ArchiveLogOpts). */
|
|
745
|
+
static archive(name, opts = {}) {
|
|
526
746
|
const dir = activeDir();
|
|
527
747
|
if (!(0, node_fs_1.existsSync)(statePath(dir))) {
|
|
528
748
|
throw new errors_1.CliError('No active test run to archive. Run an action first, or `vk run start`.', 1);
|
|
@@ -532,7 +752,7 @@ class Recorder {
|
|
|
532
752
|
throw new errors_1.CliError('Active run state is unreadable (.verikun/run/run.json is corrupt).', 3);
|
|
533
753
|
if (name)
|
|
534
754
|
state.name = name;
|
|
535
|
-
const dest = Recorder.seal(state, dir);
|
|
755
|
+
const dest = Recorder.seal(state, dir, opts);
|
|
536
756
|
return { dir: dest, xmlPath: (0, node_path_1.join)(dest, 'report.xml'), htmlPath: (0, node_path_1.join)(dest, 'report.html'), state };
|
|
537
757
|
}
|
|
538
758
|
}
|
package/dist/server.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
// through the SAME validateNode gate that guards `vk ai` model repairs, so only
|
|
12
12
|
// KNOWN_COMMANDS action verbs execute — never `ui`/`log`, never a shell. The
|
|
13
13
|
// server's driver is fixed at startup; client flags can never repoint the device.
|
|
14
|
+
// Archive-time log capture uses the dedicated /v1/logs endpoint instead.
|
|
14
15
|
// - /v1/install is a privileged management verb: auth PLUS --allow-install, body
|
|
15
16
|
// streamed to a server-generated temp path (the client supplies only an
|
|
16
17
|
// allowlisted extension — never a path), optional sha256 verification.
|
|
@@ -154,13 +155,14 @@ function buildServer(config) {
|
|
|
154
155
|
if (node.type !== 'command')
|
|
155
156
|
throw new HttpError(400, 'rejected: not a command leaf');
|
|
156
157
|
const t0 = Date.now();
|
|
157
|
-
const { code, error, step, artifacts } = await (0, cli_1.executeForServer)(node.command, node.positionals, (0, ir_1.leafToFlags)(node), config.driver, config.platform);
|
|
158
|
+
const { code, error, step, artifacts, logStart } = await (0, cli_1.executeForServer)(node.command, node.positionals, (0, ir_1.leafToFlags)(node), config.driver, config.platform);
|
|
158
159
|
(0, output_1.err)(`[server] exec ${node.command} ${node.positionals.join(' ')} → exit ${code} (${Date.now() - t0}ms)`);
|
|
159
160
|
const payload = {
|
|
160
161
|
code,
|
|
161
162
|
...(error ? { error: (0, rpc_1.describeError)(error) } : {}),
|
|
162
163
|
...(step ? { step } : {}),
|
|
163
164
|
...(artifacts && Object.keys(artifacts).length ? { artifacts: encodeArtifacts(artifacts) } : {}),
|
|
165
|
+
...(logStart ? { logStart } : {}),
|
|
164
166
|
};
|
|
165
167
|
sendJson(res, 200, payload);
|
|
166
168
|
}
|
|
@@ -169,6 +171,45 @@ function buildServer(config) {
|
|
|
169
171
|
const elements = config.driver.getElements(); // CliError(3) on dump failure → 500 below
|
|
170
172
|
sendJson(res, 200, { elements });
|
|
171
173
|
}
|
|
174
|
+
async function handleLogs(req, res) {
|
|
175
|
+
const body = await readBody(req, EXEC_BODY_CAP);
|
|
176
|
+
let parsed = {};
|
|
177
|
+
if (body.length) {
|
|
178
|
+
try {
|
|
179
|
+
parsed = JSON.parse(body.toString('utf8'));
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
throw new HttpError(400, 'invalid JSON body');
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
// Mirror the driver's --since charset gate so a remote caller cannot inject
|
|
186
|
+
// into the device shell via a crafted marker (see AdbDriver.getLogs).
|
|
187
|
+
if (parsed.since !== undefined && parsed.since !== null) {
|
|
188
|
+
if (typeof parsed.since !== 'string' || !/^[0-9 :.\-]+$/.test(parsed.since)) {
|
|
189
|
+
throw new HttpError(400, `invalid since: only a logcat timestamp (digits, space, '-', ':', '.') is allowed`);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
const lines = parsed.lines === undefined || parsed.lines === null
|
|
193
|
+
? undefined
|
|
194
|
+
: typeof parsed.lines === 'number' && Number.isFinite(parsed.lines) && parsed.lines > 0
|
|
195
|
+
? Math.floor(parsed.lines)
|
|
196
|
+
: undefined;
|
|
197
|
+
const appId = parsed.appId === undefined || parsed.appId === null
|
|
198
|
+
? undefined
|
|
199
|
+
: typeof parsed.appId === 'string' && /^[A-Za-z0-9_.-]+$/.test(parsed.appId)
|
|
200
|
+
? parsed.appId
|
|
201
|
+
: (() => {
|
|
202
|
+
throw new HttpError(400, `invalid appId '${String(parsed.appId)}'`);
|
|
203
|
+
})();
|
|
204
|
+
const logs = config.driver.getLogs({
|
|
205
|
+
...(lines !== undefined ? { lines } : {}),
|
|
206
|
+
...(parsed.since ? { since: parsed.since } : {}),
|
|
207
|
+
...(appId ? { appId } : {}),
|
|
208
|
+
...(parsed.scopedOnly ? { scopedOnly: true } : {}),
|
|
209
|
+
});
|
|
210
|
+
const payload = { logs };
|
|
211
|
+
sendJson(res, 200, payload);
|
|
212
|
+
}
|
|
172
213
|
async function handleInstall(req, res) {
|
|
173
214
|
const ext = String(req.headers['x-verikun-ext'] ?? '').toLowerCase();
|
|
174
215
|
if (ext !== 'apk' && ext !== 'ipa') {
|
|
@@ -256,6 +297,8 @@ function buildServer(config) {
|
|
|
256
297
|
return deviceEndpoint(() => handleExec(req, res));
|
|
257
298
|
if (req.method === 'POST' && path === '/v1/elements')
|
|
258
299
|
return deviceEndpoint(() => handleElements(req, res));
|
|
300
|
+
if (req.method === 'POST' && path === '/v1/logs')
|
|
301
|
+
return deviceEndpoint(() => handleLogs(req, res));
|
|
259
302
|
if (req.method === 'POST' && path === '/v1/install') {
|
|
260
303
|
if (!config.allowInstall) {
|
|
261
304
|
throw new HttpError(403, 'install is disabled on this server (start it with --allow-install)', 3);
|
package/dist/version.js
CHANGED
|
@@ -3,4 +3,4 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.VERSION = void 0;
|
|
4
4
|
// GENERATED by scripts/gen-version.mjs from package.json's "version" at build time
|
|
5
5
|
// (the `prebuild` script). Do NOT edit by hand; bump package.json instead.
|
|
6
|
-
exports.VERSION = '0.
|
|
6
|
+
exports.VERSION = '0.18.1';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "verikun",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.18.1",
|
|
4
4
|
"description": "Drive Android emulators/devices and iOS simulators for AI agents: tap, type, swipe, screenshot, and inspect the UI hierarchy by semantic identifiers — like Puppeteer for native apps.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"android",
|