verikun 0.29.1 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/verikun/SKILL.md +5 -0
- package/CHANGELOG.md +20 -0
- package/dist/agent/engine.js +15 -1
- package/dist/agent/remote.js +57 -6
- package/dist/cli.js +82 -3
- package/dist/errors.js +39 -1
- package/dist/report.js +2 -1
- package/dist/rpc.js +13 -0
- package/dist/server-http.js +5 -1
- package/dist/server.js +43 -19
- package/dist/suite.js +269 -15
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -437,6 +437,8 @@ vk ai onboarding.md --timeout 5m # tighten the run timeout (default 15m)
|
|
|
437
437
|
- **`3` also means the device toolchain is broken**, checked *before* anything is compiled
|
|
438
438
|
(missing `adb`/`idb`, no device, an ambiguous target) and again if it breaks mid-run. Treat
|
|
439
439
|
it as "fix the machine", never as a failing test — the message carries the install hint.
|
|
440
|
+
Exception: over `--server`, a `--json` result with `evicted: true` means the server's device
|
|
441
|
+
left the pool mid-run — nothing is broken; run the test again.
|
|
440
442
|
|
|
441
443
|
## Run a suite of tests (vk suite)
|
|
442
444
|
|
|
@@ -488,6 +490,9 @@ vk suite tests/ --app com.example.app --servers http://a:8391,http://b:8391
|
|
|
488
490
|
- `--concurrency N` caps how many run at once — more devices on one host can thrash it.
|
|
489
491
|
`--max-suite-cost-usd N` stops the suite once total model spend crosses it (exit `1`).
|
|
490
492
|
- A device that breaks retires; its tests move to the others. Exit `3` only when all are gone.
|
|
493
|
+
- Over `--server`, a lane with no free device waits instead of failing tests, and a test whose
|
|
494
|
+
device left the pool re-runs without spending a retry. With no device for any lane the suite
|
|
495
|
+
stops after `VERIKUN_SUITE_DEVICE_WAIT_MIN` (default 10) with exit `3` and a `notRun` list.
|
|
491
496
|
- `--ensure-device` is refused with `--devices` — start the pool with `vk devices start`.
|
|
492
497
|
|
|
493
498
|
## Drive a remote device (--server)
|
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,26 @@ All notable changes to this project are documented here. The format is based on
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.30.0] - 2026-09-26
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
- **`vk suite --server`** waits for a free device instead of failing tests that never got one, and
|
|
13
|
+
a waiting lane resumes when the pool recovers. ([#147])
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
- **`vk suite --server`** re-runs a test whose device left the pool mid-run without spending a
|
|
17
|
+
retry, up to twice; needs the matching `vk server`. ([#147])
|
|
18
|
+
- **`VERIKUN_SUITE_DEVICE_WAIT_MIN`** (default `10`) caps how long a pooled suite waits when no lane
|
|
19
|
+
gets a device, busy pools included, then exits `3`; `0` fails fast. ([#147])
|
|
20
|
+
|
|
21
|
+
### Changed
|
|
22
|
+
- **`--json` errors** name a refused `vk server` lease `NoFreeDeviceError` and an evicted run
|
|
23
|
+
`RunEvictedError` in `errorKind`; exit codes are unchanged. ([#147])
|
|
24
|
+
- **`vk ai --json`** reports a run whose device left the pool as an environment abort with
|
|
25
|
+
`evicted: true` and its report, instead of a bare error. ([#147])
|
|
26
|
+
|
|
27
|
+
[#147]: https://github.com/ddikman/verikun/issues/147
|
|
28
|
+
|
|
9
29
|
## [0.29.1] - 2026-09-26
|
|
10
30
|
|
|
11
31
|
### Fixed
|
package/dist/agent/engine.js
CHANGED
|
@@ -31,6 +31,17 @@ class GuardBlindError extends Error {
|
|
|
31
31
|
this.name = 'GuardBlindError';
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* A run the server EVICTED cannot continue: its phone left the pool, and every later call is
|
|
36
|
+
* refused the same way. So the catches below that absorb a failed READ — a guard's retry, the
|
|
37
|
+
* empty-tree fallback, a repair that could not be made — must let it through. Absorbed, a lost
|
|
38
|
+
* phone read as an absent guard, a "read found no element" FAIL or a failed repair, and a
|
|
39
|
+
* parallel suite lost the class it needs to re-run the test as a fresh run (#147).
|
|
40
|
+
*/
|
|
41
|
+
const rethrowIfEvicted = (e) => {
|
|
42
|
+
if (e instanceof errors_1.RunEvictedError)
|
|
43
|
+
throw e;
|
|
44
|
+
};
|
|
34
45
|
const describe = (leaf) => [leaf.command, ...leaf.positionals, ...leaf.flags.map((f) => (f.value === 'true' ? `--${f.name}` : `--${f.name} ${f.value}`))]
|
|
35
46
|
.join(' ')
|
|
36
47
|
.trim();
|
|
@@ -190,7 +201,8 @@ async function runPlan(plan, deps) {
|
|
|
190
201
|
try {
|
|
191
202
|
return await deps.getElements();
|
|
192
203
|
}
|
|
193
|
-
catch {
|
|
204
|
+
catch (e) {
|
|
205
|
+
rethrowIfEvicted(e);
|
|
194
206
|
return [];
|
|
195
207
|
}
|
|
196
208
|
};
|
|
@@ -256,6 +268,7 @@ async function runPlan(plan, deps) {
|
|
|
256
268
|
everRead = true;
|
|
257
269
|
}
|
|
258
270
|
catch (e) {
|
|
271
|
+
rethrowIfEvicted(e);
|
|
259
272
|
els = undefined; // transient dump failure — retry once before concluding "absent"
|
|
260
273
|
lastErr = e;
|
|
261
274
|
}
|
|
@@ -366,6 +379,7 @@ async function runPlan(plan, deps) {
|
|
|
366
379
|
repaired = node;
|
|
367
380
|
}
|
|
368
381
|
catch (e) {
|
|
382
|
+
rethrowIfEvicted(e);
|
|
369
383
|
const msg = e instanceof Error ? e.message : String(e);
|
|
370
384
|
return { status: 'fail', where, reason: `repair failed: ${msg}` };
|
|
371
385
|
}
|
package/dist/agent/remote.js
CHANGED
|
@@ -12,6 +12,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
12
12
|
exports.describeStatus = describeStatus;
|
|
13
13
|
exports.transportReason = transportReason;
|
|
14
14
|
exports.pingServer = pingServer;
|
|
15
|
+
exports.poolNote = poolNote;
|
|
15
16
|
exports.remoteDeviceOp = remoteDeviceOp;
|
|
16
17
|
exports.remoteDeviceList = remoteDeviceList;
|
|
17
18
|
exports.createRemoteBackend = createRemoteBackend;
|
|
@@ -105,9 +106,18 @@ function requestWithNodeHttp(url, method, headers, body, timeoutMs) {
|
|
|
105
106
|
/**
|
|
106
107
|
* Turn a non-2xx into the error the caller sees.
|
|
107
108
|
*
|
|
108
|
-
* The 401/409/503 arms come FIRST and
|
|
109
|
+
* The 401/409/503 arms come FIRST and never rebuild a driver error: those describe the
|
|
109
110
|
* TRANSPORT (wrong key, device leased, nothing attached), not something a driver threw, so
|
|
110
111
|
* there is no device-error identity to restore and their wording is what a user acts on.
|
|
112
|
+
* Two of them do carry a class of their own, because a parallel suite has to act on the
|
|
113
|
+
* difference (#147) — both still exit 3:
|
|
114
|
+
*
|
|
115
|
+
* - on the LEASE route (`opts.lease`), 409 and 503 are `NoFreeDeviceError`: the run never
|
|
116
|
+
* started. The route decides, not the body — every run mints a fresh token, so a lease can
|
|
117
|
+
* be refused but never evicted.
|
|
118
|
+
* - elsewhere, a 409 the server tagged `RunEvictedError` is one: the run lost its device
|
|
119
|
+
* part-way. That is the ONLY kind a 409 honours. Anything else a body carries is ignored,
|
|
120
|
+
* and an untagged 409 (an older server) stays the plain error it always was.
|
|
111
121
|
*
|
|
112
122
|
* Everything else prefers the server's `errorKind`. That field is what stops a `--server` run
|
|
113
123
|
* reading a mid-launch `NoWindowError` as a fatal environment error: the class survives the
|
|
@@ -115,16 +125,22 @@ function requestWithNodeHttp(url, method, headers, body, timeoutMs) {
|
|
|
115
125
|
* `CliError` (issue #80). No field — an older server, or a failure with no class worth
|
|
116
126
|
* naming — falls through to exactly the previous behaviour.
|
|
117
127
|
*/
|
|
118
|
-
function describeStatus(status, body, url) {
|
|
128
|
+
function describeStatus(status, body, url, opts = {}) {
|
|
119
129
|
const detail = body?.error ? `: ${body.error}` : '';
|
|
120
130
|
if (status === 401) {
|
|
121
131
|
return new errors_1.CliError(`verikun server rejected the auth key (401)${detail}. Check --auth-key / VERIKUN_SERVER_AUTH_KEY.`, 3);
|
|
122
132
|
}
|
|
123
133
|
if (status === 409) {
|
|
124
|
-
|
|
134
|
+
const busy = `verikun server device is busy (409)${detail || ' — another run holds the device; retry when it finishes'}.`;
|
|
135
|
+
if (opts.lease)
|
|
136
|
+
return new errors_1.NoFreeDeviceError(busy);
|
|
137
|
+
if (body?.errorKind === 'RunEvictedError')
|
|
138
|
+
return new errors_1.RunEvictedError(`verikun server ended this run (409)${detail}.`);
|
|
139
|
+
return new errors_1.CliError(busy, 3);
|
|
125
140
|
}
|
|
126
141
|
if (status === 503) {
|
|
127
|
-
|
|
142
|
+
const none = `verikun server has no device attached (503)${detail}.`;
|
|
143
|
+
return opts.lease ? new errors_1.NoFreeDeviceError(none) : new errors_1.CliError(none, 3);
|
|
128
144
|
}
|
|
129
145
|
// The server sends the intended exit code (usage 2 / env 3) in the body; fall
|
|
130
146
|
// back on the HTTP class when it didn't.
|
|
@@ -172,6 +188,9 @@ class RemoteTransport {
|
|
|
172
188
|
base;
|
|
173
189
|
/** One token per backend = one logical run holding the server's device lock. */
|
|
174
190
|
runToken = (0, node_crypto_1.randomUUID)();
|
|
191
|
+
/** The server has said this run was evicted. Latched, because the request that hears it is
|
|
192
|
+
* usually not the step that failed — see ExecBackend.wasEvicted. */
|
|
193
|
+
evicted = false;
|
|
175
194
|
constructor(opts) {
|
|
176
195
|
this.opts = opts;
|
|
177
196
|
this.base = trimUrl(opts.url);
|
|
@@ -214,7 +233,10 @@ class RemoteTransport {
|
|
|
214
233
|
// exactly the case a caller must not miss (an exhausted install, a dead-device read).
|
|
215
234
|
if (body?.deviceChanged)
|
|
216
235
|
this.opts.onDeviceChange?.(body.deviceChanged);
|
|
217
|
-
|
|
236
|
+
const error = describeStatus(res.status, body, url, { lease: path === '/v1/lease' });
|
|
237
|
+
if (error instanceof errors_1.RunEvictedError || body?.evicted)
|
|
238
|
+
this.evicted = true;
|
|
239
|
+
throw error;
|
|
218
240
|
}
|
|
219
241
|
const parsed = await readBody(res);
|
|
220
242
|
if (parsed === null)
|
|
@@ -245,6 +267,21 @@ async function pingServer(opts) {
|
|
|
245
267
|
}
|
|
246
268
|
return health;
|
|
247
269
|
}
|
|
270
|
+
/**
|
|
271
|
+
* What a `vk server` is serving, and what it has ruled out, in one line from one
|
|
272
|
+
* `/v1/health`. Carried on every "no free device" so the reader learns WHICH phone left and
|
|
273
|
+
* the server's own reason, not the lane slot (`host:port#1`) that happened to notice (#147).
|
|
274
|
+
* A device the server shed keeps its quarantine, which is what makes it nameable here.
|
|
275
|
+
*/
|
|
276
|
+
function poolNote(health) {
|
|
277
|
+
const serving = health.devices ?? (health.serial ? [health.serial] : []);
|
|
278
|
+
const count = health.capacity ?? serving.length;
|
|
279
|
+
const out = serving.length ? ` (${serving.join(', ')})` : '';
|
|
280
|
+
const ruledOut = health.quarantined?.length
|
|
281
|
+
? `; ruled out: ${health.quarantined.map((q) => `${q.serial} (${q.reason})`).join(', ')}`
|
|
282
|
+
: '';
|
|
283
|
+
return `pool: ${count} serving${out}${ruledOut}`;
|
|
284
|
+
}
|
|
248
285
|
/**
|
|
249
286
|
* POST /v1/devices/{start,restart,stop}. Standalone beside pingServer rather than on
|
|
250
287
|
* the ExecBackend seam: that seam is the engine's DEVICE WORK (exec/getElements/
|
|
@@ -285,6 +322,10 @@ function createRemoteBackend(opts, health) {
|
|
|
285
322
|
// A failing step is a 200, so this is the ordinary path for a mid-run device death.
|
|
286
323
|
if (res.deviceChanged)
|
|
287
324
|
opts.onDeviceChange?.(res.deviceChanged);
|
|
325
|
+
// …and for the eviction that death caused: the step keeps the phone's own error, and this
|
|
326
|
+
// response is the only one that says the run is over (see ExecResponse.evicted).
|
|
327
|
+
if (res.evicted)
|
|
328
|
+
t.evicted = true;
|
|
288
329
|
return { code: res.code, error: res.error ? (0, rpc_1.rebuildError)(res.error) : undefined };
|
|
289
330
|
};
|
|
290
331
|
return {
|
|
@@ -294,7 +335,16 @@ function createRemoteBackend(opts, health) {
|
|
|
294
335
|
// together, and a client cannot otherwise tell "old server" from "new server".
|
|
295
336
|
if (health.capacity === undefined)
|
|
296
337
|
return null;
|
|
297
|
-
|
|
338
|
+
try {
|
|
339
|
+
return await t.postJson('/v1/lease', {}, HEALTH_TIMEOUT_MS);
|
|
340
|
+
}
|
|
341
|
+
catch (e) {
|
|
342
|
+
// Say what the pool looks like. `health` was read moments ago, on the way here, and
|
|
343
|
+
// it is what names a phone that left — the refusal itself only counts devices.
|
|
344
|
+
if (e instanceof errors_1.NoFreeDeviceError)
|
|
345
|
+
throw new errors_1.NoFreeDeviceError(`${e.message} [${poolNote(health)}]`);
|
|
346
|
+
throw e;
|
|
347
|
+
}
|
|
298
348
|
},
|
|
299
349
|
async getElements() {
|
|
300
350
|
const res = await t.postJson('/v1/elements', {}, ELEMENTS_TIMEOUT_MS);
|
|
@@ -346,6 +396,7 @@ function createRemoteBackend(opts, health) {
|
|
|
346
396
|
if (code !== 0)
|
|
347
397
|
throw error ?? new errors_1.CliError(`reset (${command} ${appId}) failed on the server (exit ${code})`, 3);
|
|
348
398
|
},
|
|
399
|
+
wasEvicted: () => t.evicted,
|
|
349
400
|
async close() {
|
|
350
401
|
// Free the server's device lock so the next command (a fresh run token, e.g.
|
|
351
402
|
// `vk install` then `vk suite` in one CI job) isn't 409'd until the idle
|
package/dist/cli.js
CHANGED
|
@@ -48,6 +48,7 @@ exports.compileFromSegments = compileFromSegments;
|
|
|
48
48
|
exports.obtainPlan = obtainPlan;
|
|
49
49
|
exports.retryAfterDeviceMove = retryAfterDeviceMove;
|
|
50
50
|
exports.terminalFailure = terminalFailure;
|
|
51
|
+
exports.runAiTest = runAiTest;
|
|
51
52
|
exports.laneArgv = laneArgv;
|
|
52
53
|
exports.laneEnv = laneEnv;
|
|
53
54
|
exports.lanesFromFlags = lanesFromFlags;
|
|
@@ -2457,6 +2458,7 @@ async function prefetchArchiveLogs(backend, noLogs = false) {
|
|
|
2457
2458
|
* Run one natural-language test through a backend and return DATA — no stdout
|
|
2458
2459
|
* writes (stdout stays the caller's one result; progress streams to stderr).
|
|
2459
2460
|
* `vk ai` wraps it with its --json/report output; `vk suite` calls it per test.
|
|
2461
|
+
* Exported for the unit suite.
|
|
2460
2462
|
*/
|
|
2461
2463
|
async function runAiTest(file, opts, backend, platform, device) {
|
|
2462
2464
|
const { nl, segments } = readAiTest(file);
|
|
@@ -2495,7 +2497,19 @@ async function runAiTest(file, opts, backend, platform, device) {
|
|
|
2495
2497
|
// (exit 3 for a broken device), which is what a caller reads as an environment
|
|
2496
2498
|
// abort — the same verdict `vk suite`'s own reset failure produces.
|
|
2497
2499
|
if (opts.resetApp) {
|
|
2498
|
-
|
|
2500
|
+
try {
|
|
2501
|
+
await backend.reset(opts.resetApp);
|
|
2502
|
+
}
|
|
2503
|
+
catch (e) {
|
|
2504
|
+
// The phone this run was dealt had left the pool while idle, and the reset is the first
|
|
2505
|
+
// request to find out — the one the server marks. That is the server's eviction, not a
|
|
2506
|
+
// broken box: a suite re-runs it as a fresh run (#147). Thrown, because no run exists yet
|
|
2507
|
+
// to carry a result.
|
|
2508
|
+
if ((0, errors_1.isEnvError)(e) && !(e instanceof errors_1.RunEvictedError) && backend.wasEvicted?.()) {
|
|
2509
|
+
throw new errors_1.RunEvictedError(`the device this run was dealt had left the pool: ${e.message.split('\n')[0]}`);
|
|
2510
|
+
}
|
|
2511
|
+
throw e;
|
|
2512
|
+
}
|
|
2499
2513
|
(0, output_1.err)(`[ai] app state reset (${opts.resetApp})`);
|
|
2500
2514
|
}
|
|
2501
2515
|
// One explicit run for the whole flow (so rollover can't split the test).
|
|
@@ -2543,15 +2557,45 @@ async function runAiTest(file, opts, backend, platform, device) {
|
|
|
2543
2557
|
reason: e.message,
|
|
2544
2558
|
kind: (0, errors_1.isEnvError)(e) ? 'env' : 'fail',
|
|
2545
2559
|
});
|
|
2560
|
+
let sealed;
|
|
2546
2561
|
try {
|
|
2547
2562
|
await prefetchArchiveLogs(backend);
|
|
2548
|
-
run_1.Recorder.archive();
|
|
2563
|
+
sealed = run_1.Recorder.archive();
|
|
2549
2564
|
}
|
|
2550
2565
|
catch (sealErr) {
|
|
2551
2566
|
// Best-effort seal in an error path; surface a failure (the run state may itself be
|
|
2552
2567
|
// unreadable) but still throw the ORIGINAL error below.
|
|
2553
2568
|
(0, output_1.err)(`[ai] could not archive the run after a mid-run error (${sealErr.message})`);
|
|
2554
2569
|
}
|
|
2570
|
+
// An EVICTION is the one mid-run throw that is a RESULT, not an error. The server ended
|
|
2571
|
+
// this run because its phone left the pool — the run is not broken, it is over — and the
|
|
2572
|
+
// caller needs what a result carries: the archive (how far it got, which phone) and the
|
|
2573
|
+
// flag. A parallel suite re-runs it as a fresh run without spending a retry (#147); thrown,
|
|
2574
|
+
// it reached the suite as a bare error with no report and no device. Still exit 3.
|
|
2575
|
+
// An environment throw on a run the server evicted counts too: a worker that died mid-step
|
|
2576
|
+
// answers with an error rather than a result, and the server marks that response.
|
|
2577
|
+
if (e instanceof errors_1.RunEvictedError || ((0, errors_1.isEnvError)(e) && backend.wasEvicted?.())) {
|
|
2578
|
+
const reason = e.message.split('\n')[0];
|
|
2579
|
+
(0, output_1.err)(`[ai] ABORTED — environment: ${reason}`);
|
|
2580
|
+
if (sealed)
|
|
2581
|
+
(0, output_1.err)(`[ai] report: ${sealed.htmlPath}`);
|
|
2582
|
+
return {
|
|
2583
|
+
ok: false,
|
|
2584
|
+
cached,
|
|
2585
|
+
costUsd: Number(cost.usd().toFixed(4)),
|
|
2586
|
+
costLine: cost.summaryLine(),
|
|
2587
|
+
modelRepairs: 0,
|
|
2588
|
+
improvements: [],
|
|
2589
|
+
planSteps: plan.steps.length,
|
|
2590
|
+
runDir: sealed?.dir ?? '',
|
|
2591
|
+
reportHtml: sealed?.htmlPath ?? '',
|
|
2592
|
+
junitXml: sealed?.xmlPath ?? '',
|
|
2593
|
+
state: sealed?.state ?? null,
|
|
2594
|
+
failure: { where: 'run', reason },
|
|
2595
|
+
abortedForEnv: true,
|
|
2596
|
+
evicted: true,
|
|
2597
|
+
};
|
|
2598
|
+
}
|
|
2555
2599
|
throw e;
|
|
2556
2600
|
}
|
|
2557
2601
|
finally {
|
|
@@ -2604,6 +2648,11 @@ async function runAiTest(file, opts, backend, platform, device) {
|
|
|
2604
2648
|
...(result.abortedForBudget ? { abortedForBudget: true } : {}),
|
|
2605
2649
|
...(result.abortedForTimeout ? { abortedForTimeout: true } : {}),
|
|
2606
2650
|
...(result.abortedForEnv ? { abortedForEnv: true } : {}),
|
|
2651
|
+
// The step that was running when the phone vanished failed with the phone's own error, and
|
|
2652
|
+
// the server shed the phone while answering it: the eviction is heard only by the requests
|
|
2653
|
+
// after it (the failure evidence and the log fetch above). Asked now, and only of an
|
|
2654
|
+
// environment abort — an assertion that failed first is a regression whatever happened next.
|
|
2655
|
+
...(result.abortedForEnv && backend.wasEvicted?.() ? { evicted: true } : {}),
|
|
2607
2656
|
};
|
|
2608
2657
|
}
|
|
2609
2658
|
async function cmdAi(positionals, flags) {
|
|
@@ -2661,6 +2710,8 @@ async function cmdAi(positionals, flags) {
|
|
|
2661
2710
|
...(result.abortedForBudget ? { abortedForBudget: true } : {}),
|
|
2662
2711
|
...(result.abortedForTimeout ? { abortedForTimeout: true } : {}),
|
|
2663
2712
|
...(result.abortedForEnv ? { abortedForEnv: true } : {}),
|
|
2713
|
+
// The server ended the run because its phone left: start it again as a fresh run.
|
|
2714
|
+
...(result.evicted ? { evicted: true } : {}),
|
|
2664
2715
|
});
|
|
2665
2716
|
}
|
|
2666
2717
|
else if (result.reportHtml) {
|
|
@@ -2956,6 +3007,12 @@ function laneResult(code, parsed, detail, lane) {
|
|
|
2956
3007
|
const str = (k) => (typeof parsed?.[k] === 'string' ? parsed[k] : undefined);
|
|
2957
3008
|
const num = (k) => (typeof parsed?.[k] === 'number' ? parsed[k] : undefined);
|
|
2958
3009
|
const yes = (k) => parsed?.[k] === true;
|
|
3010
|
+
// The two lease outcomes a parallel suite must NOT read as a broken box (#147), both exit 3.
|
|
3011
|
+
// Gated on the code too: the exit code is the verdict, and a kind never overrides it.
|
|
3012
|
+
// - refused at /v1/lease: the child never started a run, so it is no attempt at all;
|
|
3013
|
+
// - evicted part-way: usually a result document (the run archived), or a thrown one.
|
|
3014
|
+
const noDevice = code === 3 && str('errorKind') === 'NoFreeDeviceError';
|
|
3015
|
+
const evicted = code === 3 && (yes('evicted') || str('errorKind') === 'RunEvictedError');
|
|
2959
3016
|
const raw = parsed?.failure;
|
|
2960
3017
|
// A budget / timeout / environment abort comes back as a bare FLAG with no `failure`
|
|
2961
3018
|
// object — the engine returns it that way, and `toSuiteResult` composes its own wording
|
|
@@ -3000,9 +3057,11 @@ function laneResult(code, parsed, detail, lane) {
|
|
|
3000
3057
|
// check a TypeError in a new code path reads as an environment failure: the suite
|
|
3001
3058
|
// probes the lane, finds it healthy, retries, and two in a row retire a perfectly good
|
|
3002
3059
|
// device via ENV_STREAK_LIMIT.
|
|
3003
|
-
...((yes('abortedForEnv') || code === 3 || code === 127) && str('errorKind') !== 'Error'
|
|
3060
|
+
...((yes('abortedForEnv') || code === 3 || code === 127) && str('errorKind') !== 'Error' && !noDevice
|
|
3004
3061
|
? { abortedForEnv: true }
|
|
3005
3062
|
: {}),
|
|
3063
|
+
...(noDevice ? { noDevice: true } : {}),
|
|
3064
|
+
...(evicted ? { evicted: true } : {}),
|
|
3006
3065
|
// Exit 2 is verikun's USAGE code — a flag the child rejected, an unreadable test, a
|
|
3007
3066
|
// payload the server refused. The serial path reaches this verdict from the thrown
|
|
3008
3067
|
// CliError (`isRetryableThrow`); across a process boundary the throw is only an exit
|
|
@@ -3215,6 +3274,21 @@ async function cmdSuiteParallel(dirArg, flags, pool) {
|
|
|
3215
3274
|
claimLanes: (used) => grantLanes(used, platform, pool.elastic, grants),
|
|
3216
3275
|
runTest: (file, lane) => runLaneTest(file, lane, flags, platform, app),
|
|
3217
3276
|
preflight: (lane) => lanePreflight(lane, flags, platform),
|
|
3277
|
+
// Is a phone free for this server lane? Asked of /v1/health, which leases nothing: a
|
|
3278
|
+
// lease request that finds nothing free lets the server take over a sibling lane's
|
|
3279
|
+
// quiet lease (see SuiteDeps.serverSlots). A server that does not answer is left for
|
|
3280
|
+
// the lane's own child to report, exactly as before.
|
|
3281
|
+
serverSlots: async (lane) => {
|
|
3282
|
+
if (!lane.server)
|
|
3283
|
+
return undefined;
|
|
3284
|
+
try {
|
|
3285
|
+
const health = await (0, remote_1.pingServer)(remoteOptsFrom(lane.server, flags));
|
|
3286
|
+
return { capacity: health.capacity ?? 1, note: (0, remote_1.poolNote)(health) };
|
|
3287
|
+
}
|
|
3288
|
+
catch {
|
|
3289
|
+
return undefined;
|
|
3290
|
+
}
|
|
3291
|
+
},
|
|
3218
3292
|
// `reset` is deliberately NOT wired: against a pooled server a reset issued from
|
|
3219
3293
|
// here would take its own lease and could land on a different device than the test
|
|
3220
3294
|
// that follows. `vk ai --reset-app` does it inside the test's own lease instead.
|
|
@@ -3700,6 +3774,11 @@ SUITE (run a directory of natural-language tests as one gated suite)
|
|
|
3700
3774
|
stops the suite once total model spend crosses
|
|
3701
3775
|
it (exit 1). One merged report either way, with
|
|
3702
3776
|
wall-clock reported apart from device time.
|
|
3777
|
+
Over --server a lane with no free device waits
|
|
3778
|
+
for one; a test whose device left the pool
|
|
3779
|
+
re-runs without spending a retry. With no device
|
|
3780
|
+
for any lane it stops after
|
|
3781
|
+
VERIKUN_SUITE_DEVICE_WAIT_MIN (default 10; exit 3).
|
|
3703
3782
|
|
|
3704
3783
|
SERVER (expose a locally-connected device to remote verikun clients)
|
|
3705
3784
|
server [--bind addr] [--port n] [--auth-key k] [--devices all|all-android|all-ios|a,b]
|
package/dist/errors.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// 2 usage error, ambiguous selector, or a device another job is driving (caller must refine)
|
|
7
7
|
// 3 environment error (adb/simctl missing, no usable device, dump failed)
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
-
exports.AmbiguousSelectorError = exports.DUMP_KILLED_MESSAGE = exports.dumpKilledMessage = exports.DumpKilledError = exports.NO_WINDOW_MESSAGE = exports.NoWindowError = exports.TransientReadError = exports.SelectorNotFoundError = exports.probeFailure = exports.envError = exports.usageError = exports.CliError = void 0;
|
|
9
|
+
exports.AmbiguousSelectorError = exports.RunEvictedError = exports.NoFreeDeviceError = exports.DUMP_KILLED_MESSAGE = exports.dumpKilledMessage = exports.DumpKilledError = exports.NO_WINDOW_MESSAGE = exports.NoWindowError = exports.TransientReadError = exports.SelectorNotFoundError = exports.probeFailure = exports.envError = exports.usageError = exports.CliError = void 0;
|
|
10
10
|
exports.isEnvError = isEnvError;
|
|
11
11
|
class CliError extends Error {
|
|
12
12
|
exitCode;
|
|
@@ -140,6 +140,44 @@ exports.dumpKilledMessage = dumpKilledMessage;
|
|
|
140
140
|
exports.DUMP_KILLED_MESSAGE = 'The UI hierarchy dump was killed before it could answer — the device reclaiming memory ' +
|
|
141
141
|
'while an app starts, or another tool holding the one UiAutomation connection. This ' +
|
|
142
142
|
'normally clears within seconds.';
|
|
143
|
+
// --- Remote leases: a run that never got a device, and one that lost its device --------
|
|
144
|
+
//
|
|
145
|
+
// Both are exit 3 like any other environment failure — the exit-code contract does not move
|
|
146
|
+
// and `isEnvError` still holds. They are classes only so a PARALLEL suite can tell them apart
|
|
147
|
+
// from a broken box across a process boundary (`errorKind` in a lane child's --json). Read as
|
|
148
|
+
// ordinary environment failures they were issue #147: a pool that lost one phone recorded
|
|
149
|
+
// failures for tests that never ran, and retired a lane the server would have fed again
|
|
150
|
+
// minutes later.
|
|
151
|
+
/**
|
|
152
|
+
* A `vk server` refused this run a device: every device is leased (409) or none is serving
|
|
153
|
+
* (503). Built ONLY for `/v1/lease`, the call a run makes before it does anything, so it
|
|
154
|
+
* always means the test never started. `vk suite` hands such a test back to its queue and
|
|
155
|
+
* waits for a device instead of recording a failure.
|
|
156
|
+
*/
|
|
157
|
+
class NoFreeDeviceError extends CliError {
|
|
158
|
+
constructor(message) {
|
|
159
|
+
super(message, 3);
|
|
160
|
+
this.name = 'NoFreeDeviceError';
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
exports.NoFreeDeviceError = NoFreeDeviceError;
|
|
164
|
+
/**
|
|
165
|
+
* The server ended this run part-way: its device left the pool, was taken over while the run
|
|
166
|
+
* sat idle, or was power-cycled — and a run is never re-homed onto another phone, because its
|
|
167
|
+
* earlier steps ran on this one. Only the server knows, so it TAGS the 409 (`errorKind`); an
|
|
168
|
+
* older server's eviction stays an ordinary environment failure.
|
|
169
|
+
*
|
|
170
|
+
* Nothing in the `vk ai` engine may absorb it: every later call is refused the same way, so a
|
|
171
|
+
* catch that treats it as a transient read turns a lost phone into an "absent" guard or a
|
|
172
|
+
* failed repair. `vk suite` re-runs the test as a fresh run without spending a retry.
|
|
173
|
+
*/
|
|
174
|
+
class RunEvictedError extends CliError {
|
|
175
|
+
constructor(message) {
|
|
176
|
+
super(message, 3);
|
|
177
|
+
this.name = 'RunEvictedError';
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
exports.RunEvictedError = RunEvictedError;
|
|
143
181
|
/** Selector matched >1 element. Exit 2. Carries the candidates so the agent runner
|
|
144
182
|
* can ask the model to disambiguate (a heal trigger) instead of aborting. */
|
|
145
183
|
class AmbiguousSelectorError extends CliError {
|
package/dist/report.js
CHANGED
|
@@ -278,7 +278,8 @@ const SUITE_STYLE = `
|
|
|
278
278
|
function suiteAttemptLinks(attempts, linkBase) {
|
|
279
279
|
const links = attempts
|
|
280
280
|
.map((a, i) => {
|
|
281
|
-
|
|
281
|
+
// Said outright, so nobody goes looking for a flake in a test whose phone was unplugged.
|
|
282
|
+
const label = `attempt ${i + 1}${a.evicted ? ' (device left the pool)' : ''}`;
|
|
282
283
|
if (!a.id)
|
|
283
284
|
return htmlEsc(label);
|
|
284
285
|
return `<a href="${htmlEsc(`${linkBase}runs/${encodeURIComponent(a.id)}/report.html`)}">${htmlEsc(label)}</a>`;
|
package/dist/rpc.js
CHANGED
|
@@ -34,6 +34,15 @@ function describeError(e) {
|
|
|
34
34
|
if (e instanceof errors_1.NoWindowError) {
|
|
35
35
|
return { kind: 'NoWindowError', name: e.name, message: e.message, exitCode: e.exitCode };
|
|
36
36
|
}
|
|
37
|
+
// Also before the CliError arm, for a different reader: `mapError` puts this kind in a
|
|
38
|
+
// lane child's --json, and it is how a parallel suite tells "never got a device" and "lost
|
|
39
|
+
// its device" from a broken box (#147). The server only ever SENDS the eviction.
|
|
40
|
+
if (e instanceof errors_1.NoFreeDeviceError) {
|
|
41
|
+
return { kind: 'NoFreeDeviceError', name: e.name, message: e.message, exitCode: e.exitCode };
|
|
42
|
+
}
|
|
43
|
+
if (e instanceof errors_1.RunEvictedError) {
|
|
44
|
+
return { kind: 'RunEvictedError', name: e.name, message: e.message, exitCode: e.exitCode };
|
|
45
|
+
}
|
|
37
46
|
if (e instanceof errors_1.CliError) {
|
|
38
47
|
return { kind: 'CliError', name: e.name, message: e.message, exitCode: e.exitCode };
|
|
39
48
|
}
|
|
@@ -51,6 +60,10 @@ function rebuildError(d) {
|
|
|
51
60
|
return new errors_1.NoWindowError(d.message);
|
|
52
61
|
case 'DumpKilledError':
|
|
53
62
|
return new errors_1.DumpKilledError(d.message);
|
|
63
|
+
case 'NoFreeDeviceError':
|
|
64
|
+
return new errors_1.NoFreeDeviceError(d.message);
|
|
65
|
+
case 'RunEvictedError':
|
|
66
|
+
return new errors_1.RunEvictedError(d.message);
|
|
54
67
|
case 'CliError':
|
|
55
68
|
return new errors_1.CliError(d.message, d.exitCode);
|
|
56
69
|
default: {
|
package/dist/server-http.js
CHANGED
|
@@ -15,6 +15,7 @@ class HttpError extends Error {
|
|
|
15
15
|
exitCode;
|
|
16
16
|
deviceChanged;
|
|
17
17
|
errorKind;
|
|
18
|
+
evicted;
|
|
18
19
|
constructor(status, message, exitCode = status === 400 || status === 404 || status === 413 ? 2 : 3,
|
|
19
20
|
/** Set when this request moved the server's device before failing — the client
|
|
20
21
|
* needs to know the ground shifted even though the answer is an error. */
|
|
@@ -23,12 +24,15 @@ class HttpError extends Error {
|
|
|
23
24
|
* HttpError is how the identity used to be lost: only `.message` and `.exitCode` were
|
|
24
25
|
* copied across, so a `NoWindowError` reached the client as a bare `CliError`. Carry it
|
|
25
26
|
* here and the outer catch can put it on the wire. */
|
|
26
|
-
errorKind
|
|
27
|
+
errorKind,
|
|
28
|
+
/** This request's failure evicted the caller's run (see ExecResponse.evicted). */
|
|
29
|
+
evicted) {
|
|
27
30
|
super(message);
|
|
28
31
|
this.status = status;
|
|
29
32
|
this.exitCode = exitCode;
|
|
30
33
|
this.deviceChanged = deviceChanged;
|
|
31
34
|
this.errorKind = errorKind;
|
|
35
|
+
this.evicted = evicted;
|
|
32
36
|
this.name = 'HttpError';
|
|
33
37
|
}
|
|
34
38
|
}
|
package/dist/server.js
CHANGED
|
@@ -891,11 +891,14 @@ function buildServer(config) {
|
|
|
891
891
|
* phone that is now gone, so continuing it elsewhere would produce a run whose steps
|
|
892
892
|
* came from two devices while the report named one. A new run token (a rerun) is served
|
|
893
893
|
* normally — this only closes the door on the run that was interrupted.
|
|
894
|
+
*
|
|
895
|
+
* Each keeps what it lost and why, because the run is TOLD: its 409 names the device, and a
|
|
896
|
+
* parallel suite's warning then names the phone that left rather than a lane slot (#147).
|
|
894
897
|
*/
|
|
895
|
-
const evicted = new
|
|
898
|
+
const evicted = new Map();
|
|
896
899
|
/**
|
|
897
900
|
* Bounded, because on a long-lived CI server every interrupted run leaves an entry and
|
|
898
|
-
* only its own `/v1/release` ever removes one. A
|
|
901
|
+
* only its own `/v1/release` ever removes one. A Map iterates in insertion order, so
|
|
899
902
|
* the front is the stalest — a client that died long ago and will never ask again.
|
|
900
903
|
*/
|
|
901
904
|
const EVICTED_CAP = 512;
|
|
@@ -906,17 +909,17 @@ function buildServer(config) {
|
|
|
906
909
|
function evict(token, why) {
|
|
907
910
|
const had = leases.get(token);
|
|
908
911
|
leases.delete(token);
|
|
909
|
-
evicted.
|
|
912
|
+
evicted.set(token, { serial: had?.serial, why });
|
|
910
913
|
// Every eviction is announced. This is the direct cause of the 409 a client then reads
|
|
911
|
-
//
|
|
912
|
-
//
|
|
913
|
-
//
|
|
914
|
-
// device that left.
|
|
914
|
+
// (a parallel suite re-runs the test as a fresh run; an older client read it as an
|
|
915
|
+
// environment failure and retired a lane over it), and it used to be the one lease
|
|
916
|
+
// transition that happened in complete silence, so a degrading run showed a burst of
|
|
917
|
+
// unexplained 409s with nothing anywhere connecting them to the device that left.
|
|
915
918
|
if (had)
|
|
916
919
|
(0, output_1.err)(`[server] lease: run ${token.slice(0, 8)}… evicted from ${had.serial} — ${why}`);
|
|
917
920
|
// `if`, not `while`: this adds exactly one entry, so at most one can be over.
|
|
918
921
|
if (evicted.size > EVICTED_CAP)
|
|
919
|
-
evicted.delete(evicted.
|
|
922
|
+
evicted.delete(evicted.keys().next().value);
|
|
920
923
|
}
|
|
921
924
|
function evictHoldersOf(serial, why) {
|
|
922
925
|
for (const [token, lease] of leases)
|
|
@@ -1025,11 +1028,14 @@ function buildServer(config) {
|
|
|
1025
1028
|
*/
|
|
1026
1029
|
let exclusive = null;
|
|
1027
1030
|
function busyError(token) {
|
|
1028
|
-
|
|
1031
|
+
const lost = token === undefined ? undefined : evicted.get(token);
|
|
1032
|
+
if (lost) {
|
|
1029
1033
|
// Named plainly, because "device is busy" would send the operator looking for a
|
|
1030
|
-
// racing job that does not exist.
|
|
1031
|
-
|
|
1032
|
-
|
|
1034
|
+
// racing job that does not exist. TAGGED, because the client cannot tell an eviction
|
|
1035
|
+
// from contention any other way: a parallel suite re-runs an evicted test as a fresh
|
|
1036
|
+
// run without spending a retry, but must never do that for a busy pool (#147).
|
|
1037
|
+
return new server_http_1.HttpError(409, `this run lost its device${lost.serial ? ` (${lost.serial})` : ''}: ${lost.why} — ` +
|
|
1038
|
+
'start a fresh run; this one cannot continue on another device', 3, undefined, 'RunEvictedError');
|
|
1033
1039
|
}
|
|
1034
1040
|
// An empty pool never reaches here: the deviceless guard in the router answers 503 for
|
|
1035
1041
|
// every route that takes a lease, and it names `lostDevice` while doing it. So `n` is
|
|
@@ -1119,7 +1125,7 @@ function buildServer(config) {
|
|
|
1119
1125
|
evict(t, why);
|
|
1120
1126
|
}
|
|
1121
1127
|
}
|
|
1122
|
-
async function handleExec(handle, req, res) {
|
|
1128
|
+
async function handleExec(handle, req, res, token) {
|
|
1123
1129
|
const body = await (0, server_http_1.readBody)(req, EXEC_BODY_CAP);
|
|
1124
1130
|
let parsed;
|
|
1125
1131
|
try {
|
|
@@ -1160,7 +1166,12 @@ function buildServer(config) {
|
|
|
1160
1166
|
}
|
|
1161
1167
|
catch (e) {
|
|
1162
1168
|
const changed = await considerFailover(e, 'exec', handle);
|
|
1163
|
-
|
|
1169
|
+
// Evicted by the failover this very failure triggered: say so on the error itself (#147).
|
|
1170
|
+
const lost = evicted.has(token);
|
|
1171
|
+
if (lost && !changed) {
|
|
1172
|
+
throw new server_http_1.HttpError(500, e.message, e instanceof errors_1.CliError ? e.exitCode : 3, undefined, (0, rpc_1.describeError)(e).kind, true);
|
|
1173
|
+
}
|
|
1174
|
+
throw changed ? new server_http_1.HttpError(500, e.message, e instanceof errors_1.CliError ? e.exitCode : 3, changed, undefined, lost || undefined) : e;
|
|
1164
1175
|
}
|
|
1165
1176
|
const { code, error, step, artifacts, logStart } = outcome;
|
|
1166
1177
|
(0, output_1.err)(`[server] ${handle.serial} exec ${node.command} ${node.positionals.join(' ')} → exit ${code} (${Date.now() - t0}ms)`);
|
|
@@ -1177,13 +1188,17 @@ function buildServer(config) {
|
|
|
1177
1188
|
code,
|
|
1178
1189
|
...(error ? { error } : {}),
|
|
1179
1190
|
...(deviceChanged ? { deviceChanged } : {}),
|
|
1191
|
+
// The failover this failure triggered shed the device and evicted this run. MEASURED on
|
|
1192
|
+
// hardware: this response is the only one that knows — the client's next request is
|
|
1193
|
+
// usually its release, which clears the mark (#147).
|
|
1194
|
+
...(evicted.has(token) ? { evicted: true } : {}),
|
|
1180
1195
|
...(step ? { step } : {}),
|
|
1181
1196
|
...(artifacts && Object.keys(artifacts).length ? { artifacts: (0, server_http_1.encodeArtifacts)(artifacts) } : {}),
|
|
1182
1197
|
...(logStart ? { logStart } : {}),
|
|
1183
1198
|
};
|
|
1184
1199
|
(0, server_http_1.sendJson)(res, 200, payload);
|
|
1185
1200
|
}
|
|
1186
|
-
async function handleElements(handle, req, res) {
|
|
1201
|
+
async function handleElements(handle, req, res, token) {
|
|
1187
1202
|
await (0, server_http_1.readBody)(req, EXEC_BODY_CAP); // drain (the body is unused; keeps keep-alive sane)
|
|
1188
1203
|
try {
|
|
1189
1204
|
const elements = await handle.elements(); // CliError(3) on dump failure → 500 below
|
|
@@ -1198,12 +1213,13 @@ function buildServer(config) {
|
|
|
1198
1213
|
// hierarchy from somewhere else is worse than an error. The client's connect probe
|
|
1199
1214
|
// re-asks after a reported move — see remote.ts's preflight.
|
|
1200
1215
|
const deviceChanged = await considerFailover(e, 'read', handle);
|
|
1201
|
-
|
|
1216
|
+
const lost = evicted.has(token);
|
|
1217
|
+
if (!deviceChanged && !lost)
|
|
1202
1218
|
throw e;
|
|
1203
1219
|
// describeError, not just .message/.exitCode: this wrap is on the path a mid-launch
|
|
1204
1220
|
// NoWindowError takes, and the engine's guard tells "still drawing" from "box broken"
|
|
1205
1221
|
// by class alone (issue #80).
|
|
1206
|
-
throw new server_http_1.HttpError(500, e.message, e instanceof errors_1.CliError ? e.exitCode : 3, deviceChanged, (0, rpc_1.describeError)(e).kind);
|
|
1222
|
+
throw new server_http_1.HttpError(500, e.message, e instanceof errors_1.CliError ? e.exitCode : 3, deviceChanged, (0, rpc_1.describeError)(e).kind, lost || undefined);
|
|
1207
1223
|
}
|
|
1208
1224
|
}
|
|
1209
1225
|
async function handleLogs(handle, req, res) {
|
|
@@ -1771,6 +1787,13 @@ function buildServer(config) {
|
|
|
1771
1787
|
(0, server_http_1.sendJson)(res, 200, body);
|
|
1772
1788
|
return;
|
|
1773
1789
|
}
|
|
1790
|
+
// A run that was EVICTED is told so first, however empty the pool has since become: it
|
|
1791
|
+
// lost its phone part-way, which is not the same as a new run finding none, and a
|
|
1792
|
+
// parallel suite re-runs only the former as a fresh run (#147). Shedding a pool's last
|
|
1793
|
+
// device is exactly when both are true at once.
|
|
1794
|
+
if (served.size === 0 && evicted.has(token) && (path === '/v1/exec' || path === '/v1/elements' || path === '/v1/logs')) {
|
|
1795
|
+
throw busyError(token);
|
|
1796
|
+
}
|
|
1774
1797
|
// A deviceless server must not silently fail every command. In-memory check, so
|
|
1775
1798
|
// the normal path is untouched. NOTE this fires only when the server NEVER
|
|
1776
1799
|
// resolved a device — one that DIED mid-run still has a non-null binding and keeps
|
|
@@ -1811,9 +1834,9 @@ function buildServer(config) {
|
|
|
1811
1834
|
return holdingLease(token, () => fn(h));
|
|
1812
1835
|
};
|
|
1813
1836
|
if (req.method === 'POST' && path === '/v1/exec')
|
|
1814
|
-
return onLeasedDevice((h) => handleExec(h, req, res));
|
|
1837
|
+
return onLeasedDevice((h) => handleExec(h, req, res, token));
|
|
1815
1838
|
if (req.method === 'POST' && path === '/v1/elements')
|
|
1816
|
-
return onLeasedDevice((h) => handleElements(h, req, res));
|
|
1839
|
+
return onLeasedDevice((h) => handleElements(h, req, res, token));
|
|
1817
1840
|
if (req.method === 'POST' && path === '/v1/logs')
|
|
1818
1841
|
return onLeasedDevice((h) => handleLogs(h, req, res));
|
|
1819
1842
|
if (req.method === 'POST' && path === '/v1/install') {
|
|
@@ -1891,6 +1914,7 @@ function buildServer(config) {
|
|
|
1891
1914
|
exitCode: mapped.exitCode,
|
|
1892
1915
|
...(errorKind ? { errorKind } : {}),
|
|
1893
1916
|
...(mapped.deviceChanged ? { deviceChanged: mapped.deviceChanged } : {}),
|
|
1917
|
+
...(mapped.evicted ? { evicted: true } : {}),
|
|
1894
1918
|
};
|
|
1895
1919
|
(0, server_http_1.sendJson)(res, mapped.status, body);
|
|
1896
1920
|
}
|
package/dist/suite.js
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// The lane itself is executed by cli.ts as a child PROCESS, because exec.ts is spawnSync
|
|
22
22
|
// throughout: tests awaited inside one process would not overlap device I/O at all.
|
|
23
23
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
24
|
+
exports.deviceWaitMs = deviceWaitMs;
|
|
24
25
|
exports.sortTestFiles = sortTestFiles;
|
|
25
26
|
exports.orderTests = orderTests;
|
|
26
27
|
exports.readDurationHints = readDurationHints;
|
|
@@ -41,12 +42,40 @@ const wait_1 = require("./wait");
|
|
|
41
42
|
const version_1 = require("./version");
|
|
42
43
|
/** The implicit single lane, so serial and parallel share one code path. */
|
|
43
44
|
const SERIAL_LANE = { id: '', label: '' };
|
|
45
|
+
/** Default for `VERIKUN_SUITE_DEVICE_WAIT_MIN`. Long enough for the server to take over a
|
|
46
|
+
* crashed client's idle lease (5 min) and for its sweep's first few readmission attempts. */
|
|
47
|
+
const DEFAULT_DEVICE_WAIT_MIN = 10;
|
|
48
|
+
/**
|
|
49
|
+
* How long a suite waits when NO lane can get a device — every phone left the pool, or other
|
|
50
|
+
* runs hold them all — before it stops with exit 3 and the unrun tests in `notRun`. `0` stops
|
|
51
|
+
* at once. A lane that waits while another lane is still working is never bounded by this:
|
|
52
|
+
* the queue drains through the working lane. Mirrors `claimTtlMs`, except that an empty
|
|
53
|
+
* value means "unset" rather than zero. Exported for the unit suite.
|
|
54
|
+
*/
|
|
55
|
+
function deviceWaitMs(env = process.env) {
|
|
56
|
+
const raw = env.VERIKUN_SUITE_DEVICE_WAIT_MIN?.trim();
|
|
57
|
+
const n = raw ? Number(raw) : NaN;
|
|
58
|
+
return (Number.isFinite(n) && n >= 0 ? n : DEFAULT_DEVICE_WAIT_MIN) * 60_000;
|
|
59
|
+
}
|
|
44
60
|
/** Gap between the two health probes below. Long enough to outlast a USB
|
|
45
61
|
* re-enumeration or a simulator relaunch, short enough not to pad a real abort. */
|
|
46
62
|
const PROBE_RETRY_MS = 1000;
|
|
47
63
|
/** How often an idle lane re-checks the queue while another lane is still working.
|
|
48
64
|
* Only reached on the tail of a suite, and only when a requeue is still possible. */
|
|
49
65
|
const IDLE_POLL_MS = 25;
|
|
66
|
+
/** A lane waiting for a device re-checks at the probe cadence, doubling up to this many
|
|
67
|
+
* cadences between checks — 30s in production, which is how quickly a readmitted phone is
|
|
68
|
+
* noticed. Derived from `probeRetryMs` so the unit suite's cadence shrinks with it. */
|
|
69
|
+
const PARK_BACKOFF_STEPS = 30;
|
|
70
|
+
/** How long a lane that is LOOKING for a device holds a test's header. Whether the look got
|
|
71
|
+
* a phone is only known when the child answers, and a refusal answers in well under a second
|
|
72
|
+
* (one health read, one lease): printed at once, every refused look announced a test that
|
|
73
|
+
* never started. */
|
|
74
|
+
const PROBE_HEADER_DELAY_MS = 2_000;
|
|
75
|
+
/** How many times ONE test may be re-run, without spending a retry, because the server
|
|
76
|
+
* evicted it. Enough for two independent blips; a phone that keeps vanishing under the same
|
|
77
|
+
* test is a problem worth a red row, so the next eviction counts as an ordinary attempt. */
|
|
78
|
+
const MAX_FREE_RERUNS = 2;
|
|
50
79
|
/**
|
|
51
80
|
* An environment-flavoured failure is only FATAL if the toolchain is STILL broken when
|
|
52
81
|
* we re-probe. This distinction is load-bearing: a transient uiautomator dump failure
|
|
@@ -183,6 +212,7 @@ function toSuiteResult(file, r, durationMs) {
|
|
|
183
212
|
...(r.planSteps === undefined ? {} : { planSteps: r.planSteps }),
|
|
184
213
|
modelRepairs: r.modelRepairs,
|
|
185
214
|
...(r.ok ? {} : { failure: failure ?? 'failed' }),
|
|
215
|
+
...(r.evicted ? { evicted: true } : {}),
|
|
186
216
|
};
|
|
187
217
|
}
|
|
188
218
|
/** Compact one attempt for the `attempts` evidence array (pure). */
|
|
@@ -193,6 +223,7 @@ function toSuiteAttempt(r) {
|
|
|
193
223
|
durationMs: r.durationMs,
|
|
194
224
|
costUsd: r.costUsd,
|
|
195
225
|
...(r.failure ? { failure: r.failure } : {}),
|
|
226
|
+
...(r.evicted ? { evicted: true } : {}),
|
|
196
227
|
};
|
|
197
228
|
}
|
|
198
229
|
/**
|
|
@@ -208,7 +239,9 @@ function mergeSuiteAttempts(attempts) {
|
|
|
208
239
|
return last;
|
|
209
240
|
const round = (n) => Number(n.toFixed(4));
|
|
210
241
|
const prior = attempts.slice(0, -1).map(toSuiteAttempt);
|
|
211
|
-
|
|
242
|
+
// An eviction is not a flake: the phone left, the test did not fail. Counting it would send
|
|
243
|
+
// someone to stabilise a test whose only fault was being mid-run when a cable came loose.
|
|
244
|
+
const flaky = last.ok && prior.some((a) => !a.ok && !a.evicted);
|
|
212
245
|
return {
|
|
213
246
|
...last,
|
|
214
247
|
durationMs: attempts.reduce((a, t) => a + t.durationMs, 0),
|
|
@@ -302,6 +335,10 @@ function laneCount(available, tests, flags) {
|
|
|
302
335
|
// Never open a lane with nothing to run — it only adds a device to the report.
|
|
303
336
|
return Math.max(1, Math.min(available, tests, n ?? available));
|
|
304
337
|
}
|
|
338
|
+
/** A wait, for a human: `10m`, `4.2m`, `12s`. */
|
|
339
|
+
function duration(ms) {
|
|
340
|
+
return ms >= 60_000 ? `${Math.round(ms / 6_000) / 10}m` : `${Math.round(ms / 100) / 10}s`;
|
|
341
|
+
}
|
|
305
342
|
/** Consecutive environment failures that retire a lane even when the probe says the
|
|
306
343
|
* box is fine. The backstop for a POOL: `deps.preflight` on a remote lane can only ask
|
|
307
344
|
* whether the SERVER answers, so one dead device behind a healthy server would
|
|
@@ -344,6 +381,18 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
344
381
|
let spentUsd = 0;
|
|
345
382
|
let retiredLanes = 0;
|
|
346
383
|
let started = 0;
|
|
384
|
+
/**
|
|
385
|
+
* Attempts that travel WITH a file when a lane hands it back for want of a device, so the
|
|
386
|
+
* lane that picks it up carries on where it stopped: the same attempt count, the same
|
|
387
|
+
* evidence, the same free re-runs used. Deleted the moment a lane takes the file again.
|
|
388
|
+
*/
|
|
389
|
+
const carried = new Map();
|
|
390
|
+
/** Lease slots this suite has in flight, per server — our own half of the device gate. */
|
|
391
|
+
const reserved = new Map();
|
|
392
|
+
/** When an attempt last got a device and finished. With no lane busy, a suite whose last
|
|
393
|
+
* progress is `waitMs` old is waiting on a device that is not coming. */
|
|
394
|
+
let lastProgressAt = Date.now();
|
|
395
|
+
const waitMs = deps.deviceWaitMs ?? deviceWaitMs();
|
|
347
396
|
const tag = (lane) => (parallel && lane.label ? `[suite ${lane.label}]` : '[suite]');
|
|
348
397
|
async function resetApp(lane, label) {
|
|
349
398
|
// Returns the abort reason when the lane should stop (confirmed env break during reset).
|
|
@@ -376,21 +425,62 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
376
425
|
(0, output_1.err)(`${tag(lane)} WARN ${warn}`);
|
|
377
426
|
await (0, wait_1.sleep)((deps.probeRetryMs ?? PROBE_RETRY_MS) * (attempt + 1));
|
|
378
427
|
}
|
|
379
|
-
/** Every attempt at one file, on one lane.
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
//
|
|
383
|
-
//
|
|
384
|
-
|
|
385
|
-
const
|
|
428
|
+
/** Every attempt at one file, on one lane. `probing`: the lane is waiting for a device, and
|
|
429
|
+
* this dispatch is a look — see PROBE_HEADER_DELAY_MS. */
|
|
430
|
+
async function runOneTest(file, lane, probing = false) {
|
|
431
|
+
// A SERVER lane is a slot, not a device: whether a phone is free for it is the server's
|
|
432
|
+
// answer, asked before every dispatch (see SuiteDeps.serverSlots). Keyed on the wiring,
|
|
433
|
+
// never on `parallel` — `--concurrency 1` against a pooled server still leases per test.
|
|
434
|
+
const gated = !!(deps.serverSlots && lane.server);
|
|
435
|
+
const prior = carried.get(file);
|
|
436
|
+
carried.delete(file);
|
|
437
|
+
let announced = false;
|
|
438
|
+
const announce = () => {
|
|
439
|
+
if (announced)
|
|
440
|
+
return;
|
|
441
|
+
announced = true;
|
|
442
|
+
// Bounded: a lane that retires re-queues its file, so a plain dispatch counter can
|
|
443
|
+
// print "(9/8)". Clamp rather than drop the position — it is the only progress signal
|
|
444
|
+
// a parallel run gives, and an occasional repeat reads better than a lie. A file picked
|
|
445
|
+
// back up after a hand-back was counted when it first ran.
|
|
446
|
+
if (prior)
|
|
447
|
+
(0, output_1.err)(`${tag(lane)} ── ${file} (continued) ──`);
|
|
448
|
+
else
|
|
449
|
+
(0, output_1.err)(`${tag(lane)} ── (${Math.min(++started, files.length)}/${files.length}) ${file} ──`);
|
|
450
|
+
};
|
|
451
|
+
// A gated lane says nothing until it has a device, or a lane waiting for one would print
|
|
452
|
+
// a header every time it looked.
|
|
453
|
+
if (!gated)
|
|
454
|
+
announce();
|
|
455
|
+
const attemptRows = prior?.rows ?? [];
|
|
456
|
+
let freeReruns = prior?.freeReruns ?? 0;
|
|
457
|
+
/** The attempt whose "retry k/N" line a gated lane has printed — the header is said once
|
|
458
|
+
* per test, the retry line once per attempt. */
|
|
459
|
+
let saidFor = -1;
|
|
386
460
|
let envFlavoured = false;
|
|
387
461
|
const merge = () => attemptRows.length ? mergeSuiteAttempts(attemptRows) : undefined;
|
|
388
|
-
|
|
462
|
+
/** Did any attempt on THIS dispatch get a device and run? */
|
|
463
|
+
let ranHere = false;
|
|
464
|
+
/** No device for this attempt: nothing is recorded and nothing is spent — not a retry, not
|
|
465
|
+
* a streak tick — and whatever came before rides back to the queue with the file (#147). */
|
|
466
|
+
const handBack = (attempt, reason) => {
|
|
467
|
+
if (attemptRows.length || attempt > 0 || freeReruns > 0)
|
|
468
|
+
carried.set(file, { rows: attemptRows, attempt, freeReruns });
|
|
469
|
+
// A dispatch that never ran never held a place in the run order.
|
|
470
|
+
if (announced && !prior && !attemptRows.length)
|
|
471
|
+
started -= 1;
|
|
472
|
+
return { handBack: reason, ...(ranHere ? { ran: true } : {}) };
|
|
473
|
+
};
|
|
474
|
+
for (let attempt = prior?.attempt ?? 0; attempt <= retries; attempt++) {
|
|
389
475
|
// The last attempt is where a retryable failure becomes the verdict: a confirmed
|
|
390
476
|
// env break retires the lane, anything else stands as this test's failed row.
|
|
391
477
|
const lastAttempt = attempt === retries;
|
|
392
|
-
|
|
393
|
-
|
|
478
|
+
const retryLine = () => {
|
|
479
|
+
if (attempt > 0)
|
|
480
|
+
(0, output_1.err)(`${tag(lane)} retry ${attempt}/${retries} for ${file}`);
|
|
481
|
+
};
|
|
482
|
+
if (!gated)
|
|
483
|
+
retryLine();
|
|
394
484
|
// Re-isolate before EVERY attempt — between tests and between retries alike.
|
|
395
485
|
const resetBreak = await resetApp(lane, attempt > 0 ? ' (retry)' : '');
|
|
396
486
|
if (resetBreak) {
|
|
@@ -401,11 +491,71 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
401
491
|
// With no attempt row this test never ran, so the caller re-queues it.
|
|
402
492
|
return { row: merge(), envBreak: `reset failed: ${resetBreak}` };
|
|
403
493
|
}
|
|
494
|
+
let slot;
|
|
495
|
+
if (gated) {
|
|
496
|
+
const server = lane.server;
|
|
497
|
+
const slots = await deps.serverSlots(lane);
|
|
498
|
+
const inUse = reserved.get(server) ?? 0;
|
|
499
|
+
// Check and reserve with NO `await` in between: two lanes waking together must not
|
|
500
|
+
// both see the one free phone and both ask for it, because the loser's lease request
|
|
501
|
+
// is exactly what lets the server take over a quiet sibling's phone.
|
|
502
|
+
if (slots && slots.capacity <= inUse)
|
|
503
|
+
return handBack(attempt, slots.note);
|
|
504
|
+
reserved.set(server, inUse + 1);
|
|
505
|
+
slot = server;
|
|
506
|
+
}
|
|
507
|
+
const say = () => {
|
|
508
|
+
announce();
|
|
509
|
+
if (saidFor === attempt)
|
|
510
|
+
return;
|
|
511
|
+
saidFor = attempt;
|
|
512
|
+
retryLine();
|
|
513
|
+
};
|
|
514
|
+
// A look says nothing until its child has plainly got a phone; any other gated dispatch
|
|
515
|
+
// announces now, before the child's own output starts streaming.
|
|
516
|
+
const held = gated && probing ? setTimeout(say, PROBE_HEADER_DELAY_MS) : undefined;
|
|
517
|
+
if (gated && !probing)
|
|
518
|
+
say();
|
|
404
519
|
const t0 = Date.now();
|
|
405
520
|
try {
|
|
406
|
-
|
|
521
|
+
let r;
|
|
522
|
+
try {
|
|
523
|
+
r = await deps.runTest((0, node_path_1.join)(dir, file), lane);
|
|
524
|
+
}
|
|
525
|
+
finally {
|
|
526
|
+
clearTimeout(held);
|
|
527
|
+
// Only the child holds a lease; the re-probes below do not, so neither does the slot.
|
|
528
|
+
if (slot) {
|
|
529
|
+
const n = (reserved.get(slot) ?? 1) - 1;
|
|
530
|
+
if (n > 0)
|
|
531
|
+
reserved.set(slot, n);
|
|
532
|
+
else
|
|
533
|
+
reserved.delete(slot);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
// Refused at the lease (the gate raced another run, or a job outside this suite holds
|
|
537
|
+
// the phones). The test never started, so this was never an attempt.
|
|
538
|
+
if (r.noDevice)
|
|
539
|
+
return handBack(attempt, r.failure?.reason ?? 'the server had no free device');
|
|
540
|
+
if (gated)
|
|
541
|
+
say(); // a look that got its phone, and finished inside the header's hold
|
|
542
|
+
ranHere = true;
|
|
543
|
+
lastProgressAt = Date.now();
|
|
407
544
|
attemptRows.push(toSuiteResult(file, r, Date.now() - t0));
|
|
408
545
|
envFlavoured = !!r.abortedForEnv;
|
|
546
|
+
// The server ended this run because its phone left the pool. Neither the test's fault
|
|
547
|
+
// nor this lane's: start it again as a fresh run without spending a retry. Only on a
|
|
548
|
+
// server lane, where each attempt is a new run token — the serial path's single token
|
|
549
|
+
// is as dead as its phone, and aborts on the re-probe below exactly as it did.
|
|
550
|
+
if (r.evicted && lane.server && freeReruns < MAX_FREE_RERUNS) {
|
|
551
|
+
freeReruns += 1;
|
|
552
|
+
const warn = `${file}: its device ${r.device ?? '(unknown)'} left the pool mid-run — re-run as a fresh run ` +
|
|
553
|
+
`(${freeReruns}/${MAX_FREE_RERUNS}, not counted toward --retries)`;
|
|
554
|
+
warnings.push(warn);
|
|
555
|
+
(0, output_1.err)(`${tag(lane)} WARN ${warn}`);
|
|
556
|
+
attempt -= 1;
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
409
559
|
if (r.abortedForEnv) {
|
|
410
560
|
const broken = await stillBroken(deps, lane);
|
|
411
561
|
if (broken) {
|
|
@@ -423,6 +573,10 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
423
573
|
break;
|
|
424
574
|
}
|
|
425
575
|
catch (e) {
|
|
576
|
+
if (gated)
|
|
577
|
+
say(); // a look whose child threw inside the header's hold still ran a test
|
|
578
|
+
ranHere = true;
|
|
579
|
+
lastProgressAt = Date.now();
|
|
426
580
|
// A test that THREW (device gone, server unreachable, bad file) still becomes a
|
|
427
581
|
// failed row — one broken test must not vaporize the suite report for the tests
|
|
428
582
|
// that already ran. Out of attempts, a confirmed env break retires the lane.
|
|
@@ -466,6 +620,47 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
466
620
|
}
|
|
467
621
|
stop ??= { reason, kind: 'environment' };
|
|
468
622
|
}
|
|
623
|
+
/**
|
|
624
|
+
* Wait for this lane's next look at the queue, re-checking every IDLE_POLL_MS whether to
|
|
625
|
+
* stop waiting altogether — so a waiting lane neither holds a finished suite open nor
|
|
626
|
+
* sleeps through its end. 'look' means try the queue again.
|
|
627
|
+
*/
|
|
628
|
+
async function waitForDevice(park) {
|
|
629
|
+
for (;;) {
|
|
630
|
+
if (stop)
|
|
631
|
+
return 'done';
|
|
632
|
+
// Nothing left to run and nobody running: the suite finished without this lane.
|
|
633
|
+
if (!queue.length && busyLanes === 0)
|
|
634
|
+
return 'done';
|
|
635
|
+
// Nobody is running, and nothing has for the whole wait: every lane is waiting on a
|
|
636
|
+
// device that is not coming. That stops the SUITE, with one reason — not one retirement
|
|
637
|
+
// per lane, each claiming its tests "move to the others" who are waiting too.
|
|
638
|
+
if (busyLanes === 0 && Date.now() - lastProgressAt >= waitMs) {
|
|
639
|
+
// One last look first, unless one has happened since the deadline: backed off, the
|
|
640
|
+
// previous look may be half the wait old, and a phone that came back in between would
|
|
641
|
+
// otherwise never be seen — the suite exiting 3 to say none became free.
|
|
642
|
+
if (queue.length && park.lastLookAt < lastProgressAt + waitMs)
|
|
643
|
+
return 'look';
|
|
644
|
+
stop ??= {
|
|
645
|
+
reason: `no device became free${waitMs > 0 ? ` for ${duration(waitMs)}` : ''} — ${park.reason}`,
|
|
646
|
+
kind: 'environment',
|
|
647
|
+
};
|
|
648
|
+
return 'done';
|
|
649
|
+
}
|
|
650
|
+
if (queue.length && Date.now() >= park.nextAt)
|
|
651
|
+
return 'look';
|
|
652
|
+
await (0, wait_1.sleep)(IDLE_POLL_MS);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
/** One warning per waiting spell, not one per look: how long, and the server's reason.
|
|
656
|
+
* `ran` is the test the lane finally got a device for — reported once it is over, which is
|
|
657
|
+
* the first moment the spell is known to have ended. */
|
|
658
|
+
function endPark(lane, park, ran) {
|
|
659
|
+
const waited = duration(Date.now() - park.since);
|
|
660
|
+
const warn = `${lane.label || 'the suite'} waited ${waited} for a free device — ${park.reason}`;
|
|
661
|
+
warnings.push(warn);
|
|
662
|
+
(0, output_1.err)(`${tag(lane)} ${ran ? `${ran} ran after this lane waited ${waited} for a free device` : `stopped waiting for a device after ${waited}`}`);
|
|
663
|
+
}
|
|
469
664
|
/**
|
|
470
665
|
* One lane, pulling from the shared queue until it is empty, the lane is retired, or
|
|
471
666
|
* the suite stops. This — not a fixed partition — is what absorbs the duration spread:
|
|
@@ -474,9 +669,23 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
474
669
|
let busyLanes = 0;
|
|
475
670
|
async function laneWorker(lane) {
|
|
476
671
|
let envStreak = 0;
|
|
672
|
+
// Set while this lane has no device to run on. A PARKED lane is not busy and holds no
|
|
673
|
+
// file: whatever it could not start went back to the front of the queue for any lane.
|
|
674
|
+
let park;
|
|
675
|
+
const cadence = Math.max(IDLE_POLL_MS, deps.probeRetryMs ?? PROBE_RETRY_MS);
|
|
477
676
|
for (;;) {
|
|
478
|
-
if (stop)
|
|
677
|
+
if (stop) {
|
|
678
|
+
if (park)
|
|
679
|
+
endPark(lane, park);
|
|
479
680
|
return;
|
|
681
|
+
}
|
|
682
|
+
if (park) {
|
|
683
|
+
if ((await waitForDevice(park)) === 'done') {
|
|
684
|
+
endPark(lane, park);
|
|
685
|
+
return;
|
|
686
|
+
}
|
|
687
|
+
park.lastLookAt = Date.now();
|
|
688
|
+
}
|
|
480
689
|
const file = queue.shift();
|
|
481
690
|
if (file === undefined) {
|
|
482
691
|
// An empty queue is NOT the end while another lane is still working: a lane that
|
|
@@ -485,8 +694,12 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
485
694
|
// is the one outcome a gate must never produce. Poll rather than signal: the
|
|
486
695
|
// wait only ever happens on the tail of a suite, and a condition variable here
|
|
487
696
|
// would be more machinery than the case is worth.
|
|
488
|
-
if (busyLanes === 0)
|
|
697
|
+
if (busyLanes === 0) {
|
|
698
|
+
// A waiting lane whose look lost the last file to a sibling still reports its wait.
|
|
699
|
+
if (park)
|
|
700
|
+
endPark(lane, park);
|
|
489
701
|
return;
|
|
702
|
+
}
|
|
490
703
|
await (0, wait_1.sleep)(IDLE_POLL_MS);
|
|
491
704
|
continue;
|
|
492
705
|
}
|
|
@@ -494,7 +707,7 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
494
707
|
let outcome;
|
|
495
708
|
const startedTest = Date.now();
|
|
496
709
|
try {
|
|
497
|
-
outcome = await runOneTest(file, lane);
|
|
710
|
+
outcome = await runOneTest(file, lane, park !== undefined);
|
|
498
711
|
}
|
|
499
712
|
catch (e) {
|
|
500
713
|
// The file is already off the queue, so an escaping throw would erase this test
|
|
@@ -507,6 +720,35 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
507
720
|
finally {
|
|
508
721
|
busyLanes -= 1;
|
|
509
722
|
}
|
|
723
|
+
if (outcome.handBack !== undefined) {
|
|
724
|
+
// No device: back to the FRONT, so whichever lane frees first runs it next and the
|
|
725
|
+
// longest-first order holds. This lane waits, backing off; it neither fails the test
|
|
726
|
+
// nor pulls the next one only to be refused again (#147).
|
|
727
|
+
queue.unshift(file);
|
|
728
|
+
const now = Date.now();
|
|
729
|
+
// A look that RAN a test was working, not waiting: that spell is over, and a new one
|
|
730
|
+
// starts — its run time is not waiting time.
|
|
731
|
+
if (park && outcome.ran) {
|
|
732
|
+
endPark(lane, park, file);
|
|
733
|
+
park = undefined;
|
|
734
|
+
}
|
|
735
|
+
if (!park) {
|
|
736
|
+
park = { since: now, reason: outcome.handBack, delayMs: cadence, nextAt: now + cadence, lastLookAt: now };
|
|
737
|
+
(0, output_1.err)(`${tag(lane)} no free device — ${outcome.handBack}; ${file} goes back to the queue and this lane waits for one`);
|
|
738
|
+
}
|
|
739
|
+
else {
|
|
740
|
+
// The LATEST reason: the pool may have changed since the spell began, and the give-up
|
|
741
|
+
// and the warning should name what the server says now.
|
|
742
|
+
park.reason = outcome.handBack;
|
|
743
|
+
park.delayMs = Math.min(park.delayMs * 2, cadence * PARK_BACKOFF_STEPS);
|
|
744
|
+
park.nextAt = now + park.delayMs;
|
|
745
|
+
}
|
|
746
|
+
continue;
|
|
747
|
+
}
|
|
748
|
+
if (park) {
|
|
749
|
+
endPark(lane, park, file);
|
|
750
|
+
park = undefined;
|
|
751
|
+
}
|
|
510
752
|
if (outcome.row) {
|
|
511
753
|
results.push(outcome.row);
|
|
512
754
|
spentUsd += outcome.row.costUsd;
|
|
@@ -526,6 +768,8 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
526
768
|
// Only a ceiling that actually STOPS something is an abort. Without the queue
|
|
527
769
|
// check, a suite whose last test tips the total over its budget reports ABORTED
|
|
528
770
|
// with an empty `notRun` and exits 1 — a red gate on a run where every test passed.
|
|
771
|
+
// (A file handed back mid-retry carries its spend with it, so that spend is counted
|
|
772
|
+
// when its row lands — at most one attempt late.)
|
|
529
773
|
if (maxSuiteCost !== undefined && spentUsd >= maxSuiteCost && queue.length > 0) {
|
|
530
774
|
stop ??= {
|
|
531
775
|
reason: `suite cost ceiling $${maxSuiteCost} reached (spent $${spentUsd.toFixed(4)})`,
|
|
@@ -546,6 +790,16 @@ async function cmdSuite(dirArg, flags, deps) {
|
|
|
546
790
|
warnings.push(warn);
|
|
547
791
|
(0, output_1.err)(`[suite] WARN ${warn}`);
|
|
548
792
|
}
|
|
793
|
+
// A file handed back with attempts already on it RAN, so it is a row — it is only still
|
|
794
|
+
// queued because the suite stopped before any lane could take it again, and a file must be
|
|
795
|
+
// in exactly one of `tests` and `notRun`.
|
|
796
|
+
for (const [file, c] of carried) {
|
|
797
|
+
const at = queue.indexOf(file);
|
|
798
|
+
if (at < 0 || !c.rows.length)
|
|
799
|
+
continue;
|
|
800
|
+
queue.splice(at, 1);
|
|
801
|
+
results.push(mergeSuiteAttempts(c.rows));
|
|
802
|
+
}
|
|
549
803
|
// Belt and braces: work left in the queue with nothing explaining why would be a
|
|
550
804
|
// silently short suite. Every real path (retirement, budget) has already set `stop`.
|
|
551
805
|
if (!stop && queue.length) {
|
package/dist/version.js
CHANGED
|
@@ -3,4 +3,4 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.VERSION = void 0;
|
|
4
4
|
// GENERATED by scripts/gen-version.mjs from package.json's "version" at build time
|
|
5
5
|
// (the `prebuild` script). Do NOT edit by hand; bump package.json instead.
|
|
6
|
-
exports.VERSION = '0.
|
|
6
|
+
exports.VERSION = '0.30.0';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "verikun",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.30.0",
|
|
4
4
|
"description": "Drive Android emulators/devices and iOS simulators for AI agents: tap, type, swipe, screenshot, and inspect the UI hierarchy by semantic identifiers — like Puppeteer for native apps.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"android",
|