verikun 0.24.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -452,6 +452,19 @@ device; `503` means the server has no device attached — boot one (below). To
452
452
  expose a device from THIS machine: `vk server --allow-install`
453
453
  (add `--bind <addr>` to leave loopback; auth key auto-generates if unset).
454
454
 
455
+ **If you see `[verikun] server moved device: A → B` on stderr**, the server left a
456
+ device that failed and is now on another one. What that means depends on the line:
457
+
458
+ - `— retried there` (installs only): the build DID land, on **B**. Anything you go on
459
+ to do with an explicit serial must name B, not A.
460
+ - `— this step failed on the old device; the next runs on the new one`: your step
461
+ failed on **A**. Do NOT re-run it expecting a different answer for the same reason —
462
+ the failure was real on A, and B has none of the state your flow built up. Start the
463
+ flow again from the top if you want it on B.
464
+
465
+ The server rules the bad device out until it is power-cycled;
466
+ `vk devices --server <url>` shows why in its `NOTE` column.
467
+
455
468
  ## The device is missing or wedged
456
469
 
457
470
  ```sh
package/CHANGELOG.md CHANGED
@@ -6,6 +6,25 @@ All notable changes to this project are documented here. The format is based on
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.25.0] - 2026-08-21
10
+
11
+ ### Added
12
+ - **`vk server`**: moves to another attached healthy device when the bound one fails; only a pinned `--device` disables it. ([#99])
13
+ - **`vk server --allow-failover[=serials]`**: re-enable failover on a pinned server, and bound where it may go.
14
+ - **`vk server --no-failover`** / **`VERIKUN_NO_FAILOVER`**: disable failover outright.
15
+ - **`/v1/health`**: adds `failoverEnabled` and `quarantined`, so a client can see which devices the server ruled out.
16
+ - **`vk install --server`**: reports the device the build landed on when it differs from the one bound.
17
+
18
+ ### Changed
19
+ - **`vk install --server`**: a device-attributable install failure retries on another healthy device.
20
+ - **`/v1/exec`**, **`/v1/elements`**: report `deviceChanged` when the server moved device; the failing step is never replayed.
21
+
22
+ ### Fixed
23
+ - **`/v1/logs`**: served the startup device's logs after a rebind instead of the bound device's.
24
+ - **`/v1/health`**: reported the startup device's read path after a rebind.
25
+
26
+ [#99]: https://github.com/ddikman/verikun/issues/99
27
+
9
28
  ## [0.24.0] - 2026-08-20
10
29
 
11
30
  ### Added
package/README.md CHANGED
@@ -8,7 +8,7 @@
8
8
  - **Puppeteer for native mobile** — a thin wrapper over native Android and iOS automation runners with zero runtime dependencies.
9
9
  - **Natural-language tests** — `vk ai <file>`: runs plain-English tests, compiled once and replayed model-free (~$0), calling a model only to self-heal a drifted step. [What that costs](https://ddikman.github.io/verikun/reference/cost/), and how the `--max-cost-usd` ceiling bounds it.
10
10
  - **Self-improving** — the agent runner will provide prescriptive improvements to existing scripts to help stabilise flakiness for future runs.
11
- - **CI-ready** — `vk suite` runs a folder of tests as one gated pass/fail run; `vk server` exposes a real device over an authenticated tunnel so a disposable CI runner (no phone attached) can still drive it.
11
+ - **CI-ready** — `vk suite` runs a folder of tests as one gated pass/fail run; `vk server` exposes a real device over an authenticated tunnel so a disposable CI runner (no phone attached) can still drive it, and moves to another attached device if that one goes bad.
12
12
 
13
13
  ```
14
14
  $ vk ui
@@ -89,8 +89,14 @@ class RemoteTransport {
89
89
  finally {
90
90
  clearTimeout(timer);
91
91
  }
92
- if (!res.ok)
93
- throw describeStatus(res.status, await readBody(res), url);
92
+ if (!res.ok) {
93
+ const body = await readBody(res);
94
+ // Before throwing: a failing request may still have moved the device, and that is
95
+ // exactly the case a caller must not miss (an exhausted install, a dead-device read).
96
+ if (body?.deviceChanged)
97
+ this.opts.onDeviceChange?.(body.deviceChanged);
98
+ throw describeStatus(res.status, body, url);
99
+ }
94
100
  const parsed = await readBody(res);
95
101
  if (parsed === null)
96
102
  throw new errors_1.CliError(`verikun server at ${url} returned a non-JSON response`, 3);
@@ -157,6 +163,9 @@ function createRemoteBackend(opts, health) {
157
163
  const res = await t.postJson('/v1/exec', req, EXEC_TIMEOUT_MS);
158
164
  if (record && res.step)
159
165
  opts.onStep?.(res.step, decodeArtifacts(res.artifacts), res.logStart);
166
+ // A failing step is a 200, so this is the ordinary path for a mid-run device death.
167
+ if (res.deviceChanged)
168
+ opts.onDeviceChange?.(res.deviceChanged);
160
169
  return { code: res.code, error: res.error ? (0, rpc_1.rebuildError)(res.error) : undefined };
161
170
  };
162
171
  return {
@@ -184,11 +193,15 @@ function createRemoteBackend(opts, health) {
184
193
  throw new errors_1.CliError(`install: cannot read '${appPath}' (${e.message})`, 2);
185
194
  }
186
195
  const sha256 = (0, node_crypto_1.createHash)('sha256').update(buf).digest('hex');
187
- await t.request('POST', '/v1/install', buf, INSTALL_TIMEOUT_MS, {
196
+ const res = await t.request('POST', '/v1/install', buf, INSTALL_TIMEOUT_MS, {
188
197
  'content-type': 'application/octet-stream',
189
198
  'x-verikun-ext': ext,
190
199
  'x-verikun-sha256': sha256,
191
200
  });
201
+ // Install is the one operation the server replays elsewhere, so a move here means
202
+ // the build DID land — on a different device than the one we started with.
203
+ if (res.deviceChanged)
204
+ opts.onDeviceChange?.(res.deviceChanged);
192
205
  },
193
206
  async reset(appId) {
194
207
  // Between-test housekeeping (vk suite): the step is deliberately NOT spliced
package/dist/args.js CHANGED
@@ -49,10 +49,12 @@ const BOOLEAN = new Set([
49
49
  'no-sleep-when-idle',
50
50
  'allow-install',
51
51
  'allow-unsafe-anonymous',
52
- // NOT here on purpose: 'allow-device-control' and 'ensure-device' are tri-state
53
- // (absent / on / on-with-a-value), and the inline `--flag=value` branch is checked
54
- // BEFORE this set listing them would make `--allow-device-control=Pixel_6` parse
55
- // fine but `--ensure-device Pixel_6` silently become a boolean plus a positional.
52
+ 'no-failover',
53
+ // NOT here on purpose: 'allow-device-control', 'allow-failover' and 'ensure-device'
54
+ // are tri-state (absent / on / on-with-a-value), and the inline `--flag=value` branch
55
+ // is checked BEFORE this set listing them would make `--allow-device-control=Pixel_6`
56
+ // parse fine but `--ensure-device Pixel_6` silently become a boolean plus a positional.
57
+ // Its opposite, 'no-failover', never takes a value, so it DOES belong above.
56
58
  // Selector state modifiers (STATE_ATTRS in ui/selector.ts) and their negations.
57
59
  // `enabled` was missing here until 0.15.0, and the omission was not cosmetic: a
58
60
  // non-BOOLEAN flag swallows the next token, so `vk tap --enabled @submit` bound
package/dist/cli.js CHANGED
@@ -50,6 +50,7 @@ exports.chooseLogOpts = chooseLogOpts;
50
50
  exports.evalAssert = evalAssert;
51
51
  exports.tokenizeLine = tokenizeLine;
52
52
  exports.withBatchGlobals = withBatchGlobals;
53
+ exports.retryAfterDeviceMove = retryAfterDeviceMove;
53
54
  exports.serverFromFlags = serverFromFlags;
54
55
  exports.ensureDeviceTarget = ensureDeviceTarget;
55
56
  exports.terminalFailure = terminalFailure;
@@ -1999,6 +2000,38 @@ async function obtainPlan(key, file, opts, cost, provider) {
1999
2000
  }
2000
2001
  return { plan: compiled.plan, cached: false };
2001
2002
  }
2003
+ // --- execution backend (local driver vs remote `vk server`) -----------------
2004
+ //
2005
+ // `vk ai`, `vk suite`, and `vk install` run their device work through an
2006
+ // ExecBackend. Local wraps one shared Driver; remote speaks HTTP to a `vk server`
2007
+ // beside the device (--server / VERIKUN_SERVER), where each validated leaf is ONE
2008
+ // round-trip (the auto-wait loop stays server-side). In remote mode the server
2009
+ // owns the device: its /v1/health platform+serial supersede the client's
2010
+ // --platform/--device, and no local driver is ever built.
2011
+ /**
2012
+ * Run a remote read; if it failed AND the server reported it moved device, ask once more.
2013
+ *
2014
+ * This is the ONE place a failed remote read is re-asked, and the narrowness is the point.
2015
+ * It is only ever wired to `preflight` — the connect probe at `vk ai`/`vk suite` startup and
2016
+ * the suite's between-tests health check — where nothing has run yet on either device. A
2017
+ * mid-flow read is never retried: the new device has none of the state the flow built up, so
2018
+ * its screen would answer a different question than the one being asked.
2019
+ *
2020
+ * Gating on the move (rather than retrying every failure) also keeps the connect probe's
2021
+ * fail-fast property: a device that is simply broken still fails on the first try.
2022
+ *
2023
+ * Exported solely so the unit suite can reach it.
2024
+ */
2025
+ async function retryAfterDeviceMove(read, moved) {
2026
+ try {
2027
+ return await read();
2028
+ }
2029
+ catch (e) {
2030
+ if (!moved())
2031
+ throw e;
2032
+ return read();
2033
+ }
2034
+ }
2002
2035
  /** The `--server` URL, or VERIKUN_SERVER. Exported-shape helper so `resolveBackend`
2003
2036
  * and `vk devices --server` can never disagree about what "remote" means. */
2004
2037
  function serverFromFlags(flags) {
@@ -2171,9 +2204,14 @@ async function resolveBackend(platform, device, flags) {
2171
2204
  },
2172
2205
  platform,
2173
2206
  device,
2207
+ moves: [],
2174
2208
  };
2175
2209
  }
2176
2210
  let runCtx = { platform, device };
2211
+ const moves = [];
2212
+ /** Set by the last move; the preflight below reads it to decide whether re-asking is
2213
+ * warranted, then clears it. */
2214
+ let movedDuringCall;
2177
2215
  const opts = {
2178
2216
  url: server,
2179
2217
  authKey: (0, args_1.flagStr)(flags, 'auth-key') || process.env.VERIKUN_SERVER_AUTH_KEY || undefined,
@@ -2181,6 +2219,19 @@ async function resolveBackend(platform, device, flags) {
2181
2219
  // is identical to a local run's. logStart travels from the server's device clock
2182
2220
  // so archive-time / vk log scoping works without a local driver.
2183
2221
  onStep: (step, artifacts, logStart) => run_1.Recorder.appendForeignStep(step, artifacts, { ...runCtx, logStart }),
2222
+ onDeviceChange: (c) => {
2223
+ moves.push(c);
2224
+ movedDuringCall = c;
2225
+ // Re-point the run context, so steps after the move are attributed to the device
2226
+ // that actually ran them. This makes `rolloverReason` seal the device-A run and
2227
+ // open a fresh one for B — intended: since a step is never replayed, no single run
2228
+ // can contain steps from two devices, and a report that claimed otherwise would lie.
2229
+ runCtx = { ...runCtx, device: c.to };
2230
+ (0, output_1.err)(`[verikun] server moved device: ${c.from} → ${c.to} (${c.reason})` +
2231
+ (c.retried
2232
+ ? ' — retried there'
2233
+ : ' — this step failed on the old device; the next runs on the new one'));
2234
+ },
2184
2235
  };
2185
2236
  let health = await (0, remote_1.pingServer)(opts); // fails fast (exit 3) on a bad URL or key
2186
2237
  // `--ensure-device` boots BEFORE runCtx is fixed: resolveBackend bakes the serial
@@ -2188,7 +2239,15 @@ async function resolveBackend(platform, device, flags) {
2188
2239
  // attribute every spliced step to a device that didn't exist yet.
2189
2240
  health = await ensureRemoteDevice(health, opts, server, flags);
2190
2241
  runCtx = { platform: health.platform, device: health.serial ?? undefined };
2191
- (0, output_1.err)(`[verikun] server ${server}: ${health.platform} · device ${health.serial ?? '(none)'} · verikun ${health.version}`);
2242
+ (0, output_1.err)(`[verikun] server ${server}: ${health.platform} · device ${health.serial ?? '(none)'} · verikun ${health.version}` +
2243
+ (health.failoverEnabled ? ' · failover: on' : ''));
2244
+ // A pool the server has already ruled out explains a lot of otherwise-baffling
2245
+ // behaviour ("why is it on THAT phone?"), so say it once, up front.
2246
+ if (health.quarantined?.length) {
2247
+ (0, output_1.err)(`[verikun] server has ruled out ${health.quarantined.length} device(s):`);
2248
+ for (const q of health.quarantined)
2249
+ (0, output_1.err)(`[verikun] ${q.serial} ${q.reason}`);
2250
+ }
2192
2251
  // Say the read path once, here. Reads execute server-side, so this is the only end of the
2193
2252
  // connection that knows it — and without it a companion that had silently stood down was
2194
2253
  // indistinguishable from one that never engaged, for a whole suite (issue #77). An older
@@ -2206,7 +2265,8 @@ async function resolveBackend(platform, device, flags) {
2206
2265
  // healthy and keep grinding. One dump is the cheap call that actually proves it.
2207
2266
  preflight: async () => {
2208
2267
  await (0, remote_1.pingServer)(opts);
2209
- await remote.getElements();
2268
+ movedDuringCall = undefined;
2269
+ await retryAfterDeviceMove(() => remote.getElements(), () => movedDuringCall !== undefined);
2210
2270
  },
2211
2271
  // Hierarchy only: the server exposes no screenshot route, so a remote run's
2212
2272
  // engine failure archives without a picture. Honest degrade over a protocol
@@ -2223,6 +2283,7 @@ async function resolveBackend(platform, device, flags) {
2223
2283
  platform: health.platform,
2224
2284
  device: health.serial ?? undefined,
2225
2285
  remote: { url: server, version: health.version, reads: health.reads },
2286
+ moves,
2226
2287
  };
2227
2288
  }
2228
2289
  /**
@@ -2501,7 +2562,7 @@ async function cmdInstall(positionals, flags) {
2501
2562
  if (!(0, node_fs_1.existsSync)(path))
2502
2563
  throw new errors_1.CliError(`install: '${appPath}' does not exist`, 2);
2503
2564
  const platform = platformFromFlags(flags);
2504
- const { backend, remote } = await resolveBackend(platform, deviceFromFlags(flags, platform), flags);
2565
+ const { backend, remote, moves } = await resolveBackend(platform, deviceFromFlags(flags, platform), flags);
2505
2566
  (0, output_1.err)(`[verikun] installing ${appPath}${remote ? ` via ${remote.url}` : ''}…`);
2506
2567
  try {
2507
2568
  await backend.install(path);
@@ -2509,10 +2570,16 @@ async function cmdInstall(positionals, flags) {
2509
2570
  finally {
2510
2571
  await backend.close?.();
2511
2572
  }
2512
- if ((0, args_1.flagBool)(flags, 'json'))
2513
- (0, output_1.json)({ installed: appPath, ...(remote ? { server: remote.url } : {}) });
2514
- else
2515
- (0, output_1.out)(`installed ${appPath}`);
2573
+ // Where it LANDED, not just that it landed: after a failover that is a different
2574
+ // device than the one the run started against, and a caller acting on the old serial
2575
+ // (`adb -s … shell am start`) would be driving a phone without the build.
2576
+ const moved = moves.length ? moves[moves.length - 1] : undefined;
2577
+ if ((0, args_1.flagBool)(flags, 'json')) {
2578
+ (0, output_1.json)({ installed: appPath, ...(remote ? { server: remote.url } : {}), ...(moved ? { deviceChanged: moved } : {}) });
2579
+ }
2580
+ else {
2581
+ (0, output_1.out)(`installed ${appPath}${moved ? ` on ${moved.to}` : ''}`);
2582
+ }
2516
2583
  return 0;
2517
2584
  }
2518
2585
  // ---------------------------------------------------------------------------
@@ -2961,7 +3028,7 @@ SUITE (run a directory of natural-language tests as one gated suite)
2961
3028
 
2962
3029
  SERVER (expose a locally-connected device to remote verikun clients)
2963
3030
  server [--bind addr] [--port n] [--auth-key k] [--allow-install]
2964
- [--allow-device-control[=names]]
3031
+ [--allow-device-control[=names]] [--allow-failover[=serials]|--no-failover]
2965
3032
  [--allow-unsafe-anonymous] Serve THIS machine's device over HTTP+JSON for
2966
3033
  \`vk ai/suite/install --server <url>\`. Only
2967
3034
  verikun's validated action grammar is runnable
@@ -2979,6 +3046,12 @@ SERVER (expose a locally-connected device to remote verikun clients)
2979
3046
  the flag the server also starts even when no device is attached, so a client can
2980
3047
  boot one: \`vk devices start|stop|restart [name] --server <url>\`, or add
2981
3048
  --ensure-device[=name] to ai/suite/install to boot once before the first step.
3049
+ Failover is ON by default: if the bound device cannot serve a request, the server
3050
+ moves to another attached, healthy, unclaimed one and rules the bad one out until it
3051
+ is power-cycled. An install is retried there; a mid-run step is NOT — it fails on the
3052
+ device it ran on, and the next request lands on the healthy one. Passing --device
3053
+ pins the binding and turns this off; --allow-failover[=serials] turns it back on (and
3054
+ bounds where it may go), --no-failover / VERIKUN_NO_FAILOVER disables it outright.
2982
3055
 
2983
3056
  ENVIRONMENT
2984
3057
  devices [--all] [--json] List attached devices/simulators, and which job is
@@ -0,0 +1,230 @@
1
+ "use strict";
2
+ // Should `vk server` move off the device it is bound to, and which device next?
3
+ //
4
+ // PURE — no fs, no spawn, no timers, no Driver. It classifies strings some driver has
5
+ // already produced and filters a list somebody else enumerated, so the whole matrix is
6
+ // unit-testable with no device. Platform-agnostic by design, like `device/settings.ts`
7
+ // and `device/claims.ts`; the probing and the rebinding live in `server.ts`.
8
+ //
9
+ // THE POLARITY IS THE DESIGN, so read this before touching the tables.
10
+ //
11
+ // `install X onto Y` has exactly two operands, so a failure is about the FILE or about
12
+ // the DEVICE — there is no third thing. The file-attributable set is small, closed and
13
+ // decade-stable (a parser's verdict on a byte sequence is identical on every device).
14
+ // The device-attributable set is open-ended, OEM-specific and unknowable in advance —
15
+ // the failure that prompted this (issue #99) carried no `INSTALL_FAILED_*` code at all,
16
+ // just a raw `java.io.IOException: Requested internal only, but not enough space`.
17
+ //
18
+ // So the ENUMERABLE side is the one we enumerate, and the default falls the other way:
19
+ // an install failure moves unless it matches the artifact denylist. The named
20
+ // device-state strings below are a FAST PATH and documentation, never the gate —
21
+ // deleting one changes the reason text, never the decision. That is what makes this
22
+ // survive phrasings nobody has met yet, and it is the property `tests/failover.test.ts`
23
+ // pins by feeding the classifier pure gibberish and asserting it still moves.
24
+ //
25
+ // `exec`/`elements` keep the OPPOSITE default, and that asymmetry is deliberate rather
26
+ // than sloppy: it is the same "which operand is at fault?" question with a different
27
+ // answer. There the operand is the app under test, and the exit-3 population is
28
+ // dominated by transient device noise (a flaky uiautomator dump, NoWindowError, a
29
+ // keyguard read). Moving on those would rotate the pool on ordinary flake, so that arm
30
+ // moves only on an unreachable device or one a probe confirms is dead.
31
+ Object.defineProperty(exports, "__esModule", { value: true });
32
+ exports.classifyFailure = classifyFailure;
33
+ exports.classifyInstallFailure = classifyInstallFailure;
34
+ exports.failoverCandidates = failoverCandidates;
35
+ const errors_1 = require("../errors");
36
+ // --- the artifact denylist: the ONLY thing that blocks an install failover -----
37
+ //
38
+ // Every entry is a property of the FILE. A parser verdict on a byte sequence is the
39
+ // same on every device, so a second attempt elsewhere is a guaranteed second failure.
40
+ const ARTIFACT_RULES = [
41
+ // One prefix covers all ten INSTALL_PARSE_FAILED_* variants (NOT_APK, BAD_MANIFEST,
42
+ // NO_CERTIFICATES, INCONSISTENT_CERTIFICATES, MANIFEST_MALFORMED, …).
43
+ [/INSTALL_PARSE_FAILED_(\w+)/, 'the APK does not parse'],
44
+ [/INSTALL_FAILED_INVALID_APK/, 'the APK is not a valid package'],
45
+ [/INSTALL_FAILED_INVALID_URI/, 'the install path is not valid'],
46
+ [/INSTALL_FAILED_PACKAGE_CHANGED/, 'the APK changed between staging and commit (corrupt upload)'],
47
+ [/INSTALL_FAILED_TEST_ONLY/, 'the APK is marked test-only (needs `adb install -t`)'],
48
+ // Dexopt CAN OOM, but the dominant cause is a bad build, and a wrong move costs a
49
+ // full install per device and still ends red. Flip it if the field disagrees.
50
+ [/INSTALL_FAILED_DEXOPT/, 'the APK failed dexopt (usually a bad build)'],
51
+ // The server's OWN temp file, not the device's storage — a host bug. Moving would
52
+ // burn the whole pool on a problem no device can fix.
53
+ [/adb: failed to stat|can't find '[^']*\.(?:apk|ipa)'/i, 'the server could not read the uploaded file'],
54
+ ];
55
+ /** Clears by itself; the device is not exhausted. */
56
+ const TRANSIENT_RULES = [[/INSTALL_FAILED_ABORTED/, 'the install session was aborted']];
57
+ // --- fast paths: name the reason and skip the probe. NEVER the gate. ----------
58
+ const UNREACHABLE_RULES = [
59
+ [/device (?:'[^']*' )?not found/i, 'the device is not attached'],
60
+ [/no devices\/emulators found/i, 'no device is attached'],
61
+ [/device offline/i, 'the device is offline'],
62
+ [/device (?:still authorizing|unauthorized)/i, 'the device is not authorized'],
63
+ [/error: closed|protocol fault|failed to get feature set/i, 'the adb connection dropped'],
64
+ [/is not ready \(/, 'the device is not ready'],
65
+ ];
66
+ const DEVICE_STATE_RULES = [
67
+ // Both spellings of a full disk. The second is the one from #99, and it arrives with
68
+ // NO INSTALL_FAILED_* code — which is precisely why the denylist, not this list, is
69
+ // what decides. Keep both: they only shape the message.
70
+ [/INSTALL_FAILED_INSUFFICIENT_STORAGE/, 'the device is out of space'],
71
+ [/Requested internal only, but not enough space|not enough space/i, 'the device is out of space'],
72
+ [/INSTALL_FAILED_UPDATE_INCOMPATIBLE/, 'a differently-signed build of this package is installed on the device'],
73
+ [/INSTALL_FAILED_VERSION_DOWNGRADE/, 'the device holds a newer build of this package'],
74
+ [/INSTALL_FAILED_ALREADY_EXISTS/, 'the package is already installed on the device'],
75
+ [/INSTALL_FAILED_DUPLICATE_PERMISSION/, 'another app on the device declares one of these permissions'],
76
+ [/INSTALL_FAILED_CONFLICTING_PROVIDER/, 'another app on the device owns one of these provider authorities'],
77
+ [/INSTALL_FAILED_UID_CHANGED|INSTALL_FAILED_SHARED_USER_INCOMPATIBLE/, "the existing install's identity on the device conflicts"],
78
+ [/INSTALL_FAILED_USER_RESTRICTED/, 'this device or profile disallows installs'],
79
+ [/INSTALL_FAILED_MEDIA_UNAVAILABLE/, "the device's storage is not mounted"],
80
+ [/INSTALL_FAILED_VERIFICATION_(?:FAILURE|TIMEOUT)/, "the device's package verifier rejected the build"],
81
+ // Device-RELATIVE, so another device genuinely takes it.
82
+ [/INSTALL_FAILED_MISSING_SHARED_LIBRARY/, 'the device lacks a shared library this build needs'],
83
+ [/INSTALL_FAILED_NO_MATCHING_ABIS/, "the build has no native code for this device's ABI"],
84
+ [/INSTALL_FAILED_OLDER_SDK/, "the build needs a newer Android than this device runs"],
85
+ [/INSTALL_FAILED_INTERNAL_ERROR/, "the device's package manager failed internally"],
86
+ ];
87
+ /** adb/idb itself is missing or broken. Matched on the wording `exec.ts` and the tool
88
+ * probes produce — a hint string is the reliable marker, since the detail varies. */
89
+ const TOOLCHAIN_RULES = [
90
+ [/was not found on PATH/i, 'the device toolchain is not installed'],
91
+ [/install the Android platform-tools/i, 'adb is missing or broken'],
92
+ [/brew install idb-companion|xcode-select --install/i, 'the iOS toolchain is missing or broken'],
93
+ ];
94
+ /** First matching rule, or undefined. */
95
+ function firstMatch(message, rules) {
96
+ for (const [pattern, reason] of rules) {
97
+ const m = pattern.exec(message);
98
+ if (m)
99
+ return m[0].startsWith('INSTALL_') ? `${reason} (${m[0]})` : reason;
100
+ }
101
+ return undefined;
102
+ }
103
+ /** The exit code a thrown value carries, or 3 for a non-CliError (matching `run()`). */
104
+ function exitCodeOf(e) {
105
+ return e instanceof errors_1.CliError ? e.exitCode : 3;
106
+ }
107
+ const messageOf = (e) => (e instanceof Error ? e.message : String(e ?? ''));
108
+ /**
109
+ * The arms share everything except what an unrecognised exit-3 means, so they share
110
+ * this and differ only in `fallback`.
111
+ */
112
+ function classify(e, fallback) {
113
+ // Identity first, never message text: NoWindowError is exit 3 and its wording could
114
+ // plausibly be matched by another rule, and getting this one wrong means rotating the
115
+ // pool every time an app is mid-launch.
116
+ if (e instanceof errors_1.NoWindowError) {
117
+ return { move: false, kind: 'transient', reason: 'the app has not drawn yet — this clears on its own' };
118
+ }
119
+ const code = exitCodeOf(e);
120
+ if (code === 0 || code === 1)
121
+ return { move: false, kind: 'app', reason: 'the app failed, not the device' };
122
+ if (code === 2)
123
+ return { move: false, kind: 'usage', reason: 'the request was refused, not the device' };
124
+ const message = messageOf(e);
125
+ const toolchain = firstMatch(message, TOOLCHAIN_RULES);
126
+ if (toolchain)
127
+ return { move: false, kind: 'toolchain', reason: toolchain };
128
+ const transient = firstMatch(message, TRANSIENT_RULES);
129
+ if (transient)
130
+ return { move: false, kind: 'transient', reason: transient };
131
+ const unreachable = firstMatch(message, UNREACHABLE_RULES);
132
+ if (unreachable)
133
+ return { move: true, kind: 'unreachable', reason: unreachable };
134
+ return { ...fallback };
135
+ }
136
+ /** Nothing matched, and the message has no opinion. Ask the device instead of guessing. */
137
+ const UNKNOWN_STAY = {
138
+ move: false,
139
+ kind: 'unknown',
140
+ reason: 'the device may still be fine — asking it',
141
+ probe: true,
142
+ unclassified: true,
143
+ };
144
+ /**
145
+ * The generic arm: `exec`, `elements`, and anything that is not an install.
146
+ *
147
+ * STAY unless the device is provably unreachable, or a probe says so. Accepts `unknown`
148
+ * so a `catch (e)` binding passes straight in, mirroring `isEnvError`.
149
+ */
150
+ function classifyFailure(e) {
151
+ return classify(e, UNKNOWN_STAY);
152
+ }
153
+ /**
154
+ * The install arm: MOVE unless the failure is provably about the FILE.
155
+ *
156
+ * `handleInstall` knows it just ran an install and calls this explicitly — sniffing the
157
+ * message to pick an arm would couple the classifier to a message format that is free
158
+ * to change.
159
+ */
160
+ function classifyInstallFailure(e) {
161
+ const code = exitCodeOf(e);
162
+ // Only an environment failure is ever the device's fault; a usage error (a rejected
163
+ // extension, an unreadable path) is the caller's and no device fixes it.
164
+ if (code === 3 && !(e instanceof errors_1.NoWindowError)) {
165
+ const message = messageOf(e);
166
+ // Same order as `classify` below, so the two arms can only ever differ in their
167
+ // DEFAULT — which is the one difference between them that is meant to exist.
168
+ const artifact = firstMatch(message, ARTIFACT_RULES);
169
+ if (artifact)
170
+ return { move: false, kind: 'artifact', reason: artifact };
171
+ const toolchain = firstMatch(message, TOOLCHAIN_RULES);
172
+ if (toolchain)
173
+ return { move: false, kind: 'toolchain', reason: toolchain };
174
+ const transient = firstMatch(message, TRANSIENT_RULES);
175
+ if (transient)
176
+ return { move: false, kind: 'transient', reason: transient };
177
+ const unreachable = firstMatch(message, UNREACHABLE_RULES);
178
+ if (unreachable)
179
+ return { move: true, kind: 'unreachable', reason: unreachable };
180
+ const named = firstMatch(message, DEVICE_STATE_RULES);
181
+ if (named)
182
+ return { move: true, kind: 'device-state', reason: named };
183
+ // The inversion. Not in the denylist ⇒ it is about the device, even though we have
184
+ // never seen this wording. Bounded by MAX_FAILOVER_HOPS, and the caller reports the
185
+ // FIRST device's error on exhaustion, so a wrong guess costs time, not diagnosis.
186
+ return { move: true, kind: 'device-state', reason: 'the device could not install this build', unclassified: true };
187
+ }
188
+ return classify(e, UNKNOWN_STAY);
189
+ }
190
+ /**
191
+ * Which attached devices could take over, in preference order.
192
+ *
193
+ * Listing order is preserved for the same reason `selectAndClaim` documents it: a device
194
+ * that worked last time is the one most likeliest to work now, and round-robining would
195
+ * spread a flaky run across the whole pool.
196
+ *
197
+ * Claim status is deliberately NOT a filter — deciding "this one is free" and then
198
+ * claiming it is the exact read-then-write race `device/claims.ts` exists to prevent.
199
+ * The caller claims as it walks.
200
+ */
201
+ function failoverCandidates(devices, opts) {
202
+ const excluded = new Set(opts.exclude.filter(Boolean));
203
+ const allow = opts.allow ?? [];
204
+ return devices.filter((d) => {
205
+ // An unbooted AVD is { serial: '', state: 'shutdown' } and has no adb address.
206
+ // Failover is lateral, never upward: booting is `vk devices start`'s job.
207
+ if (!d.serial)
208
+ return false;
209
+ if (!isUsableState(d.state))
210
+ return false;
211
+ if (excluded.has(d.serial))
212
+ return false;
213
+ // Name OR serial: an operator writes `Pixel_6_API_34` for an AVD and a raw serial
214
+ // for a phone, and being forced to know which is which is a papercut with teeth.
215
+ if (allow.length && !allow.includes(d.serial) && !(d.name && allow.includes(d.name)))
216
+ return false;
217
+ return true;
218
+ });
219
+ }
220
+ /**
221
+ * Is this device drivable right now? The two platforms spell it differently.
222
+ *
223
+ * Deliberately DUPLICATED from `isRunning` in drivers/lifecycle.ts rather than imported:
224
+ * that module imports adb.ts and ios.ts, so importing it here would drag both platform
225
+ * backends into a module whose whole value is being pure. One line of duplication is the
226
+ * cheaper side of that trade.
227
+ */
228
+ function isUsableState(state) {
229
+ return state === 'device' || state === 'booted';
230
+ }
package/dist/server.js CHANGED
@@ -34,6 +34,7 @@
34
34
  Object.defineProperty(exports, "__esModule", { value: true });
35
35
  exports.buildServer = buildServer;
36
36
  exports.parseDeviceControl = parseDeviceControl;
37
+ exports.parseFailover = parseFailover;
37
38
  exports.cmdServer = cmdServer;
38
39
  const node_http_1 = require("node:http");
39
40
  const node_crypto_1 = require("node:crypto");
@@ -47,6 +48,7 @@ const errors_1 = require("./errors");
47
48
  const drivers_1 = require("./drivers");
48
49
  const manager_1 = require("./companion/manager");
49
50
  const claims_1 = require("./device/claims");
51
+ const failover_1 = require("./device/failover");
50
52
  const lifecycle_1 = require("./drivers/lifecycle");
51
53
  const output_1 = require("./output");
52
54
  const ir_1 = require("./agent/ir");
@@ -76,6 +78,15 @@ const INSTALL_BODY_CAP = 512 * 1024 * 1024; // 512 MB app build
76
78
  // to survive a client-side compile/repair pause, short enough that a crashed
77
79
  // caller doesn't wedge the device.
78
80
  const LOCK_IDLE_MS = 5 * 60 * 1000;
81
+ // How many times ONE request may move device. 2 moves = 3 devices tried, which sits
82
+ // comfortably inside the client's 15-minute install ceiling at ~1 minute an install,
83
+ // while a farm of ten wedged emulators cannot burn ten installs inside one request.
84
+ const MAX_FAILOVER_HOPS = 2;
85
+ // Gap between the two probes that separate a momentary blip from a dead device. Mirrors
86
+ // suite.ts's stillBroken, and for the same reason: a flaky dump also surfaces as exit 3,
87
+ // so acting on one probe would rotate the pool on ordinary flake.
88
+ const PROBE_RETRY_MS = 1000;
89
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
79
90
  // Deliberately below the client's 5-minute ceiling, so a slow boot is reported by the
80
91
  // side that knows WHY ("did not finish booting within 240s") rather than as a generic
81
92
  // client-side abort.
@@ -84,13 +95,20 @@ const SERVER_BOOT_TIMEOUT_MS = 4 * 60 * 1000;
84
95
  class HttpError extends Error {
85
96
  status;
86
97
  exitCode;
87
- constructor(status, message, exitCode = status === 400 || status === 404 || status === 413 ? 2 : 3) {
98
+ deviceChanged;
99
+ constructor(status, message, exitCode = status === 400 || status === 404 || status === 413 ? 2 : 3,
100
+ /** Set when this request moved the server's device before failing — the client
101
+ * needs to know the ground shifted even though the answer is an error. */
102
+ deviceChanged) {
88
103
  super(message);
89
104
  this.status = status;
90
105
  this.exitCode = exitCode;
106
+ this.deviceChanged = deviceChanged;
91
107
  this.name = 'HttpError';
92
108
  }
93
109
  }
110
+ /** Errors here are multi-line (detail + hint); logs and reasons want the headline. */
111
+ const firstLine = (m) => m.split('\n')[0].trim();
94
112
  function sendJson(res, status, payload) {
95
113
  const body = JSON.stringify(payload);
96
114
  res.writeHead(status, { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) });
@@ -181,6 +199,8 @@ function buildServer(config) {
181
199
  const sha = (s) => (0, node_crypto_1.createHash)('sha256').update(s, 'utf8').digest();
182
200
  const lifecycle = config.lifecycle ?? realLifecycle;
183
201
  const makeDriver = config.makeDriver ?? drivers_1.getDriver;
202
+ const claimOpts = config.claimOpts ?? {};
203
+ const claimEnv = claimOpts.env ?? process.env;
184
204
  // The ONE piece of server state a request can change, and only via /v1/devices/*.
185
205
  // Everything in `config` is startup policy and is never written. Grep `bound =`
186
206
  // to find every rebind.
@@ -190,11 +210,170 @@ function buildServer(config) {
190
210
  // on a different port would leave a permanently dead instance. Always rebind to the
191
211
  // CONCRETE serial the lifecycle layer returned — never undefined, which would
192
212
  // auto-resolve and could silently latch onto a different attached device.
193
- const rebind = (serial) => {
194
- bound = serial === null ? { driver: bound.driver, serial: null } : { driver: makeDriver(config.platform, serial), serial };
213
+ //
214
+ // `driver` is passed when the caller has ALREADY built and probed one (failover), so
215
+ // the instance that answered the probe is the instance we go on to use.
216
+ const rebind = (serial, driver) => {
217
+ bound =
218
+ serial === null
219
+ ? { driver: bound.driver, serial: null }
220
+ : { driver: driver ?? makeDriver(config.platform, serial), serial };
195
221
  config.onRebind?.(serial);
196
222
  (0, output_1.err)(`[server] device: ${config.platform} · ${serial ?? '(none)'}`);
197
223
  };
224
+ // --- failover ---------------------------------------------------------------
225
+ //
226
+ // Devices this server has ruled out, and why. In-memory, PROCESS-LIFETIME, no TTL: a
227
+ // TTL would silently re-try a device that ran out of disk ten minutes ago and burn
228
+ // another full install on it, on a schedule nobody can see — precisely the minutes
229
+ // this feature exists to save. A power cycle is the fix, so a successful
230
+ // /v1/devices/{start,restart,stop} is what clears an entry (see handleDeviceOp).
231
+ const quarantine = new Map();
232
+ const quarantineDevice = (serial, reason) => {
233
+ if (!serial || quarantine.has(serial))
234
+ return;
235
+ quarantine.set(serial, { reason, at: Date.now() });
236
+ (0, output_1.err)(`[server] failover: ${serial} quarantined (${reason})`);
237
+ };
238
+ /** For /v1/health and the exhaustion message. */
239
+ const quarantineList = () => [...quarantine.entries()].map(([serial, q]) => ({ serial, reason: q.reason }));
240
+ /**
241
+ * Move to another healthy device. Returns the serial moved to, or null when none
242
+ * remains (which is not an error here — the caller reports the ORIGINAL failure).
243
+ *
244
+ * The walk order is load-bearing: claim-new -> probe -> commit -> release-old.
245
+ * Releasing the old claim first would leave this server bound to a device it no longer
246
+ * holds, and another job on the host would take it mid-request.
247
+ */
248
+ const pickFailoverDevice = () => {
249
+ const policy = config.failover;
250
+ if (!policy)
251
+ return null;
252
+ const from = bound.serial;
253
+ // lifecycle.list is the SAME source /v1/devices answers from, so what a client can
254
+ // see and where the server will actually go cannot drift. bound.driver is not: it
255
+ // may be pointed at a corpse.
256
+ let seen = [];
257
+ try {
258
+ seen = lifecycle.list(config.platform);
259
+ }
260
+ catch (e) {
261
+ (0, output_1.err)(`[server] failover: cannot enumerate devices (${firstLine(e.message)})`);
262
+ return null;
263
+ }
264
+ const candidates = (0, failover_1.failoverCandidates)(seen, {
265
+ exclude: [...(from ? [from] : []), ...quarantine.keys()],
266
+ allow: policy.allowedTargets,
267
+ });
268
+ if (!candidates.length)
269
+ return null;
270
+ (0, output_1.err)(`[server] failover: ${candidates.length} candidate(s) — ${candidates.map((d) => d.serial).join(', ')}`);
271
+ for (const c of candidates) {
272
+ // Claim BEFORE probing: deciding "this one is free" and then taking it is the
273
+ // read-then-write race device/claims.ts exists to prevent.
274
+ if ((0, claims_1.claimsEnabled)(claimEnv) && !(0, claims_1.claimDevice)(c.serial, config.platform, claimOpts).ok) {
275
+ (0, output_1.err)(`[server] failover: ${c.serial} is held by another job — skipping`);
276
+ continue;
277
+ }
278
+ const driver = makeDriver(config.platform, c.serial);
279
+ try {
280
+ // ONE probe per candidate, against TWO for the bound device on the unknown path.
281
+ // Deliberate: a false negative here just moves to the next candidate, while a
282
+ // false positive there quarantines a device that was fine.
283
+ driver.preflight();
284
+ }
285
+ catch (e) {
286
+ if ((0, claims_1.claimsEnabled)(claimEnv))
287
+ (0, claims_1.releaseClaim)(c.serial, { ...claimOpts, mineOnly: true });
288
+ quarantineDevice(c.serial, `probe failed (${firstLine(e.message)})`);
289
+ continue;
290
+ }
291
+ (0, output_1.err)(`[server] failover: ${c.serial} probe ok — moving`);
292
+ if (from) {
293
+ // Hand back the old device's ONE UiAutomation connection, or it stays held for
294
+ // up to 15 minutes with nothing on the host able to explain why. Never throws.
295
+ (0, manager_1.releaseCompanionOn)(from);
296
+ if ((0, claims_1.claimsEnabled)(claimEnv))
297
+ (0, claims_1.releaseClaim)(from, { ...claimOpts, mineOnly: true });
298
+ }
299
+ rebind(c.serial, driver);
300
+ return c.serial;
301
+ }
302
+ return null;
303
+ };
304
+ /** Why no move happened, in a form worth putting in front of an operator. */
305
+ const exhaustedNote = () => {
306
+ const rows = quarantineList().map((q) => ` ${q.serial} ${q.reason}`);
307
+ return (`\n[failover] no working device remains${rows.length ? `; ruled out:\n${rows.join('\n')}` : ''}` +
308
+ '\n[failover] clear one with `vk devices restart <name> --server <url>`, or fix it and restart the server');
309
+ };
310
+ /** Announce an unrecognised move, so real-world strings reach a CI log and can be
311
+ * promoted into device/failover.ts's tables deliberately rather than guessed at. */
312
+ const noteVerdict = (v, e, what) => {
313
+ if (v.unclassified && v.move) {
314
+ (0, output_1.err)(`[server] failover: unclassified ${what} failure, treating as device-attributable — ${firstLine(e.message)}`);
315
+ }
316
+ };
317
+ /**
318
+ * Is the bound device actually gone? Two probes a second apart, because that gap is the
319
+ * only thing separating a USB re-enumeration or a mid-`launch --clear` gap from a dead
320
+ * box — and quarantining a healthy device is the expensive mistake here. Returns the
321
+ * reason when dead, undefined when it was a blip.
322
+ */
323
+ const boundDeviceIsDead = async () => {
324
+ let last = '';
325
+ for (let attempt = 0; attempt < 2; attempt++) {
326
+ if (attempt > 0)
327
+ await sleep(PROBE_RETRY_MS);
328
+ try {
329
+ bound.driver.preflight();
330
+ return undefined;
331
+ }
332
+ catch (e) {
333
+ last = firstLine(e.message);
334
+ }
335
+ }
336
+ return last || 'the device stopped answering';
337
+ };
338
+ /**
339
+ * A non-install operation failed. Move off the device if it is genuinely at fault —
340
+ * but NEVER replay the operation there.
341
+ *
342
+ * That restraint is the whole point. A `vk ai` step twelve deep presupposes the eleven
343
+ * before it ran on THIS device; device B's app is at whatever an earlier run left
344
+ * behind. Replaying would either find something matching and go green (a false green
345
+ * that ships a regression) or wake the repair model against the wrong screen. So the
346
+ * failing operation still fails, honestly, with the ORIGINAL device's error — and it is
347
+ * the NEXT request that benefits from the move.
348
+ *
349
+ * Returns the change to report, or undefined when we stayed put.
350
+ */
351
+ const considerFailover = async (e, what) => {
352
+ if (!config.failover)
353
+ return undefined;
354
+ const verdict = (0, failover_1.classifyFailure)(e);
355
+ let reason = verdict.reason;
356
+ if (!verdict.move) {
357
+ // Only an unrecognised exit 3 earns a probe; `transient` and `toolchain` set
358
+ // probe:false precisely so a mid-launch gap or a missing adb cannot become a move.
359
+ if (!verdict.probe)
360
+ return undefined;
361
+ const dead = await boundDeviceIsDead();
362
+ if (!dead)
363
+ return undefined; // a blip — the test rerun is the right answer, not a new device
364
+ reason = dead;
365
+ noteVerdict({ ...verdict, move: true }, e, what);
366
+ }
367
+ (0, output_1.err)(`[server] ${what}: FAILED on ${bound.serial ?? '(none)'} — ${reason}`);
368
+ const from = bound.serial ?? '(none)';
369
+ quarantineDevice(bound.serial, reason);
370
+ const to = pickFailoverDevice();
371
+ if (!to) {
372
+ (0, output_1.err)(`[server] failover: nothing healthier to move to — staying on ${from}`);
373
+ return undefined;
374
+ }
375
+ return { from, to, reason, retried: false };
376
+ };
198
377
  const authorized = (req) => {
199
378
  if (!config.authKey)
200
379
  return true; // --allow-unsafe-anonymous
@@ -249,9 +428,13 @@ function buildServer(config) {
249
428
  const t0 = Date.now();
250
429
  const { code, error, step, artifacts, logStart } = await (0, cli_1.executeForServer)(node.command, node.positionals, (0, ir_1.leafToFlags)(node), bound.driver, config.platform);
251
430
  (0, output_1.err)(`[server] exec ${node.command} ${node.positionals.join(' ')} → exit ${code} (${Date.now() - t0}ms)`);
431
+ // The step keeps its own verdict whatever we decide here: the error below is the one
432
+ // THIS device produced, never a replay's. Only the binding moves.
433
+ const deviceChanged = code !== 0 && error ? await considerFailover(error, 'exec') : undefined;
252
434
  const payload = {
253
435
  code,
254
436
  ...(error ? { error: (0, rpc_1.describeError)(error) } : {}),
437
+ ...(deviceChanged ? { deviceChanged } : {}),
255
438
  ...(step ? { step } : {}),
256
439
  ...(artifacts && Object.keys(artifacts).length ? { artifacts: encodeArtifacts(artifacts) } : {}),
257
440
  ...(logStart ? { logStart } : {}),
@@ -260,8 +443,20 @@ function buildServer(config) {
260
443
  }
261
444
  async function handleElements(req, res) {
262
445
  await readBody(req, EXEC_BODY_CAP); // drain (the body is unused; keeps keep-alive sane)
263
- const elements = bound.driver.getElements(); // CliError(3) on dump failure → 500 below
264
- sendJson(res, 200, { elements });
446
+ try {
447
+ const elements = bound.driver.getElements(); // CliError(3) on dump failure → 500 below
448
+ sendJson(res, 200, { elements });
449
+ }
450
+ catch (e) {
451
+ // Move if the device is at fault, but NEVER answer with the new device's screen:
452
+ // this is the engine's `if-present` guard input and its repair context, and a
453
+ // hierarchy from somewhere else is worse than an error. The client's connect probe
454
+ // re-asks after a reported move — see remote.ts's preflight.
455
+ const deviceChanged = await considerFailover(e, 'read');
456
+ if (!deviceChanged)
457
+ throw e;
458
+ throw new HttpError(500, e.message, e instanceof errors_1.CliError ? e.exitCode : 3, deviceChanged);
459
+ }
265
460
  }
266
461
  async function handleLogs(req, res) {
267
462
  const body = await readBody(req, EXEC_BODY_CAP);
@@ -293,7 +488,10 @@ function buildServer(config) {
293
488
  : (() => {
294
489
  throw new HttpError(400, `invalid appId '${String(parsed.appId)}'`);
295
490
  })();
296
- const logs = config.driver.getLogs({
491
+ // The BOUND driver, never config.driver: after a rebind (device control, or a
492
+ // failover) the startup driver is pinned to a serial that may be gone, and logs
493
+ // are evidence about the run that just failed — serving another device's is a lie.
494
+ const logs = bound.driver.getLogs({
297
495
  ...(lines !== undefined ? { lines } : {}),
298
496
  ...(parsed.since ? { since: parsed.since } : {}),
299
497
  ...(appId ? { appId } : {}),
@@ -302,6 +500,49 @@ function buildServer(config) {
302
500
  const payload = { logs };
303
501
  sendJson(res, 200, payload);
304
502
  }
503
+ /**
504
+ * Install, moving to another device when THIS one is at fault.
505
+ *
506
+ * Install is the one operation safe to REPLAY elsewhere: it is idempotent, carries no
507
+ * app session, and the uploaded bytes are still on the server's disk, so a retry costs
508
+ * one more `adb install` and no re-upload. Every other endpoint rebinds without
509
+ * replaying — see handleExec.
510
+ *
511
+ * On exhaustion it throws the FIRST device's error, never the last. That inversion is
512
+ * what makes move-by-default safe: a wrong move costs time, not the diagnosis.
513
+ */
514
+ function installWithFailover(tmpPath) {
515
+ let change;
516
+ let moves = 0;
517
+ let firstError;
518
+ for (let hop = 0;; hop++) {
519
+ const from = bound.serial ?? '(none)';
520
+ try {
521
+ bound.driver.install(tmpPath);
522
+ return { change, moves };
523
+ }
524
+ catch (e) {
525
+ if (firstError === undefined)
526
+ firstError = e;
527
+ const verdict = (0, failover_1.classifyInstallFailure)(e);
528
+ (0, output_1.err)(`[server] install: FAILED on ${from} — ${verdict.reason}`);
529
+ noteVerdict(verdict, e, 'install');
530
+ // The artifact is broken / the caller is wrong / failover is off / we are out of
531
+ // hops: report the first failure unchanged, exactly as before this feature.
532
+ if (!verdict.move || !config.failover || hop >= MAX_FAILOVER_HOPS)
533
+ throw firstError;
534
+ quarantineDevice(bound.serial, verdict.reason);
535
+ const to = pickFailoverDevice();
536
+ if (!to) {
537
+ const original = firstError instanceof Error ? firstError.message : String(firstError);
538
+ throw new HttpError(500, `${original}${exhaustedNote()}`, 3, change);
539
+ }
540
+ change = { from, to, reason: verdict.reason, retried: true };
541
+ moves++;
542
+ (0, output_1.err)(`[server] install: retrying on ${to}…`);
543
+ }
544
+ }
545
+ }
305
546
  async function handleInstall(req, res) {
306
547
  const ext = String(req.headers['x-verikun-ext'] ?? '').toLowerCase();
307
548
  if (ext !== 'apk' && ext !== 'ipa') {
@@ -333,9 +574,10 @@ function buildServer(config) {
333
574
  throw new HttpError(400, `sha256 mismatch: upload arrived corrupted (got ${digest.slice(0, 12)}…, expected ${expected.slice(0, 12)}…)`);
334
575
  }
335
576
  (0, output_1.err)(`[server] install: received ${size} bytes (.${ext}), installing…`);
336
- bound.driver.install(tmpPath);
337
- (0, output_1.err)('[server] install: done');
338
- sendJson(res, 200, { ok: true, bytes: size, sha256: digest });
577
+ const { change, moves } = installWithFailover(tmpPath);
578
+ (0, output_1.err)(`[server] install: done${change ? ` on ${change.to} (after ${moves} move${moves === 1 ? '' : 's'})` : ''}`);
579
+ const body = { ok: true, bytes: size, sha256: digest, ...(change ? { deviceChanged: change } : {}) };
580
+ sendJson(res, 200, body);
339
581
  }
340
582
  finally {
341
583
  try {
@@ -425,6 +667,12 @@ function buildServer(config) {
425
667
  rebind(serial);
426
668
  result = { ok: true, platform: config.platform, serial, changed: started, durationMs: Date.now() - t0 };
427
669
  }
670
+ // A power cycle IS the fix for a quarantined device, and performing one is the
671
+ // assertion that it worked. Clear by both keys: a client names an AVD, the
672
+ // lifecycle layer answers with a serial.
673
+ for (const key of [result.serial, target])
674
+ if (key)
675
+ quarantine.delete(key);
428
676
  (0, output_1.err)(`[server] device ${op} → ${result.serial ?? '(none)'} (${result.durationMs}ms, changed=${result.changed})`);
429
677
  sendJson(res, 200, result);
430
678
  }
@@ -436,7 +684,7 @@ function buildServer(config) {
436
684
  if (config.authKey && req.headers.authorization && !authorized(req)) {
437
685
  throw new HttpError(401, 'invalid auth key');
438
686
  }
439
- const reads = safeHierarchySource(config.driver);
687
+ const reads = bound.serial === null ? null : safeHierarchySource(bound.driver);
440
688
  const health = {
441
689
  ok: true,
442
690
  version: version_1.VERSION,
@@ -446,6 +694,11 @@ function buildServer(config) {
446
694
  ...(reads ? { reads } : {}),
447
695
  deviceControlEnabled: config.deviceControl !== undefined,
448
696
  deviceNamingEnabled: (config.deviceControl?.allowedTargets.length ?? 0) > 0,
697
+ failoverEnabled: config.failover !== undefined,
698
+ // Omitted when empty, so a CI job can assert on its ABSENCE. Unauthenticated
699
+ // like the rest of health, which is what makes "is the pool ok?" answerable
700
+ // without holding a run token.
701
+ ...(quarantine.size ? { quarantined: quarantineList() } : {}),
449
702
  // Derived from `serial` right here, so the two can never drift apart.
450
703
  deviceState: bound.serial === null ? 'none' : 'ready',
451
704
  };
@@ -498,13 +751,20 @@ function buildServer(config) {
498
751
  // Who is driving what, so a client can see "is it free" before committing to a run
499
752
  // rather than discovering it as a 409 mid-suite. Read-only, exactly like the local
500
753
  // listing — asking must never take a claim.
501
- if ((0, claims_1.claimsEnabled)()) {
754
+ if ((0, claims_1.claimsEnabled)(claimEnv)) {
502
755
  for (const d of seen) {
503
- const claim = (0, claims_1.summarize)(d.serial);
756
+ const claim = (0, claims_1.summarize)(d.serial, claimOpts);
504
757
  if (claim)
505
758
  d.claim = claim;
506
759
  }
507
760
  }
761
+ // `note` is the existing optional-caveat column formatDeviceTable already renders,
762
+ // so `vk devices --server` shows this with no wire change.
763
+ for (const d of seen) {
764
+ const q = quarantine.get(d.serial);
765
+ if (q)
766
+ d.note = `quarantined: ${q.reason}`;
767
+ }
508
768
  const body = {
509
769
  devices: policy.allowedTargets.length
510
770
  ? seen.filter((d) => d.serial === bound.serial || policy.allowedTargets.includes(d.name ?? ''))
@@ -549,7 +809,11 @@ function buildServer(config) {
549
809
  ? new HttpError(e.exitCode === 2 ? 400 : 500, e.message, e.exitCode)
550
810
  : new HttpError(500, e.message || 'internal error', 3);
551
811
  if (!res.headersSent) {
552
- const body = { error: mapped.message, exitCode: mapped.exitCode };
812
+ const body = {
813
+ error: mapped.message,
814
+ exitCode: mapped.exitCode,
815
+ ...(mapped.deviceChanged ? { deviceChanged: mapped.deviceChanged } : {}),
816
+ };
553
817
  sendJson(res, mapped.status, body);
554
818
  }
555
819
  else {
@@ -587,9 +851,59 @@ function parseDeviceControl(flags) {
587
851
  }
588
852
  return { allowedTargets: names };
589
853
  }
854
+ /**
855
+ * Decide whether this server may move off a device that fails. PURE — exported for unit
856
+ * tests, and it takes `pinned`/`env` explicitly rather than reading `process.env` so the
857
+ * whole truth table is assertable.
858
+ *
859
+ * Precedence, and each step earns its place:
860
+ * 1. An explicit OFF wins over everything — a kill switch you can override is not one.
861
+ * `VERIKUN_NO_FAILOVER` mirrors `VERIKUN_NO_CLAIM`: host-level policy for an operator
862
+ * who cannot change every command line. It is announced at startup, so it can never
863
+ * silently explain a server that "won't fail over".
864
+ * 2. `--allow-failover[=names]` turns it on, and OVERRIDES a `--device` pin — two flags
865
+ * that appear to disagree are resolved by the later, more specific one, loudly.
866
+ * 3. A `--device` pin turns it off. The operator named the device; honour that.
867
+ * 4. Otherwise ON, unbounded. See FailoverPolicy for why that is the honest default.
868
+ *
869
+ * `flagBool` is deliberately NOT used for `allow-failover`, for the same reason as
870
+ * `parseDeviceControl`: it returns FALSE for `--allow-failover=emulator-5556`, silently
871
+ * disabling the feature for the exact spelling that bounds it.
872
+ */
873
+ function parseFailover(flags, opts = {}) {
874
+ const raw = flags['allow-failover'];
875
+ const asked = raw !== undefined && raw !== false;
876
+ const refused = (0, args_1.flagBool)(flags, 'no-failover');
877
+ if (asked && refused) {
878
+ throw new errors_1.CliError('--allow-failover and --no-failover contradict each other — pass one.', 2);
879
+ }
880
+ if (refused)
881
+ return { why: 'disabled (--no-failover)' };
882
+ if ((opts.env ?? process.env).VERIKUN_NO_FAILOVER) {
883
+ return { why: 'disabled (VERIKUN_NO_FAILOVER)' };
884
+ }
885
+ if (asked) {
886
+ if (raw === true || raw === 'true') {
887
+ return { policy: { allowedTargets: [] }, why: 'ENABLED · any attached device on this host (--allow-failover)' };
888
+ }
889
+ const names = String(raw)
890
+ .split(',')
891
+ .map((t) => t.trim())
892
+ .filter(Boolean);
893
+ if (!names.length) {
894
+ throw new errors_1.CliError('--allow-failover=<serials> needs a comma-separated list of device serials or AVD/simulator names ' +
895
+ '(or pass a bare --allow-failover to permit any attached device).', 2);
896
+ }
897
+ return { policy: { allowedTargets: names }, why: `ENABLED · may move to: ${names.join(', ')}` };
898
+ }
899
+ if (opts.pinned) {
900
+ return { why: 'disabled (--device pins the binding; pass --allow-failover to permit moving)' };
901
+ }
902
+ return { policy: { allowedTargets: [] }, why: 'ENABLED · any attached device on this host' };
903
+ }
590
904
  async function cmdServer(positionals, flags) {
591
905
  if (positionals.length > 0) {
592
- throw new errors_1.CliError(`server: unexpected argument '${positionals[0]}'. Usage: verikun server [--bind addr] [--port n] [--auth-key k] [--allow-install] [--allow-device-control[=names]] [--allow-unsafe-anonymous]`, 2);
906
+ throw new errors_1.CliError(`server: unexpected argument '${positionals[0]}'. Usage: verikun server [--bind addr] [--port n] [--auth-key k] [--allow-install] [--allow-device-control[=names]] [--allow-failover[=serials]|--no-failover] [--allow-unsafe-anonymous]`, 2);
593
907
  }
594
908
  const platform = (0, cli_1.platformFromFlags)(flags);
595
909
  const device = (0, cli_1.deviceFromFlags)(flags, platform);
@@ -597,6 +911,8 @@ async function cmdServer(positionals, flags) {
597
911
  const port = (0, args_1.flagNum)(flags, 'port') ?? DEFAULT_PORT;
598
912
  const allowInstall = (0, args_1.flagBool)(flags, 'allow-install');
599
913
  const deviceControl = parseDeviceControl(flags);
914
+ // `device` is --device || VERIKUN_DEVICE || ANDROID_SERIAL: an env pin is still a pin.
915
+ const failover = parseFailover(flags, { pinned: device !== undefined });
600
916
  const anonymous = (0, args_1.flagBool)(flags, 'allow-unsafe-anonymous');
601
917
  // The env var is the documented channel for the key (keeps it out of argv/ps).
602
918
  let authKey = (0, args_1.flagStr)(flags, 'auth-key') || process.env.VERIKUN_SERVER_AUTH_KEY || undefined;
@@ -663,7 +979,7 @@ async function cmdServer(positionals, flags) {
663
979
  // shutdown has to release the companion on whatever is bound then.
664
980
  let boundSerial = serial;
665
981
  const server = buildServer({
666
- driver, platform, serial, authKey, allowInstall, deviceControl,
982
+ driver, platform, serial, authKey, allowInstall, deviceControl, failover: failover.policy,
667
983
  onRebind: (s) => { boundSerial = s; },
668
984
  });
669
985
  return new Promise((resolve, reject) => {
@@ -685,6 +1001,16 @@ async function cmdServer(positionals, flags) {
685
1001
  if (deviceControl) {
686
1002
  (0, output_1.err)('[server] NOTE: an authenticated client can now power-cycle AND erase this device.');
687
1003
  }
1004
+ (0, output_1.err)(`[server] failover: ${failover.why}`);
1005
+ // Two flags that appear to disagree. Permitted rather than refused — `--device X`
1006
+ // alongside the other --allow-* flags is straight out of the docs, so refusing
1007
+ // would break the commonest shape — but never silently: a bare --allow-failover
1008
+ // means the pin governs only the INITIAL binding.
1009
+ if (device && failover.policy && failover.policy.allowedTargets.length === 0) {
1010
+ (0, output_1.err)(`[server] WARNING: --device ${device} pins only the INITIAL binding — a bare --allow-failover`);
1011
+ (0, output_1.err)('[server] permits moving to any other attached device on this host. Pass');
1012
+ (0, output_1.err)('[server] --allow-failover=<serials> to bound where it may go.');
1013
+ }
688
1014
  if (generated) {
689
1015
  (0, output_1.err)('[server] auth key generated for this session — clients pass it via VERIKUN_SERVER_AUTH_KEY or --auth-key:');
690
1016
  (0, output_1.err)(`[server] ${authKey}`);
package/dist/version.js CHANGED
@@ -3,4 +3,4 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.VERSION = void 0;
4
4
  // GENERATED by scripts/gen-version.mjs from package.json's "version" at build time
5
5
  // (the `prebuild` script). Do NOT edit by hand; bump package.json instead.
6
- exports.VERSION = '0.24.0';
6
+ exports.VERSION = '0.25.0';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "verikun",
3
- "version": "0.24.0",
3
+ "version": "0.25.0",
4
4
  "description": "Drive Android emulators/devices and iOS simulators for AI agents: tap, type, swipe, screenshot, and inspect the UI hierarchy by semantic identifiers — like Puppeteer for native apps.",
5
5
  "keywords": [
6
6
  "android",