verikun 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +435 -0
- package/dist/agent/cache.js +128 -0
- package/dist/agent/claude.js +144 -0
- package/dist/agent/cost.js +100 -0
- package/dist/agent/engine.js +205 -0
- package/dist/agent/grammar.js +80 -0
- package/dist/agent/ir.js +212 -0
- package/dist/agent/provider.js +2 -0
- package/dist/args.js +102 -0
- package/dist/bin/verikun.js +8 -0
- package/dist/cli.js +1298 -0
- package/dist/drivers/adb.js +300 -0
- package/dist/drivers/index.js +13 -0
- package/dist/drivers/simctl.js +156 -0
- package/dist/errors.js +51 -0
- package/dist/exec.js +42 -0
- package/dist/image.js +212 -0
- package/dist/output.js +43 -0
- package/dist/report.js +223 -0
- package/dist/run.js +434 -0
- package/dist/types.js +5 -0
- package/dist/ui/android-parse.js +149 -0
- package/dist/ui/format.js +71 -0
- package/dist/ui/selector.js +117 -0
- package/dist/version.js +6 -0
- package/package.json +53 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ClaudeProvider = void 0;
|
|
4
|
+
const errors_1 = require("../errors");
|
|
5
|
+
const format_1 = require("../ui/format");
|
|
6
|
+
const ir_1 = require("./ir");
|
|
7
|
+
const grammar_1 = require("./grammar");
|
|
8
|
+
// The one v1 provider: Anthropic's Messages API over Node's built-in fetch — no SDK,
|
|
9
|
+
// honoring the repo's zero-runtime-dependency rule. Structured output guarantees a
|
|
10
|
+
// schema-valid plan; the stable grammar prefix is cache_control'd; 429/5xx retry with
|
|
11
|
+
// backoff (no SDK to do it for us). The LLM runs here ONLY on compile + repair.
|
|
12
|
+
const API_URL = 'https://api.anthropic.com/v1/messages';
|
|
13
|
+
const ANTHROPIC_VERSION = '2023-06-01';
|
|
14
|
+
// Per-request wall-clock cap so a stalled connection can't hang the run past its
|
|
15
|
+
// --timeout deadline (the engine checks the deadline only between calls, not during one).
|
|
16
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 120_000;
|
|
17
|
+
// effort (output_config.effort) is rejected by Haiku 4.5; only send it for models
|
|
18
|
+
// that accept it.
|
|
19
|
+
const EFFORT_MODELS = new Set(['claude-opus-4-8', 'claude-sonnet-4-6', 'claude-fable-5']);
|
|
20
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
21
|
+
const backoffMs = (attempt) => Math.min(1000 * 2 ** (attempt - 1), 15000);
|
|
22
|
+
class ClaudeProvider {
|
|
23
|
+
opts;
|
|
24
|
+
constructor(opts) {
|
|
25
|
+
this.opts = opts;
|
|
26
|
+
}
|
|
27
|
+
async compile(input) {
|
|
28
|
+
const parts = [];
|
|
29
|
+
if (input.pkg)
|
|
30
|
+
parts.push(`App package: ${input.pkg}`);
|
|
31
|
+
parts.push(`Platform: ${input.platform}`);
|
|
32
|
+
if (input.seed) {
|
|
33
|
+
parts.push('A plan compiled for a PREVIOUS build of this app follows. Reuse it where the test still holds; ' +
|
|
34
|
+
'change only what the test now requires. PRIOR PLAN:\n' +
|
|
35
|
+
JSON.stringify(input.seed, null, 2));
|
|
36
|
+
}
|
|
37
|
+
parts.push('NATURAL-LANGUAGE TEST:\n' + input.nl);
|
|
38
|
+
const { json, usage } = await this.call(grammar_1.GRAMMAR, parts.join('\n\n'), ir_1.PLAN_JSON_SCHEMA, 8192);
|
|
39
|
+
return { plan: (0, ir_1.parsePlan)(json), usage };
|
|
40
|
+
}
|
|
41
|
+
async repair(ctx) {
|
|
42
|
+
const parts = [
|
|
43
|
+
'FAILED STEP: ' + JSON.stringify(ctx.failedStep),
|
|
44
|
+
'FAILURE: ' + ctx.reason,
|
|
45
|
+
];
|
|
46
|
+
if (ctx.candidates && ctx.candidates.length) {
|
|
47
|
+
parts.push(`The selector matched ${ctx.candidates.length} elements (ambiguous) — pick a more specific selector for the SAME intended element, or give_up if none of them is it.`);
|
|
48
|
+
}
|
|
49
|
+
parts.push('CURRENT SCREEN:\n' + (0, format_1.formatCompact)(ctx.hierarchy));
|
|
50
|
+
const { json, usage } = await this.call(grammar_1.REPAIR_GRAMMAR, parts.join('\n\n'), ir_1.REPAIR_DECISION_JSON_SCHEMA, 1024);
|
|
51
|
+
const decision = (json ?? {});
|
|
52
|
+
if (decision.decision === 'give_up') {
|
|
53
|
+
return {
|
|
54
|
+
replaceStep: null,
|
|
55
|
+
declineReason: decision.reason?.trim() || 'no element on the current screen matches the step intent',
|
|
56
|
+
usage,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
// Hand the proposed leaf back UNVALIDATED — engine.ts validates every repair against
|
|
60
|
+
// the grammar before splicing (it is the execution trust boundary and can't assume a
|
|
61
|
+
// provider validated). A missing/invalid step is rejected there as a failed repair.
|
|
62
|
+
return { replaceStep: (decision.step ?? null), usage };
|
|
63
|
+
}
|
|
64
|
+
async call(system, user, schema, maxTokens) {
|
|
65
|
+
const outputConfig = { format: { type: 'json_schema', schema } };
|
|
66
|
+
if (this.opts.effort && EFFORT_MODELS.has(this.opts.model))
|
|
67
|
+
outputConfig.effort = this.opts.effort;
|
|
68
|
+
const res = await this.fetchWithRetry({
|
|
69
|
+
model: this.opts.model,
|
|
70
|
+
max_tokens: maxTokens,
|
|
71
|
+
// The grammar is the large, stable prefix — cache it so repeat calls in a
|
|
72
|
+
// repair-heavy session read it at ~0.1x instead of full input price.
|
|
73
|
+
system: [{ type: 'text', text: system, cache_control: { type: 'ephemeral' } }],
|
|
74
|
+
messages: [{ role: 'user', content: user }],
|
|
75
|
+
output_config: outputConfig,
|
|
76
|
+
});
|
|
77
|
+
if (res.stop_reason === 'refusal') {
|
|
78
|
+
throw new errors_1.CliError('Model refused the request (stop_reason=refusal).', 1);
|
|
79
|
+
}
|
|
80
|
+
if (res.stop_reason === 'max_tokens') {
|
|
81
|
+
throw new errors_1.CliError('Model output was truncated (max_tokens) before a complete result — raise max_tokens or shorten the test.', 1);
|
|
82
|
+
}
|
|
83
|
+
const text = (res.content ?? [])
|
|
84
|
+
.filter((b) => b.type === 'text')
|
|
85
|
+
.map((b) => b.text ?? '')
|
|
86
|
+
.join('');
|
|
87
|
+
if (!text.trim())
|
|
88
|
+
throw new errors_1.CliError('Model returned an empty response.', 1);
|
|
89
|
+
let json;
|
|
90
|
+
try {
|
|
91
|
+
json = JSON.parse(text);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
throw new errors_1.CliError('Model output was not valid JSON.', 1);
|
|
95
|
+
}
|
|
96
|
+
return { json, usage: res.usage ?? {} };
|
|
97
|
+
}
|
|
98
|
+
async fetchWithRetry(body) {
|
|
99
|
+
const maxRetries = this.opts.maxRetries ?? 4;
|
|
100
|
+
const timeoutMs = this.opts.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
101
|
+
let attempt = 0;
|
|
102
|
+
for (;;) {
|
|
103
|
+
// Abort a stalled request after timeoutMs so it can't hang forever; the abort is
|
|
104
|
+
// caught below and retried like any other network error (bounded by maxRetries).
|
|
105
|
+
const controller = new AbortController();
|
|
106
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
107
|
+
let res;
|
|
108
|
+
try {
|
|
109
|
+
res = await fetch(API_URL, {
|
|
110
|
+
method: 'POST',
|
|
111
|
+
headers: {
|
|
112
|
+
'content-type': 'application/json',
|
|
113
|
+
'x-api-key': this.opts.apiKey,
|
|
114
|
+
'anthropic-version': ANTHROPIC_VERSION,
|
|
115
|
+
},
|
|
116
|
+
body: JSON.stringify(body),
|
|
117
|
+
signal: controller.signal,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
catch (e) {
|
|
121
|
+
if (attempt++ >= maxRetries)
|
|
122
|
+
throw new errors_1.CliError(`Anthropic API request failed: ${e.message}`, 3);
|
|
123
|
+
await sleep(backoffMs(attempt));
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
clearTimeout(timer);
|
|
128
|
+
}
|
|
129
|
+
if (res.ok)
|
|
130
|
+
return (await res.json());
|
|
131
|
+
// Retry 429 + 5xx with backoff, honoring Retry-After (no SDK to do it for us).
|
|
132
|
+
if ((res.status === 429 || res.status >= 500) && attempt++ < maxRetries) {
|
|
133
|
+
const retryAfter = Number(res.headers.get('retry-after'));
|
|
134
|
+
await sleep(Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : backoffMs(attempt));
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
const errText = await res.text().catch(() => '');
|
|
138
|
+
// 401/403 = auth/permission (env); 400 = bad request (usage); else env.
|
|
139
|
+
const code = res.status === 401 || res.status === 403 ? 3 : res.status === 400 ? 2 : 3;
|
|
140
|
+
throw new errors_1.CliError(`Anthropic API error ${res.status}: ${errText.slice(0, 500)}`, code);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
exports.ClaudeProvider = ClaudeProvider;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CostTracker = exports.DEFAULT_MAX_COST_USD = exports.DEFAULT_MODEL = exports.ALLOWED_MODELS = exports.MODEL_PRICES = void 0;
|
|
4
|
+
exports.resolveModel = resolveModel;
|
|
5
|
+
exports.parseCostOverride = parseCostOverride;
|
|
6
|
+
exports.priceFor = priceFor;
|
|
7
|
+
exports.estimateCostUsd = estimateCostUsd;
|
|
8
|
+
const errors_1 = require("../errors");
|
|
9
|
+
// Per-1M-token prices (Anthropic, cached 2026-05-26). This table WILL drift as
|
|
10
|
+
// pricing changes between releases — `--cost-override <input/output>` is the escape
|
|
11
|
+
// hatch, and is authoritative when supplied. The --model allowlist is exactly the
|
|
12
|
+
// keys of this table, so the two can never disagree.
|
|
13
|
+
exports.MODEL_PRICES = {
|
|
14
|
+
'claude-haiku-4-5': { input: 1, output: 5 },
|
|
15
|
+
'claude-sonnet-4-6': { input: 3, output: 15 },
|
|
16
|
+
'claude-opus-4-8': { input: 5, output: 25 },
|
|
17
|
+
'claude-fable-5': { input: 10, output: 50 },
|
|
18
|
+
};
|
|
19
|
+
exports.ALLOWED_MODELS = Object.keys(exports.MODEL_PRICES);
|
|
20
|
+
exports.DEFAULT_MODEL = 'claude-sonnet-4-6';
|
|
21
|
+
/** Default total-run cost ceiling for `vk ai` when --max-cost-usd is not given, so a
|
|
22
|
+
* runaway compile/repair loop can't spend unbounded tokens. */
|
|
23
|
+
exports.DEFAULT_MAX_COST_USD = 3;
|
|
24
|
+
/** Validate a --model against the allowlist (unknown -> exit 2, not a raw 404). */
|
|
25
|
+
function resolveModel(model) {
|
|
26
|
+
if (!model)
|
|
27
|
+
return exports.DEFAULT_MODEL;
|
|
28
|
+
if (!exports.MODEL_PRICES[model]) {
|
|
29
|
+
throw new errors_1.CliError(`Unknown --model '${model}'. Allowed: ${exports.ALLOWED_MODELS.join(', ')}.`, 2);
|
|
30
|
+
}
|
|
31
|
+
return model;
|
|
32
|
+
}
|
|
33
|
+
/** Parse `--cost-override <input/output>` (e.g. "3/15" => $3 in / $15 out per 1M). */
|
|
34
|
+
function parseCostOverride(raw) {
|
|
35
|
+
const m = /^(\d+(?:\.\d+)?)\s*\/\s*(\d+(?:\.\d+)?)$/.exec(raw.trim());
|
|
36
|
+
if (!m)
|
|
37
|
+
throw new errors_1.CliError(`--cost-override must be <input/output> per 1M tokens, e.g. 3/15; got '${raw}'`, 2);
|
|
38
|
+
return { input: Number(m[1]), output: Number(m[2]) };
|
|
39
|
+
}
|
|
40
|
+
/** Resolve the price to use: an explicit override wins over the bundled table. */
|
|
41
|
+
function priceFor(model, override) {
|
|
42
|
+
return override ?? exports.MODEL_PRICES[model] ?? exports.MODEL_PRICES[exports.DEFAULT_MODEL];
|
|
43
|
+
}
|
|
44
|
+
// Cache reads bill at ~0.1x input; cache writes (5-min TTL) at ~1.25x input.
|
|
45
|
+
const CACHE_READ_MULT = 0.1;
|
|
46
|
+
const CACHE_WRITE_MULT = 1.25;
|
|
47
|
+
const PER_M = 1_000_000;
|
|
48
|
+
/** Estimate the USD cost of a single API response from its `usage`. */
|
|
49
|
+
function estimateCostUsd(usage, price) {
|
|
50
|
+
const input = usage.input_tokens ?? 0;
|
|
51
|
+
const output = usage.output_tokens ?? 0;
|
|
52
|
+
const cacheWrite = usage.cache_creation_input_tokens ?? 0;
|
|
53
|
+
const cacheRead = usage.cache_read_input_tokens ?? 0;
|
|
54
|
+
return ((input * price.input +
|
|
55
|
+
output * price.output +
|
|
56
|
+
cacheWrite * price.input * CACHE_WRITE_MULT +
|
|
57
|
+
cacheRead * price.input * CACHE_READ_MULT) /
|
|
58
|
+
PER_M);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Accumulates token usage across a run and exposes the running dollar estimate.
|
|
62
|
+
* `exceeded()` is the budget gate: when the estimate crosses `maxUsd`, the engine
|
|
63
|
+
* aborts the run (recording it as aborted) rather than spending unbounded tokens.
|
|
64
|
+
*/
|
|
65
|
+
class CostTracker {
|
|
66
|
+
price;
|
|
67
|
+
maxUsd;
|
|
68
|
+
cacheRead = 0;
|
|
69
|
+
compileUsd = 0;
|
|
70
|
+
repairUsd = 0;
|
|
71
|
+
constructor(price, maxUsd) {
|
|
72
|
+
this.price = price;
|
|
73
|
+
this.maxUsd = maxUsd;
|
|
74
|
+
}
|
|
75
|
+
/** Record one API response. `phase` splits compile vs repair spend for the report. */
|
|
76
|
+
add(usage, phase) {
|
|
77
|
+
this.cacheRead += usage.cache_read_input_tokens ?? 0;
|
|
78
|
+
const usd = estimateCostUsd(usage, this.price);
|
|
79
|
+
if (phase === 'compile')
|
|
80
|
+
this.compileUsd += usd;
|
|
81
|
+
else
|
|
82
|
+
this.repairUsd += usd;
|
|
83
|
+
}
|
|
84
|
+
usd() {
|
|
85
|
+
return this.compileUsd + this.repairUsd;
|
|
86
|
+
}
|
|
87
|
+
/** True once the running estimate has crossed the configured ceiling. */
|
|
88
|
+
exceeded() {
|
|
89
|
+
return this.maxUsd !== undefined && this.usd() >= this.maxUsd;
|
|
90
|
+
}
|
|
91
|
+
get budgetUsd() {
|
|
92
|
+
return this.maxUsd;
|
|
93
|
+
}
|
|
94
|
+
/** The `compile=… · repairs=… · replay=0 · cache_read=… · est $…` report line. */
|
|
95
|
+
summaryLine() {
|
|
96
|
+
const fmt = (n) => `$${n.toFixed(4)}`;
|
|
97
|
+
return `compile=${fmt(this.compileUsd)} · repairs=${fmt(this.repairUsd)} · replay=$0 · cache_read=${this.cacheRead} tok · est ${fmt(this.usd())}`;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
exports.CostTracker = CostTracker;
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_RUN_TIMEOUT_MS = void 0;
|
|
4
|
+
exports.runPlan = runPlan;
|
|
5
|
+
const selector_1 = require("../ui/selector");
|
|
6
|
+
const errors_1 = require("../errors");
|
|
7
|
+
const ir_1 = require("./ir");
|
|
8
|
+
const describe = (leaf) => [leaf.command, ...leaf.positionals, ...leaf.flags.map((f) => (f.value === 'true' ? `--${f.name}` : `--${f.name} ${f.value}`))]
|
|
9
|
+
.join(' ')
|
|
10
|
+
.trim();
|
|
11
|
+
/** A structural fingerprint of the screen: sorted id+text+type set. Used for the
|
|
12
|
+
* loop no-progress check — deliberately NOT the raw hierarchy (its node ordering
|
|
13
|
+
* is nondeterministic between identical states, which would false-trip). */
|
|
14
|
+
function structuralHash(els) {
|
|
15
|
+
return els
|
|
16
|
+
.map((e) => `${e.idShort}|${e.text}|${e.type}`)
|
|
17
|
+
.sort()
|
|
18
|
+
.join('\n');
|
|
19
|
+
}
|
|
20
|
+
function isHealable(outcome) {
|
|
21
|
+
return (!!outcome.error &&
|
|
22
|
+
(outcome.error instanceof errors_1.SelectorNotFoundError || outcome.error instanceof errors_1.AmbiguousSelectorError));
|
|
23
|
+
}
|
|
24
|
+
/** Default wall-clock ceiling for a whole `vk ai` run (overridable via --timeout). */
|
|
25
|
+
exports.DEFAULT_RUN_TIMEOUT_MS = 15 * 60 * 1000;
|
|
26
|
+
async function runPlan(plan, deps) {
|
|
27
|
+
const maxRepairs = deps.maxRepairs ?? 3;
|
|
28
|
+
const overDeadline = () => deps.deadline !== undefined && Date.now() >= deps.deadline;
|
|
29
|
+
const improvements = [];
|
|
30
|
+
let modelRepairs = 0;
|
|
31
|
+
// A UI dump can fail transiently on a real device (uiautomator throws). Treat
|
|
32
|
+
// that as "empty screen" rather than letting it abort the whole run — the rest
|
|
33
|
+
// of the codebase treats dumps as recoverable (resolveOneWaiting re-polls).
|
|
34
|
+
const safeElements = () => {
|
|
35
|
+
try {
|
|
36
|
+
return deps.getElements();
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return [];
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
const present = (selector) => {
|
|
43
|
+
// Re-fetch on a dump FAILURE (uiautomator can throw transiently) so a flaky dump at
|
|
44
|
+
// a guard check doesn't silently read as "absent" and skip a body that should run.
|
|
45
|
+
// Once a dump SUCCEEDS (even if empty) we trust it — no slow re-poll, so a genuinely
|
|
46
|
+
// absent guard still skips fast (the common if-present case).
|
|
47
|
+
let els;
|
|
48
|
+
for (let i = 0; i < 2 && els === undefined; i++) {
|
|
49
|
+
try {
|
|
50
|
+
els = deps.getElements();
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
/* transient dump failure — retry once before concluding "absent" */
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (els === undefined)
|
|
57
|
+
return false;
|
|
58
|
+
try {
|
|
59
|
+
return (0, selector_1.matchElements)(els, (0, selector_1.parseSelector)(selector)).matches.length > 0;
|
|
60
|
+
}
|
|
61
|
+
catch (e) {
|
|
62
|
+
// A guard selector that won't parse is a compiler/plan bug — surface it (then treat
|
|
63
|
+
// as not present) rather than silently skip the guarded body.
|
|
64
|
+
deps.log(`[ai] guard selector '${selector}' did not parse (${e.message}) — treating as not present`);
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
const runLeaf = (leaf) => deps.exec(leaf.command, leaf.positionals, (0, ir_1.leafToFlags)(leaf));
|
|
69
|
+
/** Execute one leaf, healing a selector miss/ambiguity via the model up to the cap.
|
|
70
|
+
* `replace` writes a repaired leaf back into the plan so it persists on green. */
|
|
71
|
+
async function execLeaf(leaf, where, replace) {
|
|
72
|
+
deps.log(`[ai] ${where}: ${describe(leaf)}`);
|
|
73
|
+
let current = leaf;
|
|
74
|
+
let outcome = await runLeaf(current);
|
|
75
|
+
let attempts = 0;
|
|
76
|
+
while (isHealable(outcome) && attempts < maxRepairs) {
|
|
77
|
+
if (deps.cost.exceeded()) {
|
|
78
|
+
deps.log(`[ai] budget ceiling reached ($${deps.cost.budgetUsd}) — aborting before another repair`);
|
|
79
|
+
return { status: 'budget' };
|
|
80
|
+
}
|
|
81
|
+
if (overDeadline()) {
|
|
82
|
+
deps.log(`[ai] run timeout reached — aborting before another repair`);
|
|
83
|
+
return { status: 'timeout' };
|
|
84
|
+
}
|
|
85
|
+
attempts++;
|
|
86
|
+
const reason = outcome.error.message.split('\n')[0];
|
|
87
|
+
const candidates = outcome.error instanceof errors_1.AmbiguousSelectorError ? outcome.error.candidates : undefined;
|
|
88
|
+
deps.log(`[ai] ${where} failed (${reason}) — asking model to repair (attempt ${attempts}/${maxRepairs})`);
|
|
89
|
+
let repaired;
|
|
90
|
+
try {
|
|
91
|
+
const { replaceStep, declineReason, usage } = await deps.provider.repair({
|
|
92
|
+
failedStep: current,
|
|
93
|
+
reason,
|
|
94
|
+
candidates,
|
|
95
|
+
hierarchy: safeElements(),
|
|
96
|
+
});
|
|
97
|
+
deps.cost.add(usage, 'repair');
|
|
98
|
+
// The model may DECLINE (null) when the current screen has no element serving
|
|
99
|
+
// this step's intent — i.e. the flow drifted to an unexpected screen. Fail
|
|
100
|
+
// terminally rather than substitute a loosely-related element, which would let
|
|
101
|
+
// a real regression pass as a false green (the bug this guards against).
|
|
102
|
+
if (replaceStep === null) {
|
|
103
|
+
return { status: 'fail', where, reason: `drifted, not repaired: ${declineReason ?? 'no element matches the step intent'}` };
|
|
104
|
+
}
|
|
105
|
+
// The SOLE validation gate for a repair: the provider hands back the model's
|
|
106
|
+
// proposed leaf UNVALIDATED (a third-party provider can't be trusted to check),
|
|
107
|
+
// so the engine validates it against the grammar BEFORE splicing. A hallucinated
|
|
108
|
+
// command is rejected here, never run (it would otherwise exit 2 + abort).
|
|
109
|
+
const node = (0, ir_1.validateNode)(replaceStep, 'repair');
|
|
110
|
+
if (node.type !== 'command')
|
|
111
|
+
throw new ir_1.InvalidPlanError('repair must be a single command step');
|
|
112
|
+
repaired = node;
|
|
113
|
+
}
|
|
114
|
+
catch (e) {
|
|
115
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
116
|
+
return { status: 'fail', where, reason: `repair failed: ${msg}` };
|
|
117
|
+
}
|
|
118
|
+
// The just-failed attempt was recorded as a failed step; downgrade it to a
|
|
119
|
+
// healed pass so a self-healed leaf doesn't surface as a failure in the
|
|
120
|
+
// report (the retry below records its own step).
|
|
121
|
+
deps.markHealed?.(`healed: ${reason} → ${describe(repaired)}`);
|
|
122
|
+
improvements.push(`${where}: "${describe(current)}" needed repair (${reason}); model replaced it with "${describe(repaired)}". ` +
|
|
123
|
+
`Update the test to "${describe(repaired)}" to skip this repair next run.`);
|
|
124
|
+
modelRepairs++;
|
|
125
|
+
current = repaired;
|
|
126
|
+
replace(current);
|
|
127
|
+
deps.log(`[ai] ${where} repaired → ${describe(current)} (retrying)`);
|
|
128
|
+
outcome = await runLeaf(current);
|
|
129
|
+
}
|
|
130
|
+
if (outcome.code === 0)
|
|
131
|
+
return { status: 'ok' };
|
|
132
|
+
if (isHealable(outcome)) {
|
|
133
|
+
return { status: 'fail', where, reason: `unresolved after ${maxRepairs} repair attempt(s): ${outcome.error.message.split('\n')[0]}` };
|
|
134
|
+
}
|
|
135
|
+
// Terminal: an assertion failure (exit 1, no throw) or an environment error.
|
|
136
|
+
const reason = outcome.error ? outcome.error.message.split('\n')[0] : `exited ${outcome.code}`;
|
|
137
|
+
return { status: 'fail', where, reason };
|
|
138
|
+
}
|
|
139
|
+
async function walkBody(body, parentWhere) {
|
|
140
|
+
for (let j = 0; j < body.length; j++) {
|
|
141
|
+
const res = await execLeaf(body[j], `${parentWhere}.body[${j}]`, (l) => (body[j] = l));
|
|
142
|
+
if (res.status !== 'ok')
|
|
143
|
+
return res;
|
|
144
|
+
}
|
|
145
|
+
return { status: 'ok' };
|
|
146
|
+
}
|
|
147
|
+
async function walkNode(node, where, replace) {
|
|
148
|
+
switch (node.type) {
|
|
149
|
+
case 'command':
|
|
150
|
+
return execLeaf(node, where, replace);
|
|
151
|
+
case 'if-present': {
|
|
152
|
+
if (present(node.selector)) {
|
|
153
|
+
deps.log(`[ai] ${where}: if-present '${node.selector}' → present, running ${node.body.length} step(s)`);
|
|
154
|
+
return walkBody(node.body, where);
|
|
155
|
+
}
|
|
156
|
+
deps.log(`[ai] ${where}: if-present '${node.selector}' → absent, skipping`);
|
|
157
|
+
return { status: 'ok' };
|
|
158
|
+
}
|
|
159
|
+
case 'repeat': {
|
|
160
|
+
let prevHash = '';
|
|
161
|
+
for (let i = 0; i < node.cap; i++) {
|
|
162
|
+
if (overDeadline()) {
|
|
163
|
+
deps.log(`[ai] ${where}: run timeout reached — stopping repeat after ${i} iteration(s)`);
|
|
164
|
+
return { status: 'timeout' };
|
|
165
|
+
}
|
|
166
|
+
if (present(node.selector)) {
|
|
167
|
+
deps.log(`[ai] ${where}: repeat reached '${node.selector}' after ${i} iteration(s)`);
|
|
168
|
+
return { status: 'ok' };
|
|
169
|
+
}
|
|
170
|
+
const hash = structuralHash(safeElements());
|
|
171
|
+
if (i > 0 && hash === prevHash) {
|
|
172
|
+
deps.log(`[ai] ${where}: repeat made no progress (screen unchanged) — stopping after ${i} iteration(s)`);
|
|
173
|
+
return { status: 'ok' };
|
|
174
|
+
}
|
|
175
|
+
prevHash = hash;
|
|
176
|
+
deps.log(`[ai] ${where}: repeat iteration ${i + 1}/${node.cap}`);
|
|
177
|
+
const res = await walkBody(node.body, `${where}#${i + 1}`);
|
|
178
|
+
if (res.status !== 'ok')
|
|
179
|
+
return res;
|
|
180
|
+
}
|
|
181
|
+
deps.log(`[ai] ${where}: repeat hit cap ${node.cap} without '${node.selector}' (continuing)`);
|
|
182
|
+
return { status: 'ok' };
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
for (let i = 0; i < plan.steps.length; i++) {
|
|
187
|
+
if (overDeadline()) {
|
|
188
|
+
deps.log(`[ai] run timeout reached before steps[${i}] — aborting`);
|
|
189
|
+
return { ok: false, plan, modelRepairs, improvements, abortedForTimeout: true };
|
|
190
|
+
}
|
|
191
|
+
const res = await walkNode(plan.steps[i], `steps[${i}]`, (l) => (plan.steps[i] = l));
|
|
192
|
+
if (res.status === 'budget') {
|
|
193
|
+
return { ok: false, plan, modelRepairs, improvements, abortedForBudget: true };
|
|
194
|
+
}
|
|
195
|
+
if (res.status === 'timeout') {
|
|
196
|
+
return { ok: false, plan, modelRepairs, improvements, abortedForTimeout: true };
|
|
197
|
+
}
|
|
198
|
+
if (res.status === 'fail') {
|
|
199
|
+
deps.log(`[ai] FAILED at ${res.where}: ${res.reason}`);
|
|
200
|
+
return { ok: false, plan, modelRepairs, improvements, failure: { where: res.where, reason: res.reason } };
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
deps.log(`[ai] all ${plan.steps.length} step(s) passed${modelRepairs ? ` (${modelRepairs} model repair(s))` : ''}`);
|
|
204
|
+
return { ok: true, plan, modelRepairs, improvements };
|
|
205
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// The command grammar handed to the model so it compiles NL into a valid plan IR.
|
|
3
|
+
// This mirrors the agent-facing contract in .claude/skills/verikun/SKILL.md; keep
|
|
4
|
+
// the two in sync (the SKILL.md is the human/source-of-truth, this is the compact
|
|
5
|
+
// runtime copy). It is the large, STABLE prefix of every compile/repair prompt, so
|
|
6
|
+
// the provider marks it cache_control: ephemeral to bill repeat calls at ~0.1x.
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.REPAIR_GRAMMAR = exports.GRAMMAR = void 0;
|
|
9
|
+
exports.GRAMMAR = `You compile a natural-language mobile UI test into a verikun PLAN — a JSON program
|
|
10
|
+
that a deterministic engine replays against a real Android/iOS device with NO further
|
|
11
|
+
model calls on the happy path. Emit ONLY the plan object matching the provided schema.
|
|
12
|
+
|
|
13
|
+
A plan has { "version": 1, "package"?, "platform"?, "steps": [...] }.
|
|
14
|
+
Each step is one of three node types:
|
|
15
|
+
|
|
16
|
+
1. COMMAND leaf — { "type":"command", "command":<name>, "positionals":[...], "flags":[{"name","value"}] }
|
|
17
|
+
A boolean flag is {"name":"clear","value":"true"}. A valued flag is {"name":"wait","value":"5s"}.
|
|
18
|
+
Available commands (verikun):
|
|
19
|
+
launch <package> [--clear] [--no-restart] — start the app; force-stops it first so a
|
|
20
|
+
rerun starts FRESH (--clear also wipes data → fresh-install;
|
|
21
|
+
--no-restart skips the force-stop, just bringing it forward)
|
|
22
|
+
stop <package> — force-stop the app
|
|
23
|
+
tap <selector> — tap the element a selector resolves to
|
|
24
|
+
text <selector> <value...> — focus a field and type value (--clear to clear first, --enter to submit)
|
|
25
|
+
type <value...> — type into the already-focused field
|
|
26
|
+
key <name> | back | home | enter
|
|
27
|
+
swipe <up|down|left|right> [--on <selector>] — scroll/swipe (up = scroll down the page)
|
|
28
|
+
assert <selector> [--text <s>] [--gone] — assert presence/text/absence (FAILS the test if false)
|
|
29
|
+
wait <selector> [--gone] [--timeout <ms>] — block until present/absent
|
|
30
|
+
screenshot — capture the screen into the report
|
|
31
|
+
|
|
32
|
+
2. IF-PRESENT — { "type":"if-present", "selector":<sel>, "body":[<command leaves>] }
|
|
33
|
+
Run body ONLY if the selector is on screen now. Use for OPTIONAL interstitials:
|
|
34
|
+
permission dialogs, "rate us" popups, cookie banners, A/B variants. This is how you
|
|
35
|
+
keep a flow from breaking when an extra screen sometimes appears.
|
|
36
|
+
|
|
37
|
+
3. REPEAT — { "type":"repeat", "selector":<sel>, "cap":<n>, "body":[<command leaves>] }
|
|
38
|
+
Repeat body until the selector appears, up to cap iterations. Use for "scroll until X
|
|
39
|
+
is visible". Always set a sane cap (e.g. 10). The engine also stops early if the screen
|
|
40
|
+
stops changing.
|
|
41
|
+
|
|
42
|
+
NESTING: control-node bodies hold COMMAND leaves only — do NOT nest if-present/repeat
|
|
43
|
+
inside another control node.
|
|
44
|
+
|
|
45
|
+
SELECTORS (the engine auto-heals case/whitespace/partial, so prefer stable identifiers):
|
|
46
|
+
@login resource-id 'login' (shorthand for id:login)
|
|
47
|
+
id:login resource-id (full, suffix, or short)
|
|
48
|
+
text:Sign in visible text (case-insensitive)
|
|
49
|
+
desc:Submit content-desc / accessibility label
|
|
50
|
+
class:Button type or class
|
|
51
|
+
"Sign in" bare string == text:Sign in
|
|
52
|
+
|
|
53
|
+
RULES:
|
|
54
|
+
- assert is for VERIFICATION only and is terminal — never use it as a step you expect to
|
|
55
|
+
fail. Put genuinely-optional UI behind if-present.
|
|
56
|
+
- Prefer resource-id / accessibility selectors over visible text where possible.
|
|
57
|
+
- Translate the test literally and minimally; do not invent steps the prose does not imply.`;
|
|
58
|
+
exports.REPAIR_GRAMMAR = `A single step in a verikun plan failed to resolve its selector against the live screen
|
|
59
|
+
(shown below). Decide between two outcomes — and be STRICT:
|
|
60
|
+
|
|
61
|
+
- "repair": the current screen genuinely contains an element that serves the SAME
|
|
62
|
+
PURPOSE as the failed step (the same control after a UI/build change, a renamed id,
|
|
63
|
+
a translated label, the same button relocated). Return it as ONE replacement command
|
|
64
|
+
leaf in "step", reusing the same command unless the screen clearly requires another.
|
|
65
|
+
|
|
66
|
+
- "give_up": the screen does NOT contain an element matching the step's intent — e.g.
|
|
67
|
+
the flow has landed on an unexpected screen, a different app, or a dead end. Return
|
|
68
|
+
"give_up" with a short "reason". The test will then FAIL, which is the CORRECT result.
|
|
69
|
+
|
|
70
|
+
"Same purpose" means the same user-facing action, NOT merely "a tappable element
|
|
71
|
+
exists". Do NOT substitute a loosely-related or convenient element (a back arrow, a
|
|
72
|
+
prominent unrelated button, a menu item that sounds similar) just to make the step
|
|
73
|
+
pass — a wrong substitution hides a real regression behind a false green. If you are
|
|
74
|
+
not confident the element does what the original step intended, choose give_up.
|
|
75
|
+
|
|
76
|
+
Emit ONLY an object matching the schema:
|
|
77
|
+
{ "decision":"repair", "step": { "type":"command","command","positionals":[...],"flags":[{"name","value"}] } }
|
|
78
|
+
{ "decision":"give_up", "reason": "<why no element on this screen matches the intent>" }
|
|
79
|
+
Prefer a stable selector (resource-id / accessibility label) visible in the hierarchy.
|
|
80
|
+
Do not invent elements that are not in the hierarchy.`;
|