verikun 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -7
- package/dist/agent/claude.js +6 -0
- package/dist/agent/cli-provider.js +6 -0
- package/dist/agent/engine.js +391 -45
- package/dist/agent/grammar.js +73 -6
- package/dist/agent/ir.js +220 -46
- package/dist/agent/lint.js +77 -0
- package/dist/agent/openai.js +6 -0
- package/dist/cli.js +124 -49
- package/dist/drivers/adb.js +41 -0
- package/dist/drivers/index.js +5 -1
- package/dist/drivers/ios.js +75 -0
- package/dist/errors.js +16 -1
- package/dist/report.js +16 -1
- package/dist/run.js +9 -4
- package/dist/server.js +5 -2
- package/dist/suite.js +75 -15
- package/dist/ui/selector.js +17 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/agent/grammar.js
CHANGED
|
@@ -34,13 +34,75 @@ Each step is one of three node types:
|
|
|
34
34
|
permission dialogs, "rate us" popups, cookie banners, A/B variants. This is how you
|
|
35
35
|
keep a flow from breaking when an extra screen sometimes appears.
|
|
36
36
|
|
|
37
|
-
3. REPEAT — { "type":"repeat", "selector":<sel>, "cap":<n>, "body":[<
|
|
38
|
-
Repeat body
|
|
39
|
-
is visible". Always set a sane cap (e.g.
|
|
40
|
-
stops changing.
|
|
37
|
+
3. REPEAT — { "type":"repeat", "selector":<sel>, "cap":<n>, "body":[<nodes>] }
|
|
38
|
+
Repeat body UNTIL the selector appears, up to cap iterations. Use for "scroll until X
|
|
39
|
+
is visible", or "keep answering until the results screen". Always set a sane cap (e.g.
|
|
40
|
+
10). The engine also stops early if the screen stops changing. A repeat that finishes
|
|
41
|
+
without its selector ever appearing FAILS the test — it did not do its job.
|
|
41
42
|
|
|
42
|
-
|
|
43
|
-
|
|
43
|
+
4. WHEN — { "type":"when", "branches":[{ "selector":<sel>, "body":[<nodes>] }, ...],
|
|
44
|
+
"else":[<nodes>] (optional) }
|
|
45
|
+
Ordered n-way dispatch: the FIRST branch whose selector is on screen runs, and only it.
|
|
46
|
+
Use when a screen is one of several KINDS that each need different handling —
|
|
47
|
+
"the question is multiple-choice, or match-the-pairs, or arrange-the-words".
|
|
48
|
+
If no branch matches and there is no "else", the test FAILS (the app showed something
|
|
49
|
+
this test does not handle — that is a real result, not something to skip past).
|
|
50
|
+
Use "else": [] to say explicitly "if none match, do nothing".
|
|
51
|
+
WHEN vs IF-PRESENT: if-present = "this may or may not be there, carry on either way".
|
|
52
|
+
when = "it is one of these; if it is none of them, that is a failure".
|
|
53
|
+
|
|
54
|
+
5. WHILE-PRESENT — { "type":"while-present", "selector":<sel>, "bind":<name>,
|
|
55
|
+
"cap":<n>, "body":[<nodes>] }
|
|
56
|
+
Repeat body WHILE the selector is present. With "bind", the named counter starts at 0
|
|
57
|
+
and increments after each iteration, and you reference it as {{ctx.<name>}} inside the
|
|
58
|
+
selector and the body. This is how you walk an index-addressed list whose LENGTH you
|
|
59
|
+
cannot know when compiling:
|
|
60
|
+
{ "type":"while-present", "selector":"id:word_bubble_container_id_{{ctx.i}}",
|
|
61
|
+
"bind":"i", "cap":20,
|
|
62
|
+
"body":[ { "type":"command","command":"tap",
|
|
63
|
+
"positionals":["id:word_bubble_container_id_{{ctx.i}}"],"flags":[] } ] }
|
|
64
|
+
|
|
65
|
+
6. READ — { "type":"read", "selector":<sel>, "field":"text"|"desc"|"id"|"idShort",
|
|
66
|
+
"into":<name> }
|
|
67
|
+
Capture a value off the live screen into {{ctx.<name>}} for a later step to use. Use it
|
|
68
|
+
when the test must ACT ON a value it cannot know in advance — e.g. read the correct
|
|
69
|
+
answer's text, then type that text into a field.
|
|
70
|
+
|
|
71
|
+
PLACEHOLDERS — any positional or flag value, and any control-node selector, may contain:
|
|
72
|
+
{{ctx.NAME}} a value stored by read, or a while-present counter
|
|
73
|
+
{{env.NAME}} an environment variable (use for credentials; never inline a secret)
|
|
74
|
+
{{uuid}} a fresh id, generated once per run (same value everywhere in the run)
|
|
75
|
+
{{timestamp}} epoch ms, once per run {{run_id}} this run's id
|
|
76
|
+
Use {{uuid}} when the test needs data that must be unique per run, e.g. a signup email
|
|
77
|
+
like "user-{{uuid}}@example.com" — never a hard-coded literal, which collides on rerun.
|
|
78
|
+
|
|
79
|
+
NESTING: control nodes may nest ONE level — a control node inside a control node, whose
|
|
80
|
+
body is command leaves. A repeat containing a when is the shape for "until the flow ends,
|
|
81
|
+
handle whichever screen is showing", and it is legal. Three levels is not.
|
|
82
|
+
EXCEPT: if-present and while-present may go one level deeper (their bodies are leaves), so
|
|
83
|
+
repeat { when { while-present { tap ... } } } — walk an index-addressed list
|
|
84
|
+
repeat { when { if-present { tap ... } } } — an optional step inside a branch
|
|
85
|
+
are both legal. Use them rather than approximating.
|
|
86
|
+
In particular: "if X appears, tap it" inside a branch is an if-present. Do NOT turn it
|
|
87
|
+
into a bare wait + tap — wait FAILS the test when X never appears, and "if" means it
|
|
88
|
+
might not. That mistake reads as a passing plan and fails on the first run where the app
|
|
89
|
+
skips that step.
|
|
90
|
+
|
|
91
|
+
Inside a "repeat until X" loop, GUARD a tap whose target is the thing that brings X about:
|
|
92
|
+
repeat until <form> { if-present <button> { tap <button> } ; screenshot }
|
|
93
|
+
Once the transition starts, <button> is gone — so on the final iteration an unguarded tap
|
|
94
|
+
misses and fails the whole run, even though the loop did its job. The guard makes that
|
|
95
|
+
last lap a no-op instead of an error. Apply this whenever the prose says "tap ... until"
|
|
96
|
+
or "repeatedly until".
|
|
97
|
+
|
|
98
|
+
Do NOT flatten a branch into an unconditional sequence: emitting the taps for ALL the
|
|
99
|
+
kinds of screen one after another is wrong — on any given iteration most of them are not
|
|
100
|
+
there, and the test will fail on the first one that is missing. Use when.
|
|
101
|
+
|
|
102
|
+
Do NOT hard-code a run of indices you were not told the length of. If the prose says
|
|
103
|
+
"tap each pair", "until every pair is matched", or "tap the bubbles in order", the COUNT
|
|
104
|
+
varies per run — emit a while-present over {{ctx.i}} rather than tap _0, _1, _2, _3.
|
|
105
|
+
A hard-coded list is right only when the prose states the exact count.
|
|
44
106
|
|
|
45
107
|
SELECTORS (the engine auto-heals case/whitespace/partial, so prefer stable identifiers):
|
|
46
108
|
@login resource-id 'login' (shorthand for id:login)
|
|
@@ -51,6 +113,11 @@ SELECTORS (the engine auto-heals case/whitespace/partial, so prefer stable ident
|
|
|
51
113
|
"Sign in" bare string == text:Sign in
|
|
52
114
|
|
|
53
115
|
RULES:
|
|
116
|
+
- --enabled on a tap makes it match only a control that is ACTIONABLE right now, and (with
|
|
117
|
+
auto-wait) wait until it becomes so. Use it for any button that the app disables until
|
|
118
|
+
something else is done — a Check/Submit/Continue that only lights up once an answer is
|
|
119
|
+
selected or a form is valid. Without it the step taps a dead control, does nothing, and
|
|
120
|
+
the failure surfaces later as a confusing timeout on the NEXT step.
|
|
54
121
|
- assert is for VERIFICATION only and is terminal — never use it as a step you expect to
|
|
55
122
|
fail. Put genuinely-optional UI behind if-present.
|
|
56
123
|
- Prefer resource-id / accessibility selectors over visible text where possible.
|
package/dist/agent/ir.js
CHANGED
|
@@ -18,10 +18,25 @@
|
|
|
18
18
|
// structured-output schemas can't express an arbitrary-key object. A pure boolean
|
|
19
19
|
// flag is value:"true" (flagBool reads 'true' as true; flagStr reads the string).
|
|
20
20
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
-
exports.InvalidPlanError = exports.REPAIR_DECISION_JSON_SCHEMA = exports.PLAN_JSON_SCHEMA = exports.DEFAULT_LOOP_CAP = exports.KNOWN_COMMANDS = void 0;
|
|
21
|
+
exports.InvalidPlanError = exports.REPAIR_DECISION_JSON_SCHEMA = exports.PLAN_JSON_SCHEMA = exports.DEFAULT_LOOP_CAP = exports.KNOWN_COMMANDS = exports.isControlNode = exports.MAX_CONTROL_DEPTH = void 0;
|
|
22
|
+
exports.bodiesOf = bodiesOf;
|
|
22
23
|
exports.validateNode = validateNode;
|
|
23
24
|
exports.parsePlan = parsePlan;
|
|
24
25
|
exports.leafToFlags = leafToFlags;
|
|
26
|
+
/** Max control-node nesting. 1 = a control node at top level whose body is leaves
|
|
27
|
+
* (the v1 rule). 2 = a control node inside that body, whose own body is leaves —
|
|
28
|
+
* i.e. `repeat { when { … } }`, the loop-that-branches shape a real dynamic flow
|
|
29
|
+
* needs. The schema is hand-unrolled to exactly this depth, so raising it means
|
|
30
|
+
* editing controlSchema()'s nesting as well as this constant. */
|
|
31
|
+
exports.MAX_CONTROL_DEPTH = 2;
|
|
32
|
+
const isControlNode = (n) => n.type === 'if-present' || n.type === 'when' || n.type === 'repeat' || n.type === 'while-present';
|
|
33
|
+
exports.isControlNode = isControlNode;
|
|
34
|
+
/** Every body a control node owns (a `when` owns one per branch, plus `else`). */
|
|
35
|
+
function bodiesOf(n) {
|
|
36
|
+
if (n.type === 'when')
|
|
37
|
+
return [...n.branches.map((b) => b.body), ...(n.else ? [n.else] : [])];
|
|
38
|
+
return [n.body];
|
|
39
|
+
}
|
|
25
40
|
/** Commands a leaf step is allowed to carry — the agent-emittable ACTION/assertion verbs
|
|
26
41
|
* the grammar offers, a SUBSET of cli.ts's dispatch (inspection/diagnostic commands are
|
|
27
42
|
* excluded; see the note in the set). The engine validates every step — including a
|
|
@@ -47,10 +62,115 @@ exports.KNOWN_COMMANDS = new Set([
|
|
|
47
62
|
exports.DEFAULT_LOOP_CAP = 25;
|
|
48
63
|
/**
|
|
49
64
|
* JSON Schema for `output_config.format` so the model returns a guaranteed-valid
|
|
50
|
-
* Plan. Deliberately NON-RECURSIVE:
|
|
51
|
-
*
|
|
52
|
-
*
|
|
65
|
+
* Plan. Deliberately NON-RECURSIVE: instead of a `Plan -> node -> Plan` cycle
|
|
66
|
+
* (structured output rejects recursive schemas) the control levels are HAND-UNROLLED
|
|
67
|
+
* to MAX_CONTROL_DEPTH. Raising the depth means adding another unrolled level here.
|
|
53
68
|
*/
|
|
69
|
+
const stepItems = (bodyItems) => ({
|
|
70
|
+
anyOf: [
|
|
71
|
+
leafSchema(),
|
|
72
|
+
readSchema(),
|
|
73
|
+
{
|
|
74
|
+
type: 'object',
|
|
75
|
+
additionalProperties: false,
|
|
76
|
+
required: ['type', 'selector', 'body'],
|
|
77
|
+
properties: {
|
|
78
|
+
type: { type: 'string', enum: ['if-present'] },
|
|
79
|
+
selector: { type: 'string' },
|
|
80
|
+
body: { type: 'array', items: bodyItems },
|
|
81
|
+
},
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
type: 'object',
|
|
85
|
+
additionalProperties: false,
|
|
86
|
+
required: ['type', 'branches'],
|
|
87
|
+
properties: {
|
|
88
|
+
type: { type: 'string', enum: ['when'] },
|
|
89
|
+
branches: {
|
|
90
|
+
type: 'array',
|
|
91
|
+
items: {
|
|
92
|
+
type: 'object',
|
|
93
|
+
additionalProperties: false,
|
|
94
|
+
required: ['selector', 'body'],
|
|
95
|
+
properties: { selector: { type: 'string' }, body: { type: 'array', items: bodyItems } },
|
|
96
|
+
},
|
|
97
|
+
},
|
|
98
|
+
else: { type: 'array', items: bodyItems },
|
|
99
|
+
},
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
type: 'object',
|
|
103
|
+
additionalProperties: false,
|
|
104
|
+
required: ['type', 'selector', 'cap', 'body'],
|
|
105
|
+
properties: {
|
|
106
|
+
type: { type: 'string', enum: ['repeat'] },
|
|
107
|
+
selector: { type: 'string' },
|
|
108
|
+
cap: { type: 'integer' },
|
|
109
|
+
body: { type: 'array', items: bodyItems },
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
type: 'object',
|
|
114
|
+
additionalProperties: false,
|
|
115
|
+
required: ['type', 'selector', 'cap', 'body'],
|
|
116
|
+
properties: {
|
|
117
|
+
type: { type: 'string', enum: ['while-present'] },
|
|
118
|
+
selector: { type: 'string' },
|
|
119
|
+
bind: { type: 'string' },
|
|
120
|
+
cap: { type: 'integer' },
|
|
121
|
+
body: { type: 'array', items: bodyItems },
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
],
|
|
125
|
+
});
|
|
126
|
+
/** The innermost level: leaves, reads, and the two SIMPLE control nodes — `if-present`
|
|
127
|
+
* and `while-present` — restricted to leaf-only bodies so they cannot open a further
|
|
128
|
+
* level.
|
|
129
|
+
*
|
|
130
|
+
* Both were added from measured failures, and both are the same failure: when the model
|
|
131
|
+
* needs a construct one level deeper than the grammar allows, it does not give up — it
|
|
132
|
+
* DEGRADES to an unconditional approximation, which is exactly the bug #33 is about.
|
|
133
|
+
* - Without `while-present` here, a compile of the dynamic-lesson repro produced
|
|
134
|
+
* `repeat { when { … } }` correctly but HARD-CODED the index lists inside each branch
|
|
135
|
+
* (pairs 0..3, bubbles 0..5) — broken the moment a question has a different count.
|
|
136
|
+
* - Without `if-present` here, the same compile turned the prose "IF a Continue button
|
|
137
|
+
* appears, tap it" into an unconditional `wait id:continue_button_id` inside every
|
|
138
|
+
* branch. The app auto-advances and shows no Continue, so all three runs failed on
|
|
139
|
+
* that wait — a conditional flattened into an assertion, one level down.
|
|
140
|
+
*
|
|
141
|
+
* A full third level for every node would ~3x a 28KB schema (bad for OpenAI strict
|
|
142
|
+
* limits, worse for cursor, which injects the schema as prompt text). Granting it to the
|
|
143
|
+
* two leaf-bodied nodes is the cheap, bounded version. `when` and `repeat` stay at
|
|
144
|
+
* MAX_CONTROL_DEPTH: they are the nodes whose bodies want to be deep, and allowing them
|
|
145
|
+
* here is what would actually cost the schema. */
|
|
146
|
+
const innerControls = () => {
|
|
147
|
+
const leafBody = { type: 'array', items: { anyOf: [leafSchema(), readSchema()] } };
|
|
148
|
+
return [
|
|
149
|
+
{
|
|
150
|
+
type: 'object',
|
|
151
|
+
additionalProperties: false,
|
|
152
|
+
required: ['type', 'selector', 'body'],
|
|
153
|
+
properties: {
|
|
154
|
+
type: { type: 'string', enum: ['if-present'] },
|
|
155
|
+
selector: { type: 'string' },
|
|
156
|
+
body: leafBody,
|
|
157
|
+
},
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
type: 'object',
|
|
161
|
+
additionalProperties: false,
|
|
162
|
+
required: ['type', 'selector', 'cap', 'body'],
|
|
163
|
+
properties: {
|
|
164
|
+
type: { type: 'string', enum: ['while-present'] },
|
|
165
|
+
selector: { type: 'string' },
|
|
166
|
+
bind: { type: 'string' },
|
|
167
|
+
cap: { type: 'integer' },
|
|
168
|
+
body: leafBody,
|
|
169
|
+
},
|
|
170
|
+
},
|
|
171
|
+
];
|
|
172
|
+
};
|
|
173
|
+
const LEAF_ONLY_ITEMS = { anyOf: [leafSchema(), readSchema(), ...innerControls()] };
|
|
54
174
|
exports.PLAN_JSON_SCHEMA = {
|
|
55
175
|
type: 'object',
|
|
56
176
|
additionalProperties: false,
|
|
@@ -61,32 +181,7 @@ exports.PLAN_JSON_SCHEMA = {
|
|
|
61
181
|
platform: { type: 'string', enum: ['android', 'ios'] },
|
|
62
182
|
steps: {
|
|
63
183
|
type: 'array',
|
|
64
|
-
items:
|
|
65
|
-
anyOf: [
|
|
66
|
-
leafSchema(),
|
|
67
|
-
{
|
|
68
|
-
type: 'object',
|
|
69
|
-
additionalProperties: false,
|
|
70
|
-
required: ['type', 'selector', 'body'],
|
|
71
|
-
properties: {
|
|
72
|
-
type: { type: 'string', enum: ['if-present'] },
|
|
73
|
-
selector: { type: 'string' },
|
|
74
|
-
body: { type: 'array', items: leafSchema() },
|
|
75
|
-
},
|
|
76
|
-
},
|
|
77
|
-
{
|
|
78
|
-
type: 'object',
|
|
79
|
-
additionalProperties: false,
|
|
80
|
-
required: ['type', 'selector', 'cap', 'body'],
|
|
81
|
-
properties: {
|
|
82
|
-
type: { type: 'string', enum: ['repeat'] },
|
|
83
|
-
selector: { type: 'string' },
|
|
84
|
-
cap: { type: 'integer' },
|
|
85
|
-
body: { type: 'array', items: leafSchema() },
|
|
86
|
-
},
|
|
87
|
-
},
|
|
88
|
-
],
|
|
89
|
-
},
|
|
184
|
+
items: stepItems(stepItems(LEAF_ONLY_ITEMS)),
|
|
90
185
|
},
|
|
91
186
|
},
|
|
92
187
|
};
|
|
@@ -107,6 +202,19 @@ exports.REPAIR_DECISION_JSON_SCHEMA = {
|
|
|
107
202
|
reason: { type: 'string' },
|
|
108
203
|
},
|
|
109
204
|
};
|
|
205
|
+
function readSchema() {
|
|
206
|
+
return {
|
|
207
|
+
type: 'object',
|
|
208
|
+
additionalProperties: false,
|
|
209
|
+
required: ['type', 'selector', 'field', 'into'],
|
|
210
|
+
properties: {
|
|
211
|
+
type: { type: 'string', enum: ['read'] },
|
|
212
|
+
selector: { type: 'string' },
|
|
213
|
+
field: { type: 'string', enum: ['text', 'desc', 'id', 'idShort'] },
|
|
214
|
+
into: { type: 'string' },
|
|
215
|
+
},
|
|
216
|
+
};
|
|
217
|
+
}
|
|
110
218
|
function leafSchema() {
|
|
111
219
|
return {
|
|
112
220
|
type: 'object',
|
|
@@ -141,11 +249,49 @@ function isFlagSpecArray(v) {
|
|
|
141
249
|
return (Array.isArray(v) &&
|
|
142
250
|
v.every((f) => f && typeof f === 'object' && typeof f.name === 'string' && typeof f.value === 'string'));
|
|
143
251
|
}
|
|
144
|
-
|
|
145
|
-
|
|
252
|
+
const READ_FIELDS = new Set(['text', 'desc', 'id', 'idShort']);
|
|
253
|
+
/** ctx keys are interpolated into selectors as {{ctx.NAME}}; keep them boring so a
|
|
254
|
+
* name can never smuggle regex/template metacharacters into a selector. */
|
|
255
|
+
const CTX_NAME_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
256
|
+
/** Validate a single node (used for both compile output and a spliced repair).
|
|
257
|
+
*
|
|
258
|
+
* `depth` is how many control nodes enclose this one. It is the ONLY thing bounding
|
|
259
|
+
* nesting at runtime, and it must agree with PLAN_JSON_SCHEMA's hand-unrolled levels
|
|
260
|
+
* — the schema stops a compliant model, this stops everything else (a repair, a
|
|
261
|
+
* hand-edited plan, a poisoned cache entry). */
|
|
262
|
+
function validateNode(node, where, depth = 0) {
|
|
146
263
|
if (!node || typeof node !== 'object')
|
|
147
264
|
throw new InvalidPlanError(`${where}: not an object`);
|
|
148
265
|
const n = node;
|
|
266
|
+
/** Validate one control body. `allowEmpty` is true only for `when`'s `else`. */
|
|
267
|
+
const validateBody = (raw, label, allowEmpty = false) => {
|
|
268
|
+
if (!Array.isArray(raw))
|
|
269
|
+
throw new InvalidPlanError(`${label}: body must be an array`);
|
|
270
|
+
if (raw.length === 0 && !allowEmpty) {
|
|
271
|
+
// Silence must never read as success — same rule parsePlan applies to the plan.
|
|
272
|
+
throw new InvalidPlanError(`${label}: body is empty (a control body must do something; use when's "else": [] to mean "do nothing")`);
|
|
273
|
+
}
|
|
274
|
+
return raw.map((b, i) => validateNode(b, `${label}[${i}]`, depth + 1));
|
|
275
|
+
};
|
|
276
|
+
/** `if-present` and `while-present` are allowed one level deeper than the branching
|
|
277
|
+
* nodes: they are the conditional and the index-walk an innermost branch body needs,
|
|
278
|
+
* and because their own bodies are leaves they cannot open a further level. Blocking
|
|
279
|
+
* them here does not make the model simplify — it makes it emit an unconditional
|
|
280
|
+
* approximation instead. See LEAF_ONLY_ITEMS. */
|
|
281
|
+
const guardDepth = (extra = 0) => {
|
|
282
|
+
const limit = exports.MAX_CONTROL_DEPTH + extra;
|
|
283
|
+
if (depth >= limit) {
|
|
284
|
+
throw new InvalidPlanError(`${where}: control nodes may nest ${limit} deep at most (got ${depth + 1}) — flatten this or lift it to an outer step`);
|
|
285
|
+
}
|
|
286
|
+
};
|
|
287
|
+
const selectorOf = () => {
|
|
288
|
+
if (typeof n.selector !== 'string' || !n.selector)
|
|
289
|
+
throw new InvalidPlanError(`${where}: ${n.type} needs a selector`);
|
|
290
|
+
return n.selector;
|
|
291
|
+
};
|
|
292
|
+
// OpenAI/codex strict mode forces every property into `required` and nullifies the
|
|
293
|
+
// optional ones, so an absent `else`/`bind` arrives as null rather than missing.
|
|
294
|
+
const optional = (v, pick) => v === undefined || v === null ? undefined : pick(v);
|
|
149
295
|
switch (n.type) {
|
|
150
296
|
case 'command': {
|
|
151
297
|
if (typeof n.command !== 'string' || !exports.KNOWN_COMMANDS.has(n.command)) {
|
|
@@ -158,22 +304,50 @@ function validateNode(node, where) {
|
|
|
158
304
|
throw new InvalidPlanError(`${where}: flags must be {name,value}[]`);
|
|
159
305
|
return { type: 'command', command: n.command, positionals: n.positionals, flags: n.flags };
|
|
160
306
|
}
|
|
161
|
-
case '
|
|
162
|
-
|
|
163
|
-
if (typeof n.
|
|
164
|
-
throw new InvalidPlanError(`${where}: ${
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
307
|
+
case 'read': {
|
|
308
|
+
const selector = selectorOf();
|
|
309
|
+
if (typeof n.field !== 'string' || !READ_FIELDS.has(n.field)) {
|
|
310
|
+
throw new InvalidPlanError(`${where}: read needs field one of ${[...READ_FIELDS].join('|')} (got ${JSON.stringify(n.field)})`);
|
|
311
|
+
}
|
|
312
|
+
if (typeof n.into !== 'string' || !CTX_NAME_RE.test(n.into)) {
|
|
313
|
+
throw new InvalidPlanError(`${where}: read "into" must be a plain identifier (got ${JSON.stringify(n.into)})`);
|
|
314
|
+
}
|
|
315
|
+
return { type: 'read', selector, field: n.field, into: n.into };
|
|
316
|
+
}
|
|
317
|
+
case 'if-present': {
|
|
318
|
+
guardDepth(1);
|
|
319
|
+
return { type: 'if-present', selector: selectorOf(), body: validateBody(n.body, `${where}.body`) };
|
|
320
|
+
}
|
|
321
|
+
case 'when': {
|
|
322
|
+
guardDepth();
|
|
323
|
+
if (!Array.isArray(n.branches) || n.branches.length === 0) {
|
|
324
|
+
throw new InvalidPlanError(`${where}: when needs at least one branch`);
|
|
325
|
+
}
|
|
326
|
+
const branches = n.branches.map((b, i) => {
|
|
327
|
+
const br = b;
|
|
328
|
+
if (!br || typeof br !== 'object' || typeof br.selector !== 'string' || !br.selector) {
|
|
329
|
+
throw new InvalidPlanError(`${where}.branches[${i}]: needs a selector`);
|
|
330
|
+
}
|
|
331
|
+
return { selector: br.selector, body: validateBody(br.body, `${where}.branches[${i}].body`) };
|
|
172
332
|
});
|
|
173
|
-
|
|
174
|
-
|
|
333
|
+
const els = optional(n.else, (v) => validateBody(v, `${where}.else`, true));
|
|
334
|
+
return { type: 'when', branches, ...(els ? { else: els } : {}) };
|
|
335
|
+
}
|
|
336
|
+
case 'repeat':
|
|
337
|
+
case 'while-present': {
|
|
338
|
+
guardDepth(n.type === 'while-present' ? 1 : 0);
|
|
339
|
+
const selector = selectorOf();
|
|
175
340
|
const cap = typeof n.cap === 'number' && n.cap > 0 ? Math.floor(n.cap) : exports.DEFAULT_LOOP_CAP;
|
|
176
|
-
|
|
341
|
+
const body = validateBody(n.body, `${where}.body`);
|
|
342
|
+
if (n.type === 'repeat')
|
|
343
|
+
return { type: 'repeat', selector, cap, body };
|
|
344
|
+
const bind = optional(n.bind, (v) => {
|
|
345
|
+
if (typeof v !== 'string' || !CTX_NAME_RE.test(v)) {
|
|
346
|
+
throw new InvalidPlanError(`${where}: while-present "bind" must be a plain identifier (got ${JSON.stringify(v)})`);
|
|
347
|
+
}
|
|
348
|
+
return v;
|
|
349
|
+
});
|
|
350
|
+
return { type: 'while-present', selector, cap, body, ...(bind ? { bind } : {}) };
|
|
177
351
|
}
|
|
178
352
|
default:
|
|
179
353
|
throw new InvalidPlanError(`${where}: unknown node type ${JSON.stringify(n.type)}`);
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Compile-fidelity lint: does the plan the model produced still say what the prose said?
|
|
3
|
+
//
|
|
4
|
+
// The model is a compiler, and this is the compiler's own sanity check. It exists because
|
|
5
|
+
// compilation is NONDETERMINISTIC in a way that is invisible until a run fails, and two
|
|
6
|
+
// failure modes showed up repeatedly against a real suite:
|
|
7
|
+
//
|
|
8
|
+
// 1. An explicit directive silently vanishes. The same prose ("Launch the app WITH ITS
|
|
9
|
+
// DATA CLEARED so it starts logged-out") compiled to `launch <pkg> --clear` on one
|
|
10
|
+
// run and plain `launch <pkg>` on the next. The plan looked fine and the test failed
|
|
11
|
+
// several steps later, on a screen that only appears when you are already logged in.
|
|
12
|
+
// 2. Conditional prose compiles to an unconditional step. "IF a Continue button appears,
|
|
13
|
+
// tap it" becomes `wait` + `tap`, which FAILS on every run where the app skips it.
|
|
14
|
+
//
|
|
15
|
+
// Both are cheap to detect and cheap to fix: hand the finding back to the model and let it
|
|
16
|
+
// compile once more. That is far better than the alternative, which is a plan that is
|
|
17
|
+
// quietly wrong and burns a device run to say so.
|
|
18
|
+
//
|
|
19
|
+
// Deliberately CONSERVATIVE — a false positive costs a wasted recompile, so every rule
|
|
20
|
+
// requires a fairly unambiguous phrase and checks for a specific structural counterpart.
|
|
21
|
+
// It never edits the plan; the model gets the feedback and stays the author.
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
exports.lintPlan = lintPlan;
|
|
24
|
+
const ir_1 = require("./ir");
|
|
25
|
+
/** Walk every node in the plan, including control-node bodies. */
|
|
26
|
+
function* walk(nodes) {
|
|
27
|
+
for (const n of nodes) {
|
|
28
|
+
yield n;
|
|
29
|
+
if ((0, ir_1.isControlNode)(n))
|
|
30
|
+
for (const body of (0, ir_1.bodiesOf)(n))
|
|
31
|
+
yield* walk(body);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const hasControlNode = (plan) => {
|
|
35
|
+
for (const n of walk(plan.steps))
|
|
36
|
+
if ((0, ir_1.isControlNode)(n))
|
|
37
|
+
return true;
|
|
38
|
+
return false;
|
|
39
|
+
};
|
|
40
|
+
const hasLeafWithFlag = (plan, command, flag) => {
|
|
41
|
+
for (const n of walk(plan.steps)) {
|
|
42
|
+
if (n.type === 'command' && n.command === command && n.flags.some((f) => f.name === flag))
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
return false;
|
|
46
|
+
};
|
|
47
|
+
/** Phrases that state the app must start from a clean slate. Kept tight on purpose:
|
|
48
|
+
* "clear" alone is far too common (clearing a text field, a clear button). */
|
|
49
|
+
const FRESH_START_RE = /\b(data cleared|cleared data|clear its data|with its data cleared|freshly installed|logged[- ]out|logged out|clean state|from scratch)\b/i;
|
|
50
|
+
/** Phrases that make a step conditional. These are the ones that produced an
|
|
51
|
+
* unconditional step in practice; vaguer hedges ("should", "normally") are excluded. */
|
|
52
|
+
const CONDITIONAL_RE = /\b(if (?:a|an|the|any)\b|may appear|might appear|sometimes|optionally|if present|if shown|if it appears|dismiss any)\b/i;
|
|
53
|
+
/**
|
|
54
|
+
* Check a compiled plan against the prose it came from.
|
|
55
|
+
*
|
|
56
|
+
* @param nl the natural-language test, verbatim
|
|
57
|
+
* @param plan the plan the model just produced
|
|
58
|
+
* @returns findings; empty means the plan is consistent with the prose
|
|
59
|
+
*/
|
|
60
|
+
function lintPlan(nl, plan) {
|
|
61
|
+
const findings = [];
|
|
62
|
+
if (FRESH_START_RE.test(nl) && !hasLeafWithFlag(plan, 'launch', 'clear')) {
|
|
63
|
+
findings.push({
|
|
64
|
+
message: 'The test says the app must start from cleared data / logged out, but no step launches it with --clear. ' +
|
|
65
|
+
'Emit `launch <package> --clear` so the run does not inherit the previous run\'s session.',
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
if (CONDITIONAL_RE.test(nl) && !hasControlNode(plan)) {
|
|
69
|
+
findings.push({
|
|
70
|
+
message: 'The test describes something that may or may not appear ("if ...", "may appear", "dismiss any ..."), ' +
|
|
71
|
+
'but the plan has no if-present/when node — every step is unconditional. An unconditional step for ' +
|
|
72
|
+
'optional UI fails on every run where that UI does not show. Put the optional part behind if-present ' +
|
|
73
|
+
'(skip when absent), or behind when (when the screen is one of several known kinds).',
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
return findings;
|
|
77
|
+
}
|
package/dist/agent/openai.js
CHANGED
|
@@ -116,6 +116,12 @@ class OpenAiProvider {
|
|
|
116
116
|
JSON.stringify(input.seed, null, 2));
|
|
117
117
|
}
|
|
118
118
|
parts.push('NATURAL-LANGUAGE TEST:\n' + input.nl);
|
|
119
|
+
if (input.retryFeedback) {
|
|
120
|
+
// Last, so it is the freshest thing in context: a previous compile of this same
|
|
121
|
+
// test lost something the prose stated. Naming it beats hoping the retry differs.
|
|
122
|
+
parts.push('YOUR PREVIOUS ATTEMPT AT THIS TEST WAS REJECTED. Fix this and emit the whole plan again:\n' +
|
|
123
|
+
input.retryFeedback);
|
|
124
|
+
}
|
|
119
125
|
// Generous completion budget: on reasoning models the plan JSON shares this ceiling
|
|
120
126
|
// with reasoning tokens, so leave headroom (a 'length' finish is surfaced as an error).
|
|
121
127
|
const { json, usage } = await this.call(grammar_1.GRAMMAR, parts.join('\n\n'), ir_1.PLAN_JSON_SCHEMA, 16384);
|