pi-ui-extend 1.0.23 → 1.0.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +5 -1
  2. package/dist/app/app.d.ts +1 -0
  3. package/dist/app/app.js +12 -0
  4. package/dist/app/constants.d.ts +1 -1
  5. package/dist/app/constants.js +1 -1
  6. package/dist/app/icons.d.ts +1 -0
  7. package/dist/app/icons.js +2 -0
  8. package/dist/app/input/input-controller.d.ts +2 -0
  9. package/dist/app/input/input-controller.js +8 -5
  10. package/dist/app/rendering/render-controller.js +2 -0
  11. package/dist/app/rendering/status-line-renderer.d.ts +4 -1
  12. package/dist/app/rendering/status-line-renderer.js +12 -0
  13. package/dist/app/runtime.d.ts +1 -8
  14. package/dist/app/runtime.js +0 -39
  15. package/dist/app/screen/mouse-controller.d.ts +4 -1
  16. package/dist/app/screen/mouse-controller.js +9 -0
  17. package/dist/app/session/agent-pause-controller.d.ts +30 -0
  18. package/dist/app/session/agent-pause-controller.js +178 -0
  19. package/dist/app/session/session-event-controller.d.ts +1 -0
  20. package/dist/app/session/session-event-controller.js +22 -15
  21. package/dist/app/session/session-lifecycle-controller.d.ts +1 -0
  22. package/dist/app/session/session-lifecycle-controller.js +1 -0
  23. package/dist/app/types.d.ts +10 -0
  24. package/dist/config.js +1 -1
  25. package/dist/default-pix-config.js +2 -2
  26. package/external/pi-tools-suite/README.md +1 -1
  27. package/external/pi-tools-suite/docs/browser-qa-subagent.md +34 -2
  28. package/external/pi-tools-suite/package.json +3 -3
  29. package/external/pi-tools-suite/src/async-subagents/async-subagents.sample.jsonc +24 -18
  30. package/external/pi-tools-suite/src/async-subagents/core/config.ts +14 -3
  31. package/external/pi-tools-suite/src/async-subagents/core/process.ts +19 -0
  32. package/external/pi-tools-suite/src/async-subagents/core/spawn.ts +97 -17
  33. package/external/pi-tools-suite/src/async-subagents/core/stop.ts +4 -2
  34. package/external/pi-tools-suite/src/async-subagents/private-skills/browser-qa/SKILL.md +108 -13
  35. package/external/pi-tools-suite/src/async-subagents/private-skills/browser-qa/references/qa-design.md +80 -0
  36. package/external/pi-tools-suite/src/async-subagents/private-skills/browser-qa/references/qa-flow.example.jsonc +54 -1
  37. package/external/pi-tools-suite/src/async-subagents/private-skills/browser-qa/scripts/browser-qa-runner.mjs +952 -76
  38. package/external/pi-tools-suite/src/coding-discipline/index.ts +2 -94
  39. package/external/pi-tools-suite/src/dcp/index.ts +21 -0
  40. package/external/pi-tools-suite/src/dcp/pruner-compression-blocks.ts +92 -0
  41. package/external/pi-tools-suite/src/default-pi-tools-suite-config.ts +21 -16
  42. package/external/pi-tools-suite/src/todo/index.ts +86 -46
  43. package/external/pi-tools-suite/src/todo/todo.ts +1 -1
  44. package/package.json +4 -4
@@ -6,7 +6,7 @@ import { selectSuitableToolsForModel } from "../../lib/tool-args.js";
6
6
  import { validateBasename } from "./paths.js";
7
7
  import { getPiInvocation } from "./pi-invocation.js";
8
8
  import { writePromptFile } from "./prompt.js";
9
- import { terminateProcess } from "./process.js";
9
+ import { terminateProcessTree } from "./process.js";
10
10
  import { getAgentSessionDir, SUBAGENT_PARENT_SESSION_FILE, SUBAGENT_RETURN_SESSION_FILE, SUBAGENT_SESSION_FILE, writeParentSessionLink, writeSessionFileLink } from "./sessions.js";
11
11
  import { getAgentState } from "./state.js";
12
12
  import { writeStructuredResult } from "./structured-result.js";
@@ -31,6 +31,7 @@ const AGENT_TIMEOUT_KILL_GRACE_MS = 5_000;
31
31
  const AGENT_SETTLED_TERMINATE_GRACE_MS = 50;
32
32
  const AGENT_SETTLED_COMPLETION_FALLBACK_MS = 1_000;
33
33
  const EXIT_STDIO_FLUSH_GRACE_MS = 10;
34
+ const PROGRESS_LOG_MAX_BYTES = 1024 * 1024;
34
35
 
35
36
  export function shouldPersistSubagentSessions(env: NodeJS.ProcessEnv = process.env): boolean {
36
37
  return isTruthyEnv(env.ASYNC_SUBAGENTS_ENABLE_SESSIONS);
@@ -136,11 +137,16 @@ export function spawnAgent(
136
137
  const invocation = getPiInvocation(piArgs);
137
138
  const stderrStream = createDeferredFileWriter(stderrFile, logLimits.stderrMaxBytes, "stderr.log");
138
139
  const transcriptStream = createBoundedFileWriter(transcriptFile, logLimits.eventsMaxBytes, "events.jsonl");
140
+ const progressStream = createBoundedFileWriter(path.join(agentDir, "progress.jsonl"), PROGRESS_LOG_MAX_BYTES, "progress.jsonl");
141
+ const writeProgress = (stage: string, details: Record<string, unknown> = {}) => {
142
+ progressStream.write(serializeJsonLine({ at: isoNow(), stage, ...details }));
143
+ };
139
144
 
140
145
  const proc = spawn(invocation.command, invocation.args, {
141
146
  cwd,
142
147
  env: subagentEnvironment(process.env, task.subagentType === "browser-qa" ? agentDir : undefined),
143
148
  stdio: ["pipe", "pipe", "pipe"],
149
+ detached: process.platform !== "win32",
144
150
  });
145
151
  proc.stdin.on("error", (error: NodeJS.ErrnoException) => {
146
152
  if (error.code === "EPIPE") return;
@@ -156,18 +162,33 @@ export function spawnAgent(
156
162
  let timeoutKillTimer: NodeJS.Timeout | undefined;
157
163
  let agentSettledKillTimer: NodeJS.Timeout | undefined;
158
164
  let agentSettledCompletionFallbackTimer: NodeJS.Timeout | undefined;
165
+ let processExitTreeKillTimer: NodeJS.Timeout | undefined;
159
166
  let exitFinalizationTimer: NodeJS.Timeout | undefined;
167
+ let processTermination: { code: number | null; signal: NodeJS.Signals | null } | undefined;
168
+ let stdoutEnded = false;
160
169
  const suppressedRpcEventCounts = new Map<string, number>();
170
+ const scheduleProcessTreeKill = (reason: string) => {
171
+ if (processExitTreeKillTimer) return;
172
+ processExitTreeKillTimer = setTimeout(() => {
173
+ try {
174
+ writeProgress("shutdown_signal", { reason, signal: "SIGKILL" });
175
+ terminateChildProcessTree(proc, "SIGKILL");
176
+ } catch {
177
+ /* process group may already be gone */
178
+ }
179
+ }, AGENT_SETTLED_COMPLETION_FALLBACK_MS);
180
+ processExitTreeKillTimer.unref?.();
181
+ };
161
182
 
162
183
  const notifyComplete = (exitCode: number) => {
163
184
  if (completionNotified) return;
164
185
  if (exitCode !== 0) shouldKeepStderr = true;
165
186
  completionNotified = true;
166
187
  if (timeoutTimer) clearTimeout(timeoutTimer);
167
- if (timeoutKillTimer) clearTimeout(timeoutKillTimer);
168
188
  if (agentSettledKillTimer) clearTimeout(agentSettledKillTimer);
169
- if (agentSettledCompletionFallbackTimer) clearTimeout(agentSettledCompletionFallbackTimer);
170
189
  if (exitFinalizationTimer) clearTimeout(exitFinalizationTimer);
190
+ writeProgress("completed", { exitCode });
191
+ progressStream.end();
171
192
  if (!fs.existsSync(agentDir)) {
172
193
  onComplete?.({
173
194
  runDir,
@@ -207,6 +228,7 @@ export function spawnAgent(
207
228
 
208
229
  const finalizeCompletion = (code: number | null, signal: NodeJS.Signals | null) => {
209
230
  if (completionNotified) return;
231
+ if (!timeoutKillTimer && !agentSettledCompletionFallbackTimer) scheduleProcessTreeKill("process_exit");
210
232
  writeSuppressedRpcEventSummary(transcriptStream, suppressedRpcEventCounts);
211
233
  const exitCode = resolveAgentExitCode({
212
234
  timedOut,
@@ -233,19 +255,21 @@ export function spawnAgent(
233
255
  agentSettledKillTimer = setTimeout(() => {
234
256
  agentSettledKillTimer = undefined;
235
257
  try {
236
- terminateChildProcess(proc, "SIGTERM");
258
+ writeProgress("shutdown_signal", { reason: "agent_settled", signal: "SIGTERM" });
259
+ terminateChildProcessTree(proc, "SIGTERM");
237
260
  } catch {
238
261
  /* process may have exited before the graceful termination timer fired */
239
262
  }
240
263
  }, AGENT_SETTLED_TERMINATE_GRACE_MS);
241
264
  agentSettledKillTimer.unref?.();
242
265
  agentSettledCompletionFallbackTimer = setTimeout(() => {
243
- if (completionNotified) return;
244
266
  try {
245
- terminateChildProcess(proc, "SIGKILL");
267
+ writeProgress("shutdown_signal", { reason: "agent_settled", signal: "SIGKILL" });
268
+ terminateChildProcessTree(proc, "SIGKILL");
246
269
  } catch {
247
270
  /* process may already be gone */
248
271
  }
272
+ if (completionNotified) return;
249
273
  proc.stdin.destroy();
250
274
  proc.stdout?.destroy();
251
275
  proc.stderr?.destroy();
@@ -261,6 +285,7 @@ export function spawnAgent(
261
285
  if (completionNotified) return;
262
286
  timedOut = true;
263
287
  const timeoutMessage = `Sub-agent timed out after ${Math.round(timeoutMs / 1000)} seconds.`;
288
+ writeProgress("timeout", { timeoutMs });
264
289
  if (fs.existsSync(agentDir)) {
265
290
  fs.writeFileSync(path.join(agentDir, "timeout_ms"), String(timeoutMs), "utf-8");
266
291
  fs.writeFileSync(path.join(agentDir, "timed_out_at"), isoNow(), "utf-8");
@@ -269,14 +294,15 @@ export function spawnAgent(
269
294
  stderrStream.write(`${timeoutMessage}\n`);
270
295
  }
271
296
  try {
272
- terminateChildProcess(proc, "SIGTERM");
297
+ writeProgress("shutdown_signal", { reason: "timeout", signal: "SIGTERM" });
298
+ terminateChildProcessTree(proc, "SIGTERM");
273
299
  } catch {
274
300
  /* process may have exited between the timer and signal */
275
301
  }
276
302
  timeoutKillTimer = setTimeout(() => {
277
- if (completionNotified) return;
278
303
  try {
279
- terminateChildProcess(proc, "SIGKILL");
304
+ writeProgress("shutdown_signal", { reason: "timeout", signal: "SIGKILL" });
305
+ terminateChildProcessTree(proc, "SIGKILL");
280
306
  } catch {
281
307
  /* process may have exited after SIGTERM */
282
308
  }
@@ -297,6 +323,8 @@ export function spawnAgent(
297
323
  const event = JSON.parse(line) as RpcEventRecord;
298
324
  const storedEvent = compactRpcEventForTranscript(event, Buffer.byteLength(line, "utf8"));
299
325
  if (storedEvent) transcriptStream.write(serializeJsonLine(storedEvent));
326
+ const progressEvent = compactRpcEventForProgress(event);
327
+ if (progressEvent) writeProgress("rpc_event", progressEvent);
300
328
  onRpcEvent?.(event);
301
329
  const sessionFile = extractSessionFileFromEvent(event);
302
330
  if (sessionFile) writeSessionFileLink(agentDir, sessionFile);
@@ -307,8 +335,13 @@ export function spawnAgent(
307
335
  if (event.type === "response" && event.command === "prompt" && event.success === false) {
308
336
  const errorText = typeof event.error === "string" ? event.error : "RPC prompt failed";
309
337
  fs.writeFileSync(path.join(agentDir, "result.md"), errorText, "utf-8");
338
+ try {
339
+ terminateChildProcessTree(proc, "SIGTERM");
340
+ } catch {
341
+ /* process may have exited immediately after emitting the failure */
342
+ }
343
+ scheduleProcessTreeKill("prompt_failed");
310
344
  notifyComplete(1);
311
- terminateChildProcess(proc, "SIGTERM");
312
345
  return;
313
346
  }
314
347
  if (event.type === "agent_end") {
@@ -359,10 +392,29 @@ export function spawnAgent(
359
392
  },
360
393
  });
361
394
 
362
- proc.once("exit", (code, signal) => {
363
- exitFinalizationTimer = setTimeout(() => finalizeCompletion(code, signal), EXIT_STDIO_FLUSH_GRACE_MS);
395
+ // Bun on Windows may emit the ChildProcess `close` event before the stdout
396
+ // Readable has delivered its final buffered `data`/`end` events. Gate process
397
+ // finalization on stdout itself so a trailing RPC failure cannot be mistaken
398
+ // for a clean exit.
399
+ const finalizeAfterStdout = () => {
400
+ if (!processTermination || !stdoutEnded || completionNotified || exitFinalizationTimer) return;
401
+ const { code, signal } = processTermination;
402
+ exitFinalizationTimer = setTimeout(
403
+ () => finalizeCompletion(code, signal),
404
+ EXIT_STDIO_FLUSH_GRACE_MS,
405
+ );
364
406
  exitFinalizationTimer.unref?.();
407
+ };
408
+ const recordProcessTermination = (code: number | null, signal: NodeJS.Signals | null) => {
409
+ processTermination ??= { code, signal };
410
+ finalizeAfterStdout();
411
+ };
412
+ proc.stdout.once("end", () => {
413
+ stdoutEnded = true;
414
+ finalizeAfterStdout();
365
415
  });
416
+ proc.once("exit", recordProcessTermination);
417
+ proc.once("close", recordProcessTermination);
366
418
 
367
419
  proc.once("error", (error) => {
368
420
  const message = String(error);
@@ -376,6 +428,11 @@ export function spawnAgent(
376
428
  notifyComplete(1);
377
429
  });
378
430
 
431
+ const pid = proc.pid!;
432
+ fs.writeFileSync(path.join(agentDir, "pid"), String(pid), "utf-8");
433
+ if (process.platform !== "win32") fs.writeFileSync(path.join(agentDir, "process_group"), String(pid), "utf-8");
434
+ writeProgress("spawned", { pid });
435
+
379
436
  proc.stdin.write([
380
437
  serializeJsonLine({
381
438
  id: "sub_get_state",
@@ -388,6 +445,7 @@ export function spawnAgent(
388
445
  ...(promptImages ? { images: promptImages } : {}),
389
446
  }),
390
447
  ].join(""));
448
+ writeProgress("prompt_sent");
391
449
  // Keep stdin open while the RPC prompt is running. pi RPC mode treats stdin
392
450
  // EOF as a shutdown request, while the prompt command itself is handled
393
451
  // asynchronously after preflight. Closing stdin here can therefore terminate
@@ -395,9 +453,6 @@ export function spawnAgent(
395
453
  // producing exit 0 with no result.md. The child is terminated explicitly after agent_settled,
396
454
  // timeout, stop, or process error.
397
455
 
398
- const pid = proc.pid!;
399
- fs.writeFileSync(path.join(agentDir, "pid"), String(pid), "utf-8");
400
-
401
456
  return { pid, agentDir, process: proc };
402
457
  }
403
458
 
@@ -423,9 +478,9 @@ function getAntigravityAuthExtensionPath(): string {
423
478
  return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "antigravity-auth", "index.ts");
424
479
  }
425
480
 
426
- function terminateChildProcess(proc: ChildProcess, signal: NodeJS.Signals): void {
481
+ function terminateChildProcessTree(proc: ChildProcess, signal: NodeJS.Signals): void {
427
482
  if (proc.pid) {
428
- terminateProcess(proc.pid, signal as "SIGTERM" | "SIGINT" | "SIGKILL");
483
+ terminateProcessTree(proc.pid, signal as "SIGTERM" | "SIGINT" | "SIGKILL");
429
484
  return;
430
485
  }
431
486
  proc.kill(signal);
@@ -603,6 +658,31 @@ function compactRpcEventForTranscript(event: RpcEventRecord, originalBytes: numb
603
658
  return { type: event.type, bytes: originalBytes };
604
659
  }
605
660
 
661
+ function compactRpcEventForProgress(event: RpcEventRecord): Record<string, unknown> | undefined {
662
+ if (event.type === "message_update" || event.type === "tool_execution_update") return undefined;
663
+ if (event.type === "response") {
664
+ return stripUndefined({
665
+ type: event.type,
666
+ command: typeof event.command === "string" ? event.command : undefined,
667
+ success: typeof event.success === "boolean" ? event.success : undefined,
668
+ });
669
+ }
670
+ if (event.type === "tool_execution_start" || event.type === "tool_execution_end") {
671
+ return stripUndefined({
672
+ type: event.type,
673
+ toolName: typeof event.toolName === "string" ? event.toolName : undefined,
674
+ });
675
+ }
676
+ if (event.type === "message_end") {
677
+ return stripUndefined({
678
+ type: event.type,
679
+ role: isRecord(event.message) && typeof event.message.role === "string" ? event.message.role : undefined,
680
+ stopReason: isRecord(event.message) && typeof event.message.stopReason === "string" ? event.message.stopReason : undefined,
681
+ });
682
+ }
683
+ return { type: event.type };
684
+ }
685
+
606
686
  function suppressedRpcEventType(line: string): string | undefined {
607
687
  if (line.includes('"type":"message_update"') || line.includes('"type": "message_update"')) return "message_update";
608
688
  if (line.includes('"type":"tool_execution_update"') || line.includes('"type": "tool_execution_update"')) return "tool_execution_update";
@@ -1,7 +1,7 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
3
  import { getAgentState, getRunState } from "./state.js";
4
- import { terminateProcess } from "./process.js";
4
+ import { terminateProcess, terminateProcessTree } from "./process.js";
5
5
  import { writeStructuredResult } from "./structured-result.js";
6
6
  import type { AgentState } from "./types.js";
7
7
  import { isoNow } from "./utils.js";
@@ -70,7 +70,9 @@ function stopAgent(runDir: string, agent: AgentState, signal: StopSignal): StopA
70
70
  }
71
71
 
72
72
  try {
73
- terminateProcess(agent.pid, signal);
73
+ const ownsProcessGroup = fs.existsSync(path.join(runDir, agent.id, "process_group"));
74
+ if (ownsProcessGroup) terminateProcessTree(agent.pid, signal);
75
+ else terminateProcess(agent.pid, signal);
74
76
  markStopped(runDir, agent.id, signal);
75
77
  return {
76
78
  ...result,
@@ -15,6 +15,13 @@ Never read, print, grep, copy, or edit credential values from
15
15
 
16
16
  ## Workflow
17
17
 
18
+ Treat target discovery as a 30-second preflight and invoke the runner within 45
19
+ seconds of starting. Use at most one runner invocation unless the task explicitly
20
+ requests multiple auth profiles. If you cannot identify a reachable target and
21
+ a supported deterministic assertion inside that preflight, return a structured
22
+ `BLOCKED` result immediately. Do not consume the launcher budget on further
23
+ source reading, server polling, capability probing, or retries.
24
+
18
25
  1. Resolve `scripts/browser-qa-runner.mjs` relative to this skill.
19
26
  2. Use the launcher-provided `$PI_SUBAGENT_AGENT_DIR/browser-qa/` workspace.
20
27
  The launcher creates its private `flows/` directory and the runner rejects
@@ -30,8 +37,9 @@ Never read, print, grep, copy, or edit credential values from
30
37
  safe profile traits make the choice unambiguous, or a public run proves that
31
38
  the requested page requires login.
32
39
  5. Inspect the target code and write a declarative JSONC flow under
33
- `$PI_SUBAGENT_AGENT_DIR/browser-qa/flows/`. Never put credentials or
34
- executable JavaScript in it.
40
+ `$PI_SUBAGENT_AGENT_DIR/browser-qa/flows/`. Never put credentials or raw
41
+ executable JavaScript in it. The `evaluate` action exposes only the safe
42
+ operations documented below; it does not accept expressions or scripts.
35
43
  6. Run public QA with
36
44
  `node <runner> run --base-url <url> --flow <flow.jsonc>`. The URL's exact
37
45
  origin becomes the fail-closed allowlist. Only for authenticated QA, add
@@ -39,7 +47,7 @@ Never read, print, grep, copy, or edit credential values from
39
47
  Profile id, URL, and flow path are non-secret; never pass credentials as
40
48
  arguments or environment variables.
41
49
  7. Report deterministic assertions and every artifact returned by the runner.
42
- For each screenshot, video, or trace, emit a separate clickable Markdown
50
+ For each screenshot, video, trace, or retained download, emit a separate clickable Markdown
43
51
  link using its `uri` and also show its absolute `path`. Do this for failed
44
52
  runs too whenever `artifacts` is present; never report only `evidenceDir`.
45
53
  Visual inspection supplements assertions; it does not replace them.
@@ -56,9 +64,10 @@ Never read, print, grep, copy, or edit credential values from
56
64
  plus accessible `name`; `label`; `placeholder`; visible `text`; CSS only as a
57
65
  last resort. Use `exact: true` when similar elements could make a match
58
66
  ambiguous.
59
- - Let locator actions auto-wait. Use `waitFor` for an explicit UI state and use
60
- `waitForTimeout` only for a short, unavoidable animation/debounce—not as a
61
- substitute for an assertion. Set flow `timeoutMs` only as high as the target
67
+ - Let locator actions auto-wait. Assertions retry until the flow timeout, so
68
+ prefer them over a preceding sleep. Use `waitFor` for an explicit setup state
69
+ and `waitForTimeout` only for short input settling or an unavoidable
70
+ animation/debounce. Set flow `timeoutMs` only as high as the target
62
71
  legitimately needs.
63
72
  - Place `authRejectedIf` immediately after navigation or any transition that
64
73
  may reveal expired authentication.
@@ -71,22 +80,97 @@ ambiguous failure, or deciding what evidence proves the result.
71
80
 
72
81
  ## Flow contract
73
82
 
74
- The flow is `{ "steps": [...] }` with at most 100 steps. Supported actions:
83
+ The flow is `{ "steps": [...] }`, no larger than 16 MiB, with at most 100
84
+ steps. The larger bound exists only for bounded in-memory upload payloads.
85
+ Supported actions:
75
86
 
76
87
  - navigation: `goto`, `reload`, `waitFor`, `waitForTimeout`
77
88
  - interaction: `click`, `doubleClick`, `hover`, `fill`, `press`, `check`,
78
- `uncheck`, `selectOption`
89
+ `uncheck`, `selectOption`, `wheel`, `evaluate`, `dragTo`, `uploadFiles`,
90
+ `openPopup`, `download`
79
91
  - assertions: `assertVisible`, `assertHidden`, `assertEnabled`,
80
92
  `assertDisabled`, `assertChecked`, `assertUnchecked`, `assertText`,
81
- `assertValue`, `assertCount`, `assertURL`
93
+ `assertValue`, `assertAttribute`, `assertCount`, `assertURL`,
94
+ `assertDOMMetric`
82
95
  - evidence/auth: `screenshot`, `authRejectedIf`
83
96
 
84
97
  Locators accept one of `testId`, `role` (plus optional `name`), `label`,
85
98
  `placeholder`, `text`, or `css`; add `exact: true` where useful. String
86
99
  assertions require exactly one of `equals` or `includes`.
100
+ `assertAttribute` additionally requires a bounded `attribute` name and is
101
+ useful for `aria-*`, `data-*`, `href`, and similar observable state. All
102
+ assertions retry until `timeoutMs` and report generic failures without exposing
103
+ the actual text, value, or attribute content.
104
+
105
+ Set an optional top-level `viewport` with integer `width` and `height` from
106
+ `320×240` through `3840×2160`; the default is `1280×720`. The same dimensions
107
+ are used for the browser viewport and recorded video.
108
+
109
+ The top-level `environment` may set `locale`, `timezoneId`, `colorScheme`, and
110
+ `reducedMotion`. Defaults are deterministic: `en-US`, `UTC`, `light`, and
111
+ `reduce`. Color scheme accepts `light`, `dark`, or `no-preference`; reduced
112
+ motion accepts `reduce` or `no-preference`. The resolved environment is returned
113
+ in the result alongside the viewport.
114
+
115
+ Triggering interactions (`click`, `doubleClick`, `press`, `check`, `uncheck`,
116
+ and `selectOption`) may declare race-free expectations that are armed before
117
+ the interaction:
118
+
119
+ - `expectResponse`: exact origin-relative `path` (without query/fragment),
120
+ uppercase `method`, and integer `status` from 100 through 599;
121
+ - `expectDialog`: `type` (`alert`, `beforeunload`, `confirm`, or `prompt`), a
122
+ nested `message` matcher with exactly one of `equals`/`includes`, and boolean
123
+ `accept`.
124
+
125
+ The runner never records response bodies, headers, URLs, actual dialog text, or
126
+ prompt defaults in observations or failure reasons. A mismatching dialog is
127
+ dismissed so it cannot deadlock the browser.
128
+
129
+ `dragTo` requires a source `locator` and `dropTarget`, with optional bounded
130
+ `sourcePosition` and `dropPosition` `{ x, y }`. `uploadFiles` accepts only
131
+ in-memory entries `{ name, mimeType, base64 }`; up to 10 files, 5 MiB each and
132
+ 10 MiB total. An empty array clears the input. Filesystem paths and directories
133
+ are not supported.
134
+
135
+ `download` atomically clicks its locator and requires a nested `filename`
136
+ matcher. `maxBytes` defaults to 5 MiB and is capped at 25 MiB. Downloads are
137
+ deleted after validation unless `retain: true` and a safe `name` are supplied;
138
+ retained bytes appear in `artifacts.downloads` under a runner-generated `.bin`
139
+ name. The runner cancels while its private copy grows past `maxBytes`, but this
140
+ is an evidence-retention bound rather than a network-bandwidth guarantee because
141
+ the browser may already hold temporary bytes. Retained bytes are scanned for
142
+ configured authentication before publication. The actual server filename is
143
+ never placed in diagnostics.
144
+
145
+ `openPopup` atomically clicks a locator, captures a same-origin popup, and stores
146
+ it under a safe `name` (maximum three). Target later actions with
147
+ `{ "target": { "type": "popup", "name": "..." } }`. Target same-origin
148
+ iframes with `{ "target": { "type": "frame", "locator": { ... } } }`; add
149
+ `page` with a popup name for a frame inside that popup. Frame origin is checked
150
+ from its live document before every scoped step. Cross-origin popups/frames are
151
+ rejected. Popup recordings are returned as separate video artifacts.
152
+
153
+ `wheel` accepts finite `deltaX`/`deltaY` values and requires at least one
154
+ non-zero delta. With a locator, the runner hovers that element before sending
155
+ the wheel input. Safe `evaluate` operations are:
156
+
157
+ - `scrollTo`: optional locator plus numeric `x`/`y` or the string `"max"`;
158
+ - `scrollBy`: optional locator plus numeric `deltaX`/`deltaY`;
159
+ - `metrics`: optional locator plus an optional safe `name`; values are returned
160
+ in the runner's `observations` array.
161
+
162
+ Element metrics are `scrollLeft`, `scrollTop`, `scrollWidth`, `scrollHeight`,
163
+ `clientWidth`, `clientHeight`, `x`, `y`, `width`, and `height`. Page metrics are
164
+ `scrollX`, `scrollY`, `scrollWidth`, `scrollHeight`, `clientWidth`,
165
+ `clientHeight`, `viewportWidth`, and `viewportHeight`. Use `assertDOMMetric`
166
+ with a `metric` and exactly one of `equals`, `greaterThan`,
167
+ `greaterThanOrEqual`, `lessThan`, or `lessThanOrEqual` for a deterministic
168
+ oracle. Raw JavaScript remains intentionally unsupported.
87
169
 
88
170
  ```jsonc
89
171
  {
172
+ "viewport": { "width": 844, "height": 847 },
173
+ "environment": { "locale": "en-GB", "timezoneId": "Europe/London", "colorScheme": "dark" },
90
174
  "steps": [
91
175
  { "action": "goto", "path": "/settings" },
92
176
  { "action": "authRejectedIf", "urlIncludes": "/login" },
@@ -94,7 +178,18 @@ assertions require exactly one of `equals` or `includes`.
94
178
  "action": "assertVisible",
95
179
  "locator": { "role": "heading", "name": "Settings", "exact": true }
96
180
  },
97
- { "action": "click", "locator": { "testId": "save-settings" } },
181
+ { "action": "wheel", "locator": { "css": ".settings-panel" }, "deltaY": 500 },
182
+ {
183
+ "action": "assertDOMMetric",
184
+ "locator": { "css": ".settings-panel" },
185
+ "metric": "scrollTop",
186
+ "greaterThan": 0
187
+ },
188
+ {
189
+ "action": "click",
190
+ "locator": { "testId": "save-settings" },
191
+ "expectResponse": { "path": "/api/settings", "method": "PUT", "status": 200 }
192
+ },
98
193
  { "action": "assertText", "locator": { "testId": "toast" }, "includes": "Saved" },
99
194
  { "action": "screenshot", "name": "settings-saved" }
100
195
  ]
@@ -104,7 +199,7 @@ assertions require exactly one of `equals` or `includes`.
104
199
  For multiple profiles, invoke the runner separately. Every invocation gets an
105
200
  isolated browser context and exclusive evidence directory; the runner closes
106
201
  all owned browser resources on success and failure. Flows, screenshots, video,
107
- sanitized traces, and runner result manifests remain under
202
+ sanitized traces, retained downloads, and runner result manifests remain under
108
203
  `$PI_SUBAGENT_AGENT_DIR/browser-qa/` so normal sub-agent shutdown or cleanup
109
204
  deletes them with the run directory. For form auth, recording starts on the login
110
205
  page and includes field filling and submission; password inputs remain masked,
@@ -132,8 +227,8 @@ claim that browser QA passed based on source inspection, unit tests, or a build.
132
227
 
133
228
  After any runner invocation that actually performed browser testing, include
134
229
  all non-empty `artifacts.screenshots`, `artifacts.videos`, and
135
- `artifacts.traces` groups in the final response. These links are mandatory so
136
- the user can open the evidence directly.
230
+ `artifacts.traces`, and `artifacts.downloads` groups in the final response.
231
+ These links are mandatory so the user can open the evidence directly.
137
232
 
138
233
  See `references/qa-auth.example.jsonc`, `references/qa-flow.example.jsonc`, and
139
234
  `references/qa-design.md`.
@@ -47,6 +47,86 @@ observable intermediate state. Sleeping longer hides races instead of proving
47
47
  behavior. If a normal operation legitimately needs more time, adjust the flow's
48
48
  `timeoutMs` rather than inserting repeated sleeps.
49
49
 
50
+ All assertion actions retry until that timeout. This makes an action followed
51
+ directly by `assertText`, `assertVisible`, `assertURL`, or another assertion
52
+ safe for asynchronously rendered outcomes. Use `assertAttribute` for
53
+ observable state such as `aria-expanded`, `aria-invalid`, or `data-state`
54
+ instead of reading DOM state through executable JavaScript.
55
+
56
+ ## Responsive and scrolling scenarios
57
+
58
+ Set the flow's top-level `viewport` whenever the bug depends on a breakpoint or
59
+ available height. Assert `viewportWidth` or `viewportHeight` with
60
+ `assertDOMMetric` when the dimensions themselves are part of the proof; the
61
+ runner also includes the applied viewport in its result.
62
+
63
+ Use `wheel` to reproduce real pointer-wheel input. Add a locator when the wheel
64
+ must target a nested scrolling container—the runner hovers it before sending
65
+ the input. Because browser scrolling may be scheduled after the wheel event,
66
+ wait only for a short known settling interval when a direct metric assertion is
67
+ otherwise racy.
68
+
69
+ Use safe `evaluate` `scrollTo`/`scrollBy` operations for deterministic setup or
70
+ to distinguish input handling from layout behavior. Use the `metrics` operation
71
+ to retain a named page/element snapshot in result `observations`, and use
72
+ `assertDOMMetric` for pass/fail. Raw JavaScript expressions are intentionally
73
+ excluded: flows remain declarative and cannot inspect authentication storage or
74
+ execute arbitrary same-origin requests.
75
+
76
+ ## Deterministic browser environment
77
+
78
+ Set top-level `environment` when locale, timezone, color scheme, or motion
79
+ preferences can change the behavior. The runner otherwise uses stable defaults
80
+ (`en-US`, `UTC`, `light`, `reduce`) instead of inheriting host settings. Prefer
81
+ asserting product-visible copy or state derived from those settings; do not use
82
+ screenshot pixels as the only oracle.
83
+
84
+ ## Causal network and dialog expectations
85
+
86
+ Put `expectResponse` or `expectDialog` on the interaction that causes the event.
87
+ The runner arms both listeners before the interaction, avoiding the race in a
88
+ separate “click, then wait” sequence. Response expectations deliberately match
89
+ only an exact allowlisted-origin pathname, HTTP method, and status. This proves
90
+ that a matching request started and received a response within the action
91
+ window without retaining its URL query, headers, or body.
92
+
93
+ Dialog expectations match a fixed dialog type and exact/included message, then
94
+ accept or dismiss it declaratively. A mismatch is dismissed before the step
95
+ fails so the page cannot freeze. Actual event metadata is never included in
96
+ failure diagnostics. Do not place secrets in expected messages or response
97
+ paths even though the runner keeps diagnostics generic.
98
+
99
+ ## Drag, upload, and download scenarios
100
+
101
+ Use `dragTo` for native DOM drag/drop and assert the resulting product state.
102
+ Optional source/drop positions are relative bounded coordinates. Canvas-only,
103
+ OS-native, or custom synthetic-event drag protocols remain unsupported; do not
104
+ work around that with executable JavaScript.
105
+
106
+ Uploads are memory-only base64 payloads declared in the flow. This intentionally
107
+ prevents a flow from selecting arbitrary project files, credential config, or
108
+ directories. Keep fixtures minimal and non-secret. An empty file list clears a
109
+ file input.
110
+
111
+ Use the atomic `download` action rather than clicking a download link directly.
112
+ Always match the suggested filename and choose a tight `maxBytes`. Retain a
113
+ download only when its contents are needed as evidence; otherwise the runner
114
+ deletes it after validation. Retained downloads use generated private names,
115
+ not server-provided paths, and are scanned for configured authentication before
116
+ publication. `maxBytes` bounds the runner's private evidence copy and triggers
117
+ cancellation while it grows, but it is not a network-bandwidth guarantee: the
118
+ browser can receive temporary bytes before cancellation.
119
+
120
+ ## Same-origin frames and popups
121
+
122
+ Use a scoped `target` for iframe or named popup interactions. The runner checks
123
+ the live frame origin before every scoped step and checks a popup after it loads;
124
+ both must remain in `allowedOrigins`. This supports embedded application UI and
125
+ same-origin auxiliary windows without opening a route around the network guard.
126
+ Cross-origin login, payment, and third-party widgets remain intentionally out of
127
+ scope. Each popup adds a separate private video artifact, so open only the
128
+ windows needed for the proof.
129
+
50
130
  ## Authentication transitions
51
131
 
52
132
  Add `authRejectedIf` directly after initial navigation and after transitions
@@ -1,5 +1,12 @@
1
1
  {
2
2
  "timeoutMs": 15000,
3
+ "viewport": { "width": 844, "height": 847 },
4
+ "environment": {
5
+ "locale": "en-GB",
6
+ "timezoneId": "Europe/London",
7
+ "colorScheme": "dark",
8
+ "reducedMotion": "reduce"
9
+ },
3
10
  "steps": [
4
11
  { "action": "goto", "path": "/settings" },
5
12
  { "action": "authRejectedIf", "urlIncludes": "/login" },
@@ -12,8 +19,54 @@
12
19
  "locator": { "label": "Display name", "exact": true },
13
20
  "value": "QA Example"
14
21
  },
15
- { "action": "click", "locator": { "testId": "save-settings" } },
22
+ { "action": "evaluate", "operation": "scrollTo", "locator": { "css": ".settings-panel" }, "y": 0 },
23
+ { "action": "wheel", "locator": { "css": ".settings-panel" }, "deltaY": 400 },
24
+ { "action": "assertDOMMetric", "locator": { "css": ".settings-panel" }, "metric": "scrollTop", "greaterThan": 0 },
25
+ { "action": "evaluate", "operation": "metrics", "locator": { "css": ".settings-panel" }, "name": "settings-scroll" },
26
+ {
27
+ "action": "dragTo",
28
+ "locator": { "testId": "available-widget" },
29
+ "dropTarget": { "testId": "enabled-widgets" }
30
+ },
31
+ {
32
+ "action": "uploadFiles",
33
+ "locator": { "label": "Import settings", "exact": true },
34
+ "files": [
35
+ { "name": "settings.json", "mimeType": "application/json", "base64": "eyJ0aGVtZSI6ImRhcmsifQ==" }
36
+ ]
37
+ },
38
+ {
39
+ "action": "click",
40
+ "locator": { "testId": "save-settings" },
41
+ "expectResponse": { "path": "/api/settings", "method": "PUT", "status": 200 },
42
+ "expectDialog": {
43
+ "type": "confirm",
44
+ "message": { "equals": "Save these settings?" },
45
+ "accept": true
46
+ }
47
+ },
48
+ { "action": "assertAttribute", "locator": { "testId": "save-settings" }, "attribute": "aria-busy", "equals": "false" },
16
49
  { "action": "assertText", "locator": { "testId": "toast" }, "includes": "Saved" },
50
+ { "action": "openPopup", "locator": { "testId": "preview-settings" }, "name": "preview" },
51
+ {
52
+ "action": "assertText",
53
+ "target": { "type": "popup", "name": "preview" },
54
+ "locator": { "role": "heading", "name": "Settings preview", "exact": true },
55
+ "equals": "Settings preview"
56
+ },
57
+ {
58
+ "action": "assertVisible",
59
+ "target": { "type": "frame", "locator": { "testId": "settings-help" } },
60
+ "locator": { "role": "heading", "name": "Settings help", "exact": true }
61
+ },
62
+ {
63
+ "action": "download",
64
+ "locator": { "testId": "export-settings" },
65
+ "filename": { "equals": "settings.json" },
66
+ "maxBytes": 1048576,
67
+ "retain": true,
68
+ "name": "settings-export"
69
+ },
17
70
  { "action": "screenshot", "name": "settings-saved" }
18
71
  ]
19
72
  }