swipium 2.0.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +32 -20
  3. package/THREAT_MODEL.md +109 -21
  4. package/dist/automationGen/run.js.map +1 -1
  5. package/dist/cli/init.js +101 -11
  6. package/dist/cli/init.js.map +1 -1
  7. package/dist/cli/verify.js +2 -2
  8. package/dist/cli/verify.js.map +1 -1
  9. package/dist/consent/consent.js +365 -17
  10. package/dist/consent/consent.js.map +1 -1
  11. package/dist/context/projectRoot.js +9 -4
  12. package/dist/context/projectRoot.js.map +1 -1
  13. package/dist/context/protocolEra.js +15 -0
  14. package/dist/context/protocolEra.js.map +1 -0
  15. package/dist/drivers/DirectDriver.js +5 -2
  16. package/dist/drivers/DirectDriver.js.map +1 -1
  17. package/dist/featureTesting/executionBootstrap.js +85 -77
  18. package/dist/featureTesting/executionBootstrap.js.map +1 -1
  19. package/dist/flows/run.js +9 -5
  20. package/dist/flows/run.js.map +1 -1
  21. package/dist/lib/abortScope.js +36 -3
  22. package/dist/lib/abortScope.js.map +1 -1
  23. package/dist/lib/android.js +15 -6
  24. package/dist/lib/android.js.map +1 -1
  25. package/dist/lib/codexEnv.js +112 -0
  26. package/dist/lib/codexEnv.js.map +1 -0
  27. package/dist/lib/logger.js +18 -0
  28. package/dist/lib/logger.js.map +1 -1
  29. package/dist/lib/result.js +188 -10
  30. package/dist/lib/result.js.map +1 -1
  31. package/dist/lib/schemaHash.js +20 -31
  32. package/dist/lib/schemaHash.js.map +1 -1
  33. package/dist/lib/simctl.js +102 -3
  34. package/dist/lib/simctl.js.map +1 -1
  35. package/dist/lib/toolSchema.js +143 -0
  36. package/dist/lib/toolSchema.js.map +1 -0
  37. package/dist/lib/wda.js +5 -2
  38. package/dist/lib/wda.js.map +1 -1
  39. package/dist/mobileAudit/runner.js +12 -4
  40. package/dist/mobileAudit/runner.js.map +1 -1
  41. package/dist/oracle/failures.js +2 -2
  42. package/dist/oracle/failures.js.map +1 -1
  43. package/dist/orchestration/testThis/execute.js +21 -4
  44. package/dist/orchestration/testThis/execute.js.map +1 -1
  45. package/dist/orchestration/testThis/pipeline.js +2 -0
  46. package/dist/orchestration/testThis/pipeline.js.map +1 -1
  47. package/dist/orchestration/testThis/plan.js.map +1 -1
  48. package/dist/server.js +564 -139
  49. package/dist/server.js.map +1 -1
  50. package/dist/services/prepareAndroid.js +3 -2
  51. package/dist/services/prepareAndroid.js.map +1 -1
  52. package/dist/services/prepareIos.js +31 -4
  53. package/dist/services/prepareIos.js.map +1 -1
  54. package/dist/services/smoke.js +38 -4
  55. package/dist/services/smoke.js.map +1 -1
  56. package/dist/session/processRegistry.js +12 -1
  57. package/dist/session/processRegistry.js.map +1 -1
  58. package/dist/snapshot/parse.js +64 -9
  59. package/dist/snapshot/parse.js.map +1 -1
  60. package/dist/snapshot/present.js +14 -4
  61. package/dist/snapshot/present.js.map +1 -1
  62. package/dist/snapshot/settle.js +14 -3
  63. package/dist/snapshot/settle.js.map +1 -1
  64. package/dist/tools/act.js +694 -670
  65. package/dist/tools/act.js.map +1 -1
  66. package/dist/tools/agent.js +15 -16
  67. package/dist/tools/agent.js.map +1 -1
  68. package/dist/tools/appControl.js +4 -4
  69. package/dist/tools/appControl.js.map +1 -1
  70. package/dist/tools/appMap.js +11 -13
  71. package/dist/tools/appMap.js.map +1 -1
  72. package/dist/tools/build.js +4 -6
  73. package/dist/tools/build.js.map +1 -1
  74. package/dist/tools/bundletool.js +3 -3
  75. package/dist/tools/bundletool.js.map +1 -1
  76. package/dist/tools/clearOverlay.js +2 -3
  77. package/dist/tools/clearOverlay.js.map +1 -1
  78. package/dist/tools/device.js +4 -3
  79. package/dist/tools/device.js.map +1 -1
  80. package/dist/tools/doctor.js +23 -11
  81. package/dist/tools/doctor.js.map +1 -1
  82. package/dist/tools/explore.js +15 -21
  83. package/dist/tools/explore.js.map +1 -1
  84. package/dist/tools/featureTesting.js +22 -6
  85. package/dist/tools/featureTesting.js.map +1 -1
  86. package/dist/tools/firstRun.js +4 -5
  87. package/dist/tools/firstRun.js.map +1 -1
  88. package/dist/tools/flow.js +13 -17
  89. package/dist/tools/flow.js.map +1 -1
  90. package/dist/tools/flowRepair.js +3 -5
  91. package/dist/tools/flowRepair.js.map +1 -1
  92. package/dist/tools/generate.js +10 -11
  93. package/dist/tools/generate.js.map +1 -1
  94. package/dist/tools/getArtifact.js +114 -10
  95. package/dist/tools/getArtifact.js.map +1 -1
  96. package/dist/tools/health.js +2 -1
  97. package/dist/tools/health.js.map +1 -1
  98. package/dist/tools/ios.js +35 -13
  99. package/dist/tools/ios.js.map +1 -1
  100. package/dist/tools/issues.js +8 -8
  101. package/dist/tools/issues.js.map +1 -1
  102. package/dist/tools/jobs.js +26 -10
  103. package/dist/tools/jobs.js.map +1 -1
  104. package/dist/tools/metro.js +4 -5
  105. package/dist/tools/metro.js.map +1 -1
  106. package/dist/tools/mobileAudit.js +11 -10
  107. package/dist/tools/mobileAudit.js.map +1 -1
  108. package/dist/tools/note.js +2 -3
  109. package/dist/tools/note.js.map +1 -1
  110. package/dist/tools/prepareIosTarget.js +102 -31
  111. package/dist/tools/prepareIosTarget.js.map +1 -1
  112. package/dist/tools/prepareTarget.js +3 -5
  113. package/dist/tools/prepareTarget.js.map +1 -1
  114. package/dist/tools/report.js +4 -5
  115. package/dist/tools/report.js.map +1 -1
  116. package/dist/tools/resolveArtifact.js +2 -3
  117. package/dist/tools/resolveArtifact.js.map +1 -1
  118. package/dist/tools/resolveTarget.js +3 -6
  119. package/dist/tools/resolveTarget.js.map +1 -1
  120. package/dist/tools/screenRecord.js +2 -3
  121. package/dist/tools/screenRecord.js.map +1 -1
  122. package/dist/tools/screenshot.js +2 -2
  123. package/dist/tools/screenshot.js.map +1 -1
  124. package/dist/tools/smoke.js +4 -2
  125. package/dist/tools/smoke.js.map +1 -1
  126. package/dist/tools/snapshot.js +6 -7
  127. package/dist/tools/snapshot.js.map +1 -1
  128. package/dist/tools/startSession.js +12 -16
  129. package/dist/tools/startSession.js.map +1 -1
  130. package/dist/tools/suite.js +2 -4
  131. package/dist/tools/suite.js.map +1 -1
  132. package/dist/tools/testSuite.js +13 -19
  133. package/dist/tools/testSuite.js.map +1 -1
  134. package/dist/tools/testThis.js +28 -13
  135. package/dist/tools/testThis.js.map +1 -1
  136. package/dist/tools/visual.js +7 -9
  137. package/dist/tools/visual.js.map +1 -1
  138. package/dist/tools/wait.js +130 -21
  139. package/dist/tools/wait.js.map +1 -1
  140. package/dist/tools/wda.js +199 -60
  141. package/dist/tools/wda.js.map +1 -1
  142. package/dist/version.js +1 -1
  143. package/docs/README.md +4 -4
  144. package/docs/ci-reports.md +39 -7
  145. package/docs/concepts.md +37 -25
  146. package/docs/flows.md +1 -1
  147. package/docs/mcp-server.md +281 -76
  148. package/docs/physical-devices.md +4 -4
  149. package/docs/tools.md +59 -30
  150. package/package.json +4 -3
package/dist/tools/act.js CHANGED
@@ -3,7 +3,7 @@
3
3
  // to settle, then returns the post-action snapshot + a deterministic health check. So one
4
4
  // call both acts AND observes the result.
5
5
  import { z } from 'zod';
6
- import { qaOk, qaError, qaStop, unknownSessionError, cancelledResult } from '../lib/result.js';
6
+ import { qaOk, qaError, qaStop, qaAnnotate, unknownSessionError, cancelledResult } from '../lib/result.js';
7
7
  import { parseSnapshot, signature } from '../snapshot/parse.js';
8
8
  import { presentElements } from '../snapshot/present.js';
9
9
  import { obstructionAt } from '../snapshot/overlays.js';
@@ -18,7 +18,7 @@ import { center, resolveTarget, setsEqual } from '../core/target.js';
18
18
  import { recordableNativeSelector, recordableTap } from '../flows/generate.js';
19
19
  import { classifyFlowDriverError } from '../flows/run.js';
20
20
  import { structuredSignature } from '../explore/signatures.js';
21
- import { isAbortError, runWithSignal } from '../lib/abortScope.js';
21
+ import { isAbortError, runWithSignal, sleepOrCancel, throwIfCancelled } from '../lib/abortScope.js';
22
22
  import { SECRET_VAR_NAME } from '../flows/schema.js';
23
23
  /** Drop the (already-handled) native selector before generic ref/text/id/coords resolution. */
24
24
  function stripSelector(t) {
@@ -48,15 +48,17 @@ async function awaitIme(d, capMs, floorMs = 0) {
48
48
  // new field having focus yet, so give RN a floor to move focus before typing.
49
49
  const rest = floorMs - (Date.now() - started);
50
50
  if (rest > 0)
51
- await new Promise((r) => setTimeout(r, rest));
51
+ await sleepOrCancel(rest);
52
52
  return;
53
53
  }
54
- await new Promise((r) => setTimeout(r, Math.min(100, Math.max(1, deadline - Date.now()))));
54
+ await sleepOrCancel(Math.min(100, Math.max(1, deadline - Date.now())));
55
55
  }
56
56
  }
57
- catch {
57
+ catch (e) {
58
+ if (isAbortError(e))
59
+ throw e; // cancelled: unwind, the action's catch returns CANCELLED
58
60
  // imeShown unsupported here: keep the original fixed-sleep behavior
59
- await new Promise((r) => setTimeout(r, Math.max(0, deadline - Date.now())));
61
+ await sleepOrCancel(Math.max(0, deadline - Date.now()));
60
62
  }
61
63
  }
62
64
  /** Values to scrub from driver error text: session secrets + the value just typed, in raw AND
@@ -146,6 +148,8 @@ function stateLabel(n) {
146
148
  * (after a navigation "diff" = every new element + every old one listed as removed, larger
147
149
  * than "full"). */
148
150
  export const DIFF_FULL_FALLBACK_RATIO = 0.5;
151
+ /** qa_act timeoutMs ceiling (wait / settle cap); larger values are clamped with a note. */
152
+ export const ACT_TIMEOUT_MAX_MS = 50_000;
149
153
  /** Only `${SWIPIUM_*}` placeholders are expanded in typed text (anything else stays literal). */
150
154
  const INPUT_PLACEHOLDER_RE = /\$\{(SWIPIUM_[A-Z0-9_]+)\}/g;
151
155
  /** Env-sourced placeholder values whose NAME looks secret join the redaction set. Uses the shared
@@ -388,10 +392,10 @@ function recordableNativeTarget(session, native) {
388
392
  export function registerAct(server, sessions) {
389
393
  server.registerTool('qa_act', {
390
394
  title: 'Act on the screen',
391
- description: 'Perform one UI action, wait for the screen to settle, and observe: changed/settled, snapshot quality, a health check, and ' +
392
- 'post-action elements per observe. action: tap, type, clear, swipe, scroll, press, open_url, wait (each param names its ' +
393
- 'actions). Target @eN refs from qa_snapshot (invalid after navigation), text, id, a native selector (WDA iOS), or x/y. A field ' +
394
- 'hidden under the keyboard is handled (hide + re-resolve; KEYBOARD_OBSTRUCTION if still covered). See docs/tools.md#qa_act.',
395
+ description: 'One UI action (tap, type, clear, swipe, scroll, press, open_url, wait), then wait for the screen to settle and ' +
396
+ 'report changed/settled, snapshot quality, health, and new elements (observe). Target: an @eN ref from qa_snapshot, ' +
397
+ 'text, id, a native selector (WDA iOS), or x/y. Each param names the actions it applies to. Details: ' +
398
+ 'docs/tools.md#qa_act.',
395
399
  inputSchema: {
396
400
  sessionId: z.string(),
397
401
  action: z.enum(['tap', 'type', 'clear', 'swipe', 'scroll', 'press', 'open_url', 'wait']),
@@ -430,7 +434,11 @@ export function registerAct(server, sessions) {
430
434
  })
431
435
  .optional()
432
436
  .describe('wait: {settled:true} (default) or an element.'),
433
- timeoutMs: z.number().optional().describe('wait: max ms (default 8000); others: settle-wait cap.'),
437
+ timeoutMs: z
438
+ .number()
439
+ .min(0)
440
+ .optional()
441
+ .describe('wait: max ms (default 8000, max 50000; larger values are clamped); others: settle-wait cap.'),
434
442
  observe: z
435
443
  .enum(['diff', 'full', 'none'])
436
444
  .optional()
@@ -440,733 +448,749 @@ export function registerAct(server, sessions) {
440
448
  // Cancellation (MCP notifications/cancelled): the call's signal is scoped to THIS call
441
449
  // (abortScope): an in-flight adb child / WDA request is aborted without touching a
442
450
  // concurrently running job's own cancellation.
443
- async (args, extra) => runWithSignal(extra?.signal, async () => {
444
- const { sessionId, action } = args;
445
- // Single validation layer for the per-action field contract (see REQUIRED_BY_ACTION).
446
- const invalid = missingRequiredField(action, args);
447
- if (invalid)
448
- return invalid;
449
- const session = sessions.get(sessionId);
450
- if (!session)
451
- return unknownSessionError(sessionId);
452
- const { driver: d, blocked } = await getDriver(session);
453
- if (!d) {
454
- // H6: a device is online but refused (physical) or not ready (still booting).
455
- return (blockedDeviceResult(blocked) ??
456
- qaError({
457
- what: 'No device attached to this session',
458
- changedState: false,
459
- retrySafe: true,
460
- failureCode: 'NO_DEVICE',
461
- nextSteps: ['Call qa_prepare_target first.'],
462
- }));
463
- }
464
- if (d.kind === 'simulator') {
465
- return qaError({
466
- what: 'Structured interaction (tap/type/swipe) is not available on the iOS simulator backend',
467
- changedState: false,
468
- retrySafe: false,
469
- failureCode: 'BACKEND_UNSUPPORTED',
470
- nextSteps: [
471
- 'Attach WebDriverAgent with qa_wda for structured tap/type/snapshot. Without WDA, locate targets with qa_visual (mode:"find_text" OCR or mode:"find_image" return tappable device coordinates; tap:true taps via idb when installed), navigate via qa_ios deep links, and verify with qa_visual mode:"assert" or mode:"diff".',
472
- ],
473
- });
474
- }
475
- // Budget gate (review §4.1 / Rec 4): refuse new work once the session budget is spent.
476
- // `wait` is exempt from the action/screenshot caps (it's synchronization, not an
477
- // action), but it is NOT exempt from the TIME budget, otherwise repeated waits could
478
- // burn the clock indefinitely.
479
- const stopReason = sessions.budgetStop(session);
480
- if (stopReason && (action !== 'wait' || /time budget/.test(stopReason))) {
481
- return qaStop(stopReason, { counters: session.counters, mode: session.mode });
482
- }
483
- const fail = (what, changedState) => qaError({
484
- what,
485
- changedState,
486
- retrySafe: true,
487
- failureCode: targetErrorCode(what),
488
- nextSteps: ['Run qa_snapshot to see the current screen, then retry.'],
489
- });
490
- // ---- wait is its own path (no settle/health afterward) ----
491
- if (action === 'wait') {
492
- const timeoutMs = args.timeoutMs ?? 8000;
493
- if (args.for?.settled || !args.for) {
494
- const s = await settle(d, { timeoutMs });
495
- const post = parseSnapshot(s.xml);
496
- session.lastSnapshot = {
497
- fullByRef: post.fullByRef,
498
- signatures: new Set(post.elements.map(signature)),
499
- allNodes: post.allNodes,
500
- };
501
- const { elements: shown, rendered, omitted } = presentElements(post.elements, makeRedactor(session.secrets));
502
- return qaOk({ action, settled: s.settled, quality: post.quality.verdict, elementsOmitted: omitted, elements: shown }, `wait(settled)=${s.settled}\n\n${rendered}`, { textOmit: ['elements'] });
503
- }
504
- const deadline = Date.now() + timeoutMs;
505
- const want = args.for;
506
- const native = want?.selector ?? null;
507
- if (native) {
508
- if (!d.existsBySelector)
509
- return qaError({
510
- what: `${native.using} waits require backend-native selector support`,
451
+ async (rawArgs, ctx) => {
452
+ // timeoutMs is clamped (not rejected) like qa_job_status waitMs, with a note on the result.
453
+ const clampedFrom = rawArgs.timeoutMs != null && rawArgs.timeoutMs > ACT_TIMEOUT_MAX_MS ? rawArgs.timeoutMs : undefined;
454
+ const args = clampedFrom != null ? { ...rawArgs, timeoutMs: ACT_TIMEOUT_MAX_MS } : rawArgs;
455
+ const res = await runWithSignal(ctx?.mcpReq.signal, async () => {
456
+ const { sessionId, action } = args;
457
+ // Single validation layer for the per-action field contract (see REQUIRED_BY_ACTION).
458
+ const invalid = missingRequiredField(action, args);
459
+ if (invalid)
460
+ return invalid;
461
+ const session = sessions.get(sessionId);
462
+ if (!session)
463
+ return unknownSessionError(sessionId);
464
+ const { driver: d, blocked } = await getDriver(session);
465
+ if (!d) {
466
+ // H6: a device is online but refused (physical) or not ready (still booting).
467
+ return (blockedDeviceResult(blocked) ??
468
+ qaError({
469
+ what: 'No device attached to this session',
511
470
  changedState: false,
512
- retrySafe: false,
513
- failureCode: 'BACKEND_UNSUPPORTED',
514
- nextSteps: ['Use a WDA-backed iOS session, or wait by text/id/ref on this backend.'],
515
- });
516
- while (Date.now() < deadline) {
517
- if (await d.existsBySelector(native.using, native.value)) {
518
- return qaOk({ action, found: true, selector: want?.selector, via: 'native-selector' }, `wait: found ${native.using}=${native.value}`);
519
- }
520
- await new Promise((r) => setTimeout(r, 400));
521
- }
471
+ retrySafe: true,
472
+ failureCode: 'NO_DEVICE',
473
+ nextSteps: ['Call qa_prepare_target first.'],
474
+ }));
475
+ }
476
+ if (d.kind === 'simulator') {
522
477
  return qaError({
523
- what: `wait timed out (${timeoutMs}ms) for ${JSON.stringify(want)}`,
478
+ what: 'Structured interaction (tap/type/swipe) is not available on the iOS simulator backend',
524
479
  changedState: false,
525
- retrySafe: true,
526
- failureCode: 'ELEMENT_NOT_FOUND',
527
- nextSteps: ['Re-check the native selector value, or run qa_snapshot to inspect the current screen.'],
480
+ retrySafe: false,
481
+ failureCode: 'BACKEND_UNSUPPORTED',
482
+ nextSteps: [
483
+ 'Attach WebDriverAgent with qa_wda for structured tap/type/snapshot. Without WDA, locate targets with qa_visual (mode:"find_text" OCR or mode:"find_image" return tappable device coordinates; tap:true taps via idb when installed), navigate via qa_ios deep links, and verify with qa_visual mode:"assert" or mode:"diff".',
484
+ ],
528
485
  });
529
486
  }
530
- while (Date.now() < deadline) {
531
- const parsed = parseSnapshot(await d.dumpXml());
532
- session.lastSnapshot = {
533
- fullByRef: parsed.fullByRef,
534
- signatures: new Set(parsed.elements.map(signature)),
535
- allNodes: parsed.allNodes,
536
- };
537
- const hit = parsed.elements.find((e) => (want.ref && e.ref === want.ref) ||
538
- (want.id && e.id === want.id) ||
539
- (want.text &&
540
- (e.text?.toLowerCase().includes(want.text.toLowerCase()) || e.label?.toLowerCase().includes(want.text.toLowerCase()))));
541
- if (hit)
542
- return qaOk({ action, found: true, ref: hit.ref }, `wait: found ${hit.ref}`);
543
- await new Promise((r) => setTimeout(r, 400));
487
+ // Budget gate (review §4.1 / Rec 4): refuse new work once the session budget is spent.
488
+ // `wait` is exempt from the action/screenshot caps (it's synchronization, not an
489
+ // action), but it is NOT exempt from the TIME budget, otherwise repeated waits could
490
+ // burn the clock indefinitely.
491
+ const stopReason = sessions.budgetStop(session);
492
+ if (stopReason && (action !== 'wait' || /time budget/.test(stopReason))) {
493
+ return qaStop(stopReason, { counters: session.counters, mode: session.mode });
544
494
  }
545
- return qaError({
546
- what: `wait timed out (${timeoutMs}ms) for ${JSON.stringify(want)}`,
547
- changedState: false,
495
+ const fail = (what, changedState) => qaError({
496
+ what,
497
+ changedState,
548
498
  retrySafe: true,
549
- failureCode: 'ELEMENT_NOT_FOUND',
550
- nextSteps: ['Re-snapshot; the element may use different text/id.'],
499
+ failureCode: targetErrorCode(what),
500
+ nextSteps: ['Run qa_snapshot to see the current screen, then retry.'],
551
501
  });
552
- }
553
- // Phase timing (P1.6): mark the first real action so the report can split setup vs active.
554
- sessions.milestone(session, 'first_action');
555
- const preSigs = session.lastSnapshot?.signatures ?? new Set();
556
- // F: positions too. A scroll/swipe can move content without adding/removing elements.
557
- const prePositions = positionalFingerprint(session.lastSnapshot?.fullByRef);
558
- // Toggle/selection state of the pre-action screen (a Switch flip changes no signature).
559
- const preState = stateByIdentity(session.lastSnapshot?.fullByRef);
560
- // The value actually typed (after ${SWIPIUM_*} expansion), redacted from results/errors.
561
- let typedValue;
562
- // untilVisible's last probe dump. Seeds the post-action settle (no redundant dump).
563
- let settleSeed;
564
- // I: set when the soft keyboard was hidden because it covered the target.
565
- let keyboardHidden = false;
566
- // Observation mode is fixed at action start; diff needs a pre-action baseline.
567
- const observe = args.observe ?? (session.lastSnapshot ? 'diff' : 'full');
568
- let meta = {};
569
- // remembered so a no-change tap can be retried as a longer press (RN tap quirk)
570
- // imeUp: the soft keyboard was shown at tap time, so a re-press could hit a key (H5).
571
- let tapRetry;
572
- // action-IR step to record once the action succeeds (built here while lastSnapshot is
573
- // still the PRE-navigation screen, so a tapped @ref still resolves to its label).
574
- let toRecord;
575
- // Non-fatal caveats for the result (unknown keyboard area, recovered WDA session, …).
576
- const warnings = [];
577
- try {
578
- switch (action) {
579
- case 'tap': {
580
- const native = args.target?.selector ?? null;
502
+ // ---- wait is its own path (no settle/health afterward) ----
503
+ if (action === 'wait') {
504
+ // A cancelled wait (notifications/cancelled) stops polling at once and is not a failure.
505
+ try {
506
+ const timeoutMs = args.timeoutMs ?? 8000;
507
+ if (args.for?.settled || !args.for) {
508
+ const s = await settle(d, { timeoutMs });
509
+ const post = parseSnapshot(s.xml);
510
+ session.lastSnapshot = {
511
+ fullByRef: post.fullByRef,
512
+ signatures: new Set(post.elements.map(signature)),
513
+ allNodes: post.allNodes,
514
+ };
515
+ const { payload: shown, rendered, omitted } = presentElements(post.elements, makeRedactor(session.secrets));
516
+ return qaOk({ action, settled: s.settled, quality: post.quality.verdict, elementsOmitted: omitted, elements: shown }, `wait(settled)=${s.settled}\n\n${rendered}`, { textOmit: ['elements'] });
517
+ }
518
+ const deadline = Date.now() + timeoutMs;
519
+ const want = args.for;
520
+ const native = want?.selector ?? null;
581
521
  if (native) {
582
- if (!d.tapBySelector)
522
+ if (!d.existsBySelector)
583
523
  return qaError({
584
- what: `${native.using} selectors require backend-native selector support`,
524
+ what: `${native.using} waits require backend-native selector support`,
585
525
  changedState: false,
586
526
  retrySafe: false,
587
527
  failureCode: 'BACKEND_UNSUPPORTED',
588
- nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
528
+ nextSteps: ['Use a WDA-backed iOS session, or wait by text/id/ref on this backend.'],
589
529
  });
590
- await d.tapBySelector(native.using, native.value);
591
- meta = { selector: args.target?.selector, via: 'native-selector' };
592
- toRecord = { action: 'tap', ...recordableNativeTarget(session, native) };
593
- break;
594
- }
595
- const resolved = await resolveTarget(session, stripSelector(args.target));
596
- if ('error' in resolved)
597
- return fail(resolved.error, false);
598
- let t = resolved;
599
- // H5: keyboard obstruction (the IME window is not in the app's UI dump, so the
600
- // overlay check below can't see it). Coordinate taps are taken as deliberate.
601
- let imeUp = false;
602
- if (!args.ignoreOverlay && t.via !== 'coords') {
603
- const g = await guardKeyboard(session, d, t, stripSelector(args.target));
604
- if ('result' in g)
605
- return g.result;
606
- t = g.t;
607
- imeUp = g.imeUp;
608
- if (g.warning)
609
- warnings.push(g.warning);
610
- if (g.hidKeyboard)
611
- keyboardHidden = true;
612
- }
613
- else {
614
- imeUp = await d.imeShown().catch(() => false);
615
- }
616
- // Overlay obstruction check (CR4): if another element is drawn over the target
617
- // point, return a structured blockedByOverlay instead of tapping blindly.
618
- if (!args.ignoreOverlay && t.via !== 'coords' && session.lastSnapshot?.allNodes) {
619
- // works for ref AND selector taps: t.ref is the resolved @eN in either case
620
- const node = t.ref ? session.lastSnapshot.fullByRef.get(t.ref) : undefined;
621
- const obs = obstructionAt(session.lastSnapshot.allNodes, node, t.x, t.y);
622
- if (obs.obstructed) {
623
- return qaError({
624
- what: `Target at (${t.x},${t.y}) is obstructed by ${obs.by?.cls?.split('.').pop()}${obs.by?.text ? ` "${obs.by.text}"` : ''}`,
625
- changedState: false,
626
- retrySafe: true,
627
- failureCode: 'OVERLAY_OBSTRUCTION',
628
- nextSteps: [
629
- 'Call qa_clear_overlay (auto, or hide_keyboard/minimize_logbox), then retry. Or pass ignoreOverlay:true to tap anyway.',
630
- ],
631
- }, { blockedByOverlay: true, obstructedBy: obs.by });
530
+ while (Date.now() < deadline) {
531
+ throwIfCancelled();
532
+ if (await d.existsBySelector(native.using, native.value)) {
533
+ return qaOk({ action, found: true, selector: want?.selector, via: 'native-selector' }, `wait: found ${native.using}=${native.value}`);
534
+ }
535
+ await sleepOrCancel(400);
632
536
  }
633
- }
634
- // Coordinate taps default to a short press (RN often ignores instant taps);
635
- // ref/selector taps stay instant unless durationMs is given.
636
- const isCoord = t.via === 'coords';
637
- const durationMs = args.durationMs ?? (isCoord ? 100 : undefined);
638
- if (durationMs)
639
- await d.pressXY(t.x, t.y, durationMs);
640
- else
641
- await d.tapXY(t.x, t.y);
642
- tapRetry = { x: t.x, y: t.y, instant: !durationMs, imeUp };
643
- meta = { tappedAt: [t.x, t.y], via: t.via, ...(durationMs ? { durationMs } : {}) };
644
- toRecord = { action: 'tap', ...recordableTap(session, stripSelector(args.target), t) };
645
- break;
646
- }
647
- case 'type': {
648
- // presence enforced by missingRequiredField
649
- // P1: `${SWIPIUM_*}` placeholders expand from session inputs (qa_continue_from_blocker) or the env,
650
- // so credentials never have to pass through the agent transcript.
651
- const secretVars = new Set(session.inputs.filter((i) => i.secret).map((i) => i.varName));
652
- const expanded = expandInputPlaceholders(args.text, { values: session.inputValues, secretVars });
653
- if (expanded.missing.length) {
654
537
  return qaError({
655
- what: `No value for ${expanded.missing.map((v) => `\${${v}}`).join(', ')}. Nothing was typed.`,
538
+ what: `wait timed out (${timeoutMs}ms) for ${JSON.stringify(want)}`,
656
539
  changedState: false,
657
540
  retrySafe: true,
658
- failureCode: 'MISSING_TEST_DATA',
659
- nextSteps: [
660
- 'Provide it via qa_continue_from_blocker (needs_input credentials) or set the env var for the Swipium server, then retry. Only ${SWIPIUM_*} placeholders are expanded.',
661
- ],
541
+ failureCode: 'ELEMENT_NOT_FOUND',
542
+ nextSteps: ['Re-check the native selector value, or run qa_snapshot to inspect the current screen.'],
662
543
  });
663
544
  }
664
- for (const v of expanded.secretValues)
665
- session.secrets.add(v);
666
- const text = expanded.text;
667
- typedValue = text;
668
- const recordText = recordableTypedText(args.text, text, expanded.vars, session.inputValues);
669
- // D: validate deliverability BEFORE any focus tap / clear. A refused value must
670
- // leave the field (and the device) exactly as it was.
671
- const deliverable = d.canDeliverText?.(text);
672
- if (deliverable && !deliverable.ok) {
673
- return qaError({
674
- what: `${deliverable.reason} Nothing was tapped or cleared.`,
675
- changedState: false,
676
- retrySafe: false,
677
- failureCode: 'TEXT_INPUT_UNSUPPORTED',
678
- nextSteps: ['Use ASCII-safe text on this backend (adb `input text`), or type it on a Unicode-safe backend.'],
679
- });
545
+ while (Date.now() < deadline) {
546
+ throwIfCancelled();
547
+ const parsed = parseSnapshot(await d.dumpXml());
548
+ session.lastSnapshot = {
549
+ fullByRef: parsed.fullByRef,
550
+ signatures: new Set(parsed.elements.map(signature)),
551
+ allNodes: parsed.allNodes,
552
+ };
553
+ const hit = parsed.elements.find((e) => (want.ref && e.ref === want.ref) ||
554
+ (want.id && e.id === want.id) ||
555
+ (want.text &&
556
+ (e.text?.toLowerCase().includes(want.text.toLowerCase()) || e.label?.toLowerCase().includes(want.text.toLowerCase()))));
557
+ if (hit)
558
+ return qaOk({ action, found: true, ref: hit.ref }, `wait: found ${hit.ref}`);
559
+ await sleepOrCancel(400);
680
560
  }
681
- // A: a value that is/contains a registered secret is recorded as secret even when the
682
- // target field is not a secure one (checked before this call can register it).
683
- const knownSecret = matchesSessionSecret(session.secrets, text);
684
- const native = args.target?.selector ?? null;
685
- if (native) {
686
- if (!d.typeBySelector)
561
+ return qaError({
562
+ what: `wait timed out (${timeoutMs}ms) for ${JSON.stringify(want)}`,
563
+ changedState: false,
564
+ retrySafe: true,
565
+ failureCode: 'ELEMENT_NOT_FOUND',
566
+ nextSteps: ['Re-snapshot; the element may use different text/id.'],
567
+ });
568
+ }
569
+ catch (e) {
570
+ if (isAbortError(e))
571
+ return cancelledResult('wait cancelled: the call was aborted before the condition held');
572
+ throw e;
573
+ }
574
+ }
575
+ // Phase timing (P1.6): mark the first real action so the report can split setup vs active.
576
+ sessions.milestone(session, 'first_action');
577
+ const preSigs = session.lastSnapshot?.signatures ?? new Set();
578
+ // F: positions too. A scroll/swipe can move content without adding/removing elements.
579
+ const prePositions = positionalFingerprint(session.lastSnapshot?.fullByRef);
580
+ // Toggle/selection state of the pre-action screen (a Switch flip changes no signature).
581
+ const preState = stateByIdentity(session.lastSnapshot?.fullByRef);
582
+ // The value actually typed (after ${SWIPIUM_*} expansion), redacted from results/errors.
583
+ let typedValue;
584
+ // untilVisible's last probe dump. Seeds the post-action settle (no redundant dump).
585
+ let settleSeed;
586
+ // I: set when the soft keyboard was hidden because it covered the target.
587
+ let keyboardHidden = false;
588
+ // Observation mode is fixed at action start; diff needs a pre-action baseline.
589
+ const observe = args.observe ?? (session.lastSnapshot ? 'diff' : 'full');
590
+ let meta = {};
591
+ // remembered so a no-change tap can be retried as a longer press (RN tap quirk)
592
+ // imeUp: the soft keyboard was shown at tap time, so a re-press could hit a key (H5).
593
+ let tapRetry;
594
+ // action-IR step to record once the action succeeds (built here while lastSnapshot is
595
+ // still the PRE-navigation screen, so a tapped @ref still resolves to its label).
596
+ let toRecord;
597
+ // Non-fatal caveats for the result (unknown keyboard area, recovered WDA session, …).
598
+ const warnings = [];
599
+ try {
600
+ switch (action) {
601
+ case 'tap': {
602
+ const native = args.target?.selector ?? null;
603
+ if (native) {
604
+ if (!d.tapBySelector)
605
+ return qaError({
606
+ what: `${native.using} selectors require backend-native selector support`,
607
+ changedState: false,
608
+ retrySafe: false,
609
+ failureCode: 'BACKEND_UNSUPPORTED',
610
+ nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
611
+ });
612
+ await d.tapBySelector(native.using, native.value);
613
+ meta = { selector: args.target?.selector, via: 'native-selector' };
614
+ toRecord = { action: 'tap', ...recordableNativeTarget(session, native) };
615
+ break;
616
+ }
617
+ const resolved = await resolveTarget(session, stripSelector(args.target));
618
+ if ('error' in resolved)
619
+ return fail(resolved.error, false);
620
+ let t = resolved;
621
+ // H5: keyboard obstruction (the IME window is not in the app's UI dump, so the
622
+ // overlay check below can't see it). Coordinate taps are taken as deliberate.
623
+ let imeUp = false;
624
+ if (!args.ignoreOverlay && t.via !== 'coords') {
625
+ const g = await guardKeyboard(session, d, t, stripSelector(args.target));
626
+ if ('result' in g)
627
+ return g.result;
628
+ t = g.t;
629
+ imeUp = g.imeUp;
630
+ if (g.warning)
631
+ warnings.push(g.warning);
632
+ if (g.hidKeyboard)
633
+ keyboardHidden = true;
634
+ }
635
+ else {
636
+ imeUp = await d.imeShown().catch(() => false);
637
+ }
638
+ // Overlay obstruction check (CR4): if another element is drawn over the target
639
+ // point, return a structured blockedByOverlay instead of tapping blindly.
640
+ if (!args.ignoreOverlay && t.via !== 'coords' && session.lastSnapshot?.allNodes) {
641
+ // works for ref AND selector taps: t.ref is the resolved @eN in either case
642
+ const node = t.ref ? session.lastSnapshot.fullByRef.get(t.ref) : undefined;
643
+ const obs = obstructionAt(session.lastSnapshot.allNodes, node, t.x, t.y);
644
+ if (obs.obstructed) {
645
+ return qaError({
646
+ what: `Target at (${t.x},${t.y}) is obstructed by ${obs.by?.cls?.split('.').pop()}${obs.by?.text ? ` "${obs.by.text}"` : ''}`,
647
+ changedState: false,
648
+ retrySafe: true,
649
+ failureCode: 'OVERLAY_OBSTRUCTION',
650
+ nextSteps: [
651
+ 'Call qa_clear_overlay (auto, or hide_keyboard/minimize_logbox), then retry. Or pass ignoreOverlay:true to tap anyway.',
652
+ ],
653
+ }, { blockedByOverlay: true, obstructedBy: obs.by });
654
+ }
655
+ }
656
+ // Coordinate taps default to a short press (RN often ignores instant taps);
657
+ // ref/selector taps stay instant unless durationMs is given.
658
+ const isCoord = t.via === 'coords';
659
+ const durationMs = args.durationMs ?? (isCoord ? 100 : undefined);
660
+ if (durationMs)
661
+ await d.pressXY(t.x, t.y, durationMs);
662
+ else
663
+ await d.tapXY(t.x, t.y);
664
+ tapRetry = { x: t.x, y: t.y, instant: !durationMs, imeUp };
665
+ meta = { tappedAt: [t.x, t.y], via: t.via, ...(durationMs ? { durationMs } : {}) };
666
+ toRecord = { action: 'tap', ...recordableTap(session, stripSelector(args.target), t) };
667
+ break;
668
+ }
669
+ case 'type': {
670
+ // presence enforced by missingRequiredField
671
+ // P1: `${SWIPIUM_*}` placeholders expand from session inputs (qa_continue_from_blocker) or the env,
672
+ // so credentials never have to pass through the agent transcript.
673
+ const secretVars = new Set(session.inputs.filter((i) => i.secret).map((i) => i.varName));
674
+ const expanded = expandInputPlaceholders(args.text, { values: session.inputValues, secretVars });
675
+ if (expanded.missing.length) {
687
676
  return qaError({
688
- what: `${native.using} selectors require backend-native selector support`,
677
+ what: `No value for ${expanded.missing.map((v) => `\${${v}}`).join(', ')}. Nothing was typed.`,
689
678
  changedState: false,
690
- retrySafe: false,
691
- failureCode: 'BACKEND_UNSUPPORTED',
692
- nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
679
+ retrySafe: true,
680
+ failureCode: 'MISSING_TEST_DATA',
681
+ nextSteps: [
682
+ 'Provide it via qa_continue_from_blocker (needs_input credentials) or set the env var for the Swipium server, then retry. Only ${SWIPIUM_*} placeholders are expanded.',
683
+ ],
693
684
  });
694
- const replace = (args.mode ?? 'replace') === 'replace';
695
- if (replace && !d.clearBySelector)
685
+ }
686
+ for (const v of expanded.secretValues)
687
+ session.secrets.add(v);
688
+ const text = expanded.text;
689
+ typedValue = text;
690
+ const recordText = recordableTypedText(args.text, text, expanded.vars, session.inputValues);
691
+ // D: validate deliverability BEFORE any focus tap / clear. A refused value must
692
+ // leave the field (and the device) exactly as it was.
693
+ const deliverable = d.canDeliverText?.(text);
694
+ if (deliverable && !deliverable.ok) {
696
695
  return qaError({
697
- what: `replace-mode typing by ${native.using} requires backend-native clear support`,
696
+ what: `${deliverable.reason} Nothing was tapped or cleared.`,
698
697
  changedState: false,
699
698
  retrySafe: false,
700
- failureCode: 'BACKEND_UNSUPPORTED',
701
- nextSteps: ['Use append mode, or attach a WDA backend that supports element clear.'],
699
+ failureCode: 'TEXT_INPUT_UNSUPPORTED',
700
+ nextSteps: ['Use ASCII-safe text on this backend (adb `input text`), or type it on a Unicode-safe backend.'],
702
701
  });
703
- // This path must capture secrets exactly like the generic path below, or a
704
- // password typed by native selector is recorded VERBATIM into generated flows.
705
- // Heuristic first (selector value looks secret, same SECRET_RE the generic
706
- // resolver applies to ids), then the real signal: probe the resolved element's
707
- // type (XCUIElementTypeSecureTextField). A failing probe keeps the heuristic verdict.
708
- let secure = isSecureNode({ id: native.value, desc: '', attrs: {} });
709
- if (!secure && d.isSecureBySelector) {
710
- try {
711
- secure = await d.isSecureBySelector(native.using, native.value);
702
+ }
703
+ // A: a value that is/contains a registered secret is recorded as secret even when the
704
+ // target field is not a secure one (checked before this call can register it).
705
+ const knownSecret = matchesSessionSecret(session.secrets, text);
706
+ const native = args.target?.selector ?? null;
707
+ if (native) {
708
+ if (!d.typeBySelector)
709
+ return qaError({
710
+ what: `${native.using} selectors require backend-native selector support`,
711
+ changedState: false,
712
+ retrySafe: false,
713
+ failureCode: 'BACKEND_UNSUPPORTED',
714
+ nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
715
+ });
716
+ const replace = (args.mode ?? 'replace') === 'replace';
717
+ if (replace && !d.clearBySelector)
718
+ return qaError({
719
+ what: `replace-mode typing by ${native.using} requires backend-native clear support`,
720
+ changedState: false,
721
+ retrySafe: false,
722
+ failureCode: 'BACKEND_UNSUPPORTED',
723
+ nextSteps: ['Use append mode, or attach a WDA backend that supports element clear.'],
724
+ });
725
+ // This path must capture secrets exactly like the generic path below, or a
726
+ // password typed by native selector is recorded VERBATIM into generated flows.
727
+ // Heuristic first (selector value looks secret, same SECRET_RE the generic
728
+ // resolver applies to ids), then the real signal: probe the resolved element's
729
+ // type (XCUIElementTypeSecureTextField). A failing probe keeps the heuristic verdict.
730
+ let secure = isSecureNode({ id: native.value, desc: '', attrs: {} });
731
+ if (!secure && d.isSecureBySelector) {
732
+ try {
733
+ secure = await d.isSecureBySelector(native.using, native.value);
734
+ }
735
+ catch {
736
+ // probe unavailable (older WDA / element churn): heuristic alone decides
737
+ }
712
738
  }
713
- catch {
714
- // probe unavailable (older WDA / element churn): heuristic alone decides
739
+ if (secure) {
740
+ session.secrets.add(text);
741
+ sessions.markAuth(session, { loginPerformed: true, loginPerformedAt: Date.now() });
742
+ sessions.milestone(session, 'login_performed');
743
+ }
744
+ if (replace)
745
+ await d.clearBySelector(native.using, native.value);
746
+ await d.typeBySelector(native.using, native.value, text);
747
+ const recordSecret = secure || knownSecret;
748
+ meta = {
749
+ typedChars: text.length,
750
+ // The response never echoes the value; `redacted`/`secret` flag a value treated as
751
+ // secret (registered for scrubbing, recorded only as a placeholder).
752
+ ...(recordSecret ? { redacted: true, secret: true } : {}),
753
+ ...(expanded.vars.length ? { placeholders: expanded.vars } : {}),
754
+ mode: args.mode ?? 'replace',
755
+ via: 'native-selector',
756
+ selector: args.target?.selector,
757
+ submit: !!args.submit,
758
+ };
759
+ // Never store a secret's value in the IR; secrets become a ${VAR} at generate time.
760
+ {
761
+ const nativeTarget = recordableNativeTarget(session, native);
762
+ toRecord = {
763
+ action: 'type',
764
+ ...nativeTarget,
765
+ secret: recordSecret,
766
+ // a secret is recorded only as its ${VAR} placeholder (never the value)
767
+ text: recordSecret ? (recordText !== text ? recordText : undefined) : recordText,
768
+ exportability: recordSecret ? 'needs-human-data' : nativeTarget.exportability,
769
+ };
715
770
  }
771
+ if (args.submit)
772
+ await d.pressKey('enter');
773
+ break;
716
774
  }
717
- if (secure) {
775
+ const resolved = await resolveTarget(session, stripSelector(args.target));
776
+ if ('error' in resolved)
777
+ return fail(resolved.error, false);
778
+ let t = resolved;
779
+ // H5: focusing a field hidden under the keyboard would type a stray key character.
780
+ const g = await guardKeyboard(session, d, t, stripSelector(args.target));
781
+ if ('result' in g)
782
+ return g.result;
783
+ t = g.t;
784
+ if (g.warning)
785
+ warnings.push(g.warning);
786
+ if (g.hidKeyboard)
787
+ keyboardHidden = true;
788
+ // Typing into a secure field > remember the value so it's scrubbed everywhere, and
789
+ // record that a login was performed (auth-state reporting, P1.5).
790
+ if (t.secure) {
718
791
  session.secrets.add(text);
719
792
  sessions.markAuth(session, { loginPerformed: true, loginPerformedAt: Date.now() });
720
793
  sessions.milestone(session, 'login_performed');
721
794
  }
722
- if (replace)
723
- await d.clearBySelector(native.using, native.value);
724
- await d.typeBySelector(native.using, native.value, text);
725
- const recordSecret = secure || knownSecret;
795
+ await d.tapXY(t.x, t.y); // focus + raise IME (real touch)
796
+ await awaitIme(d, 700, g.imeUp ? IME_HOP_FLOOR_MS : 0);
797
+ if ((args.mode ?? 'replace') === 'replace')
798
+ await d.clearFocusedText(t.textLen);
799
+ await d.inputText(text);
800
+ if (args.submit)
801
+ await d.pressKey('enter');
802
+ // Never echo the typed value. It may be a password/OTP/email/token and would
803
+ // leak into the agent transcript + artifacts (sensitive-mode).
804
+ const recordSecret = !!t.secure || knownSecret;
726
805
  meta = {
727
806
  typedChars: text.length,
728
- // The response never echoes the value; `redacted`/`secret` flag a value treated as
729
- // secret (registered for scrubbing, recorded only as a placeholder).
730
807
  ...(recordSecret ? { redacted: true, secret: true } : {}),
731
808
  ...(expanded.vars.length ? { placeholders: expanded.vars } : {}),
732
809
  mode: args.mode ?? 'replace',
733
- via: 'native-selector',
734
- selector: args.target?.selector,
810
+ via: t.via,
735
811
  submit: !!args.submit,
736
812
  };
737
813
  // Never store a secret's value in the IR; secrets become a ${VAR} at generate time.
738
814
  {
739
- const nativeTarget = recordableNativeTarget(session, native);
815
+ const targetRecord = recordableTap(session, stripSelector(args.target), t);
740
816
  toRecord = {
741
817
  action: 'type',
742
- ...nativeTarget,
818
+ selector: targetRecord.selector,
819
+ selectorKind: targetRecord.selectorKind,
743
820
  secret: recordSecret,
744
821
  // a secret is recorded only as its ${VAR} placeholder (never the value)
745
822
  text: recordSecret ? (recordText !== text ? recordText : undefined) : recordText,
746
- exportability: recordSecret ? 'needs-human-data' : nativeTarget.exportability,
823
+ exportability: recordSecret ? 'needs-human-data' : targetRecord.exportability,
824
+ provenance: targetRecord.provenance,
747
825
  };
748
826
  }
749
- if (args.submit)
750
- await d.pressKey('enter');
751
827
  break;
752
828
  }
753
- const resolved = await resolveTarget(session, stripSelector(args.target));
754
- if ('error' in resolved)
755
- return fail(resolved.error, false);
756
- let t = resolved;
757
- // H5: focusing a field hidden under the keyboard would type a stray key character.
758
- const g = await guardKeyboard(session, d, t, stripSelector(args.target));
759
- if ('result' in g)
760
- return g.result;
761
- t = g.t;
762
- if (g.warning)
763
- warnings.push(g.warning);
764
- if (g.hidKeyboard)
765
- keyboardHidden = true;
766
- // Typing into a secure field > remember the value so it's scrubbed everywhere, and
767
- // record that a login was performed (auth-state reporting, P1.5).
768
- if (t.secure) {
769
- session.secrets.add(text);
770
- sessions.markAuth(session, { loginPerformed: true, loginPerformedAt: Date.now() });
771
- sessions.milestone(session, 'login_performed');
772
- }
773
- await d.tapXY(t.x, t.y); // focus + raise IME (real touch)
774
- await awaitIme(d, 700, g.imeUp ? IME_HOP_FLOOR_MS : 0);
775
- if ((args.mode ?? 'replace') === 'replace')
829
+ case 'clear': {
830
+ const native = args.target?.selector ?? null;
831
+ if (native) {
832
+ if (!d.clearBySelector)
833
+ return qaError({
834
+ what: `${native.using} selectors require backend-native clear support`,
835
+ changedState: false,
836
+ retrySafe: false,
837
+ failureCode: 'BACKEND_UNSUPPORTED',
838
+ nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
839
+ });
840
+ await d.clearBySelector(native.using, native.value);
841
+ meta = { cleared: true, via: 'native-selector', selector: args.target?.selector };
842
+ toRecord = { action: 'clear', ...recordableNativeTarget(session, native) };
843
+ break;
844
+ }
845
+ const resolved = await resolveTarget(session, stripSelector(args.target));
846
+ if ('error' in resolved)
847
+ return fail(resolved.error, false);
848
+ const g = await guardKeyboard(session, d, resolved, stripSelector(args.target));
849
+ if ('result' in g)
850
+ return g.result;
851
+ const t = g.t;
852
+ if (g.warning)
853
+ warnings.push(g.warning);
854
+ if (g.hidKeyboard)
855
+ keyboardHidden = true;
856
+ await d.tapXY(t.x, t.y); // focus + raise IME (real touch)
857
+ await awaitIme(d, 400, g.imeUp ? IME_HOP_FLOOR_MS : 0);
776
858
  await d.clearFocusedText(t.textLen);
777
- await d.inputText(text);
778
- if (args.submit)
779
- await d.pressKey('enter');
780
- // Never echo the typed value. It may be a password/OTP/email/token and would
781
- // leak into the agent transcript + artifacts (sensitive-mode).
782
- const recordSecret = !!t.secure || knownSecret;
783
- meta = {
784
- typedChars: text.length,
785
- ...(recordSecret ? { redacted: true, secret: true } : {}),
786
- ...(expanded.vars.length ? { placeholders: expanded.vars } : {}),
787
- mode: args.mode ?? 'replace',
788
- via: t.via,
789
- submit: !!args.submit,
790
- };
791
- // Never store a secret's value in the IR; secrets become a ${VAR} at generate time.
792
- {
793
- const targetRecord = recordableTap(session, stripSelector(args.target), t);
794
- toRecord = {
795
- action: 'type',
796
- selector: targetRecord.selector,
797
- selectorKind: targetRecord.selectorKind,
798
- secret: recordSecret,
799
- // a secret is recorded only as its ${VAR} placeholder (never the value)
800
- text: recordSecret ? (recordText !== text ? recordText : undefined) : recordText,
801
- exportability: recordSecret ? 'needs-human-data' : targetRecord.exportability,
802
- provenance: targetRecord.provenance,
803
- };
804
- }
805
- break;
806
- }
807
- case 'clear': {
808
- const native = args.target?.selector ?? null;
809
- if (native) {
810
- if (!d.clearBySelector)
811
- return qaError({
812
- what: `${native.using} selectors require backend-native clear support`,
813
- changedState: false,
814
- retrySafe: false,
815
- failureCode: 'BACKEND_UNSUPPORTED',
816
- nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
817
- });
818
- await d.clearBySelector(native.using, native.value);
819
- meta = { cleared: true, via: 'native-selector', selector: args.target?.selector };
820
- toRecord = { action: 'clear', ...recordableNativeTarget(session, native) };
859
+ meta = { cleared: true, via: t.via };
860
+ toRecord = { action: 'clear', ...recordableTap(session, stripSelector(args.target), t) };
821
861
  break;
822
862
  }
823
- const resolved = await resolveTarget(session, stripSelector(args.target));
824
- if ('error' in resolved)
825
- return fail(resolved.error, false);
826
- const g = await guardKeyboard(session, d, resolved, stripSelector(args.target));
827
- if ('result' in g)
828
- return g.result;
829
- const t = g.t;
830
- if (g.warning)
831
- warnings.push(g.warning);
832
- if (g.hidKeyboard)
833
- keyboardHidden = true;
834
- await d.tapXY(t.x, t.y); // focus + raise IME (real touch)
835
- await awaitIme(d, 400, g.imeUp ? IME_HOP_FLOOR_MS : 0);
836
- await d.clearFocusedText(t.textLen);
837
- meta = { cleared: true, via: t.via };
838
- toRecord = { action: 'clear', ...recordableTap(session, stripSelector(args.target), t) };
839
- break;
840
- }
841
- case 'swipe': {
842
- const direction = args.direction; // presence enforced by missingRequiredField
843
- // Derive the gesture from the real screen size. WDA swipes are in POINTS
844
- // (≤~440pt wide), so the old fixed 540/1200 constants were off-screen on iOS; the
845
- // legacy constants survive only inside the shared fallback for screenSize()===null.
846
- // A supplied target that fails to resolve is a structured error (not a
847
- // silent default swipe), and a resolved point is used verbatim (0 is a legitimate
848
- // coordinate). Endpoints keep an ~8% inset clear of iOS system-gesture zones.
849
- const size = await d.screenSize().catch(() => null);
850
- let vec;
851
- if (args.target) {
852
- const start = await resolveTarget(session, stripSelector(args.target));
853
- if ('error' in start)
854
- return fail(start.error, false);
855
- vec = swipeFromPoint(size, { x: start.x, y: start.y }, direction, 0.5, GESTURE_EDGE_INSET);
856
- }
857
- else {
858
- vec = swipeVector(size, direction, 'center', 0.5, GESTURE_EDGE_INSET);
859
- }
860
- await d.swipe(vec[0], vec[1], vec[2], vec[3], 300);
861
- meta = { direction };
862
- toRecord = { action: 'swipe', direction, exportability: 'coordinate' };
863
- break;
864
- }
865
- case 'scroll': {
866
- const max = args.maxScrolls ?? 8;
867
- // scroll down = FINGER swipes up (same for the other directions where finger ==
868
- // content axis). Screen-relative + clamped vector, size fetched once.
869
- const finger = { down: 'up', up: 'down', left: 'left', right: 'right' };
870
- const size = await d.screenSize().catch(() => null);
871
- const dir = args.direction; // presence enforced by missingRequiredField
872
- const u = args.untilVisible;
873
- // A match only counts as FOUND when its center is on screen and not under the soft
874
- // keyboard: a row just crossing the bottom edge is "in the tree" but a center tap on it
875
- // would land off-screen / on a key, so keep swiping instead.
876
- const kb = u ? await keyboardArea(d) : undefined;
877
- const probe = async () => {
878
- const xml = await d.dumpXml();
879
- const at = Date.now();
880
- const parsed = parseSnapshot(xml);
881
- const [sw, sh] = size ? [size.width, size.height] : parsed.screen;
882
- const hit = parsed.elements.some((e) => {
883
- const match = (u?.id && e.id === u.id) ||
884
- (u?.text &&
885
- (e.text?.toLowerCase().includes(u.text.toLowerCase()) || e.label?.toLowerCase().includes(u.text.toLowerCase())));
886
- if (!match)
887
- return false;
888
- const c = center(e.bounds);
889
- const onScreen = !(sw > 0 && sh > 0) || (c.x >= 0 && c.y >= 0 && c.x < sw && c.y < sh);
890
- return onScreen && !inRect(kb?.rect, c.x, c.y);
891
- });
892
- // Positions are part of the signature: a list that moved but still shows the same
893
- // labels is not at its end (the label-only signature reported endOfList mid-list).
894
- const sig = parsed.elements
895
- .map((e) => `${signature(e)}@${e.bounds.join(',')}`)
896
- .sort()
897
- .join('\n');
898
- settleSeed = { xml, at };
899
- return { hit, sig, nodes: parsed.allNodes };
900
- };
901
- // B: anchor the swipe INSIDE the largest scrollable container (a swipe that starts on
902
- // a sticky app bar moves nothing); screen center only when no container is known.
903
- let anchorNodes = session.lastSnapshot?.allNodes;
904
- if (!anchorNodes && !u)
905
- anchorNodes = await d.dumpXml().then((x) => parseSnapshot(x).allNodes, () => undefined);
906
- let anchoredIn = 'screen';
907
- const nextVec = () => {
908
- const rect = largestScrollableRect(anchorNodes, size);
909
- anchoredIn = rect ? 'scrollable' : 'screen';
910
- return rect
911
- ? swipeInRect(rect, finger[dir], size, 0.6, SCROLL_CONTAINER_INSET, GESTURE_EDGE_INSET)
912
- : swipeVector(size, finger[dir], 'center', 0.6, GESTURE_EDGE_INSET);
913
- };
914
- let found = false;
915
- let endOfList = false;
916
- let swipes = 0;
917
- let prevSig;
918
- // Already visible? Then don't swipe at all (it could scroll the target away).
919
- if (u) {
920
- const first = await probe();
921
- found = first.hit;
922
- prevSig = first.sig;
923
- anchorNodes = first.nodes;
924
- }
925
- // C: a plain scroll is exactly ONE swipe; only untilVisible loops (up to maxScrolls).
926
- const limit = u ? max : 1;
927
- for (let i = 0; i < limit && !found; i++) {
928
- const vec = nextVec();
863
+ case 'swipe': {
864
+ const direction = args.direction; // presence enforced by missingRequiredField
865
+ // Derive the gesture from the real screen size. WDA swipes are in POINTS
866
+ // (≤~440pt wide), so the old fixed 540/1200 constants were off-screen on iOS; the
867
+ // legacy constants survive only inside the shared fallback for screenSize()===null.
868
+ // A supplied target that fails to resolve is a structured error (not a
869
+ // silent default swipe), and a resolved point is used verbatim (0 is a legitimate
870
+ // coordinate). Endpoints keep an ~8% inset clear of iOS system-gesture zones.
871
+ const size = await d.screenSize().catch(() => null);
872
+ let vec;
873
+ if (args.target) {
874
+ const start = await resolveTarget(session, stripSelector(args.target));
875
+ if ('error' in start)
876
+ return fail(start.error, false);
877
+ vec = swipeFromPoint(size, { x: start.x, y: start.y }, direction, 0.5, GESTURE_EDGE_INSET);
878
+ }
879
+ else {
880
+ vec = swipeVector(size, direction, 'center', 0.5, GESTURE_EDGE_INSET);
881
+ }
929
882
  await d.swipe(vec[0], vec[1], vec[2], vec[3], 300);
930
- swipes++;
883
+ meta = { direction };
884
+ toRecord = { action: 'swipe', direction, exportability: 'coordinate' };
885
+ break;
886
+ }
887
+ case 'scroll': {
888
+ const max = args.maxScrolls ?? 8;
889
+ // scroll down = FINGER swipes up (same for the other directions where finger ==
890
+ // content axis). Screen-relative + clamped vector, size fetched once.
891
+ const finger = { down: 'up', up: 'down', left: 'left', right: 'right' };
892
+ const size = await d.screenSize().catch(() => null);
893
+ const dir = args.direction; // presence enforced by missingRequiredField
894
+ const u = args.untilVisible;
895
+ // A match only counts as FOUND when its center is on screen and not under the soft
896
+ // keyboard: a row just crossing the bottom edge is "in the tree" but a center tap on it
897
+ // would land off-screen / on a key, so keep swiping instead.
898
+ const kb = u ? await keyboardArea(d) : undefined;
899
+ const probe = async () => {
900
+ const xml = await d.dumpXml();
901
+ const at = Date.now();
902
+ const parsed = parseSnapshot(xml);
903
+ const [sw, sh] = size ? [size.width, size.height] : parsed.screen;
904
+ const hit = parsed.elements.some((e) => {
905
+ const match = (u?.id && e.id === u.id) ||
906
+ (u?.text &&
907
+ (e.text?.toLowerCase().includes(u.text.toLowerCase()) || e.label?.toLowerCase().includes(u.text.toLowerCase())));
908
+ if (!match)
909
+ return false;
910
+ const c = center(e.bounds);
911
+ const onScreen = !(sw > 0 && sh > 0) || (c.x >= 0 && c.y >= 0 && c.x < sw && c.y < sh);
912
+ return onScreen && !inRect(kb?.rect, c.x, c.y);
913
+ });
914
+ // Positions are part of the signature: a list that moved but still shows the same
915
+ // labels is not at its end (the label-only signature reported endOfList mid-list).
916
+ const sig = parsed.elements
917
+ .map((e) => `${signature(e)}@${e.bounds.join(',')}`)
918
+ .sort()
919
+ .join('\n');
920
+ settleSeed = { xml, at };
921
+ return { hit, sig, nodes: parsed.allNodes };
922
+ };
923
+ // B: anchor the swipe INSIDE the largest scrollable container (a swipe that starts on
924
+ // a sticky app bar moves nothing); screen center only when no container is known.
925
+ let anchorNodes = session.lastSnapshot?.allNodes;
926
+ if (!anchorNodes && !u)
927
+ anchorNodes = await d.dumpXml().then((x) => parseSnapshot(x).allNodes, () => undefined);
928
+ let anchoredIn = 'screen';
929
+ const nextVec = () => {
930
+ const rect = largestScrollableRect(anchorNodes, size);
931
+ anchoredIn = rect ? 'scrollable' : 'screen';
932
+ return rect
933
+ ? swipeInRect(rect, finger[dir], size, 0.6, SCROLL_CONTAINER_INSET, GESTURE_EDGE_INSET)
934
+ : swipeVector(size, finger[dir], 'center', 0.6, GESTURE_EDGE_INSET);
935
+ };
936
+ let found = false;
937
+ let endOfList = false;
938
+ let swipes = 0;
939
+ let prevSig;
940
+ // Already visible? Then don't swipe at all (it could scroll the target away).
931
941
  if (u) {
932
- await new Promise((r) => setTimeout(r, 400));
933
- const now = await probe();
934
- anchorNodes = now.nodes;
935
- found = now.hit;
936
- if (found)
937
- break;
938
- // Screen identical after a swipe > end of the list; more swipes can't help.
939
- if (now.sig === prevSig) {
940
- endOfList = true;
941
- break;
942
+ const first = await probe();
943
+ found = first.hit;
944
+ prevSig = first.sig;
945
+ anchorNodes = first.nodes;
946
+ }
947
+ // C: a plain scroll is exactly ONE swipe; only untilVisible loops (up to maxScrolls).
948
+ const limit = u ? max : 1;
949
+ for (let i = 0; i < limit && !found; i++) {
950
+ const vec = nextVec();
951
+ await d.swipe(vec[0], vec[1], vec[2], vec[3], 300);
952
+ swipes++;
953
+ if (u) {
954
+ await new Promise((r) => setTimeout(r, 400));
955
+ const now = await probe();
956
+ anchorNodes = now.nodes;
957
+ found = now.hit;
958
+ if (found)
959
+ break;
960
+ // Screen identical after a swipe > end of the list; more swipes can't help.
961
+ if (now.sig === prevSig) {
962
+ endOfList = true;
963
+ break;
964
+ }
965
+ prevSig = now.sig;
942
966
  }
943
- prevSig = now.sig;
944
967
  }
968
+ meta = {
969
+ direction: dir,
970
+ swipes,
971
+ ...(swipes ? { anchoredIn } : {}),
972
+ untilVisibleFound: u ? found : undefined,
973
+ ...(endOfList ? { endOfList: true } : {}),
974
+ };
975
+ toRecord = {
976
+ action: 'scroll',
977
+ direction: dir,
978
+ selector: args.untilVisible?.text,
979
+ exportability: args.untilVisible?.text ? 'semantic' : 'coordinate',
980
+ };
981
+ break;
982
+ }
983
+ case 'press': {
984
+ const key = args.key; // presence enforced by missingRequiredField
985
+ await d.pressKey(key);
986
+ meta = { key };
987
+ // iOS has no back key: WdaDriver taps the nav-bar back button or edge-swipes. Say which.
988
+ if (key === 'back' && d.kind === 'wda') {
989
+ const via = d.lastBackVia;
990
+ if (via)
991
+ meta.backVia = via;
992
+ }
993
+ toRecord = { action: 'press', key, exportability: 'semantic' };
994
+ break;
945
995
  }
946
- meta = {
947
- direction: dir,
948
- swipes,
949
- ...(swipes ? { anchoredIn } : {}),
950
- untilVisibleFound: u ? found : undefined,
951
- ...(endOfList ? { endOfList: true } : {}),
952
- };
953
- toRecord = {
954
- action: 'scroll',
955
- direction: dir,
956
- selector: args.untilVisible?.text,
957
- exportability: args.untilVisible?.text ? 'semantic' : 'coordinate',
958
- };
959
- break;
960
- }
961
- case 'press': {
962
- const key = args.key; // presence enforced by missingRequiredField
963
- await d.pressKey(key);
964
- meta = { key };
965
- // iOS has no back key: WdaDriver taps the nav-bar back button or edge-swipes. Say which.
966
- if (key === 'back' && d.kind === 'wda') {
967
- const via = d.lastBackVia;
968
- if (via)
969
- meta.backVia = via;
996
+ case 'open_url': {
997
+ const url = args.url; // presence enforced by missingRequiredField
998
+ await d.openUrl(url);
999
+ meta = { url };
1000
+ toRecord = { action: 'open_url', url, exportability: 'semantic' };
1001
+ break;
970
1002
  }
971
- toRecord = { action: 'press', key, exportability: 'semantic' };
972
- break;
973
- }
974
- case 'open_url': {
975
- const url = args.url; // presence enforced by missingRequiredField
976
- await d.openUrl(url);
977
- meta = { url };
978
- toRecord = { action: 'open_url', url, exportability: 'semantic' };
979
- break;
980
1003
  }
981
1004
  }
982
- }
983
- catch (e) {
984
- // Cancelled mid-action: not a driver failure (no WDA_UNREACHABLE / SNAPSHOT_FAILED tool
985
- // error, no finding). The action may have partly run, so changedState stays true.
986
- if (isAbortError(e))
987
- return cancelledResult(`Action "${action}" cancelled: the call was aborted before it finished`, true);
988
- const msg = String(e?.message ?? e);
989
- const failureCode = /BACKEND_UNSUPPORTED|not supported by the WDA backend/.test(msg)
990
- ? 'BACKEND_UNSUPPORTED'
991
- : classifyFlowDriverError(e);
992
- // H2: driver errors can echo argv/stderr. Scrub known secrets AND the value just typed.
993
- const redactErr = makeRedactor(errorSecrets(session, action === 'type' ? (typedValue ?? args.text) : undefined));
994
- return qaError({
995
- what: `Action "${action}" failed: ${redactErr(String(e)) ?? ''}`,
996
- changedState: true,
997
- retrySafe: !['WDA_SESSION_FAILED', 'UNKNOWN'].includes(failureCode),
998
- failureCode,
999
- nextSteps: failureCode === 'UNKNOWN'
1000
- ? ['Confirm the device is online and re-snapshot.']
1001
- : ['Use the failureCode to choose recovery, then re-snapshot before retrying.'],
1002
- });
1003
- }
1004
- // count this as an action (wait already returned earlier)
1005
- sessions.bump(session, 'actions');
1006
- // Record the action into the IR for qa_generate target:"flow" (the action definitely happened here;
1007
- // recorded now so a later post-snapshot failure doesn't lose the step).
1008
- if (toRecord) {
1009
- const sc = recordingScreenContext(session);
1010
- sessions.addRecordedAction(session, { at: Date.now(), ...sc, ...toRecord });
1011
- }
1012
- // Post-action observation is wrapped so a settle/dump/parse failure (e.g. the
1013
- // looping-animation case) returns a Swipium-shaped result, never a raw MCP error.
1014
- try {
1015
- // settle > observe > health (seeded with untilVisible's last probe; it was taken after the
1016
- // last swipe, so re-dumping it first would be pure latency)
1017
- let s = await settle(d, { timeoutMs: args.timeoutMs ?? 8000, ...(settleSeed ? { seed: settleSeed } : {}) });
1018
- // Cancelled while observing: an empty/aborted dump is not evidence (no WDA_UNREACHABLE
1019
- // health finding, no visual-fallback switch).
1020
- if (isAbortError(undefined))
1021
- return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
1022
- let post = parseSnapshot(s.xml);
1023
- let postSigs = new Set(post.elements.map(signature));
1024
- // F: for gestures that move content, a bounds shift counts as a change too.
1025
- const positional = action === 'scroll' || action === 'swipe';
1026
- const movedContent = () => positional && prePositions !== undefined && prePositions !== positionalFingerprint(post.fullByRef);
1027
- // Toggle state counts: a Switch/Checkbox flip keeps every signature (role|label|id|text).
1028
- let toggled = stateChangedRefs(preState, post.fullByRef);
1029
- let changed = !setsEqual(preSigs, postSigs) || movedContent() || toggled.length > 0;
1030
- // No-change retry (review §4.5/§4.7): an instant tap that did nothing is often the RN
1031
- // tap quirk. Retry ONCE as a longer press before believing it's blocked.
1032
- let retriedAsPress = false;
1033
- // Never when the keyboard was up at tap time: the retry would re-press a point the IME
1034
- // may own and type a second stray character (H5).
1035
- if (!changed && action === 'tap' && tapRetry?.instant && !tapRetry.imeUp) {
1036
- await d.pressXY(tapRetry.x, tapRetry.y, 120);
1037
- retriedAsPress = true;
1038
- s = await settle(d, { timeoutMs: args.timeoutMs ?? 8000 });
1039
- post = parseSnapshot(s.xml);
1040
- postSigs = new Set(post.elements.map(signature));
1041
- toggled = stateChangedRefs(preState, post.fullByRef);
1042
- changed = !setsEqual(preSigs, postSigs) || movedContent() || toggled.length > 0;
1043
- }
1044
- session.lastSnapshot = { fullByRef: post.fullByRef, signatures: postSigs, allNodes: post.allNodes };
1045
- // A structured dump succeeded > a visual-fallback session is structured again (per-screen).
1046
- const modeRecovered = s.xml ? sessions.noteStructuredDump(session) : false;
1047
- const health = await checkHealth(d, session.appId, s.xml, { nodes: post.allNodes });
1048
- if (health.cancelled)
1049
- return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
1050
- // Track no-change actions for the budget / no-op-loop detector.
1051
- if (!changed && (action === 'tap' || action === 'swipe' || action === 'scroll' || action === 'press')) {
1052
- sessions.bump(session, 'noChangeActions');
1053
- }
1054
- const budgetReached = sessions.budgetStop(session);
1055
- // Sensitive-mode present: mask secure fields + scrub known secrets AND the just-typed
1056
- // value (covers non-secure fields like email for this immediate response).
1057
- const redact = makeRedactor([...session.secrets, ...(action === 'type' && typedValue ? [typedValue] : [])]);
1058
- // Record non-info findings for qa_report (deterministic bug trail) + app-error screenshot.
1059
- await recordHealthFindings(sessions, session, health.findings, d, health.foreground);
1060
- const banner = `${action} ${JSON.stringify(meta)} > changed=${changed}${retriedAsPress ? ' (retried as press)' : ''} ` +
1061
- `settled=${s.settled} quality=${post.quality.verdict} native=${health.nativeHealthy ? 'ok' : health.nativeStatus} app=${health.appStatus}` +
1062
- (budgetReached ? `\n⏹ budget reached: ${budgetReached}, call qa_report.` : '') +
1063
- (modeRecovered ? '\nmode: structured again (a UI tree dump succeeded; visual-fallback cleared)' : '') +
1064
- (!changed && retriedAsPress
1065
- ? `\nNo change even after a press retry. Likely wrong coords / disabled element / overlay / auth wall.`
1066
- : '');
1067
- const findings = health.findings.length
1068
- ? '\n' +
1069
- health.findings
1070
- .map((f) => `[${f.severity}] ${f.layer ?? '?'}/${f.kind}: ${f.detail}${f.evidence ? `: "${f.evidence}"` : ''}`)
1071
- .join('\n')
1072
- : '';
1073
- // `observe` changes ONLY the element presentation below. Everything above
1074
- // (lastSnapshot bookkeeping, press retry, health recording, counters, budget lines)
1075
- // is identical in all modes.
1076
- let elementPayload = {};
1077
- let elementsText = '';
1078
- const added = observe === 'diff' ? post.elements.filter((e) => !preSigs.has(signature(e))) : [];
1079
- // Mostly-new screen (navigation): a diff would list every new element AND every old one
1080
- // as removed, bigger than the full list. Return the (capped) full list + removedCount.
1081
- const diffAsFull = observe === 'diff' && post.elements.length > 0 && added.length > post.elements.length * DIFF_FULL_FALLBACK_RATIO;
1082
- if (observe === 'diff' && !diffAsFull) {
1083
- const removed = [...preSigs].filter((sig) => !postSigs.has(sig)).map((sig) => redact(sig) ?? sig);
1084
- const { elements: addedShown, rendered: renderedAdded, omitted } = presentElements(added, redact);
1085
- const unchangedElements = post.elements.length - added.length;
1086
- const hint = `${unchangedElements} unchanged element(s) not shown; pass observe:"full" to see the whole screen.`;
1087
- elementPayload = { elementsOmitted: omitted, elements: addedShown, removed, unchangedElements, hint };
1088
- elementsText =
1089
- `\n\nDIFF vs pre-action: +${added.length} / -${removed.length}` +
1090
- (added.length ? `\n${renderedAdded}` : '') +
1091
- (removed.length ? `\nremoved: ${removed.join(' | ')}` : '') +
1092
- `\n${hint}`;
1093
- }
1094
- else if (diffAsFull) {
1095
- const removedCount = [...preSigs].filter((sig) => !postSigs.has(sig)).length;
1096
- const { elements: outElements, rendered, omitted } = presentElements(post.elements, redact);
1097
- elementPayload = {
1098
- diffAsFull: true,
1099
- elementsOmitted: omitted,
1100
- elements: outElements,
1101
- addedCount: added.length,
1102
- removedCount,
1103
- };
1104
- elementsText =
1105
- `\n\nNEW SCREEN (${added.length}/${post.elements.length} elements new, ${removedCount} previous gone), full list:` +
1106
- `\n${rendered}`;
1005
+ catch (e) {
1006
+ // Cancelled mid-action: not a driver failure (no WDA_UNREACHABLE / SNAPSHOT_FAILED tool
1007
+ // error, no finding). The action may have partly run, so changedState stays true.
1008
+ if (isAbortError(e))
1009
+ return cancelledResult(`Action "${action}" cancelled: the call was aborted before it finished`, true);
1010
+ const msg = String(e?.message ?? e);
1011
+ const failureCode = /BACKEND_UNSUPPORTED|not supported by the WDA backend/.test(msg)
1012
+ ? 'BACKEND_UNSUPPORTED'
1013
+ : classifyFlowDriverError(e);
1014
+ // H2: driver errors can echo argv/stderr. Scrub known secrets AND the value just typed.
1015
+ const redactErr = makeRedactor(errorSecrets(session, action === 'type' ? (typedValue ?? args.text) : undefined));
1016
+ return qaError({
1017
+ what: `Action "${action}" failed: ${redactErr(String(e)) ?? ''}`,
1018
+ changedState: true,
1019
+ retrySafe: !['WDA_SESSION_FAILED', 'UNKNOWN'].includes(failureCode),
1020
+ failureCode,
1021
+ nextSteps: failureCode === 'UNKNOWN'
1022
+ ? ['Confirm the device is online and re-snapshot.']
1023
+ : ['Use the failureCode to choose recovery, then re-snapshot before retrying.'],
1024
+ });
1107
1025
  }
1108
- else if (observe === 'full') {
1109
- const { elements: outElements, rendered, omitted } = presentElements(post.elements, redact);
1110
- elementPayload = { elementsOmitted: omitted, elements: outElements };
1111
- elementsText = `\n\n${rendered}`;
1026
+ // count this as an action (wait already returned earlier)
1027
+ sessions.bump(session, 'actions');
1028
+ // Record the action into the IR for qa_generate target:"flow" (the action definitely happened here;
1029
+ // recorded now so a later post-snapshot failure doesn't lose the step).
1030
+ if (toRecord) {
1031
+ const sc = recordingScreenContext(session);
1032
+ sessions.addRecordedAction(session, { at: Date.now(), ...sc, ...toRecord });
1112
1033
  }
1113
- else {
1114
- const hint = `${post.elements.length} element(s) not shown (observe:"none"); pass observe:"full" or run qa_snapshot to see the screen.`;
1115
- elementPayload = { hint };
1116
- elementsText = `\n\n${hint}`;
1034
+ // Post-action observation is wrapped so a settle/dump/parse failure (e.g. the
1035
+ // looping-animation case) returns a Swipium-shaped result, never a raw MCP error.
1036
+ try {
1037
+ // settle > observe > health (seeded with untilVisible's last probe; it was taken after the
1038
+ // last swipe, so re-dumping it first would be pure latency)
1039
+ let s = await settle(d, { timeoutMs: args.timeoutMs ?? 8000, ...(settleSeed ? { seed: settleSeed } : {}) });
1040
+ // Cancelled while observing: an empty/aborted dump is not evidence (no WDA_UNREACHABLE
1041
+ // health finding, no visual-fallback switch).
1042
+ if (isAbortError(undefined))
1043
+ return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
1044
+ let post = parseSnapshot(s.xml);
1045
+ let postSigs = new Set(post.elements.map(signature));
1046
+ // F: for gestures that move content, a bounds shift counts as a change too.
1047
+ const positional = action === 'scroll' || action === 'swipe';
1048
+ const movedContent = () => positional && prePositions !== undefined && prePositions !== positionalFingerprint(post.fullByRef);
1049
+ // Toggle state counts: a Switch/Checkbox flip keeps every signature (role|label|id|text).
1050
+ let toggled = stateChangedRefs(preState, post.fullByRef);
1051
+ let changed = !setsEqual(preSigs, postSigs) || movedContent() || toggled.length > 0;
1052
+ // No-change retry (review §4.5/§4.7): an instant tap that did nothing is often the RN
1053
+ // tap quirk. Retry ONCE as a longer press before believing it's blocked.
1054
+ let retriedAsPress = false;
1055
+ // Never when the keyboard was up at tap time: the retry would re-press a point the IME
1056
+ // may own and type a second stray character (H5).
1057
+ if (!changed && action === 'tap' && tapRetry?.instant && !tapRetry.imeUp) {
1058
+ await d.pressXY(tapRetry.x, tapRetry.y, 120);
1059
+ retriedAsPress = true;
1060
+ s = await settle(d, { timeoutMs: args.timeoutMs ?? 8000 });
1061
+ post = parseSnapshot(s.xml);
1062
+ postSigs = new Set(post.elements.map(signature));
1063
+ toggled = stateChangedRefs(preState, post.fullByRef);
1064
+ changed = !setsEqual(preSigs, postSigs) || movedContent() || toggled.length > 0;
1065
+ }
1066
+ session.lastSnapshot = { fullByRef: post.fullByRef, signatures: postSigs, allNodes: post.allNodes };
1067
+ // A structured dump succeeded > a visual-fallback session is structured again (per-screen).
1068
+ const modeRecovered = s.xml ? sessions.noteStructuredDump(session) : false;
1069
+ const health = await checkHealth(d, session.appId, s.xml, { nodes: post.allNodes });
1070
+ if (health.cancelled)
1071
+ return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
1072
+ // Track no-change actions for the budget / no-op-loop detector.
1073
+ if (!changed && (action === 'tap' || action === 'swipe' || action === 'scroll' || action === 'press')) {
1074
+ sessions.bump(session, 'noChangeActions');
1075
+ }
1076
+ const budgetReached = sessions.budgetStop(session);
1077
+ // Sensitive-mode present: mask secure fields + scrub known secrets AND the just-typed
1078
+ // value (covers non-secure fields like email for this immediate response).
1079
+ const redact = makeRedactor([...session.secrets, ...(action === 'type' && typedValue ? [typedValue] : [])]);
1080
+ // Record non-info findings for qa_report (deterministic bug trail) + app-error screenshot.
1081
+ await recordHealthFindings(sessions, session, health.findings, d, health.foreground);
1082
+ const banner = `${action} ${JSON.stringify(meta)} > changed=${changed}${retriedAsPress ? ' (retried as press)' : ''} ` +
1083
+ `settled=${s.settled} quality=${post.quality.verdict} native=${health.nativeHealthy ? 'ok' : health.nativeStatus} app=${health.appStatus}` +
1084
+ (budgetReached ? `\n⏹ budget reached: ${budgetReached}, call qa_report.` : '') +
1085
+ (modeRecovered ? '\nmode: structured again (a UI tree dump succeeded; visual-fallback cleared)' : '') +
1086
+ (!changed && retriedAsPress
1087
+ ? `\nNo change even after a press retry. Likely wrong coords / disabled element / overlay / auth wall.`
1088
+ : '');
1089
+ const findings = health.findings.length
1090
+ ? '\n' +
1091
+ health.findings
1092
+ .map((f) => `[${f.severity}] ${f.layer ?? '?'}/${f.kind}: ${f.detail}${f.evidence ? `: "${f.evidence}"` : ''}`)
1093
+ .join('\n')
1094
+ : '';
1095
+ // `observe` changes ONLY the element presentation below. Everything above
1096
+ // (lastSnapshot bookkeeping, press retry, health recording, counters, budget lines)
1097
+ // is identical in all modes.
1098
+ let elementPayload = {};
1099
+ let elementsText = '';
1100
+ const added = observe === 'diff' ? post.elements.filter((e) => !preSigs.has(signature(e))) : [];
1101
+ // Mostly-new screen (navigation): a diff would list every new element AND every old one
1102
+ // as removed, bigger than the full list. Return the (capped) full list + removedCount.
1103
+ const diffAsFull = observe === 'diff' && post.elements.length > 0 && added.length > post.elements.length * DIFF_FULL_FALLBACK_RATIO;
1104
+ if (observe === 'diff' && !diffAsFull) {
1105
+ const removed = [...preSigs].filter((sig) => !postSigs.has(sig)).map((sig) => redact(sig) ?? sig);
1106
+ const { payload: addedShown, rendered: renderedAdded, omitted } = presentElements(added, redact);
1107
+ const unchangedElements = post.elements.length - added.length;
1108
+ const hint = `${unchangedElements} unchanged element(s) not shown; pass observe:"full" to see the whole screen.`;
1109
+ elementPayload = { elementsOmitted: omitted, elements: addedShown, removed, unchangedElements, hint };
1110
+ elementsText =
1111
+ `\n\nDIFF vs pre-action: +${added.length} / -${removed.length}` +
1112
+ (added.length ? `\n${renderedAdded}` : '') +
1113
+ (removed.length ? `\nremoved: ${removed.join(' | ')}` : '') +
1114
+ `\n${hint}`;
1115
+ }
1116
+ else if (diffAsFull) {
1117
+ const removedCount = [...preSigs].filter((sig) => !postSigs.has(sig)).length;
1118
+ const { payload: outElements, rendered, omitted } = presentElements(post.elements, redact);
1119
+ elementPayload = {
1120
+ diffAsFull: true,
1121
+ elementsOmitted: omitted,
1122
+ elements: outElements,
1123
+ addedCount: added.length,
1124
+ removedCount,
1125
+ };
1126
+ elementsText =
1127
+ `\n\nNEW SCREEN (${added.length}/${post.elements.length} elements new, ${removedCount} previous gone), full list:` +
1128
+ `\n${rendered}`;
1129
+ }
1130
+ else if (observe === 'full') {
1131
+ const { payload: outElements, rendered, omitted } = presentElements(post.elements, redact);
1132
+ elementPayload = { elementsOmitted: omitted, elements: outElements };
1133
+ elementsText = `\n\n${rendered}`;
1134
+ }
1135
+ else {
1136
+ const hint = `${post.elements.length} element(s) not shown (observe:"none"); pass observe:"full" or run qa_snapshot to see the screen.`;
1137
+ elementPayload = { hint };
1138
+ elementsText = `\n\n${hint}`;
1139
+ }
1140
+ if (toggled.length && observe !== 'none') {
1141
+ const lines = toggled.map((ref) => {
1142
+ const n = post.fullByRef.get(ref);
1143
+ const name = isSecureNode(n) ? '«secure»' : (redact(n.desc || n.text) ?? '');
1144
+ return `${ref} ${JSON.stringify(name)} ${stateLabel(n)}`;
1145
+ });
1146
+ elementPayload = { ...elementPayload, stateChanged: lines };
1147
+ elementsText += `\nstate changed: ${lines.join(' | ')}`;
1148
+ }
1149
+ // The WDA driver transparently re-creates a reaped session (without relaunching the app);
1150
+ // the agent must know the screen may not be where it left it.
1151
+ if (d.consumeSessionRecovered?.())
1152
+ warnings.push(WDA_SESSION_RECOVERED_WARNING);
1153
+ return qaOk({
1154
+ action,
1155
+ ...meta,
1156
+ ...(keyboardHidden ? { keyboardHidden: true } : {}),
1157
+ ...(modeRecovered ? { modeRecovered: true, mode: 'structured' } : {}),
1158
+ changed,
1159
+ retriedAsPress,
1160
+ settled: s.settled,
1161
+ quality: post.quality.verdict,
1162
+ health,
1163
+ counters: session.counters,
1164
+ ...(budgetReached ? { budgetReached } : {}),
1165
+ observe,
1166
+ ...elementPayload,
1167
+ ...(warnings.length ? { warnings } : {}),
1168
+ }, `${banner}${keyboardHidden ? '\nNote: the soft keyboard covered the target and was hidden first (keyboardHidden:true).' : ''}` +
1169
+ `${findings}${warnings.map((w) => `\n⚠ ${w}`).join('')}${elementsText}`, { textOmit: ['elements', 'removed', 'hint', 'stateChanged'] });
1117
1170
  }
1118
- if (toggled.length && observe !== 'none') {
1119
- const lines = toggled.map((ref) => {
1120
- const n = post.fullByRef.get(ref);
1121
- const name = isSecureNode(n) ? '«secure»' : (redact(n.desc || n.text) ?? '');
1122
- return `${ref} ${JSON.stringify(name)} ${stateLabel(n)}`;
1171
+ catch (e) {
1172
+ if (isAbortError(e))
1173
+ return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
1174
+ // The action ran; observing the result failed (often a UI that never reaches idle).
1175
+ // visual-fallback is per-screen: the next successful qa_snapshot / qa_act dump clears it.
1176
+ const idle = /idle|dump|hierarchy/i.test(String(e));
1177
+ if (idle)
1178
+ sessions.setMode(session, 'visual-fallback');
1179
+ const redactErr = makeRedactor(errorSecrets(session, action === 'type' ? (typedValue ?? args.text) : undefined));
1180
+ return qaError({
1181
+ what: `Action "${action}" ran, but observing the result failed: ${redactErr(String(e)) ?? ''}`,
1182
+ changedState: true,
1183
+ retrySafe: false,
1184
+ failureCode: classifyFlowDriverError(e, 'SNAPSHOT_FAILED'),
1185
+ nextSteps: idle
1186
+ ? [
1187
+ 'Switched to visual-fallback for this screen (cleared by the next successful qa_snapshot / qa_act dump). Use qa_screenshot; qa_check_health still works.',
1188
+ ]
1189
+ : ['Re-check the device is online, then qa_screenshot / qa_check_health.'],
1123
1190
  });
1124
- elementPayload = { ...elementPayload, stateChanged: lines };
1125
- elementsText += `\nstate changed: ${lines.join(' | ')}`;
1126
1191
  }
1127
- // The WDA driver transparently re-creates a reaped session (without relaunching the app);
1128
- // the agent must know the screen may not be where it left it.
1129
- if (d.consumeSessionRecovered?.())
1130
- warnings.push(WDA_SESSION_RECOVERED_WARNING);
1131
- return qaOk({
1132
- action,
1133
- ...meta,
1134
- ...(keyboardHidden ? { keyboardHidden: true } : {}),
1135
- ...(modeRecovered ? { modeRecovered: true, mode: 'structured' } : {}),
1136
- changed,
1137
- retriedAsPress,
1138
- settled: s.settled,
1139
- quality: post.quality.verdict,
1140
- health,
1141
- counters: session.counters,
1142
- ...(budgetReached ? { budgetReached } : {}),
1143
- observe,
1144
- ...elementPayload,
1145
- ...(warnings.length ? { warnings } : {}),
1146
- }, `${banner}${keyboardHidden ? '\nNote: the soft keyboard covered the target and was hidden first (keyboardHidden:true).' : ''}` +
1147
- `${findings}${warnings.map((w) => `\n⚠ ${w}`).join('')}${elementsText}`, { textOmit: ['elements', 'removed', 'hint', 'stateChanged'] });
1148
- }
1149
- catch (e) {
1150
- if (isAbortError(e))
1151
- return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
1152
- // The action ran; observing the result failed (often a UI that never reaches idle).
1153
- // visual-fallback is per-screen: the next successful qa_snapshot / qa_act dump clears it.
1154
- const idle = /idle|dump|hierarchy/i.test(String(e));
1155
- if (idle)
1156
- sessions.setMode(session, 'visual-fallback');
1157
- const redactErr = makeRedactor(errorSecrets(session, action === 'type' ? (typedValue ?? args.text) : undefined));
1158
- return qaError({
1159
- what: `Action "${action}" ran, but observing the result failed: ${redactErr(String(e)) ?? ''}`,
1160
- changedState: true,
1161
- retrySafe: false,
1162
- failureCode: classifyFlowDriverError(e, 'SNAPSHOT_FAILED'),
1163
- nextSteps: idle
1164
- ? [
1165
- 'Switched to visual-fallback for this screen (cleared by the next successful qa_snapshot / qa_act dump). Use qa_screenshot; qa_check_health still works.',
1166
- ]
1167
- : ['Re-check the device is online, then qa_screenshot / qa_check_health.'],
1168
- });
1169
- }
1170
- }));
1192
+ });
1193
+ return clampedFrom != null ? qaAnnotate(res, [`timeoutMs ${clampedFrom} clamped to ${ACT_TIMEOUT_MAX_MS}.`]) : res;
1194
+ });
1171
1195
  }
1172
1196
  //# sourceMappingURL=act.js.map