swipium 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/README.md +32 -20
- package/THREAT_MODEL.md +109 -21
- package/dist/automationGen/run.js.map +1 -1
- package/dist/cli/init.js +101 -11
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/verify.js +2 -2
- package/dist/cli/verify.js.map +1 -1
- package/dist/consent/consent.js +365 -17
- package/dist/consent/consent.js.map +1 -1
- package/dist/context/projectRoot.js +9 -4
- package/dist/context/projectRoot.js.map +1 -1
- package/dist/context/protocolEra.js +15 -0
- package/dist/context/protocolEra.js.map +1 -0
- package/dist/drivers/DirectDriver.js +5 -2
- package/dist/drivers/DirectDriver.js.map +1 -1
- package/dist/featureTesting/executionBootstrap.js +85 -77
- package/dist/featureTesting/executionBootstrap.js.map +1 -1
- package/dist/flows/run.js +9 -5
- package/dist/flows/run.js.map +1 -1
- package/dist/lib/abortScope.js +36 -3
- package/dist/lib/abortScope.js.map +1 -1
- package/dist/lib/android.js +15 -6
- package/dist/lib/android.js.map +1 -1
- package/dist/lib/codexEnv.js +112 -0
- package/dist/lib/codexEnv.js.map +1 -0
- package/dist/lib/logger.js +18 -0
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/result.js +188 -10
- package/dist/lib/result.js.map +1 -1
- package/dist/lib/schemaHash.js +20 -31
- package/dist/lib/schemaHash.js.map +1 -1
- package/dist/lib/simctl.js +102 -3
- package/dist/lib/simctl.js.map +1 -1
- package/dist/lib/toolSchema.js +143 -0
- package/dist/lib/toolSchema.js.map +1 -0
- package/dist/lib/wda.js +5 -2
- package/dist/lib/wda.js.map +1 -1
- package/dist/mobileAudit/runner.js +12 -4
- package/dist/mobileAudit/runner.js.map +1 -1
- package/dist/oracle/failures.js +2 -2
- package/dist/oracle/failures.js.map +1 -1
- package/dist/orchestration/testThis/execute.js +21 -4
- package/dist/orchestration/testThis/execute.js.map +1 -1
- package/dist/orchestration/testThis/pipeline.js +2 -0
- package/dist/orchestration/testThis/pipeline.js.map +1 -1
- package/dist/orchestration/testThis/plan.js.map +1 -1
- package/dist/report/export.js +1 -1
- package/dist/report/export.js.map +1 -1
- package/dist/server.js +564 -139
- package/dist/server.js.map +1 -1
- package/dist/services/prepareAndroid.js +3 -2
- package/dist/services/prepareAndroid.js.map +1 -1
- package/dist/services/prepareIos.js +31 -4
- package/dist/services/prepareIos.js.map +1 -1
- package/dist/services/smoke.js +38 -4
- package/dist/services/smoke.js.map +1 -1
- package/dist/session/processRegistry.js +12 -1
- package/dist/session/processRegistry.js.map +1 -1
- package/dist/snapshot/parse.js +64 -9
- package/dist/snapshot/parse.js.map +1 -1
- package/dist/snapshot/present.js +14 -4
- package/dist/snapshot/present.js.map +1 -1
- package/dist/snapshot/settle.js +14 -3
- package/dist/snapshot/settle.js.map +1 -1
- package/dist/tools/act.js +694 -670
- package/dist/tools/act.js.map +1 -1
- package/dist/tools/agent.js +15 -16
- package/dist/tools/agent.js.map +1 -1
- package/dist/tools/appControl.js +4 -4
- package/dist/tools/appControl.js.map +1 -1
- package/dist/tools/appMap.js +11 -13
- package/dist/tools/appMap.js.map +1 -1
- package/dist/tools/build.js +4 -6
- package/dist/tools/build.js.map +1 -1
- package/dist/tools/bundletool.js +3 -3
- package/dist/tools/bundletool.js.map +1 -1
- package/dist/tools/clearOverlay.js +2 -3
- package/dist/tools/clearOverlay.js.map +1 -1
- package/dist/tools/device.js +4 -3
- package/dist/tools/device.js.map +1 -1
- package/dist/tools/doctor.js +23 -11
- package/dist/tools/doctor.js.map +1 -1
- package/dist/tools/explore.js +15 -21
- package/dist/tools/explore.js.map +1 -1
- package/dist/tools/featureTesting.js +22 -6
- package/dist/tools/featureTesting.js.map +1 -1
- package/dist/tools/firstRun.js +4 -5
- package/dist/tools/firstRun.js.map +1 -1
- package/dist/tools/flow.js +13 -17
- package/dist/tools/flow.js.map +1 -1
- package/dist/tools/flowRepair.js +3 -5
- package/dist/tools/flowRepair.js.map +1 -1
- package/dist/tools/generate.js +10 -11
- package/dist/tools/generate.js.map +1 -1
- package/dist/tools/getArtifact.js +114 -10
- package/dist/tools/getArtifact.js.map +1 -1
- package/dist/tools/health.js +2 -1
- package/dist/tools/health.js.map +1 -1
- package/dist/tools/ios.js +35 -13
- package/dist/tools/ios.js.map +1 -1
- package/dist/tools/issues.js +8 -8
- package/dist/tools/issues.js.map +1 -1
- package/dist/tools/jobs.js +26 -10
- package/dist/tools/jobs.js.map +1 -1
- package/dist/tools/metro.js +4 -5
- package/dist/tools/metro.js.map +1 -1
- package/dist/tools/mobileAudit.js +11 -10
- package/dist/tools/mobileAudit.js.map +1 -1
- package/dist/tools/note.js +2 -3
- package/dist/tools/note.js.map +1 -1
- package/dist/tools/prepareIosTarget.js +102 -31
- package/dist/tools/prepareIosTarget.js.map +1 -1
- package/dist/tools/prepareTarget.js +3 -5
- package/dist/tools/prepareTarget.js.map +1 -1
- package/dist/tools/report.js +4 -5
- package/dist/tools/report.js.map +1 -1
- package/dist/tools/resolveArtifact.js +2 -3
- package/dist/tools/resolveArtifact.js.map +1 -1
- package/dist/tools/resolveTarget.js +3 -6
- package/dist/tools/resolveTarget.js.map +1 -1
- package/dist/tools/screenRecord.js +2 -3
- package/dist/tools/screenRecord.js.map +1 -1
- package/dist/tools/screenshot.js +2 -2
- package/dist/tools/screenshot.js.map +1 -1
- package/dist/tools/smoke.js +4 -2
- package/dist/tools/smoke.js.map +1 -1
- package/dist/tools/snapshot.js +6 -7
- package/dist/tools/snapshot.js.map +1 -1
- package/dist/tools/startSession.js +12 -16
- package/dist/tools/startSession.js.map +1 -1
- package/dist/tools/suite.js +2 -4
- package/dist/tools/suite.js.map +1 -1
- package/dist/tools/testSuite.js +13 -19
- package/dist/tools/testSuite.js.map +1 -1
- package/dist/tools/testThis.js +28 -13
- package/dist/tools/testThis.js.map +1 -1
- package/dist/tools/visual.js +7 -9
- package/dist/tools/visual.js.map +1 -1
- package/dist/tools/wait.js +130 -21
- package/dist/tools/wait.js.map +1 -1
- package/dist/tools/wda.js +199 -60
- package/dist/tools/wda.js.map +1 -1
- package/dist/version.js +1 -1
- package/docs/README.md +4 -4
- package/docs/ci-reports.md +39 -7
- package/docs/concepts.md +37 -25
- package/docs/flows.md +1 -1
- package/docs/mcp-server.md +281 -76
- package/docs/physical-devices.md +4 -4
- package/docs/tools.md +59 -30
- package/package.json +4 -3
package/dist/tools/act.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// to settle, then returns the post-action snapshot + a deterministic health check. So one
|
|
4
4
|
// call both acts AND observes the result.
|
|
5
5
|
import { z } from 'zod';
|
|
6
|
-
import { qaOk, qaError, qaStop, unknownSessionError, cancelledResult } from '../lib/result.js';
|
|
6
|
+
import { qaOk, qaError, qaStop, qaAnnotate, unknownSessionError, cancelledResult } from '../lib/result.js';
|
|
7
7
|
import { parseSnapshot, signature } from '../snapshot/parse.js';
|
|
8
8
|
import { presentElements } from '../snapshot/present.js';
|
|
9
9
|
import { obstructionAt } from '../snapshot/overlays.js';
|
|
@@ -18,7 +18,7 @@ import { center, resolveTarget, setsEqual } from '../core/target.js';
|
|
|
18
18
|
import { recordableNativeSelector, recordableTap } from '../flows/generate.js';
|
|
19
19
|
import { classifyFlowDriverError } from '../flows/run.js';
|
|
20
20
|
import { structuredSignature } from '../explore/signatures.js';
|
|
21
|
-
import { isAbortError, runWithSignal } from '../lib/abortScope.js';
|
|
21
|
+
import { isAbortError, runWithSignal, sleepOrCancel, throwIfCancelled } from '../lib/abortScope.js';
|
|
22
22
|
import { SECRET_VAR_NAME } from '../flows/schema.js';
|
|
23
23
|
/** Drop the (already-handled) native selector before generic ref/text/id/coords resolution. */
|
|
24
24
|
function stripSelector(t) {
|
|
@@ -48,15 +48,17 @@ async function awaitIme(d, capMs, floorMs = 0) {
|
|
|
48
48
|
// new field having focus yet, so give RN a floor to move focus before typing.
|
|
49
49
|
const rest = floorMs - (Date.now() - started);
|
|
50
50
|
if (rest > 0)
|
|
51
|
-
await
|
|
51
|
+
await sleepOrCancel(rest);
|
|
52
52
|
return;
|
|
53
53
|
}
|
|
54
|
-
await
|
|
54
|
+
await sleepOrCancel(Math.min(100, Math.max(1, deadline - Date.now())));
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
|
-
catch {
|
|
57
|
+
catch (e) {
|
|
58
|
+
if (isAbortError(e))
|
|
59
|
+
throw e; // cancelled: unwind, the action's catch returns CANCELLED
|
|
58
60
|
// imeShown unsupported here: keep the original fixed-sleep behavior
|
|
59
|
-
await
|
|
61
|
+
await sleepOrCancel(Math.max(0, deadline - Date.now()));
|
|
60
62
|
}
|
|
61
63
|
}
|
|
62
64
|
/** Values to scrub from driver error text: session secrets + the value just typed, in raw AND
|
|
@@ -146,6 +148,8 @@ function stateLabel(n) {
|
|
|
146
148
|
* (after a navigation "diff" = every new element + every old one listed as removed, larger
|
|
147
149
|
* than "full"). */
|
|
148
150
|
export const DIFF_FULL_FALLBACK_RATIO = 0.5;
|
|
151
|
+
/** qa_act timeoutMs ceiling (wait / settle cap); larger values are clamped with a note. */
|
|
152
|
+
export const ACT_TIMEOUT_MAX_MS = 50_000;
|
|
149
153
|
/** Only `${SWIPIUM_*}` placeholders are expanded in typed text (anything else stays literal). */
|
|
150
154
|
const INPUT_PLACEHOLDER_RE = /\$\{(SWIPIUM_[A-Z0-9_]+)\}/g;
|
|
151
155
|
/** Env-sourced placeholder values whose NAME looks secret join the redaction set. Uses the shared
|
|
@@ -388,10 +392,10 @@ function recordableNativeTarget(session, native) {
|
|
|
388
392
|
export function registerAct(server, sessions) {
|
|
389
393
|
server.registerTool('qa_act', {
|
|
390
394
|
title: 'Act on the screen',
|
|
391
|
-
description: '
|
|
392
|
-
'
|
|
393
|
-
'
|
|
394
|
-
'
|
|
395
|
+
description: 'One UI action (tap, type, clear, swipe, scroll, press, open_url, wait), then wait for the screen to settle and ' +
|
|
396
|
+
'report changed/settled, snapshot quality, health, and new elements (observe). Target: an @eN ref from qa_snapshot, ' +
|
|
397
|
+
'text, id, a native selector (WDA iOS), or x/y. Each param names the actions it applies to. Details: ' +
|
|
398
|
+
'docs/tools.md#qa_act.',
|
|
395
399
|
inputSchema: {
|
|
396
400
|
sessionId: z.string(),
|
|
397
401
|
action: z.enum(['tap', 'type', 'clear', 'swipe', 'scroll', 'press', 'open_url', 'wait']),
|
|
@@ -430,7 +434,11 @@ export function registerAct(server, sessions) {
|
|
|
430
434
|
})
|
|
431
435
|
.optional()
|
|
432
436
|
.describe('wait: {settled:true} (default) or an element.'),
|
|
433
|
-
timeoutMs: z
|
|
437
|
+
timeoutMs: z
|
|
438
|
+
.number()
|
|
439
|
+
.min(0)
|
|
440
|
+
.optional()
|
|
441
|
+
.describe('wait: max ms (default 8000, max 50000; larger values are clamped); others: settle-wait cap.'),
|
|
434
442
|
observe: z
|
|
435
443
|
.enum(['diff', 'full', 'none'])
|
|
436
444
|
.optional()
|
|
@@ -440,733 +448,749 @@ export function registerAct(server, sessions) {
|
|
|
440
448
|
// Cancellation (MCP notifications/cancelled): the call's signal is scoped to THIS call
|
|
441
449
|
// (abortScope): an in-flight adb child / WDA request is aborted without touching a
|
|
442
450
|
// concurrently running job's own cancellation.
|
|
443
|
-
async (
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
const
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
}));
|
|
463
|
-
}
|
|
464
|
-
if (d.kind === 'simulator') {
|
|
465
|
-
return qaError({
|
|
466
|
-
what: 'Structured interaction (tap/type/swipe) is not available on the iOS simulator backend',
|
|
467
|
-
changedState: false,
|
|
468
|
-
retrySafe: false,
|
|
469
|
-
failureCode: 'BACKEND_UNSUPPORTED',
|
|
470
|
-
nextSteps: [
|
|
471
|
-
'Attach WebDriverAgent with qa_wda for structured tap/type/snapshot. Without WDA, locate targets with qa_visual (mode:"find_text" OCR or mode:"find_image" return tappable device coordinates; tap:true taps via idb when installed), navigate via qa_ios deep links, and verify with qa_visual mode:"assert" or mode:"diff".',
|
|
472
|
-
],
|
|
473
|
-
});
|
|
474
|
-
}
|
|
475
|
-
// Budget gate (review §4.1 / Rec 4): refuse new work once the session budget is spent.
|
|
476
|
-
// `wait` is exempt from the action/screenshot caps (it's synchronization, not an
|
|
477
|
-
// action), but it is NOT exempt from the TIME budget, otherwise repeated waits could
|
|
478
|
-
// burn the clock indefinitely.
|
|
479
|
-
const stopReason = sessions.budgetStop(session);
|
|
480
|
-
if (stopReason && (action !== 'wait' || /time budget/.test(stopReason))) {
|
|
481
|
-
return qaStop(stopReason, { counters: session.counters, mode: session.mode });
|
|
482
|
-
}
|
|
483
|
-
const fail = (what, changedState) => qaError({
|
|
484
|
-
what,
|
|
485
|
-
changedState,
|
|
486
|
-
retrySafe: true,
|
|
487
|
-
failureCode: targetErrorCode(what),
|
|
488
|
-
nextSteps: ['Run qa_snapshot to see the current screen, then retry.'],
|
|
489
|
-
});
|
|
490
|
-
// ---- wait is its own path (no settle/health afterward) ----
|
|
491
|
-
if (action === 'wait') {
|
|
492
|
-
const timeoutMs = args.timeoutMs ?? 8000;
|
|
493
|
-
if (args.for?.settled || !args.for) {
|
|
494
|
-
const s = await settle(d, { timeoutMs });
|
|
495
|
-
const post = parseSnapshot(s.xml);
|
|
496
|
-
session.lastSnapshot = {
|
|
497
|
-
fullByRef: post.fullByRef,
|
|
498
|
-
signatures: new Set(post.elements.map(signature)),
|
|
499
|
-
allNodes: post.allNodes,
|
|
500
|
-
};
|
|
501
|
-
const { elements: shown, rendered, omitted } = presentElements(post.elements, makeRedactor(session.secrets));
|
|
502
|
-
return qaOk({ action, settled: s.settled, quality: post.quality.verdict, elementsOmitted: omitted, elements: shown }, `wait(settled)=${s.settled}\n\n${rendered}`, { textOmit: ['elements'] });
|
|
503
|
-
}
|
|
504
|
-
const deadline = Date.now() + timeoutMs;
|
|
505
|
-
const want = args.for;
|
|
506
|
-
const native = want?.selector ?? null;
|
|
507
|
-
if (native) {
|
|
508
|
-
if (!d.existsBySelector)
|
|
509
|
-
return qaError({
|
|
510
|
-
what: `${native.using} waits require backend-native selector support`,
|
|
451
|
+
async (rawArgs, ctx) => {
|
|
452
|
+
// timeoutMs is clamped (not rejected) like qa_job_status waitMs, with a note on the result.
|
|
453
|
+
const clampedFrom = rawArgs.timeoutMs != null && rawArgs.timeoutMs > ACT_TIMEOUT_MAX_MS ? rawArgs.timeoutMs : undefined;
|
|
454
|
+
const args = clampedFrom != null ? { ...rawArgs, timeoutMs: ACT_TIMEOUT_MAX_MS } : rawArgs;
|
|
455
|
+
const res = await runWithSignal(ctx?.mcpReq.signal, async () => {
|
|
456
|
+
const { sessionId, action } = args;
|
|
457
|
+
// Single validation layer for the per-action field contract (see REQUIRED_BY_ACTION).
|
|
458
|
+
const invalid = missingRequiredField(action, args);
|
|
459
|
+
if (invalid)
|
|
460
|
+
return invalid;
|
|
461
|
+
const session = sessions.get(sessionId);
|
|
462
|
+
if (!session)
|
|
463
|
+
return unknownSessionError(sessionId);
|
|
464
|
+
const { driver: d, blocked } = await getDriver(session);
|
|
465
|
+
if (!d) {
|
|
466
|
+
// H6: a device is online but refused (physical) or not ready (still booting).
|
|
467
|
+
return (blockedDeviceResult(blocked) ??
|
|
468
|
+
qaError({
|
|
469
|
+
what: 'No device attached to this session',
|
|
511
470
|
changedState: false,
|
|
512
|
-
retrySafe:
|
|
513
|
-
failureCode: '
|
|
514
|
-
nextSteps: ['
|
|
515
|
-
});
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
return qaOk({ action, found: true, selector: want?.selector, via: 'native-selector' }, `wait: found ${native.using}=${native.value}`);
|
|
519
|
-
}
|
|
520
|
-
await new Promise((r) => setTimeout(r, 400));
|
|
521
|
-
}
|
|
471
|
+
retrySafe: true,
|
|
472
|
+
failureCode: 'NO_DEVICE',
|
|
473
|
+
nextSteps: ['Call qa_prepare_target first.'],
|
|
474
|
+
}));
|
|
475
|
+
}
|
|
476
|
+
if (d.kind === 'simulator') {
|
|
522
477
|
return qaError({
|
|
523
|
-
what:
|
|
478
|
+
what: 'Structured interaction (tap/type/swipe) is not available on the iOS simulator backend',
|
|
524
479
|
changedState: false,
|
|
525
|
-
retrySafe:
|
|
526
|
-
failureCode: '
|
|
527
|
-
nextSteps: [
|
|
480
|
+
retrySafe: false,
|
|
481
|
+
failureCode: 'BACKEND_UNSUPPORTED',
|
|
482
|
+
nextSteps: [
|
|
483
|
+
'Attach WebDriverAgent with qa_wda for structured tap/type/snapshot. Without WDA, locate targets with qa_visual (mode:"find_text" OCR or mode:"find_image" return tappable device coordinates; tap:true taps via idb when installed), navigate via qa_ios deep links, and verify with qa_visual mode:"assert" or mode:"diff".',
|
|
484
|
+
],
|
|
528
485
|
});
|
|
529
486
|
}
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
};
|
|
537
|
-
const hit = parsed.elements.find((e) => (want.ref && e.ref === want.ref) ||
|
|
538
|
-
(want.id && e.id === want.id) ||
|
|
539
|
-
(want.text &&
|
|
540
|
-
(e.text?.toLowerCase().includes(want.text.toLowerCase()) || e.label?.toLowerCase().includes(want.text.toLowerCase()))));
|
|
541
|
-
if (hit)
|
|
542
|
-
return qaOk({ action, found: true, ref: hit.ref }, `wait: found ${hit.ref}`);
|
|
543
|
-
await new Promise((r) => setTimeout(r, 400));
|
|
487
|
+
// Budget gate (review §4.1 / Rec 4): refuse new work once the session budget is spent.
|
|
488
|
+
// `wait` is exempt from the action/screenshot caps (it's synchronization, not an
|
|
489
|
+
// action), but it is NOT exempt from the TIME budget, otherwise repeated waits could
|
|
490
|
+
// burn the clock indefinitely.
|
|
491
|
+
const stopReason = sessions.budgetStop(session);
|
|
492
|
+
if (stopReason && (action !== 'wait' || /time budget/.test(stopReason))) {
|
|
493
|
+
return qaStop(stopReason, { counters: session.counters, mode: session.mode });
|
|
544
494
|
}
|
|
545
|
-
|
|
546
|
-
what
|
|
547
|
-
changedState
|
|
495
|
+
const fail = (what, changedState) => qaError({
|
|
496
|
+
what,
|
|
497
|
+
changedState,
|
|
548
498
|
retrySafe: true,
|
|
549
|
-
failureCode:
|
|
550
|
-
nextSteps: ['
|
|
499
|
+
failureCode: targetErrorCode(what),
|
|
500
|
+
nextSteps: ['Run qa_snapshot to see the current screen, then retry.'],
|
|
551
501
|
});
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
let tapRetry;
|
|
572
|
-
// action-IR step to record once the action succeeds (built here while lastSnapshot is
|
|
573
|
-
// still the PRE-navigation screen, so a tapped @ref still resolves to its label).
|
|
574
|
-
let toRecord;
|
|
575
|
-
// Non-fatal caveats for the result (unknown keyboard area, recovered WDA session, …).
|
|
576
|
-
const warnings = [];
|
|
577
|
-
try {
|
|
578
|
-
switch (action) {
|
|
579
|
-
case 'tap': {
|
|
580
|
-
const native = args.target?.selector ?? null;
|
|
502
|
+
// ---- wait is its own path (no settle/health afterward) ----
|
|
503
|
+
if (action === 'wait') {
|
|
504
|
+
// A cancelled wait (notifications/cancelled) stops polling at once and is not a failure.
|
|
505
|
+
try {
|
|
506
|
+
const timeoutMs = args.timeoutMs ?? 8000;
|
|
507
|
+
if (args.for?.settled || !args.for) {
|
|
508
|
+
const s = await settle(d, { timeoutMs });
|
|
509
|
+
const post = parseSnapshot(s.xml);
|
|
510
|
+
session.lastSnapshot = {
|
|
511
|
+
fullByRef: post.fullByRef,
|
|
512
|
+
signatures: new Set(post.elements.map(signature)),
|
|
513
|
+
allNodes: post.allNodes,
|
|
514
|
+
};
|
|
515
|
+
const { payload: shown, rendered, omitted } = presentElements(post.elements, makeRedactor(session.secrets));
|
|
516
|
+
return qaOk({ action, settled: s.settled, quality: post.quality.verdict, elementsOmitted: omitted, elements: shown }, `wait(settled)=${s.settled}\n\n${rendered}`, { textOmit: ['elements'] });
|
|
517
|
+
}
|
|
518
|
+
const deadline = Date.now() + timeoutMs;
|
|
519
|
+
const want = args.for;
|
|
520
|
+
const native = want?.selector ?? null;
|
|
581
521
|
if (native) {
|
|
582
|
-
if (!d.
|
|
522
|
+
if (!d.existsBySelector)
|
|
583
523
|
return qaError({
|
|
584
|
-
what: `${native.using}
|
|
524
|
+
what: `${native.using} waits require backend-native selector support`,
|
|
585
525
|
changedState: false,
|
|
586
526
|
retrySafe: false,
|
|
587
527
|
failureCode: 'BACKEND_UNSUPPORTED',
|
|
588
|
-
nextSteps: ['Use a WDA-backed iOS session, or
|
|
528
|
+
nextSteps: ['Use a WDA-backed iOS session, or wait by text/id/ref on this backend.'],
|
|
589
529
|
});
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
if ('error' in resolved)
|
|
597
|
-
return fail(resolved.error, false);
|
|
598
|
-
let t = resolved;
|
|
599
|
-
// H5: keyboard obstruction (the IME window is not in the app's UI dump, so the
|
|
600
|
-
// overlay check below can't see it). Coordinate taps are taken as deliberate.
|
|
601
|
-
let imeUp = false;
|
|
602
|
-
if (!args.ignoreOverlay && t.via !== 'coords') {
|
|
603
|
-
const g = await guardKeyboard(session, d, t, stripSelector(args.target));
|
|
604
|
-
if ('result' in g)
|
|
605
|
-
return g.result;
|
|
606
|
-
t = g.t;
|
|
607
|
-
imeUp = g.imeUp;
|
|
608
|
-
if (g.warning)
|
|
609
|
-
warnings.push(g.warning);
|
|
610
|
-
if (g.hidKeyboard)
|
|
611
|
-
keyboardHidden = true;
|
|
612
|
-
}
|
|
613
|
-
else {
|
|
614
|
-
imeUp = await d.imeShown().catch(() => false);
|
|
615
|
-
}
|
|
616
|
-
// Overlay obstruction check (CR4): if another element is drawn over the target
|
|
617
|
-
// point, return a structured blockedByOverlay instead of tapping blindly.
|
|
618
|
-
if (!args.ignoreOverlay && t.via !== 'coords' && session.lastSnapshot?.allNodes) {
|
|
619
|
-
// works for ref AND selector taps: t.ref is the resolved @eN in either case
|
|
620
|
-
const node = t.ref ? session.lastSnapshot.fullByRef.get(t.ref) : undefined;
|
|
621
|
-
const obs = obstructionAt(session.lastSnapshot.allNodes, node, t.x, t.y);
|
|
622
|
-
if (obs.obstructed) {
|
|
623
|
-
return qaError({
|
|
624
|
-
what: `Target at (${t.x},${t.y}) is obstructed by ${obs.by?.cls?.split('.').pop()}${obs.by?.text ? ` "${obs.by.text}"` : ''}`,
|
|
625
|
-
changedState: false,
|
|
626
|
-
retrySafe: true,
|
|
627
|
-
failureCode: 'OVERLAY_OBSTRUCTION',
|
|
628
|
-
nextSteps: [
|
|
629
|
-
'Call qa_clear_overlay (auto, or hide_keyboard/minimize_logbox), then retry. Or pass ignoreOverlay:true to tap anyway.',
|
|
630
|
-
],
|
|
631
|
-
}, { blockedByOverlay: true, obstructedBy: obs.by });
|
|
530
|
+
while (Date.now() < deadline) {
|
|
531
|
+
throwIfCancelled();
|
|
532
|
+
if (await d.existsBySelector(native.using, native.value)) {
|
|
533
|
+
return qaOk({ action, found: true, selector: want?.selector, via: 'native-selector' }, `wait: found ${native.using}=${native.value}`);
|
|
534
|
+
}
|
|
535
|
+
await sleepOrCancel(400);
|
|
632
536
|
}
|
|
633
|
-
}
|
|
634
|
-
// Coordinate taps default to a short press (RN often ignores instant taps);
|
|
635
|
-
// ref/selector taps stay instant unless durationMs is given.
|
|
636
|
-
const isCoord = t.via === 'coords';
|
|
637
|
-
const durationMs = args.durationMs ?? (isCoord ? 100 : undefined);
|
|
638
|
-
if (durationMs)
|
|
639
|
-
await d.pressXY(t.x, t.y, durationMs);
|
|
640
|
-
else
|
|
641
|
-
await d.tapXY(t.x, t.y);
|
|
642
|
-
tapRetry = { x: t.x, y: t.y, instant: !durationMs, imeUp };
|
|
643
|
-
meta = { tappedAt: [t.x, t.y], via: t.via, ...(durationMs ? { durationMs } : {}) };
|
|
644
|
-
toRecord = { action: 'tap', ...recordableTap(session, stripSelector(args.target), t) };
|
|
645
|
-
break;
|
|
646
|
-
}
|
|
647
|
-
case 'type': {
|
|
648
|
-
// presence enforced by missingRequiredField
|
|
649
|
-
// P1: `${SWIPIUM_*}` placeholders expand from session inputs (qa_continue_from_blocker) or the env,
|
|
650
|
-
// so credentials never have to pass through the agent transcript.
|
|
651
|
-
const secretVars = new Set(session.inputs.filter((i) => i.secret).map((i) => i.varName));
|
|
652
|
-
const expanded = expandInputPlaceholders(args.text, { values: session.inputValues, secretVars });
|
|
653
|
-
if (expanded.missing.length) {
|
|
654
537
|
return qaError({
|
|
655
|
-
what: `
|
|
538
|
+
what: `wait timed out (${timeoutMs}ms) for ${JSON.stringify(want)}`,
|
|
656
539
|
changedState: false,
|
|
657
540
|
retrySafe: true,
|
|
658
|
-
failureCode: '
|
|
659
|
-
nextSteps: [
|
|
660
|
-
'Provide it via qa_continue_from_blocker (needs_input credentials) or set the env var for the Swipium server, then retry. Only ${SWIPIUM_*} placeholders are expanded.',
|
|
661
|
-
],
|
|
541
|
+
failureCode: 'ELEMENT_NOT_FOUND',
|
|
542
|
+
nextSteps: ['Re-check the native selector value, or run qa_snapshot to inspect the current screen.'],
|
|
662
543
|
});
|
|
663
544
|
}
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
});
|
|
545
|
+
while (Date.now() < deadline) {
|
|
546
|
+
throwIfCancelled();
|
|
547
|
+
const parsed = parseSnapshot(await d.dumpXml());
|
|
548
|
+
session.lastSnapshot = {
|
|
549
|
+
fullByRef: parsed.fullByRef,
|
|
550
|
+
signatures: new Set(parsed.elements.map(signature)),
|
|
551
|
+
allNodes: parsed.allNodes,
|
|
552
|
+
};
|
|
553
|
+
const hit = parsed.elements.find((e) => (want.ref && e.ref === want.ref) ||
|
|
554
|
+
(want.id && e.id === want.id) ||
|
|
555
|
+
(want.text &&
|
|
556
|
+
(e.text?.toLowerCase().includes(want.text.toLowerCase()) || e.label?.toLowerCase().includes(want.text.toLowerCase()))));
|
|
557
|
+
if (hit)
|
|
558
|
+
return qaOk({ action, found: true, ref: hit.ref }, `wait: found ${hit.ref}`);
|
|
559
|
+
await sleepOrCancel(400);
|
|
680
560
|
}
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
561
|
+
return qaError({
|
|
562
|
+
what: `wait timed out (${timeoutMs}ms) for ${JSON.stringify(want)}`,
|
|
563
|
+
changedState: false,
|
|
564
|
+
retrySafe: true,
|
|
565
|
+
failureCode: 'ELEMENT_NOT_FOUND',
|
|
566
|
+
nextSteps: ['Re-snapshot; the element may use different text/id.'],
|
|
567
|
+
});
|
|
568
|
+
}
|
|
569
|
+
catch (e) {
|
|
570
|
+
if (isAbortError(e))
|
|
571
|
+
return cancelledResult('wait cancelled: the call was aborted before the condition held');
|
|
572
|
+
throw e;
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
// Phase timing (P1.6): mark the first real action so the report can split setup vs active.
|
|
576
|
+
sessions.milestone(session, 'first_action');
|
|
577
|
+
const preSigs = session.lastSnapshot?.signatures ?? new Set();
|
|
578
|
+
// F: positions too. A scroll/swipe can move content without adding/removing elements.
|
|
579
|
+
const prePositions = positionalFingerprint(session.lastSnapshot?.fullByRef);
|
|
580
|
+
// Toggle/selection state of the pre-action screen (a Switch flip changes no signature).
|
|
581
|
+
const preState = stateByIdentity(session.lastSnapshot?.fullByRef);
|
|
582
|
+
// The value actually typed (after ${SWIPIUM_*} expansion), redacted from results/errors.
|
|
583
|
+
let typedValue;
|
|
584
|
+
// untilVisible's last probe dump. Seeds the post-action settle (no redundant dump).
|
|
585
|
+
let settleSeed;
|
|
586
|
+
// I: set when the soft keyboard was hidden because it covered the target.
|
|
587
|
+
let keyboardHidden = false;
|
|
588
|
+
// Observation mode is fixed at action start; diff needs a pre-action baseline.
|
|
589
|
+
const observe = args.observe ?? (session.lastSnapshot ? 'diff' : 'full');
|
|
590
|
+
let meta = {};
|
|
591
|
+
// remembered so a no-change tap can be retried as a longer press (RN tap quirk)
|
|
592
|
+
// imeUp: the soft keyboard was shown at tap time, so a re-press could hit a key (H5).
|
|
593
|
+
let tapRetry;
|
|
594
|
+
// action-IR step to record once the action succeeds (built here while lastSnapshot is
|
|
595
|
+
// still the PRE-navigation screen, so a tapped @ref still resolves to its label).
|
|
596
|
+
let toRecord;
|
|
597
|
+
// Non-fatal caveats for the result (unknown keyboard area, recovered WDA session, …).
|
|
598
|
+
const warnings = [];
|
|
599
|
+
try {
|
|
600
|
+
switch (action) {
|
|
601
|
+
case 'tap': {
|
|
602
|
+
const native = args.target?.selector ?? null;
|
|
603
|
+
if (native) {
|
|
604
|
+
if (!d.tapBySelector)
|
|
605
|
+
return qaError({
|
|
606
|
+
what: `${native.using} selectors require backend-native selector support`,
|
|
607
|
+
changedState: false,
|
|
608
|
+
retrySafe: false,
|
|
609
|
+
failureCode: 'BACKEND_UNSUPPORTED',
|
|
610
|
+
nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
|
|
611
|
+
});
|
|
612
|
+
await d.tapBySelector(native.using, native.value);
|
|
613
|
+
meta = { selector: args.target?.selector, via: 'native-selector' };
|
|
614
|
+
toRecord = { action: 'tap', ...recordableNativeTarget(session, native) };
|
|
615
|
+
break;
|
|
616
|
+
}
|
|
617
|
+
const resolved = await resolveTarget(session, stripSelector(args.target));
|
|
618
|
+
if ('error' in resolved)
|
|
619
|
+
return fail(resolved.error, false);
|
|
620
|
+
let t = resolved;
|
|
621
|
+
// H5: keyboard obstruction (the IME window is not in the app's UI dump, so the
|
|
622
|
+
// overlay check below can't see it). Coordinate taps are taken as deliberate.
|
|
623
|
+
let imeUp = false;
|
|
624
|
+
if (!args.ignoreOverlay && t.via !== 'coords') {
|
|
625
|
+
const g = await guardKeyboard(session, d, t, stripSelector(args.target));
|
|
626
|
+
if ('result' in g)
|
|
627
|
+
return g.result;
|
|
628
|
+
t = g.t;
|
|
629
|
+
imeUp = g.imeUp;
|
|
630
|
+
if (g.warning)
|
|
631
|
+
warnings.push(g.warning);
|
|
632
|
+
if (g.hidKeyboard)
|
|
633
|
+
keyboardHidden = true;
|
|
634
|
+
}
|
|
635
|
+
else {
|
|
636
|
+
imeUp = await d.imeShown().catch(() => false);
|
|
637
|
+
}
|
|
638
|
+
// Overlay obstruction check (CR4): if another element is drawn over the target
|
|
639
|
+
// point, return a structured blockedByOverlay instead of tapping blindly.
|
|
640
|
+
if (!args.ignoreOverlay && t.via !== 'coords' && session.lastSnapshot?.allNodes) {
|
|
641
|
+
// works for ref AND selector taps: t.ref is the resolved @eN in either case
|
|
642
|
+
const node = t.ref ? session.lastSnapshot.fullByRef.get(t.ref) : undefined;
|
|
643
|
+
const obs = obstructionAt(session.lastSnapshot.allNodes, node, t.x, t.y);
|
|
644
|
+
if (obs.obstructed) {
|
|
645
|
+
return qaError({
|
|
646
|
+
what: `Target at (${t.x},${t.y}) is obstructed by ${obs.by?.cls?.split('.').pop()}${obs.by?.text ? ` "${obs.by.text}"` : ''}`,
|
|
647
|
+
changedState: false,
|
|
648
|
+
retrySafe: true,
|
|
649
|
+
failureCode: 'OVERLAY_OBSTRUCTION',
|
|
650
|
+
nextSteps: [
|
|
651
|
+
'Call qa_clear_overlay (auto, or hide_keyboard/minimize_logbox), then retry. Or pass ignoreOverlay:true to tap anyway.',
|
|
652
|
+
],
|
|
653
|
+
}, { blockedByOverlay: true, obstructedBy: obs.by });
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
// Coordinate taps default to a short press (RN often ignores instant taps);
|
|
657
|
+
// ref/selector taps stay instant unless durationMs is given.
|
|
658
|
+
const isCoord = t.via === 'coords';
|
|
659
|
+
const durationMs = args.durationMs ?? (isCoord ? 100 : undefined);
|
|
660
|
+
if (durationMs)
|
|
661
|
+
await d.pressXY(t.x, t.y, durationMs);
|
|
662
|
+
else
|
|
663
|
+
await d.tapXY(t.x, t.y);
|
|
664
|
+
tapRetry = { x: t.x, y: t.y, instant: !durationMs, imeUp };
|
|
665
|
+
meta = { tappedAt: [t.x, t.y], via: t.via, ...(durationMs ? { durationMs } : {}) };
|
|
666
|
+
toRecord = { action: 'tap', ...recordableTap(session, stripSelector(args.target), t) };
|
|
667
|
+
break;
|
|
668
|
+
}
|
|
669
|
+
case 'type': {
|
|
670
|
+
// presence enforced by missingRequiredField
|
|
671
|
+
// P1: `${SWIPIUM_*}` placeholders expand from session inputs (qa_continue_from_blocker) or the env,
|
|
672
|
+
// so credentials never have to pass through the agent transcript.
|
|
673
|
+
const secretVars = new Set(session.inputs.filter((i) => i.secret).map((i) => i.varName));
|
|
674
|
+
const expanded = expandInputPlaceholders(args.text, { values: session.inputValues, secretVars });
|
|
675
|
+
if (expanded.missing.length) {
|
|
687
676
|
return qaError({
|
|
688
|
-
what:
|
|
677
|
+
what: `No value for ${expanded.missing.map((v) => `\${${v}}`).join(', ')}. Nothing was typed.`,
|
|
689
678
|
changedState: false,
|
|
690
|
-
retrySafe:
|
|
691
|
-
failureCode: '
|
|
692
|
-
nextSteps: [
|
|
679
|
+
retrySafe: true,
|
|
680
|
+
failureCode: 'MISSING_TEST_DATA',
|
|
681
|
+
nextSteps: [
|
|
682
|
+
'Provide it via qa_continue_from_blocker (needs_input credentials) or set the env var for the Swipium server, then retry. Only ${SWIPIUM_*} placeholders are expanded.',
|
|
683
|
+
],
|
|
693
684
|
});
|
|
694
|
-
|
|
695
|
-
|
|
685
|
+
}
|
|
686
|
+
for (const v of expanded.secretValues)
|
|
687
|
+
session.secrets.add(v);
|
|
688
|
+
const text = expanded.text;
|
|
689
|
+
typedValue = text;
|
|
690
|
+
const recordText = recordableTypedText(args.text, text, expanded.vars, session.inputValues);
|
|
691
|
+
// D: validate deliverability BEFORE any focus tap / clear. A refused value must
|
|
692
|
+
// leave the field (and the device) exactly as it was.
|
|
693
|
+
const deliverable = d.canDeliverText?.(text);
|
|
694
|
+
if (deliverable && !deliverable.ok) {
|
|
696
695
|
return qaError({
|
|
697
|
-
what:
|
|
696
|
+
what: `${deliverable.reason} Nothing was tapped or cleared.`,
|
|
698
697
|
changedState: false,
|
|
699
698
|
retrySafe: false,
|
|
700
|
-
failureCode: '
|
|
701
|
-
nextSteps: ['Use
|
|
699
|
+
failureCode: 'TEXT_INPUT_UNSUPPORTED',
|
|
700
|
+
nextSteps: ['Use ASCII-safe text on this backend (adb `input text`), or type it on a Unicode-safe backend.'],
|
|
702
701
|
});
|
|
703
|
-
|
|
704
|
-
//
|
|
705
|
-
//
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
702
|
+
}
|
|
703
|
+
// A: a value that is/contains a registered secret is recorded as secret even when the
|
|
704
|
+
// target field is not a secure one (checked before this call can register it).
|
|
705
|
+
const knownSecret = matchesSessionSecret(session.secrets, text);
|
|
706
|
+
const native = args.target?.selector ?? null;
|
|
707
|
+
if (native) {
|
|
708
|
+
if (!d.typeBySelector)
|
|
709
|
+
return qaError({
|
|
710
|
+
what: `${native.using} selectors require backend-native selector support`,
|
|
711
|
+
changedState: false,
|
|
712
|
+
retrySafe: false,
|
|
713
|
+
failureCode: 'BACKEND_UNSUPPORTED',
|
|
714
|
+
nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
|
|
715
|
+
});
|
|
716
|
+
const replace = (args.mode ?? 'replace') === 'replace';
|
|
717
|
+
if (replace && !d.clearBySelector)
|
|
718
|
+
return qaError({
|
|
719
|
+
what: `replace-mode typing by ${native.using} requires backend-native clear support`,
|
|
720
|
+
changedState: false,
|
|
721
|
+
retrySafe: false,
|
|
722
|
+
failureCode: 'BACKEND_UNSUPPORTED',
|
|
723
|
+
nextSteps: ['Use append mode, or attach a WDA backend that supports element clear.'],
|
|
724
|
+
});
|
|
725
|
+
// This path must capture secrets exactly like the generic path below, or a
|
|
726
|
+
// password typed by native selector is recorded VERBATIM into generated flows.
|
|
727
|
+
// Heuristic first (selector value looks secret, same SECRET_RE the generic
|
|
728
|
+
// resolver applies to ids), then the real signal: probe the resolved element's
|
|
729
|
+
// type (XCUIElementTypeSecureTextField). A failing probe keeps the heuristic verdict.
|
|
730
|
+
let secure = isSecureNode({ id: native.value, desc: '', attrs: {} });
|
|
731
|
+
if (!secure && d.isSecureBySelector) {
|
|
732
|
+
try {
|
|
733
|
+
secure = await d.isSecureBySelector(native.using, native.value);
|
|
734
|
+
}
|
|
735
|
+
catch {
|
|
736
|
+
// probe unavailable (older WDA / element churn): heuristic alone decides
|
|
737
|
+
}
|
|
712
738
|
}
|
|
713
|
-
|
|
714
|
-
|
|
739
|
+
if (secure) {
|
|
740
|
+
session.secrets.add(text);
|
|
741
|
+
sessions.markAuth(session, { loginPerformed: true, loginPerformedAt: Date.now() });
|
|
742
|
+
sessions.milestone(session, 'login_performed');
|
|
743
|
+
}
|
|
744
|
+
if (replace)
|
|
745
|
+
await d.clearBySelector(native.using, native.value);
|
|
746
|
+
await d.typeBySelector(native.using, native.value, text);
|
|
747
|
+
const recordSecret = secure || knownSecret;
|
|
748
|
+
meta = {
|
|
749
|
+
typedChars: text.length,
|
|
750
|
+
// The response never echoes the value; `redacted`/`secret` flag a value treated as
|
|
751
|
+
// secret (registered for scrubbing, recorded only as a placeholder).
|
|
752
|
+
...(recordSecret ? { redacted: true, secret: true } : {}),
|
|
753
|
+
...(expanded.vars.length ? { placeholders: expanded.vars } : {}),
|
|
754
|
+
mode: args.mode ?? 'replace',
|
|
755
|
+
via: 'native-selector',
|
|
756
|
+
selector: args.target?.selector,
|
|
757
|
+
submit: !!args.submit,
|
|
758
|
+
};
|
|
759
|
+
// Never store a secret's value in the IR; secrets become a ${VAR} at generate time.
|
|
760
|
+
{
|
|
761
|
+
const nativeTarget = recordableNativeTarget(session, native);
|
|
762
|
+
toRecord = {
|
|
763
|
+
action: 'type',
|
|
764
|
+
...nativeTarget,
|
|
765
|
+
secret: recordSecret,
|
|
766
|
+
// a secret is recorded only as its ${VAR} placeholder (never the value)
|
|
767
|
+
text: recordSecret ? (recordText !== text ? recordText : undefined) : recordText,
|
|
768
|
+
exportability: recordSecret ? 'needs-human-data' : nativeTarget.exportability,
|
|
769
|
+
};
|
|
715
770
|
}
|
|
771
|
+
if (args.submit)
|
|
772
|
+
await d.pressKey('enter');
|
|
773
|
+
break;
|
|
716
774
|
}
|
|
717
|
-
|
|
775
|
+
const resolved = await resolveTarget(session, stripSelector(args.target));
|
|
776
|
+
if ('error' in resolved)
|
|
777
|
+
return fail(resolved.error, false);
|
|
778
|
+
let t = resolved;
|
|
779
|
+
// H5: focusing a field hidden under the keyboard would type a stray key character.
|
|
780
|
+
const g = await guardKeyboard(session, d, t, stripSelector(args.target));
|
|
781
|
+
if ('result' in g)
|
|
782
|
+
return g.result;
|
|
783
|
+
t = g.t;
|
|
784
|
+
if (g.warning)
|
|
785
|
+
warnings.push(g.warning);
|
|
786
|
+
if (g.hidKeyboard)
|
|
787
|
+
keyboardHidden = true;
|
|
788
|
+
// Typing into a secure field > remember the value so it's scrubbed everywhere, and
|
|
789
|
+
// record that a login was performed (auth-state reporting, P1.5).
|
|
790
|
+
if (t.secure) {
|
|
718
791
|
session.secrets.add(text);
|
|
719
792
|
sessions.markAuth(session, { loginPerformed: true, loginPerformedAt: Date.now() });
|
|
720
793
|
sessions.milestone(session, 'login_performed');
|
|
721
794
|
}
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
795
|
+
await d.tapXY(t.x, t.y); // focus + raise IME (real touch)
|
|
796
|
+
await awaitIme(d, 700, g.imeUp ? IME_HOP_FLOOR_MS : 0);
|
|
797
|
+
if ((args.mode ?? 'replace') === 'replace')
|
|
798
|
+
await d.clearFocusedText(t.textLen);
|
|
799
|
+
await d.inputText(text);
|
|
800
|
+
if (args.submit)
|
|
801
|
+
await d.pressKey('enter');
|
|
802
|
+
// Never echo the typed value. It may be a password/OTP/email/token and would
|
|
803
|
+
// leak into the agent transcript + artifacts (sensitive-mode).
|
|
804
|
+
const recordSecret = !!t.secure || knownSecret;
|
|
726
805
|
meta = {
|
|
727
806
|
typedChars: text.length,
|
|
728
|
-
// The response never echoes the value; `redacted`/`secret` flag a value treated as
|
|
729
|
-
// secret (registered for scrubbing, recorded only as a placeholder).
|
|
730
807
|
...(recordSecret ? { redacted: true, secret: true } : {}),
|
|
731
808
|
...(expanded.vars.length ? { placeholders: expanded.vars } : {}),
|
|
732
809
|
mode: args.mode ?? 'replace',
|
|
733
|
-
via:
|
|
734
|
-
selector: args.target?.selector,
|
|
810
|
+
via: t.via,
|
|
735
811
|
submit: !!args.submit,
|
|
736
812
|
};
|
|
737
813
|
// Never store a secret's value in the IR; secrets become a ${VAR} at generate time.
|
|
738
814
|
{
|
|
739
|
-
const
|
|
815
|
+
const targetRecord = recordableTap(session, stripSelector(args.target), t);
|
|
740
816
|
toRecord = {
|
|
741
817
|
action: 'type',
|
|
742
|
-
|
|
818
|
+
selector: targetRecord.selector,
|
|
819
|
+
selectorKind: targetRecord.selectorKind,
|
|
743
820
|
secret: recordSecret,
|
|
744
821
|
// a secret is recorded only as its ${VAR} placeholder (never the value)
|
|
745
822
|
text: recordSecret ? (recordText !== text ? recordText : undefined) : recordText,
|
|
746
|
-
exportability: recordSecret ? 'needs-human-data' :
|
|
823
|
+
exportability: recordSecret ? 'needs-human-data' : targetRecord.exportability,
|
|
824
|
+
provenance: targetRecord.provenance,
|
|
747
825
|
};
|
|
748
826
|
}
|
|
749
|
-
if (args.submit)
|
|
750
|
-
await d.pressKey('enter');
|
|
751
827
|
break;
|
|
752
828
|
}
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
session.
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
829
|
+
case 'clear': {
|
|
830
|
+
const native = args.target?.selector ?? null;
|
|
831
|
+
if (native) {
|
|
832
|
+
if (!d.clearBySelector)
|
|
833
|
+
return qaError({
|
|
834
|
+
what: `${native.using} selectors require backend-native clear support`,
|
|
835
|
+
changedState: false,
|
|
836
|
+
retrySafe: false,
|
|
837
|
+
failureCode: 'BACKEND_UNSUPPORTED',
|
|
838
|
+
nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
|
|
839
|
+
});
|
|
840
|
+
await d.clearBySelector(native.using, native.value);
|
|
841
|
+
meta = { cleared: true, via: 'native-selector', selector: args.target?.selector };
|
|
842
|
+
toRecord = { action: 'clear', ...recordableNativeTarget(session, native) };
|
|
843
|
+
break;
|
|
844
|
+
}
|
|
845
|
+
const resolved = await resolveTarget(session, stripSelector(args.target));
|
|
846
|
+
if ('error' in resolved)
|
|
847
|
+
return fail(resolved.error, false);
|
|
848
|
+
const g = await guardKeyboard(session, d, resolved, stripSelector(args.target));
|
|
849
|
+
if ('result' in g)
|
|
850
|
+
return g.result;
|
|
851
|
+
const t = g.t;
|
|
852
|
+
if (g.warning)
|
|
853
|
+
warnings.push(g.warning);
|
|
854
|
+
if (g.hidKeyboard)
|
|
855
|
+
keyboardHidden = true;
|
|
856
|
+
await d.tapXY(t.x, t.y); // focus + raise IME (real touch)
|
|
857
|
+
await awaitIme(d, 400, g.imeUp ? IME_HOP_FLOOR_MS : 0);
|
|
776
858
|
await d.clearFocusedText(t.textLen);
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
await d.pressKey('enter');
|
|
780
|
-
// Never echo the typed value. It may be a password/OTP/email/token and would
|
|
781
|
-
// leak into the agent transcript + artifacts (sensitive-mode).
|
|
782
|
-
const recordSecret = !!t.secure || knownSecret;
|
|
783
|
-
meta = {
|
|
784
|
-
typedChars: text.length,
|
|
785
|
-
...(recordSecret ? { redacted: true, secret: true } : {}),
|
|
786
|
-
...(expanded.vars.length ? { placeholders: expanded.vars } : {}),
|
|
787
|
-
mode: args.mode ?? 'replace',
|
|
788
|
-
via: t.via,
|
|
789
|
-
submit: !!args.submit,
|
|
790
|
-
};
|
|
791
|
-
// Never store a secret's value in the IR; secrets become a ${VAR} at generate time.
|
|
792
|
-
{
|
|
793
|
-
const targetRecord = recordableTap(session, stripSelector(args.target), t);
|
|
794
|
-
toRecord = {
|
|
795
|
-
action: 'type',
|
|
796
|
-
selector: targetRecord.selector,
|
|
797
|
-
selectorKind: targetRecord.selectorKind,
|
|
798
|
-
secret: recordSecret,
|
|
799
|
-
// a secret is recorded only as its ${VAR} placeholder (never the value)
|
|
800
|
-
text: recordSecret ? (recordText !== text ? recordText : undefined) : recordText,
|
|
801
|
-
exportability: recordSecret ? 'needs-human-data' : targetRecord.exportability,
|
|
802
|
-
provenance: targetRecord.provenance,
|
|
803
|
-
};
|
|
804
|
-
}
|
|
805
|
-
break;
|
|
806
|
-
}
|
|
807
|
-
case 'clear': {
|
|
808
|
-
const native = args.target?.selector ?? null;
|
|
809
|
-
if (native) {
|
|
810
|
-
if (!d.clearBySelector)
|
|
811
|
-
return qaError({
|
|
812
|
-
what: `${native.using} selectors require backend-native clear support`,
|
|
813
|
-
changedState: false,
|
|
814
|
-
retrySafe: false,
|
|
815
|
-
failureCode: 'BACKEND_UNSUPPORTED',
|
|
816
|
-
nextSteps: ['Use a WDA-backed iOS session, or target by ref/text/id/coordinates on this backend.'],
|
|
817
|
-
});
|
|
818
|
-
await d.clearBySelector(native.using, native.value);
|
|
819
|
-
meta = { cleared: true, via: 'native-selector', selector: args.target?.selector };
|
|
820
|
-
toRecord = { action: 'clear', ...recordableNativeTarget(session, native) };
|
|
859
|
+
meta = { cleared: true, via: t.via };
|
|
860
|
+
toRecord = { action: 'clear', ...recordableTap(session, stripSelector(args.target), t) };
|
|
821
861
|
break;
|
|
822
862
|
}
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
const direction = args.direction; // presence enforced by missingRequiredField
|
|
843
|
-
// Derive the gesture from the real screen size. WDA swipes are in POINTS
|
|
844
|
-
// (≤~440pt wide), so the old fixed 540/1200 constants were off-screen on iOS; the
|
|
845
|
-
// legacy constants survive only inside the shared fallback for screenSize()===null.
|
|
846
|
-
// A supplied target that fails to resolve is a structured error (not a
|
|
847
|
-
// silent default swipe), and a resolved point is used verbatim (0 is a legitimate
|
|
848
|
-
// coordinate). Endpoints keep an ~8% inset clear of iOS system-gesture zones.
|
|
849
|
-
const size = await d.screenSize().catch(() => null);
|
|
850
|
-
let vec;
|
|
851
|
-
if (args.target) {
|
|
852
|
-
const start = await resolveTarget(session, stripSelector(args.target));
|
|
853
|
-
if ('error' in start)
|
|
854
|
-
return fail(start.error, false);
|
|
855
|
-
vec = swipeFromPoint(size, { x: start.x, y: start.y }, direction, 0.5, GESTURE_EDGE_INSET);
|
|
856
|
-
}
|
|
857
|
-
else {
|
|
858
|
-
vec = swipeVector(size, direction, 'center', 0.5, GESTURE_EDGE_INSET);
|
|
859
|
-
}
|
|
860
|
-
await d.swipe(vec[0], vec[1], vec[2], vec[3], 300);
|
|
861
|
-
meta = { direction };
|
|
862
|
-
toRecord = { action: 'swipe', direction, exportability: 'coordinate' };
|
|
863
|
-
break;
|
|
864
|
-
}
|
|
865
|
-
case 'scroll': {
|
|
866
|
-
const max = args.maxScrolls ?? 8;
|
|
867
|
-
// scroll down = FINGER swipes up (same for the other directions where finger ==
|
|
868
|
-
// content axis). Screen-relative + clamped vector, size fetched once.
|
|
869
|
-
const finger = { down: 'up', up: 'down', left: 'left', right: 'right' };
|
|
870
|
-
const size = await d.screenSize().catch(() => null);
|
|
871
|
-
const dir = args.direction; // presence enforced by missingRequiredField
|
|
872
|
-
const u = args.untilVisible;
|
|
873
|
-
// A match only counts as FOUND when its center is on screen and not under the soft
|
|
874
|
-
// keyboard: a row just crossing the bottom edge is "in the tree" but a center tap on it
|
|
875
|
-
// would land off-screen / on a key, so keep swiping instead.
|
|
876
|
-
const kb = u ? await keyboardArea(d) : undefined;
|
|
877
|
-
const probe = async () => {
|
|
878
|
-
const xml = await d.dumpXml();
|
|
879
|
-
const at = Date.now();
|
|
880
|
-
const parsed = parseSnapshot(xml);
|
|
881
|
-
const [sw, sh] = size ? [size.width, size.height] : parsed.screen;
|
|
882
|
-
const hit = parsed.elements.some((e) => {
|
|
883
|
-
const match = (u?.id && e.id === u.id) ||
|
|
884
|
-
(u?.text &&
|
|
885
|
-
(e.text?.toLowerCase().includes(u.text.toLowerCase()) || e.label?.toLowerCase().includes(u.text.toLowerCase())));
|
|
886
|
-
if (!match)
|
|
887
|
-
return false;
|
|
888
|
-
const c = center(e.bounds);
|
|
889
|
-
const onScreen = !(sw > 0 && sh > 0) || (c.x >= 0 && c.y >= 0 && c.x < sw && c.y < sh);
|
|
890
|
-
return onScreen && !inRect(kb?.rect, c.x, c.y);
|
|
891
|
-
});
|
|
892
|
-
// Positions are part of the signature: a list that moved but still shows the same
|
|
893
|
-
// labels is not at its end (the label-only signature reported endOfList mid-list).
|
|
894
|
-
const sig = parsed.elements
|
|
895
|
-
.map((e) => `${signature(e)}@${e.bounds.join(',')}`)
|
|
896
|
-
.sort()
|
|
897
|
-
.join('\n');
|
|
898
|
-
settleSeed = { xml, at };
|
|
899
|
-
return { hit, sig, nodes: parsed.allNodes };
|
|
900
|
-
};
|
|
901
|
-
// B: anchor the swipe INSIDE the largest scrollable container (a swipe that starts on
|
|
902
|
-
// a sticky app bar moves nothing); screen center only when no container is known.
|
|
903
|
-
let anchorNodes = session.lastSnapshot?.allNodes;
|
|
904
|
-
if (!anchorNodes && !u)
|
|
905
|
-
anchorNodes = await d.dumpXml().then((x) => parseSnapshot(x).allNodes, () => undefined);
|
|
906
|
-
let anchoredIn = 'screen';
|
|
907
|
-
const nextVec = () => {
|
|
908
|
-
const rect = largestScrollableRect(anchorNodes, size);
|
|
909
|
-
anchoredIn = rect ? 'scrollable' : 'screen';
|
|
910
|
-
return rect
|
|
911
|
-
? swipeInRect(rect, finger[dir], size, 0.6, SCROLL_CONTAINER_INSET, GESTURE_EDGE_INSET)
|
|
912
|
-
: swipeVector(size, finger[dir], 'center', 0.6, GESTURE_EDGE_INSET);
|
|
913
|
-
};
|
|
914
|
-
let found = false;
|
|
915
|
-
let endOfList = false;
|
|
916
|
-
let swipes = 0;
|
|
917
|
-
let prevSig;
|
|
918
|
-
// Already visible? Then don't swipe at all (it could scroll the target away).
|
|
919
|
-
if (u) {
|
|
920
|
-
const first = await probe();
|
|
921
|
-
found = first.hit;
|
|
922
|
-
prevSig = first.sig;
|
|
923
|
-
anchorNodes = first.nodes;
|
|
924
|
-
}
|
|
925
|
-
// C: a plain scroll is exactly ONE swipe; only untilVisible loops (up to maxScrolls).
|
|
926
|
-
const limit = u ? max : 1;
|
|
927
|
-
for (let i = 0; i < limit && !found; i++) {
|
|
928
|
-
const vec = nextVec();
|
|
863
|
+
case 'swipe': {
|
|
864
|
+
const direction = args.direction; // presence enforced by missingRequiredField
|
|
865
|
+
// Derive the gesture from the real screen size. WDA swipes are in POINTS
|
|
866
|
+
// (≤~440pt wide), so the old fixed 540/1200 constants were off-screen on iOS; the
|
|
867
|
+
// legacy constants survive only inside the shared fallback for screenSize()===null.
|
|
868
|
+
// A supplied target that fails to resolve is a structured error (not a
|
|
869
|
+
// silent default swipe), and a resolved point is used verbatim (0 is a legitimate
|
|
870
|
+
// coordinate). Endpoints keep an ~8% inset clear of iOS system-gesture zones.
|
|
871
|
+
const size = await d.screenSize().catch(() => null);
|
|
872
|
+
let vec;
|
|
873
|
+
if (args.target) {
|
|
874
|
+
const start = await resolveTarget(session, stripSelector(args.target));
|
|
875
|
+
if ('error' in start)
|
|
876
|
+
return fail(start.error, false);
|
|
877
|
+
vec = swipeFromPoint(size, { x: start.x, y: start.y }, direction, 0.5, GESTURE_EDGE_INSET);
|
|
878
|
+
}
|
|
879
|
+
else {
|
|
880
|
+
vec = swipeVector(size, direction, 'center', 0.5, GESTURE_EDGE_INSET);
|
|
881
|
+
}
|
|
929
882
|
await d.swipe(vec[0], vec[1], vec[2], vec[3], 300);
|
|
930
|
-
|
|
883
|
+
meta = { direction };
|
|
884
|
+
toRecord = { action: 'swipe', direction, exportability: 'coordinate' };
|
|
885
|
+
break;
|
|
886
|
+
}
|
|
887
|
+
case 'scroll': {
|
|
888
|
+
const max = args.maxScrolls ?? 8;
|
|
889
|
+
// scroll down = FINGER swipes up (same for the other directions where finger ==
|
|
890
|
+
// content axis). Screen-relative + clamped vector, size fetched once.
|
|
891
|
+
const finger = { down: 'up', up: 'down', left: 'left', right: 'right' };
|
|
892
|
+
const size = await d.screenSize().catch(() => null);
|
|
893
|
+
const dir = args.direction; // presence enforced by missingRequiredField
|
|
894
|
+
const u = args.untilVisible;
|
|
895
|
+
// A match only counts as FOUND when its center is on screen and not under the soft
|
|
896
|
+
// keyboard: a row just crossing the bottom edge is "in the tree" but a center tap on it
|
|
897
|
+
// would land off-screen / on a key, so keep swiping instead.
|
|
898
|
+
const kb = u ? await keyboardArea(d) : undefined;
|
|
899
|
+
const probe = async () => {
|
|
900
|
+
const xml = await d.dumpXml();
|
|
901
|
+
const at = Date.now();
|
|
902
|
+
const parsed = parseSnapshot(xml);
|
|
903
|
+
const [sw, sh] = size ? [size.width, size.height] : parsed.screen;
|
|
904
|
+
const hit = parsed.elements.some((e) => {
|
|
905
|
+
const match = (u?.id && e.id === u.id) ||
|
|
906
|
+
(u?.text &&
|
|
907
|
+
(e.text?.toLowerCase().includes(u.text.toLowerCase()) || e.label?.toLowerCase().includes(u.text.toLowerCase())));
|
|
908
|
+
if (!match)
|
|
909
|
+
return false;
|
|
910
|
+
const c = center(e.bounds);
|
|
911
|
+
const onScreen = !(sw > 0 && sh > 0) || (c.x >= 0 && c.y >= 0 && c.x < sw && c.y < sh);
|
|
912
|
+
return onScreen && !inRect(kb?.rect, c.x, c.y);
|
|
913
|
+
});
|
|
914
|
+
// Positions are part of the signature: a list that moved but still shows the same
|
|
915
|
+
// labels is not at its end (the label-only signature reported endOfList mid-list).
|
|
916
|
+
const sig = parsed.elements
|
|
917
|
+
.map((e) => `${signature(e)}@${e.bounds.join(',')}`)
|
|
918
|
+
.sort()
|
|
919
|
+
.join('\n');
|
|
920
|
+
settleSeed = { xml, at };
|
|
921
|
+
return { hit, sig, nodes: parsed.allNodes };
|
|
922
|
+
};
|
|
923
|
+
// B: anchor the swipe INSIDE the largest scrollable container (a swipe that starts on
|
|
924
|
+
// a sticky app bar moves nothing); screen center only when no container is known.
|
|
925
|
+
let anchorNodes = session.lastSnapshot?.allNodes;
|
|
926
|
+
if (!anchorNodes && !u)
|
|
927
|
+
anchorNodes = await d.dumpXml().then((x) => parseSnapshot(x).allNodes, () => undefined);
|
|
928
|
+
let anchoredIn = 'screen';
|
|
929
|
+
const nextVec = () => {
|
|
930
|
+
const rect = largestScrollableRect(anchorNodes, size);
|
|
931
|
+
anchoredIn = rect ? 'scrollable' : 'screen';
|
|
932
|
+
return rect
|
|
933
|
+
? swipeInRect(rect, finger[dir], size, 0.6, SCROLL_CONTAINER_INSET, GESTURE_EDGE_INSET)
|
|
934
|
+
: swipeVector(size, finger[dir], 'center', 0.6, GESTURE_EDGE_INSET);
|
|
935
|
+
};
|
|
936
|
+
let found = false;
|
|
937
|
+
let endOfList = false;
|
|
938
|
+
let swipes = 0;
|
|
939
|
+
let prevSig;
|
|
940
|
+
// Already visible? Then don't swipe at all (it could scroll the target away).
|
|
931
941
|
if (u) {
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
+
const first = await probe();
|
|
943
|
+
found = first.hit;
|
|
944
|
+
prevSig = first.sig;
|
|
945
|
+
anchorNodes = first.nodes;
|
|
946
|
+
}
|
|
947
|
+
// C: a plain scroll is exactly ONE swipe; only untilVisible loops (up to maxScrolls).
|
|
948
|
+
const limit = u ? max : 1;
|
|
949
|
+
for (let i = 0; i < limit && !found; i++) {
|
|
950
|
+
const vec = nextVec();
|
|
951
|
+
await d.swipe(vec[0], vec[1], vec[2], vec[3], 300);
|
|
952
|
+
swipes++;
|
|
953
|
+
if (u) {
|
|
954
|
+
await new Promise((r) => setTimeout(r, 400));
|
|
955
|
+
const now = await probe();
|
|
956
|
+
anchorNodes = now.nodes;
|
|
957
|
+
found = now.hit;
|
|
958
|
+
if (found)
|
|
959
|
+
break;
|
|
960
|
+
// Screen identical after a swipe > end of the list; more swipes can't help.
|
|
961
|
+
if (now.sig === prevSig) {
|
|
962
|
+
endOfList = true;
|
|
963
|
+
break;
|
|
964
|
+
}
|
|
965
|
+
prevSig = now.sig;
|
|
942
966
|
}
|
|
943
|
-
prevSig = now.sig;
|
|
944
967
|
}
|
|
968
|
+
meta = {
|
|
969
|
+
direction: dir,
|
|
970
|
+
swipes,
|
|
971
|
+
...(swipes ? { anchoredIn } : {}),
|
|
972
|
+
untilVisibleFound: u ? found : undefined,
|
|
973
|
+
...(endOfList ? { endOfList: true } : {}),
|
|
974
|
+
};
|
|
975
|
+
toRecord = {
|
|
976
|
+
action: 'scroll',
|
|
977
|
+
direction: dir,
|
|
978
|
+
selector: args.untilVisible?.text,
|
|
979
|
+
exportability: args.untilVisible?.text ? 'semantic' : 'coordinate',
|
|
980
|
+
};
|
|
981
|
+
break;
|
|
982
|
+
}
|
|
983
|
+
case 'press': {
|
|
984
|
+
const key = args.key; // presence enforced by missingRequiredField
|
|
985
|
+
await d.pressKey(key);
|
|
986
|
+
meta = { key };
|
|
987
|
+
// iOS has no back key: WdaDriver taps the nav-bar back button or edge-swipes. Say which.
|
|
988
|
+
if (key === 'back' && d.kind === 'wda') {
|
|
989
|
+
const via = d.lastBackVia;
|
|
990
|
+
if (via)
|
|
991
|
+
meta.backVia = via;
|
|
992
|
+
}
|
|
993
|
+
toRecord = { action: 'press', key, exportability: 'semantic' };
|
|
994
|
+
break;
|
|
945
995
|
}
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
};
|
|
953
|
-
toRecord = {
|
|
954
|
-
action: 'scroll',
|
|
955
|
-
direction: dir,
|
|
956
|
-
selector: args.untilVisible?.text,
|
|
957
|
-
exportability: args.untilVisible?.text ? 'semantic' : 'coordinate',
|
|
958
|
-
};
|
|
959
|
-
break;
|
|
960
|
-
}
|
|
961
|
-
case 'press': {
|
|
962
|
-
const key = args.key; // presence enforced by missingRequiredField
|
|
963
|
-
await d.pressKey(key);
|
|
964
|
-
meta = { key };
|
|
965
|
-
// iOS has no back key: WdaDriver taps the nav-bar back button or edge-swipes. Say which.
|
|
966
|
-
if (key === 'back' && d.kind === 'wda') {
|
|
967
|
-
const via = d.lastBackVia;
|
|
968
|
-
if (via)
|
|
969
|
-
meta.backVia = via;
|
|
996
|
+
case 'open_url': {
|
|
997
|
+
const url = args.url; // presence enforced by missingRequiredField
|
|
998
|
+
await d.openUrl(url);
|
|
999
|
+
meta = { url };
|
|
1000
|
+
toRecord = { action: 'open_url', url, exportability: 'semantic' };
|
|
1001
|
+
break;
|
|
970
1002
|
}
|
|
971
|
-
toRecord = { action: 'press', key, exportability: 'semantic' };
|
|
972
|
-
break;
|
|
973
|
-
}
|
|
974
|
-
case 'open_url': {
|
|
975
|
-
const url = args.url; // presence enforced by missingRequiredField
|
|
976
|
-
await d.openUrl(url);
|
|
977
|
-
meta = { url };
|
|
978
|
-
toRecord = { action: 'open_url', url, exportability: 'semantic' };
|
|
979
|
-
break;
|
|
980
1003
|
}
|
|
981
1004
|
}
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
:
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
});
|
|
1003
|
-
}
|
|
1004
|
-
// count this as an action (wait already returned earlier)
|
|
1005
|
-
sessions.bump(session, 'actions');
|
|
1006
|
-
// Record the action into the IR for qa_generate target:"flow" (the action definitely happened here;
|
|
1007
|
-
// recorded now so a later post-snapshot failure doesn't lose the step).
|
|
1008
|
-
if (toRecord) {
|
|
1009
|
-
const sc = recordingScreenContext(session);
|
|
1010
|
-
sessions.addRecordedAction(session, { at: Date.now(), ...sc, ...toRecord });
|
|
1011
|
-
}
|
|
1012
|
-
// Post-action observation is wrapped so a settle/dump/parse failure (e.g. the
|
|
1013
|
-
// looping-animation case) returns a Swipium-shaped result, never a raw MCP error.
|
|
1014
|
-
try {
|
|
1015
|
-
// settle > observe > health (seeded with untilVisible's last probe; it was taken after the
|
|
1016
|
-
// last swipe, so re-dumping it first would be pure latency)
|
|
1017
|
-
let s = await settle(d, { timeoutMs: args.timeoutMs ?? 8000, ...(settleSeed ? { seed: settleSeed } : {}) });
|
|
1018
|
-
// Cancelled while observing: an empty/aborted dump is not evidence (no WDA_UNREACHABLE
|
|
1019
|
-
// health finding, no visual-fallback switch).
|
|
1020
|
-
if (isAbortError(undefined))
|
|
1021
|
-
return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
|
|
1022
|
-
let post = parseSnapshot(s.xml);
|
|
1023
|
-
let postSigs = new Set(post.elements.map(signature));
|
|
1024
|
-
// F: for gestures that move content, a bounds shift counts as a change too.
|
|
1025
|
-
const positional = action === 'scroll' || action === 'swipe';
|
|
1026
|
-
const movedContent = () => positional && prePositions !== undefined && prePositions !== positionalFingerprint(post.fullByRef);
|
|
1027
|
-
// Toggle state counts: a Switch/Checkbox flip keeps every signature (role|label|id|text).
|
|
1028
|
-
let toggled = stateChangedRefs(preState, post.fullByRef);
|
|
1029
|
-
let changed = !setsEqual(preSigs, postSigs) || movedContent() || toggled.length > 0;
|
|
1030
|
-
// No-change retry (review §4.5/§4.7): an instant tap that did nothing is often the RN
|
|
1031
|
-
// tap quirk. Retry ONCE as a longer press before believing it's blocked.
|
|
1032
|
-
let retriedAsPress = false;
|
|
1033
|
-
// Never when the keyboard was up at tap time: the retry would re-press a point the IME
|
|
1034
|
-
// may own and type a second stray character (H5).
|
|
1035
|
-
if (!changed && action === 'tap' && tapRetry?.instant && !tapRetry.imeUp) {
|
|
1036
|
-
await d.pressXY(tapRetry.x, tapRetry.y, 120);
|
|
1037
|
-
retriedAsPress = true;
|
|
1038
|
-
s = await settle(d, { timeoutMs: args.timeoutMs ?? 8000 });
|
|
1039
|
-
post = parseSnapshot(s.xml);
|
|
1040
|
-
postSigs = new Set(post.elements.map(signature));
|
|
1041
|
-
toggled = stateChangedRefs(preState, post.fullByRef);
|
|
1042
|
-
changed = !setsEqual(preSigs, postSigs) || movedContent() || toggled.length > 0;
|
|
1043
|
-
}
|
|
1044
|
-
session.lastSnapshot = { fullByRef: post.fullByRef, signatures: postSigs, allNodes: post.allNodes };
|
|
1045
|
-
// A structured dump succeeded > a visual-fallback session is structured again (per-screen).
|
|
1046
|
-
const modeRecovered = s.xml ? sessions.noteStructuredDump(session) : false;
|
|
1047
|
-
const health = await checkHealth(d, session.appId, s.xml, { nodes: post.allNodes });
|
|
1048
|
-
if (health.cancelled)
|
|
1049
|
-
return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
|
|
1050
|
-
// Track no-change actions for the budget / no-op-loop detector.
|
|
1051
|
-
if (!changed && (action === 'tap' || action === 'swipe' || action === 'scroll' || action === 'press')) {
|
|
1052
|
-
sessions.bump(session, 'noChangeActions');
|
|
1053
|
-
}
|
|
1054
|
-
const budgetReached = sessions.budgetStop(session);
|
|
1055
|
-
// Sensitive-mode present: mask secure fields + scrub known secrets AND the just-typed
|
|
1056
|
-
// value (covers non-secure fields like email for this immediate response).
|
|
1057
|
-
const redact = makeRedactor([...session.secrets, ...(action === 'type' && typedValue ? [typedValue] : [])]);
|
|
1058
|
-
// Record non-info findings for qa_report (deterministic bug trail) + app-error screenshot.
|
|
1059
|
-
await recordHealthFindings(sessions, session, health.findings, d, health.foreground);
|
|
1060
|
-
const banner = `${action} ${JSON.stringify(meta)} > changed=${changed}${retriedAsPress ? ' (retried as press)' : ''} ` +
|
|
1061
|
-
`settled=${s.settled} quality=${post.quality.verdict} native=${health.nativeHealthy ? 'ok' : health.nativeStatus} app=${health.appStatus}` +
|
|
1062
|
-
(budgetReached ? `\n⏹ budget reached: ${budgetReached}, call qa_report.` : '') +
|
|
1063
|
-
(modeRecovered ? '\nmode: structured again (a UI tree dump succeeded; visual-fallback cleared)' : '') +
|
|
1064
|
-
(!changed && retriedAsPress
|
|
1065
|
-
? `\nNo change even after a press retry. Likely wrong coords / disabled element / overlay / auth wall.`
|
|
1066
|
-
: '');
|
|
1067
|
-
const findings = health.findings.length
|
|
1068
|
-
? '\n' +
|
|
1069
|
-
health.findings
|
|
1070
|
-
.map((f) => `[${f.severity}] ${f.layer ?? '?'}/${f.kind}: ${f.detail}${f.evidence ? `: "${f.evidence}"` : ''}`)
|
|
1071
|
-
.join('\n')
|
|
1072
|
-
: '';
|
|
1073
|
-
// `observe` changes ONLY the element presentation below. Everything above
|
|
1074
|
-
// (lastSnapshot bookkeeping, press retry, health recording, counters, budget lines)
|
|
1075
|
-
// is identical in all modes.
|
|
1076
|
-
let elementPayload = {};
|
|
1077
|
-
let elementsText = '';
|
|
1078
|
-
const added = observe === 'diff' ? post.elements.filter((e) => !preSigs.has(signature(e))) : [];
|
|
1079
|
-
// Mostly-new screen (navigation): a diff would list every new element AND every old one
|
|
1080
|
-
// as removed, bigger than the full list. Return the (capped) full list + removedCount.
|
|
1081
|
-
const diffAsFull = observe === 'diff' && post.elements.length > 0 && added.length > post.elements.length * DIFF_FULL_FALLBACK_RATIO;
|
|
1082
|
-
if (observe === 'diff' && !diffAsFull) {
|
|
1083
|
-
const removed = [...preSigs].filter((sig) => !postSigs.has(sig)).map((sig) => redact(sig) ?? sig);
|
|
1084
|
-
const { elements: addedShown, rendered: renderedAdded, omitted } = presentElements(added, redact);
|
|
1085
|
-
const unchangedElements = post.elements.length - added.length;
|
|
1086
|
-
const hint = `${unchangedElements} unchanged element(s) not shown; pass observe:"full" to see the whole screen.`;
|
|
1087
|
-
elementPayload = { elementsOmitted: omitted, elements: addedShown, removed, unchangedElements, hint };
|
|
1088
|
-
elementsText =
|
|
1089
|
-
`\n\nDIFF vs pre-action: +${added.length} / -${removed.length}` +
|
|
1090
|
-
(added.length ? `\n${renderedAdded}` : '') +
|
|
1091
|
-
(removed.length ? `\nremoved: ${removed.join(' | ')}` : '') +
|
|
1092
|
-
`\n${hint}`;
|
|
1093
|
-
}
|
|
1094
|
-
else if (diffAsFull) {
|
|
1095
|
-
const removedCount = [...preSigs].filter((sig) => !postSigs.has(sig)).length;
|
|
1096
|
-
const { elements: outElements, rendered, omitted } = presentElements(post.elements, redact);
|
|
1097
|
-
elementPayload = {
|
|
1098
|
-
diffAsFull: true,
|
|
1099
|
-
elementsOmitted: omitted,
|
|
1100
|
-
elements: outElements,
|
|
1101
|
-
addedCount: added.length,
|
|
1102
|
-
removedCount,
|
|
1103
|
-
};
|
|
1104
|
-
elementsText =
|
|
1105
|
-
`\n\nNEW SCREEN (${added.length}/${post.elements.length} elements new, ${removedCount} previous gone), full list:` +
|
|
1106
|
-
`\n${rendered}`;
|
|
1005
|
+
catch (e) {
|
|
1006
|
+
// Cancelled mid-action: not a driver failure (no WDA_UNREACHABLE / SNAPSHOT_FAILED tool
|
|
1007
|
+
// error, no finding). The action may have partly run, so changedState stays true.
|
|
1008
|
+
if (isAbortError(e))
|
|
1009
|
+
return cancelledResult(`Action "${action}" cancelled: the call was aborted before it finished`, true);
|
|
1010
|
+
const msg = String(e?.message ?? e);
|
|
1011
|
+
const failureCode = /BACKEND_UNSUPPORTED|not supported by the WDA backend/.test(msg)
|
|
1012
|
+
? 'BACKEND_UNSUPPORTED'
|
|
1013
|
+
: classifyFlowDriverError(e);
|
|
1014
|
+
// H2: driver errors can echo argv/stderr. Scrub known secrets AND the value just typed.
|
|
1015
|
+
const redactErr = makeRedactor(errorSecrets(session, action === 'type' ? (typedValue ?? args.text) : undefined));
|
|
1016
|
+
return qaError({
|
|
1017
|
+
what: `Action "${action}" failed: ${redactErr(String(e)) ?? ''}`,
|
|
1018
|
+
changedState: true,
|
|
1019
|
+
retrySafe: !['WDA_SESSION_FAILED', 'UNKNOWN'].includes(failureCode),
|
|
1020
|
+
failureCode,
|
|
1021
|
+
nextSteps: failureCode === 'UNKNOWN'
|
|
1022
|
+
? ['Confirm the device is online and re-snapshot.']
|
|
1023
|
+
: ['Use the failureCode to choose recovery, then re-snapshot before retrying.'],
|
|
1024
|
+
});
|
|
1107
1025
|
}
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1026
|
+
// count this as an action (wait already returned earlier)
|
|
1027
|
+
sessions.bump(session, 'actions');
|
|
1028
|
+
// Record the action into the IR for qa_generate target:"flow" (the action definitely happened here;
|
|
1029
|
+
// recorded now so a later post-snapshot failure doesn't lose the step).
|
|
1030
|
+
if (toRecord) {
|
|
1031
|
+
const sc = recordingScreenContext(session);
|
|
1032
|
+
sessions.addRecordedAction(session, { at: Date.now(), ...sc, ...toRecord });
|
|
1112
1033
|
}
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1034
|
+
// Post-action observation is wrapped so a settle/dump/parse failure (e.g. the
|
|
1035
|
+
// looping-animation case) returns a Swipium-shaped result, never a raw MCP error.
|
|
1036
|
+
try {
|
|
1037
|
+
// settle > observe > health (seeded with untilVisible's last probe; it was taken after the
|
|
1038
|
+
// last swipe, so re-dumping it first would be pure latency)
|
|
1039
|
+
let s = await settle(d, { timeoutMs: args.timeoutMs ?? 8000, ...(settleSeed ? { seed: settleSeed } : {}) });
|
|
1040
|
+
// Cancelled while observing: an empty/aborted dump is not evidence (no WDA_UNREACHABLE
|
|
1041
|
+
// health finding, no visual-fallback switch).
|
|
1042
|
+
if (isAbortError(undefined))
|
|
1043
|
+
return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
|
|
1044
|
+
let post = parseSnapshot(s.xml);
|
|
1045
|
+
let postSigs = new Set(post.elements.map(signature));
|
|
1046
|
+
// F: for gestures that move content, a bounds shift counts as a change too.
|
|
1047
|
+
const positional = action === 'scroll' || action === 'swipe';
|
|
1048
|
+
const movedContent = () => positional && prePositions !== undefined && prePositions !== positionalFingerprint(post.fullByRef);
|
|
1049
|
+
// Toggle state counts: a Switch/Checkbox flip keeps every signature (role|label|id|text).
|
|
1050
|
+
let toggled = stateChangedRefs(preState, post.fullByRef);
|
|
1051
|
+
let changed = !setsEqual(preSigs, postSigs) || movedContent() || toggled.length > 0;
|
|
1052
|
+
// No-change retry (review §4.5/§4.7): an instant tap that did nothing is often the RN
|
|
1053
|
+
// tap quirk. Retry ONCE as a longer press before believing it's blocked.
|
|
1054
|
+
let retriedAsPress = false;
|
|
1055
|
+
// Never when the keyboard was up at tap time: the retry would re-press a point the IME
|
|
1056
|
+
// may own and type a second stray character (H5).
|
|
1057
|
+
if (!changed && action === 'tap' && tapRetry?.instant && !tapRetry.imeUp) {
|
|
1058
|
+
await d.pressXY(tapRetry.x, tapRetry.y, 120);
|
|
1059
|
+
retriedAsPress = true;
|
|
1060
|
+
s = await settle(d, { timeoutMs: args.timeoutMs ?? 8000 });
|
|
1061
|
+
post = parseSnapshot(s.xml);
|
|
1062
|
+
postSigs = new Set(post.elements.map(signature));
|
|
1063
|
+
toggled = stateChangedRefs(preState, post.fullByRef);
|
|
1064
|
+
changed = !setsEqual(preSigs, postSigs) || movedContent() || toggled.length > 0;
|
|
1065
|
+
}
|
|
1066
|
+
session.lastSnapshot = { fullByRef: post.fullByRef, signatures: postSigs, allNodes: post.allNodes };
|
|
1067
|
+
// A structured dump succeeded > a visual-fallback session is structured again (per-screen).
|
|
1068
|
+
const modeRecovered = s.xml ? sessions.noteStructuredDump(session) : false;
|
|
1069
|
+
const health = await checkHealth(d, session.appId, s.xml, { nodes: post.allNodes });
|
|
1070
|
+
if (health.cancelled)
|
|
1071
|
+
return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
|
|
1072
|
+
// Track no-change actions for the budget / no-op-loop detector.
|
|
1073
|
+
if (!changed && (action === 'tap' || action === 'swipe' || action === 'scroll' || action === 'press')) {
|
|
1074
|
+
sessions.bump(session, 'noChangeActions');
|
|
1075
|
+
}
|
|
1076
|
+
const budgetReached = sessions.budgetStop(session);
|
|
1077
|
+
// Sensitive-mode present: mask secure fields + scrub known secrets AND the just-typed
|
|
1078
|
+
// value (covers non-secure fields like email for this immediate response).
|
|
1079
|
+
const redact = makeRedactor([...session.secrets, ...(action === 'type' && typedValue ? [typedValue] : [])]);
|
|
1080
|
+
// Record non-info findings for qa_report (deterministic bug trail) + app-error screenshot.
|
|
1081
|
+
await recordHealthFindings(sessions, session, health.findings, d, health.foreground);
|
|
1082
|
+
const banner = `${action} ${JSON.stringify(meta)} > changed=${changed}${retriedAsPress ? ' (retried as press)' : ''} ` +
|
|
1083
|
+
`settled=${s.settled} quality=${post.quality.verdict} native=${health.nativeHealthy ? 'ok' : health.nativeStatus} app=${health.appStatus}` +
|
|
1084
|
+
(budgetReached ? `\n⏹ budget reached: ${budgetReached}, call qa_report.` : '') +
|
|
1085
|
+
(modeRecovered ? '\nmode: structured again (a UI tree dump succeeded; visual-fallback cleared)' : '') +
|
|
1086
|
+
(!changed && retriedAsPress
|
|
1087
|
+
? `\nNo change even after a press retry. Likely wrong coords / disabled element / overlay / auth wall.`
|
|
1088
|
+
: '');
|
|
1089
|
+
const findings = health.findings.length
|
|
1090
|
+
? '\n' +
|
|
1091
|
+
health.findings
|
|
1092
|
+
.map((f) => `[${f.severity}] ${f.layer ?? '?'}/${f.kind}: ${f.detail}${f.evidence ? `: "${f.evidence}"` : ''}`)
|
|
1093
|
+
.join('\n')
|
|
1094
|
+
: '';
|
|
1095
|
+
// `observe` changes ONLY the element presentation below. Everything above
|
|
1096
|
+
// (lastSnapshot bookkeeping, press retry, health recording, counters, budget lines)
|
|
1097
|
+
// is identical in all modes.
|
|
1098
|
+
let elementPayload = {};
|
|
1099
|
+
let elementsText = '';
|
|
1100
|
+
const added = observe === 'diff' ? post.elements.filter((e) => !preSigs.has(signature(e))) : [];
|
|
1101
|
+
// Mostly-new screen (navigation): a diff would list every new element AND every old one
|
|
1102
|
+
// as removed, bigger than the full list. Return the (capped) full list + removedCount.
|
|
1103
|
+
const diffAsFull = observe === 'diff' && post.elements.length > 0 && added.length > post.elements.length * DIFF_FULL_FALLBACK_RATIO;
|
|
1104
|
+
if (observe === 'diff' && !diffAsFull) {
|
|
1105
|
+
const removed = [...preSigs].filter((sig) => !postSigs.has(sig)).map((sig) => redact(sig) ?? sig);
|
|
1106
|
+
const { payload: addedShown, rendered: renderedAdded, omitted } = presentElements(added, redact);
|
|
1107
|
+
const unchangedElements = post.elements.length - added.length;
|
|
1108
|
+
const hint = `${unchangedElements} unchanged element(s) not shown; pass observe:"full" to see the whole screen.`;
|
|
1109
|
+
elementPayload = { elementsOmitted: omitted, elements: addedShown, removed, unchangedElements, hint };
|
|
1110
|
+
elementsText =
|
|
1111
|
+
`\n\nDIFF vs pre-action: +${added.length} / -${removed.length}` +
|
|
1112
|
+
(added.length ? `\n${renderedAdded}` : '') +
|
|
1113
|
+
(removed.length ? `\nremoved: ${removed.join(' | ')}` : '') +
|
|
1114
|
+
`\n${hint}`;
|
|
1115
|
+
}
|
|
1116
|
+
else if (diffAsFull) {
|
|
1117
|
+
const removedCount = [...preSigs].filter((sig) => !postSigs.has(sig)).length;
|
|
1118
|
+
const { payload: outElements, rendered, omitted } = presentElements(post.elements, redact);
|
|
1119
|
+
elementPayload = {
|
|
1120
|
+
diffAsFull: true,
|
|
1121
|
+
elementsOmitted: omitted,
|
|
1122
|
+
elements: outElements,
|
|
1123
|
+
addedCount: added.length,
|
|
1124
|
+
removedCount,
|
|
1125
|
+
};
|
|
1126
|
+
elementsText =
|
|
1127
|
+
`\n\nNEW SCREEN (${added.length}/${post.elements.length} elements new, ${removedCount} previous gone), full list:` +
|
|
1128
|
+
`\n${rendered}`;
|
|
1129
|
+
}
|
|
1130
|
+
else if (observe === 'full') {
|
|
1131
|
+
const { payload: outElements, rendered, omitted } = presentElements(post.elements, redact);
|
|
1132
|
+
elementPayload = { elementsOmitted: omitted, elements: outElements };
|
|
1133
|
+
elementsText = `\n\n${rendered}`;
|
|
1134
|
+
}
|
|
1135
|
+
else {
|
|
1136
|
+
const hint = `${post.elements.length} element(s) not shown (observe:"none"); pass observe:"full" or run qa_snapshot to see the screen.`;
|
|
1137
|
+
elementPayload = { hint };
|
|
1138
|
+
elementsText = `\n\n${hint}`;
|
|
1139
|
+
}
|
|
1140
|
+
if (toggled.length && observe !== 'none') {
|
|
1141
|
+
const lines = toggled.map((ref) => {
|
|
1142
|
+
const n = post.fullByRef.get(ref);
|
|
1143
|
+
const name = isSecureNode(n) ? '«secure»' : (redact(n.desc || n.text) ?? '');
|
|
1144
|
+
return `${ref} ${JSON.stringify(name)} ${stateLabel(n)}`;
|
|
1145
|
+
});
|
|
1146
|
+
elementPayload = { ...elementPayload, stateChanged: lines };
|
|
1147
|
+
elementsText += `\nstate changed: ${lines.join(' | ')}`;
|
|
1148
|
+
}
|
|
1149
|
+
// The WDA driver transparently re-creates a reaped session (without relaunching the app);
|
|
1150
|
+
// the agent must know the screen may not be where it left it.
|
|
1151
|
+
if (d.consumeSessionRecovered?.())
|
|
1152
|
+
warnings.push(WDA_SESSION_RECOVERED_WARNING);
|
|
1153
|
+
return qaOk({
|
|
1154
|
+
action,
|
|
1155
|
+
...meta,
|
|
1156
|
+
...(keyboardHidden ? { keyboardHidden: true } : {}),
|
|
1157
|
+
...(modeRecovered ? { modeRecovered: true, mode: 'structured' } : {}),
|
|
1158
|
+
changed,
|
|
1159
|
+
retriedAsPress,
|
|
1160
|
+
settled: s.settled,
|
|
1161
|
+
quality: post.quality.verdict,
|
|
1162
|
+
health,
|
|
1163
|
+
counters: session.counters,
|
|
1164
|
+
...(budgetReached ? { budgetReached } : {}),
|
|
1165
|
+
observe,
|
|
1166
|
+
...elementPayload,
|
|
1167
|
+
...(warnings.length ? { warnings } : {}),
|
|
1168
|
+
}, `${banner}${keyboardHidden ? '\nNote: the soft keyboard covered the target and was hidden first (keyboardHidden:true).' : ''}` +
|
|
1169
|
+
`${findings}${warnings.map((w) => `\n⚠ ${w}`).join('')}${elementsText}`, { textOmit: ['elements', 'removed', 'hint', 'stateChanged'] });
|
|
1117
1170
|
}
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1171
|
+
catch (e) {
|
|
1172
|
+
if (isAbortError(e))
|
|
1173
|
+
return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
|
|
1174
|
+
// The action ran; observing the result failed (often a UI that never reaches idle).
|
|
1175
|
+
// visual-fallback is per-screen: the next successful qa_snapshot / qa_act dump clears it.
|
|
1176
|
+
const idle = /idle|dump|hierarchy/i.test(String(e));
|
|
1177
|
+
if (idle)
|
|
1178
|
+
sessions.setMode(session, 'visual-fallback');
|
|
1179
|
+
const redactErr = makeRedactor(errorSecrets(session, action === 'type' ? (typedValue ?? args.text) : undefined));
|
|
1180
|
+
return qaError({
|
|
1181
|
+
what: `Action "${action}" ran, but observing the result failed: ${redactErr(String(e)) ?? ''}`,
|
|
1182
|
+
changedState: true,
|
|
1183
|
+
retrySafe: false,
|
|
1184
|
+
failureCode: classifyFlowDriverError(e, 'SNAPSHOT_FAILED'),
|
|
1185
|
+
nextSteps: idle
|
|
1186
|
+
? [
|
|
1187
|
+
'Switched to visual-fallback for this screen (cleared by the next successful qa_snapshot / qa_act dump). Use qa_screenshot; qa_check_health still works.',
|
|
1188
|
+
]
|
|
1189
|
+
: ['Re-check the device is online, then qa_screenshot / qa_check_health.'],
|
|
1123
1190
|
});
|
|
1124
|
-
elementPayload = { ...elementPayload, stateChanged: lines };
|
|
1125
|
-
elementsText += `\nstate changed: ${lines.join(' | ')}`;
|
|
1126
1191
|
}
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
warnings.push(WDA_SESSION_RECOVERED_WARNING);
|
|
1131
|
-
return qaOk({
|
|
1132
|
-
action,
|
|
1133
|
-
...meta,
|
|
1134
|
-
...(keyboardHidden ? { keyboardHidden: true } : {}),
|
|
1135
|
-
...(modeRecovered ? { modeRecovered: true, mode: 'structured' } : {}),
|
|
1136
|
-
changed,
|
|
1137
|
-
retriedAsPress,
|
|
1138
|
-
settled: s.settled,
|
|
1139
|
-
quality: post.quality.verdict,
|
|
1140
|
-
health,
|
|
1141
|
-
counters: session.counters,
|
|
1142
|
-
...(budgetReached ? { budgetReached } : {}),
|
|
1143
|
-
observe,
|
|
1144
|
-
...elementPayload,
|
|
1145
|
-
...(warnings.length ? { warnings } : {}),
|
|
1146
|
-
}, `${banner}${keyboardHidden ? '\nNote: the soft keyboard covered the target and was hidden first (keyboardHidden:true).' : ''}` +
|
|
1147
|
-
`${findings}${warnings.map((w) => `\n⚠ ${w}`).join('')}${elementsText}`, { textOmit: ['elements', 'removed', 'hint', 'stateChanged'] });
|
|
1148
|
-
}
|
|
1149
|
-
catch (e) {
|
|
1150
|
-
if (isAbortError(e))
|
|
1151
|
-
return cancelledResult(`Action "${action}" ran, but the call was cancelled before its result was observed`, true);
|
|
1152
|
-
// The action ran; observing the result failed (often a UI that never reaches idle).
|
|
1153
|
-
// visual-fallback is per-screen: the next successful qa_snapshot / qa_act dump clears it.
|
|
1154
|
-
const idle = /idle|dump|hierarchy/i.test(String(e));
|
|
1155
|
-
if (idle)
|
|
1156
|
-
sessions.setMode(session, 'visual-fallback');
|
|
1157
|
-
const redactErr = makeRedactor(errorSecrets(session, action === 'type' ? (typedValue ?? args.text) : undefined));
|
|
1158
|
-
return qaError({
|
|
1159
|
-
what: `Action "${action}" ran, but observing the result failed: ${redactErr(String(e)) ?? ''}`,
|
|
1160
|
-
changedState: true,
|
|
1161
|
-
retrySafe: false,
|
|
1162
|
-
failureCode: classifyFlowDriverError(e, 'SNAPSHOT_FAILED'),
|
|
1163
|
-
nextSteps: idle
|
|
1164
|
-
? [
|
|
1165
|
-
'Switched to visual-fallback for this screen (cleared by the next successful qa_snapshot / qa_act dump). Use qa_screenshot; qa_check_health still works.',
|
|
1166
|
-
]
|
|
1167
|
-
: ['Re-check the device is online, then qa_screenshot / qa_check_health.'],
|
|
1168
|
-
});
|
|
1169
|
-
}
|
|
1170
|
-
}));
|
|
1192
|
+
});
|
|
1193
|
+
return clampedFrom != null ? qaAnnotate(res, [`timeoutMs ${clampedFrom} clamped to ${ACT_TIMEOUT_MAX_MS}.`]) : res;
|
|
1194
|
+
});
|
|
1171
1195
|
}
|
|
1172
1196
|
//# sourceMappingURL=act.js.map
|