rnxsim 0.1.313 → 0.1.314

package/cli/commands/inspect/actions.ts

@@ -0,0 +1,622 @@
+import { waitForSootsimIdle } from './settling'
+import type { InspectBridge } from './core'
+
+export type TapFailure = 'not-found' | 'missed' | 'special'
+
+export interface TapTargetSummary {
+  nodeId: number | null
+  id: string | null
+  testID: string | null
+  text: string | null
+  type: string | null
+}
+
+export interface TapOutcome {
+  payload: any
+  result: any
+  attempts: number
+  failure?: TapFailure
+}
+
+export interface TapTextOptions {
+  exact?: boolean
+  role?: string | null
+  within?: string | null
+  minX?: number | null
+  maxX?: number | null
+  minY?: number | null
+  maxY?: number | null
+  near?: { x: number; y: number } | null
+  nth?: number | null
+  first?: boolean
+}
+
+export interface TapTimingOptions {
+  initialWaitMs?: number
+  deadlineMs?: number
+  retryWaitMs?: number
+}
+
+// the device screen a finger can actually reach, evaluated in the page. the
+// compositor keeps one canvas per surface and parks the inactive ones at zero
+// size, so measuring `canvas[data-surface-id]` reads whichever element happens
+// to come first in the DOM — `home`, sized 0x0 whenever an app is foreground —
+// which left the offscreen guard below silently disabled inside every app. the
+// engine publishes the real device spec on this snapshot instead.
+const SCREEN_EVAL = `(() => {
+  const spec = window.SootSim?.state?.engineSnapshot?.windowState?.deviceSpec
+  return spec && spec.width > 0 && spec.height > 0
+    ? { width: spec.width, height: spec.height }
+    : null
+})()`
+
+const DEFAULT_AGENT_TIMING: Required<TapTimingOptions> = {
+  initialWaitMs: 3000,
+  deadlineMs: 5000,
+  retryWaitMs: 700,
+}
+
+const DEFAULT_INTERACTIVE_TIMING: Required<TapTimingOptions> = {
+  initialWaitMs: 1200,
+  deadlineMs: 2500,
+  retryWaitMs: 700,
+}
+
+function tapTiming(agent: boolean, opts: TapTimingOptions = {}) {
+  return {
+    ...(agent ? DEFAULT_AGENT_TIMING : DEFAULT_INTERACTIVE_TIMING),
+    ...opts,
+  }
+}
+
+export function isTapSuccess(result: any): boolean {
+  if (!result || result.hit === false || result.ok === false) return false
+  if (result.requestedTargetMatched === false) return false
+  if (
+    result.pointerTapHandled === false &&
+    !result.keyboardOpened &&
+    !result.isTextInput
+  ) {
+    return false
+  }
+  return true
+}
+
+export function tapTargetFromPayload(
+  payload: any,
+  textFallback?: string,
+): TapTargetSummary {
+  return {
+    nodeId:
+      payload.target?.nodeId ?? payload.match?.nodeId ?? payload.node?.nodeId ?? null,
+    id: payload.target?.id ?? payload.match?.id ?? payload.node?.id ?? null,
+    testID:
+      payload.target?.testID ?? payload.match?.testID ?? payload.node?.testID ?? null,
+    text:
+      payload.target?.text ??
+      payload.target?.accessibilityLabel ??
+      payload.match?.text ??
+      payload.match?.accessibilityLabel ??
+      payload.node?.text ??
+      textFallback ??
+      null,
+    type: payload.target?.type ?? payload.match?.type ?? payload.node?.type ?? null,
+  }
+}
+
+export async function tapResolvedTarget(
+  bridge: InspectBridge,
+  args: {
+    textFallback?: string
+    resolve: () => Promise<any>
+    agent?: boolean
+    timing?: TapTimingOptions
+  },
+): Promise<TapOutcome> {
+  const timing = tapTiming(!!args.agent, args.timing)
+  let attempts = 0
+  let lastPayload: any = null
+  let lastResult: any = null
+
+  try {
+    await waitForSootsimIdle({
+      bridge,
+      maxMs: timing.initialWaitMs,
+      pollMs: 32,
+      stablePolls: 2,
+    })
+  } catch {
+    // best-effort readiness gate; the retry loop decides success.
+  }
+
+  const deadline = Date.now() + timing.deadlineMs
+  let lastOffscreenAt: string | null = null
+  while (Date.now() <= deadline || attempts === 0) {
+    attempts++
+    const payload = await args.resolve()
+    lastPayload = payload
+
+    if (
+      payload?.error === 'bridge-not-ready' ||
+      payload?.ambiguous ||
+      payload?.nthOutOfRange
+    ) {
+      return { payload, result: null, attempts, failure: 'special' }
+    }
+
+    if (payload && typeof payload.cx === 'number' && typeof payload.cy === 'number') {
+      // a real finger can only land within the device screen. when the
+      // resolved center sits outside it (a virtualized row scrolled away),
+      // don't dispatch — the engine would "hit" tree coordinates no touch
+      // could reach and the press would do nothing visible. report an honest
+      // offscreen miss instead so drivers scroll first rather than getting
+      // stuck trusting a phantom hit.
+      if (payload.offscreen) {
+        lastResult = {
+          hit: false,
+          reason: 'offscreen',
+          x: payload.cx,
+          y: payload.cy,
+          screen: payload.screen,
+        }
+        // stable offscreen position twice in a row is deterministic — the
+        // node is scrolled out of view, not settling in. bail early instead
+        // of retrying to the deadline.
+        const at = `${Math.round(payload.cx)},${Math.round(payload.cy)}`
+        if (lastOffscreenAt === at) {
+          return { payload, result: lastResult, attempts, failure: 'missed' }
+        }
+        lastOffscreenAt = at
+      } else {
+        lastOffscreenAt = null
+        const requestedTarget = tapTargetFromPayload(payload, args.textFallback)
+        const verification = await bridge.send({
+          type: 'evaluate',
+          code: `(async () => {
+            const t = window.__sootsimTest
+            if (
+              !t ||
+              typeof t.inspectAt !== 'function' ||
+              typeof t.resolveTapTarget !== 'function'
+            ) {
+              return { error: 'tap-target-inspection-unavailable' }
+            }
+            const requestedTarget = ${JSON.stringify(requestedTarget)}
+            const inspected = await t.inspectAt(${payload.cx}, ${payload.cy})
+            const resolved =
+              inspected && typeof inspected.nodeId === 'number'
+                ? await t.resolveTapTarget(inspected.nodeId)
+                : null
+            const node = (resolved && resolved.target) || inspected
+            const coordinateTarget = node
+              ? {
+                  nodeId: node.nodeId ?? null,
+                  id: node.id ?? null,
+                  testID: node.testID ?? null,
+                  text: node.text ?? node.accessibilityLabel ?? null,
+                  type: node.type ?? null,
+                }
+              : null
+            const coordinateAncestors = Array.isArray(inspected?.ancestors)
+              ? inspected.ancestors
+              : []
+            const ancestorMatchesRequested = coordinateAncestors.some(
+              (ancestor) =>
+                (typeof requestedTarget.nodeId === 'number' &&
+                  ancestor?.nodeId === requestedTarget.nodeId) ||
+                (requestedTarget.testID &&
+                  ancestor?.testID === requestedTarget.testID) ||
+                (requestedTarget.id && ancestor?.id === requestedTarget.id),
+            )
+            const requestedTargetMatched =
+              coordinateTarget === null
+                ? false
+                : typeof requestedTarget.nodeId === 'number' &&
+                    typeof coordinateTarget.nodeId === 'number'
+                  ? requestedTarget.nodeId === coordinateTarget.nodeId ||
+                    ancestorMatchesRequested
+                  : requestedTarget.testID && coordinateTarget.testID
+                    ? requestedTarget.testID === coordinateTarget.testID
+                      || ancestorMatchesRequested
+                    : requestedTarget.id && coordinateTarget.id
+                      ? requestedTarget.id === coordinateTarget.id
+                        || ancestorMatchesRequested
+                      : ancestorMatchesRequested
+            if (!requestedTargetMatched) {
+              return {
+                requestedTargetMatched,
+                requestedTarget,
+                coordinateTarget,
+              }
+            }
+            if (typeof t.activatePressAt !== 'function') {
+              return { error: 'tap-target-activation-unavailable' }
+            }
+            const activation = await t.activatePressAt(
+              ${payload.cx},
+              ${payload.cy},
+              requestedTarget.nodeId,
+            )
+            if (activation?.ok) {
+              window.dispatchEvent(
+                new CustomEvent('sootsim:agentAction', {
+                  detail: {
+                    type: 'tap',
+                    x: ${payload.cx},
+                    y: ${payload.cy},
+                    target: requestedTarget,
+                  },
+                }),
+              )
+            }
+            return {
+              activation,
+              requestedTargetMatched,
+              requestedTarget,
+              coordinateTarget,
+            }
+          })()`,
+        })
+        if (verification?.error) {
+          return {
+            payload,
+            result: {
+              hit: false,
+              reason: verification.error,
+              requestedTarget,
+            },
+            attempts,
+            failure: 'special',
+          }
+        }
+        if (verification?.requestedTargetMatched === false) {
+          return {
+            payload,
+            result: {
+              hit: false,
+              reason: 'target-covered',
+              ...verification,
+            },
+            attempts,
+            failure: 'missed',
+          }
+        }
+        const dispatched = verification?.activation?.tap ?? verification?.activation
+        const result =
+          dispatched && typeof dispatched === 'object'
+            ? { ...dispatched, ...verification?.activation, ...verification }
+            : { hit: !!dispatched, ...verification }
+        lastResult = result
+        if (isTapSuccess(result)) return { payload, result, attempts }
+      }
+    }
+
+    const remaining = deadline - Date.now()
+    if (remaining <= 0) break
+    try {
+      await waitForSootsimIdle({
+        bridge,
+        maxMs: Math.min(remaining, timing.retryWaitMs),
+        pollMs: 32,
+        stablePolls: 2,
+      })
+    } catch {
+      await new Promise((resolve) => setTimeout(resolve, Math.min(120, remaining)))
+    }
+  }
+
+  return {
+    payload: lastPayload,
+    result: lastResult,
+    attempts,
+    failure: lastPayload && typeof lastPayload.cx === 'number' ? 'missed' : 'not-found',
+  }
+}
+
+export async function tapCoordinates(
+  bridge: InspectBridge,
+  x: number,
+  y: number,
+  target?: TapTargetSummary,
+): Promise<any> {
+  return bridge.send({
+    type: 'tap',
+    x,
+    y,
+    ...(target ? { target } : {}),
+  })
+}
+
+export async function tapById(
+  bridge: InspectBridge,
+  query: string,
+  opts: { agent?: boolean; timing?: TapTimingOptions } = {},
+): Promise<TapOutcome> {
+  const arg = JSON.stringify(query)
+  return tapResolvedTarget(bridge, {
+    agent: opts.agent,
+    timing: opts.timing,
+    resolve: () =>
+      bridge.send({
+        type: 'evaluate',
+        code: `(async () => {
+          const t = window.__sootsimTest
+          if (!t) return null
+          const n = (await t.findByTestId(${arg})) || (await t.findById(${arg}))
+          if (!n || !n.absolutePosition || !n.layout) return { cx: null }
+          const resolved =
+            typeof n.nodeId === 'number' && typeof t.resolveTapTarget === 'function'
+              ? await t.resolveTapTarget(n.nodeId)
+              : null
+          const target = (resolved && resolved.target) || n
+          const cx =
+            resolved && typeof resolved.cx === 'number'
+              ? resolved.cx
+              : n.absolutePosition.x + (n.layout.width || 0) / 2
+          const cy =
+            resolved && typeof resolved.cy === 'number'
+              ? resolved.cy
+              : n.absolutePosition.y + (n.layout.height || 0) / 2
+          const scr = ${SCREEN_EVAL}
+          const offscreen =
+            !!scr && (cx < 0 || cy < 0 || cx > scr.width || cy > scr.height)
+          return {
+            cx,
+            cy,
+            ...(offscreen ? { offscreen: true, screen: scr } : {}),
+            match: {
+              nodeId: n.nodeId ?? null,
+              id: n.id,
+              testID: n.testID,
+              text: n.text ?? n.accessibilityLabel ?? null,
+              type: n.type,
+            },
+            target: {
+              nodeId: target.nodeId ?? null,
+              id: target.id,
+              testID: target.testID,
+              text:
+                target.text ??
+                target.accessibilityLabel ??
+                n.text ??
+                n.accessibilityLabel ??
+                null,
+              type: target.type,
+            },
+            strategy: (resolved && resolved.strategy) || 'matched-node',
+          }
+        })()`,
+      }),
+  })
+}
+
+export async function tapByText(
+  bridge: InspectBridge,
+  query: string,
+  options: TapTextOptions = {},
+  opts: { agent?: boolean; timing?: TapTimingOptions } = {},
+): Promise<TapOutcome> {
+  const filterArg = JSON.stringify({
+    query,
+    exact: !!options.exact,
+    role: options.role ?? null,
+    within: options.within ?? null,
+    minX: options.minX ?? null,
+    maxX: options.maxX ?? null,
+    minY: options.minY ?? null,
+    maxY: options.maxY ?? null,
+    near: options.near ?? null,
+    nth: options.nth ?? null,
+    first: !!options.first,
+  })
+
+  return tapResolvedTarget(bridge, {
+    agent: opts.agent,
+    timing: opts.timing,
+    textFallback: query,
+    resolve: () =>
+      bridge.send({
+        type: 'evaluate',
+        code: `(async () => {
+          const t = window.__sootsimTest
+          if (!t) return { error: 'bridge-not-ready' }
+          const F = ${filterArg}
+
+          const res = await t.queryTextCandidates({
+            query: F.query,
+            exact: !!F.exact,
+            ...(F.role ? { role: F.role } : {}),
+          })
+
+          let candidates = res.candidates || []
+
+          candidates = candidates.filter((c) => {
+            const ax = c.info.absolutePosition && c.info.absolutePosition.x
+            const ay = c.info.absolutePosition && c.info.absolutePosition.y
+            if (F.minX !== null && !(ax >= F.minX)) return false
+            if (F.maxX !== null && !(ax <= F.maxX)) return false
+            if (F.minY !== null && !(ay >= F.minY)) return false
+            if (F.maxY !== null && !(ay <= F.maxY)) return false
+            if (F.within && !c.ancestorTestIDs.includes(F.within)) return false
+            return true
+          })
+
+          if (F.near) {
+            candidates.sort((a, b) => {
+              const ax = (a.info.absolutePosition && a.info.absolutePosition.x) || 0
+              const ay = (a.info.absolutePosition && a.info.absolutePosition.y) || 0
+              const bx = (b.info.absolutePosition && b.info.absolutePosition.x) || 0
+              const by = (b.info.absolutePosition && b.info.absolutePosition.y) || 0
+              return (
+                Math.hypot(ax - F.near.x, ay - F.near.y) -
+                Math.hypot(bx - F.near.x, by - F.near.y)
+              )
+            })
+          } else {
+            candidates.sort((a, b) => {
+              const ay = (a.info.absolutePosition && a.info.absolutePosition.y) || 0
+              const by = (b.info.absolutePosition && b.info.absolutePosition.y) || 0
+              if (Math.abs(ay - by) > 2) return ay - by
+              const ax = (a.info.absolutePosition && a.info.absolutePosition.x) || 0
+              const bx = (b.info.absolutePosition && b.info.absolutePosition.x) || 0
+              return ax - bx
+            })
+          }
+
+          const total = candidates.length
+          if (total === 0) return { matched: 0, total: 0 }
+
+          let idx = 0
+          if (F.nth !== null) {
+            idx = F.nth < 0 ? total + F.nth : F.nth
+            if (idx < 0 || idx >= total) {
+              return { matched: 0, total, nthOutOfRange: true, nth: F.nth }
+            }
+          } else if (total > 1 && !F.first && !F.near) {
+            return {
+              ambiguous: true,
+              total,
+              candidates: candidates.slice(0, 10).map((c, i) => ({
+                idx: i,
+                nodeId: c.info.nodeId,
+                type: c.info.type,
+                testID: c.info.testID,
+                text: (c.info.text || '').slice(0, 60),
+                abs: c.info.absolutePosition,
+                layout: c.info.layout
+                  ? {
+                      width: Math.round(c.info.layout.width || 0),
+                      height: Math.round(c.info.layout.height || 0),
+                    }
+                  : null,
+                ancestorTestIDs: (c.ancestorTestIDs || []).slice(0, 5),
+              })),
+            }
+          }
+
+          const picked = candidates[idx]
+          const n = picked.info
+          if (!n.absolutePosition || !n.layout) return { matched: 0, total }
+
+          const resolved =
+            typeof n.nodeId === 'number' &&
+            typeof t.resolveTapTarget === 'function'
+              ? await t.resolveTapTarget(n.nodeId)
+              : null
+          const target = (resolved && resolved.target) || n
+          const cx =
+            resolved && typeof resolved.cx === 'number'
+              ? resolved.cx
+              : n.absolutePosition.x + (n.layout.width || 0) / 2
+          const cy =
+            resolved && typeof resolved.cy === 'number'
+              ? resolved.cy
+              : n.absolutePosition.y + (n.layout.height || 0) / 2
+          const scr = ${SCREEN_EVAL}
+          const offscreen =
+            !!scr && (cx < 0 || cy < 0 || cx > scr.width || cy > scr.height)
+          return {
+            cx,
+            cy,
+            ...(offscreen ? { offscreen: true, screen: scr } : {}),
+            match: {
+              nodeId: n.nodeId ?? null,
+              id: n.id,
+              testID: n.testID,
+              type: n.type,
+            },
+            target: {
+              nodeId: target.nodeId ?? null,
+              id: target.id,
+              testID: target.testID,
+              text:
+                target.text ??
+                target.accessibilityLabel ??
+                n.text ??
+                n.accessibilityLabel ??
+                null,
+              type: target.type,
+            },
+            strategy: (resolved && resolved.strategy) || 'matched-node',
+            total,
+            idx,
+          }
+        })()`,
+      }),
+  })
+}
+
+export async function tapBest(
+  bridge: InspectBridge,
+  query: string,
+  opts: { agent?: boolean; timing?: TapTimingOptions } = {},
+): Promise<TapOutcome> {
+  const arg = JSON.stringify(query)
+  return tapResolvedTarget(bridge, {
+    agent: opts.agent,
+    timing: opts.timing,
+    textFallback: query,
+    resolve: async () => {
+      const payload = await bridge.send({
+        type: 'evaluate',
+        code: `(async () => {
+          const t = window.__sootsimTest
+          if (!t) return { error: 'bridge-not-ready' }
+          const byTestId =
+            (await t.findByTestId(${arg})) || (await t.findById(${arg}))
+          if (byTestId && byTestId.absolutePosition && byTestId.layout) {
+            return {
+              strategy: 'testid',
+              node: {
+                nodeId: byTestId.nodeId ?? null,
+                id: byTestId.id,
+                testID: byTestId.testID,
+                type: byTestId.type,
+                text: byTestId.text,
+                absolutePosition: byTestId.absolutePosition,
+                layout: byTestId.layout,
+              },
+              screen: ${SCREEN_EVAL},
+            }
+          }
+          const byText = await t.findByText(${arg})
+          if (byText && byText.absolutePosition && byText.layout) {
+            return {
+              strategy: 'text',
+              node: {
+                nodeId: byText.nodeId ?? null,
+                id: byText.id,
+                testID: byText.testID,
+                type: byText.type,
+                text: byText.text,
+                absolutePosition: byText.absolutePosition,
+                layout: byText.layout,
+              },
+              screen: ${SCREEN_EVAL},
+            }
+          }
+          return { strategy: 'none' }
+        })()`,
+      })
+      if (!payload || !('node' in payload)) return payload
+      const node = payload.node
+      const cx = node.absolutePosition.x + node.layout.width / 2
+      const cy = node.absolutePosition.y + node.layout.height / 2
+      const screen = payload.screen ?? null
+      const offscreen =
+        !!screen && (cx < 0 || cy < 0 || cx > screen.width || cy > screen.height)
+      return {
+        ...payload,
+        cx,
+        cy,
+        ...(offscreen ? { offscreen: true, screen } : {}),
+        target: {
+          id: node.id,
+          testID: node.testID,
+          text: payload.strategy === 'text' ? query : node.text,
+          type: node.type,
+        },
+      }
+    },
+  })
+}