peachsim 0.1.527

package/cli/commands/inspect/actions.ts

@@ -0,0 +1,516 @@
+import { inspectTree, type InspectBridge } from './core'
+import { resolveTargetCoords, type ResolvedTargetCoords } from './resolve-target'
+import { waitForSootsimIdle } from './settling'
+import type { SimSemanticNode } from '../../../src/bridge-contract'
+import type { SootSimReplayTarget } from '@rnx/globals'
+
+export type TapFailure = 'not-found' | 'missed' | 'special'
+
+export type TapTargetSummary = SootSimReplayTarget
+
+export interface TapOutcome {
+  payload: any
+  result: any
+  attempts: number
+  failure?: TapFailure
+}
+
+export interface TapTextOptions {
+  exact?: boolean
+  role?: string | null
+  within?: string | null
+  minX?: number | null
+  maxX?: number | null
+  minY?: number | null
+  maxY?: number | null
+  near?: { x: number; y: number } | null
+  nth?: number | null
+  first?: boolean
+}
+
+export interface TapTimingOptions {
+  initialWaitMs?: number
+  deadlineMs?: number
+  retryWaitMs?: number
+}
+
+type ScreenSize = { width: number; height: number }
+
+function isScreenSize(value: unknown): value is ScreenSize {
+  if (value === null || typeof value !== 'object') return false
+  const width = Reflect.get(value, 'width')
+  const height = Reflect.get(value, 'height')
+  return (
+    typeof width === 'number' &&
+    Number.isFinite(width) &&
+    width > 0 &&
+    typeof height === 'number' &&
+    Number.isFinite(height) &&
+    height > 0
+  )
+}
+
+function viewportFromCapture(value: unknown): unknown {
+  if (value === null || typeof value !== 'object') return undefined
+  const tree = Reflect.get(value, 'tree')
+  return tree !== null && typeof tree === 'object'
+    ? Reflect.get(tree, 'viewport')
+    : undefined
+}
+
+// the semantic tree has no covering root, so its first node is often a 64x64
+// icon. capture.tree.viewport is the device layout size on local and cloud.
+//
+// a whole screen capture is the only way to ask for it today, and it is not
+// cheap: the headless runtime forces a full redraw of the tree before it walks
+// it, which measured 3.9-5.5s on a real app in a cloud box against 120-290ms
+// for every other command. that is why this runs on the failure path only.
+async function readScreen(bridge: InspectBridge): Promise<ScreenSize | null> {
+  try {
+    const viewport = viewportFromCapture(
+      await bridge.send({ type: 'capture', includeVisual: false }),
+    )
+    return isScreenSize(viewport)
+      ? { width: viewport.width, height: viewport.height }
+      : null
+  } catch {
+    return null
+  }
+}
+
+function isOffscreen(cx: number, cy: number, screen: ScreenSize): boolean {
+  return !(cx >= 0 && cy >= 0 && cx <= screen.width && cy <= screen.height)
+}
+
+const DEFAULT_AGENT_TIMING: Required<TapTimingOptions> = {
+  initialWaitMs: 3000,
+  deadlineMs: 5000,
+  retryWaitMs: 700,
+}
+
+const DEFAULT_INTERACTIVE_TIMING: Required<TapTimingOptions> = {
+  initialWaitMs: 1200,
+  deadlineMs: 2500,
+  retryWaitMs: 700,
+}
+
+function tapTiming(agent: boolean, opts: TapTimingOptions = {}) {
+  return {
+    ...(agent ? DEFAULT_AGENT_TIMING : DEFAULT_INTERACTIVE_TIMING),
+    ...opts,
+  }
+}
+
+export function isTapSuccess(result: any): boolean {
+  if (!result || result.hit === false || result.ok === false) return false
+  if (result.handled === false) return false
+  if (result.requestedTargetMatched === false) return false
+  if (
+    result.pointerTapHandled === false &&
+    !result.keyboardOpened &&
+    !result.isTextInput
+  ) {
+    return false
+  }
+  return true
+}
+
+export function tapTargetFromPayload(
+  payload: any,
+  textFallback?: string,
+): TapTargetSummary {
+  const id = payload.target?.id ?? payload.match?.id ?? payload.node?.id
+  return {
+    nodeId:
+      payload.target?.nodeId ?? payload.match?.nodeId ?? payload.node?.nodeId ?? null,
+    id: typeof id === 'string' ? id : null,
+    testID:
+      payload.target?.testID ?? payload.match?.testID ?? payload.node?.testID ?? null,
+    text:
+      payload.target?.text ??
+      payload.target?.accessibilityLabel ??
+      payload.match?.text ??
+      payload.match?.accessibilityLabel ??
+      payload.node?.text ??
+      textFallback ??
+      null,
+    type: payload.target?.type ?? payload.match?.type ?? payload.node?.type ?? null,
+  }
+}
+
+export async function tapResolvedTarget(
+  bridge: InspectBridge,
+  args: {
+    textFallback?: string
+    resolve: () => Promise<any>
+    agent?: boolean
+    timing?: TapTimingOptions
+  },
+): Promise<TapOutcome> {
+  const timing = tapTiming(!!args.agent, args.timing)
+  let attempts = 0
+  let lastPayload: any = null
+  let lastResult: any = null
+
+  // a CPU simulator only advances its animations while a command holds its
+  // clock, so a sheet that was still sliding in when the last command returned
+  // is still mid-slide now. settling first is what lets the resolved center and
+  // the hit test below describe the same tree.
+  const settleCloud = async () => {
+    try {
+      await bridge.send({ type: 'settle' })
+    } catch {
+      // best-effort readiness gate; the retry loop decides success.
+    }
+  }
+  if (bridge.plane === 'cloud') {
+    await settleCloud()
+  } else if (timing.initialWaitMs > 0) {
+    try {
+      await waitForSootsimIdle({
+        bridge,
+        maxMs: timing.initialWaitMs,
+        pollMs: 32,
+        stablePolls: 2,
+      })
+    } catch {
+      // best-effort readiness gate; the retry loop decides success.
+    }
+  }
+
+  const deadline = Date.now() + timing.deadlineMs
+  let lastOffscreenAt: string | null = null
+  // a cloud sim refuses a point outside its screen itself, and says so, so
+  // asking it for a capture first would buy nothing. every other plane still
+  // presses whatever the hit test finds at tree coordinates no finger can
+  // reach, so there the screen has to be read before dispatching.
+  const screen = bridge.plane === 'cloud' ? null : await readScreen(bridge)
+  while (Date.now() <= deadline || attempts === 0) {
+    attempts++
+    const resolved = await args.resolve()
+    const payload =
+      resolved &&
+      typeof resolved.cx === 'number' &&
+      typeof resolved.cy === 'number' &&
+      screen &&
+      isOffscreen(resolved.cx, resolved.cy, screen)
+        ? { ...resolved, offscreen: true, screen }
+        : resolved
+    lastPayload = payload
+
+    if (
+      payload?.error === 'bridge-not-ready' ||
+      payload?.ambiguous ||
+      payload?.nthOutOfRange
+    ) {
+      return { payload, result: null, attempts, failure: 'special' }
+    }
+
+    if (payload && typeof payload.cx === 'number' && typeof payload.cy === 'number') {
+      // don't dispatch a point outside the device screen: the press would do
+      // nothing a user could reproduce. report the offscreen miss instead so
+      // drivers scroll first rather than getting stuck trusting a phantom hit.
+      if (payload.offscreen) {
+        lastResult = {
+          hit: false,
+          reason: 'offscreen',
+          x: payload.cx,
+          y: payload.cy,
+          screen: payload.screen,
+        }
+        // stable offscreen position twice in a row is deterministic — the
+        // node is scrolled out of view, not settling in. bail early instead
+        // of retrying to the deadline.
+        const at = `${Math.round(payload.cx)},${Math.round(payload.cy)}`
+        if (lastOffscreenAt === at) {
+          return { payload, result: lastResult, attempts, failure: 'missed' }
+        }
+        lastOffscreenAt = at
+      } else {
+        lastOffscreenAt = null
+        const requestedTarget = tapTargetFromPayload(payload, args.textFallback)
+        const result = await tapCoordinates(
+          bridge,
+          payload.cx,
+          payload.cy,
+          requestedTarget,
+        )
+        // a cloud sim reports the offscreen refusal itself, carrying the point
+        // and the screen in its diagnostics. give it the same shape the local
+        // check produces so there is one offscreen failure to report.
+        lastResult =
+          result?.reason === 'offscreen' && result.diagnostics
+            ? {
+                ...result,
+                hit: false,
+                x: result.diagnostics.point?.x ?? payload.cx,
+                y: result.diagnostics.point?.y ?? payload.cy,
+                screen: result.diagnostics.screen,
+              }
+            : result
+        if (result?.requestedTargetMatched === false) {
+          return { payload, result, attempts, failure: 'missed' }
+        }
+        if (isTapSuccess(result)) return { payload, result, attempts }
+      }
+    }
+
+    const remaining = deadline - Date.now()
+    if (remaining <= 0) break
+    if (bridge.plane === 'cloud') {
+      if (lastResult?.reason !== 'target-moved' || attempts >= 3) break
+      await settleCloud()
+      continue
+    }
+    if (timing.retryWaitMs <= 0) break
+    try {
+      await waitForSootsimIdle({
+        bridge,
+        maxMs: Math.min(remaining, timing.retryWaitMs),
+        pollMs: 32,
+        stablePolls: 2,
+      })
+    } catch {
+      await new Promise((resolve) => setTimeout(resolve, Math.min(120, remaining)))
+    }
+  }
+
+  return {
+    payload: lastPayload,
+    result: lastResult,
+    attempts,
+    failure: lastPayload && typeof lastPayload.cx === 'number' ? 'missed' : 'not-found',
+  }
+}
+
+export async function tapCoordinates(
+  bridge: InspectBridge,
+  x: number,
+  y: number,
+  target?: TapTargetSummary,
+): Promise<any> {
+  return bridge.send({
+    type: 'tap',
+    x,
+    y,
+    ...(target ? { target } : {}),
+  })
+}
+
+function payloadFromResolved(
+  resolved: ResolvedTargetCoords,
+  match: SootSimReplayTarget,
+  extra: Record<string, unknown> = {},
+) {
+  return {
+    cx: resolved.x,
+    cy: resolved.y,
+    match: {
+      nodeId: match.nodeId ?? null,
+      id: match.testID ?? null,
+      testID: match.testID ?? null,
+      text: match.text ?? null,
+      type: match.type ?? null,
+    },
+    target: {
+      nodeId: resolved.nodeId ?? match.nodeId ?? null,
+      id: resolved.testID ?? match.testID ?? null,
+      testID: resolved.testID ?? match.testID ?? null,
+      text: resolved.text ?? match.text ?? null,
+      type: resolved.type ?? match.type ?? null,
+    },
+    strategy: 'matched-node',
+    ...extra,
+  }
+}
+
+interface TextCandidate {
+  node: SimSemanticNode
+  ancestorTestIDs: string[]
+}
+
+// a non-pressable node repeating the text of its nearest labelled ancestor,
+// when that ancestor is pressable, is the ancestor's own caption (a button and
+// its text child), so only the button is a candidate. otherwise every labelled
+// button is an ambiguous pair. a pressable descendant is never folded into its
+// ancestor: same text does not prove it shares the ancestor's tap target.
+function collectTextCandidates(
+  nodes: readonly SimSemanticNode[],
+  ancestors: string[] = [],
+  pressableOwnerText = '',
+): TextCandidate[] {
+  const out: TextCandidate[] = []
+  for (const node of nodes) {
+    const text = nodeText(node)
+    const caption = !!text && text === pressableOwnerText && !node.pressable
+    if (!caption) out.push({ node, ancestorTestIDs: ancestors })
+    if (node.children?.length) {
+      const next = node.testID ? [...ancestors, node.testID] : ancestors
+      const owner = text ? (node.pressable ? text : '') : pressableOwnerText
+      out.push(...collectTextCandidates(node.children, next, owner))
+    }
+  }
+  return out
+}
+
+function nodeText(node: SimSemanticNode): string {
+  return node.text ?? node.label ?? ''
+}
+
+export async function tapById(
+  bridge: InspectBridge,
+  query: string,
+  opts: { agent?: boolean; timing?: TapTimingOptions } = {},
+): Promise<TapOutcome> {
+  return tapResolvedTarget(bridge, {
+    agent: opts.agent,
+    timing: opts.timing,
+    resolve: async () => {
+      const resolved = await resolveTargetCoords(bridge, { mode: 'testid', value: query })
+      if (!resolved) return { cx: null }
+      return payloadFromResolved(resolved, resolved)
+    },
+  })
+}
+
+export async function tapByText(
+  bridge: InspectBridge,
+  query: string,
+  options: TapTextOptions = {},
+  opts: { agent?: boolean; timing?: TapTimingOptions } = {},
+): Promise<TapOutcome> {
+  return tapResolvedTarget(bridge, {
+    agent: opts.agent,
+    timing: opts.timing,
+    textFallback: query,
+    resolve: async () => {
+      const { tree } = await inspectTree(bridge, 50)
+      if (!Array.isArray(tree)) return { matched: 0, total: 0 }
+      const needle = options.exact ? query : query.toLowerCase()
+      let candidates = collectTextCandidates(tree).filter(({ node, ancestorTestIDs }) => {
+        const text = nodeText(node)
+        if (!text) return false
+        if (options.exact ? text !== needle : !text.toLowerCase().includes(needle)) {
+          return false
+        }
+        if (options.role && node.role !== options.role) return false
+        if (options.within && !ancestorTestIDs.includes(options.within)) return false
+        const ax = node.geometry.x
+        const ay = node.geometry.y
+        if (options.minX != null && !(ax >= options.minX)) return false
+        if (options.maxX != null && !(ax <= options.maxX)) return false
+        if (options.minY != null && !(ay >= options.minY)) return false
+        if (options.maxY != null && !(ay <= options.maxY)) return false
+        return true
+      })
+
+      if (options.near) {
+        const { x, y } = options.near
+        candidates.sort(
+          (a, b) =>
+            Math.hypot(a.node.geometry.x - x, a.node.geometry.y - y) -
+            Math.hypot(b.node.geometry.x - x, b.node.geometry.y - y),
+        )
+      } else {
+        candidates.sort((a, b) => {
+          if (Math.abs(a.node.geometry.y - b.node.geometry.y) > 2) {
+            return a.node.geometry.y - b.node.geometry.y
+          }
+          return a.node.geometry.x - b.node.geometry.x
+        })
+      }
+
+      const total = candidates.length
+      if (total === 0) return { matched: 0, total: 0 }
+
+      let idx = 0
+      if (options.nth != null) {
+        idx = options.nth < 0 ? total + options.nth : options.nth
+        if (idx < 0 || idx >= total) {
+          return { matched: 0, total, nthOutOfRange: true, nth: options.nth }
+        }
+      } else if (total > 1 && !options.first && !options.near) {
+        return {
+          ambiguous: true,
+          total,
+          candidates: candidates.slice(0, 10).map((candidate, index) => ({
+            idx: index,
+            nodeId: candidate.node.nodeId,
+            type: candidate.node.type,
+            testID: candidate.node.testID ?? null,
+            text: nodeText(candidate.node).slice(0, 60),
+            abs: { x: candidate.node.geometry.x, y: candidate.node.geometry.y },
+            layout: {
+              width: Math.round(candidate.node.geometry.width),
+              height: Math.round(candidate.node.geometry.height),
+            },
+            ancestorTestIDs: candidate.ancestorTestIDs.slice(0, 5),
+          })),
+        }
+      }
+
+      const picked = candidates[idx]
+      if (!picked) return { matched: 0, total }
+      const node = picked.node
+      let resolved = node.testID
+        ? await resolveTargetCoords(bridge, { mode: 'testid', value: node.testID })
+        : await resolveTargetCoords(bridge, {
+            mode: 'text',
+            value: nodeText(node) || query,
+          })
+      if (!resolved) {
+        resolved = {
+          x: node.geometry.x + node.geometry.width / 2,
+          y: node.geometry.y + node.geometry.height / 2,
+          testID: node.testID ?? null,
+          text: nodeText(node) || query,
+          type: node.type,
+        }
+      }
+      return payloadFromResolved(resolved, node, { total, idx })
+    },
+  })
+}
+
+export async function tapBest(
+  bridge: InspectBridge,
+  query: string,
+  opts: { agent?: boolean; timing?: TapTimingOptions } = {},
+): Promise<TapOutcome> {
+  return tapResolvedTarget(bridge, {
+    agent: opts.agent,
+    timing: opts.timing,
+    textFallback: query,
+    resolve: async () => {
+      const byId = await resolveTargetCoords(bridge, { mode: 'testid', value: query })
+      if (byId) {
+        return {
+          ...payloadFromResolved(byId, byId),
+          strategy: 'testid',
+          node: {
+            nodeId: byId.nodeId ?? null,
+            id: byId.testID ?? query,
+            testID: byId.testID ?? query,
+            type: byId.type ?? null,
+            text: byId.text ?? null,
+          },
+        }
+      }
+      const byText = await resolveTargetCoords(bridge, { mode: 'text', value: query })
+      if (byText) {
+        return {
+          ...payloadFromResolved(byText, byText),
+          strategy: 'text',
+          node: {
+            nodeId: byText.nodeId ?? null,
+            id: byText.testID ?? null,
+            testID: byText.testID ?? null,
+            type: byText.type ?? null,
+            text: byText.text ?? query,
+          },
+        }
+      }
+      return { strategy: 'none' }
+    },
+  })
+}