rnxsim 0.1.417 → 0.1.419

package/cli/commands/inspect/actions.ts

@@ -1,5 +1,7 @@
+import { inspectTree, type InspectBridge } from './core'
+import { resolveTargetCoords } from './resolve-target'
 import { waitForSootsimIdle } from './settling'
-import type { InspectBridge } from './core'
+import type { SimSemanticNode } from '../../../src/bridge-contract'
 
 export type TapFailure = 'not-found' | 'missed' | 'special'
 
@@ -37,18 +39,99 @@ export interface TapTimingOptions {
   retryWaitMs?: number
 }
 
-// the device screen a finger can actually reach, evaluated in the page. the
-// compositor keeps one canvas per surface and parks the inactive ones at zero
-// size, so measuring `canvas[data-surface-id]` reads whichever element happens
-// to come first in the DOM — `home`, sized 0x0 whenever an app is foreground —
-// which left the offscreen guard below silently disabled inside every app. the
-// engine publishes the real device spec on this snapshot instead.
-const SCREEN_EVAL = `(() => {
-  const spec = window.SootSim?.state?.engineSnapshot?.windowState?.deviceSpec
-  return spec && spec.width > 0 && spec.height > 0
-    ? { width: spec.width, height: spec.height }
-    : null
-})()`
+type ScreenSize = { width: number; height: number }
+
+function isScreenSize(value: unknown): value is ScreenSize {
+  if (value === null || typeof value !== 'object') return false
+  const width = Reflect.get(value, 'width')
+  const height = Reflect.get(value, 'height')
+  return (
+    typeof width === 'number' &&
+    Number.isFinite(width) &&
+    width > 0 &&
+    typeof height === 'number' &&
+    Number.isFinite(height) &&
+    height > 0
+  )
+}
+
+function viewportFromCapture(value: unknown): unknown {
+  if (value === null || typeof value !== 'object') return undefined
+  const tree = Reflect.get(value, 'tree')
+  return tree !== null && typeof tree === 'object'
+    ? Reflect.get(tree, 'viewport')
+    : undefined
+}
+
+// the semantic tree has no covering root, so its first node is often a 64x64
+// icon. capture.tree.viewport is the device layout size on local and cloud.
+async function readScreen(bridge: InspectBridge): Promise<ScreenSize | null> {
+  try {
+    const viewport = viewportFromCapture(
+      await bridge.send({ type: 'capture', includeVisual: false }),
+    )
+    return isScreenSize(viewport)
+      ? { width: viewport.width, height: viewport.height }
+      : null
+  } catch {
+    return null
+  }
+}
+
+function offscreenPayload(
+  cx: number,
+  cy: number,
+  screen: ScreenSize | null,
+): { offscreen: true; screen: ScreenSize } | Record<string, never> {
+  if (!screen || (cx >= 0 && cy >= 0 && cx <= screen.width && cy <= screen.height)) {
+    return {}
+  }
+  return { offscreen: true, screen }
+}
+
+function geometryContains(geometry: SimSemanticNode['geometry'], x: number, y: number) {
+  return (
+    x >= geometry.x &&
+    y >= geometry.y &&
+    x <= geometry.x + geometry.width &&
+    y <= geometry.y + geometry.height
+  )
+}
+
+function hitTestSemanticTree(
+  nodes: readonly SimSemanticNode[],
+  x: number,
+  y: number,
+  ancestors: SimSemanticNode[] = [],
+): { node: SimSemanticNode; ancestors: SimSemanticNode[] } | null {
+  for (let index = nodes.length - 1; index >= 0; index--) {
+    const node = nodes[index]
+    if (!node || !geometryContains(node.geometry, x, y)) continue
+    const childHit = node.children?.length
+      ? hitTestSemanticTree(node.children, x, y, [...ancestors, node])
+      : null
+    return childHit ?? { node, ancestors }
+  }
+  return null
+}
+
+function nodeMatchesRequested(node: SimSemanticNode, requested: TapTargetSummary) {
+  if (typeof requested.nodeId === 'number' && node.nodeId === requested.nodeId) {
+    return true
+  }
+  if (requested.testID && node.testID === requested.testID) return true
+  if (requested.id && node.testID === requested.id) return true
+  return false
+}
+
+function coveringNodeSummary(node: SimSemanticNode) {
+  return {
+    nodeId: node.nodeId,
+    testID: node.testID ?? null,
+    text: node.text ?? node.label ?? null,
+    type: node.type,
+  }
+}
 
 const DEFAULT_AGENT_TIMING: Required<TapTimingOptions> = {
   initialWaitMs: 3000,
@@ -71,6 +154,7 @@ function tapTiming(agent: boolean, opts: TapTimingOptions = {}) {
 
 export function isTapSuccess(result: any): boolean {
   if (!result || result.hit === false || result.ok === false) return false
+  if (result.handled === false) return false
   if (result.requestedTargetMatched === false) return false
   if (
     result.pointerTapHandled === false &&
@@ -118,22 +202,30 @@ export async function tapResolvedTarget(
   let lastPayload: any = null
   let lastResult: any = null
 
-  try {
-    await waitForSootsimIdle({
-      bridge,
-      maxMs: timing.initialWaitMs,
-      pollMs: 32,
-      stablePolls: 2,
-    })
-  } catch {
-    // best-effort readiness gate; the retry loop decides success.
+  const canWaitForIdle = bridge.plane !== 'cloud' && timing.initialWaitMs > 0
+  if (canWaitForIdle) {
+    try {
+      await waitForSootsimIdle({
+        bridge,
+        maxMs: timing.initialWaitMs,
+        pollMs: 32,
+        stablePolls: 2,
+      })
+    } catch {
+      // best-effort readiness gate; the retry loop decides success.
+    }
   }
 
   const deadline = Date.now() + timing.deadlineMs
   let lastOffscreenAt: string | null = null
+  const screen = await readScreen(bridge)
   while (Date.now() <= deadline || attempts === 0) {
     attempts++
-    const payload = await args.resolve()
+    const resolved = await args.resolve()
+    const payload =
+      resolved && typeof resolved.cx === 'number' && typeof resolved.cy === 'number'
+        ? { ...resolved, ...offscreenPayload(resolved.cx, resolved.cy, screen) }
+        : resolved
     lastPayload = payload
 
     if (
@@ -170,122 +262,31 @@ export async function tapResolvedTarget(
       } else {
         lastOffscreenAt = null
         const requestedTarget = tapTargetFromPayload(payload, args.textFallback)
-        const verification = await bridge.send({
-          type: 'evaluate',
-          code: `(async () => {
-            const t = window.__sootsimTest
-            if (
-              !t ||
-              typeof t.inspectAt !== 'function' ||
-              typeof t.resolveTapTarget !== 'function'
-            ) {
-              return { error: 'tap-target-inspection-unavailable' }
-            }
-            const requestedTarget = ${JSON.stringify(requestedTarget)}
-            const inspected = await t.inspectAt(${payload.cx}, ${payload.cy})
-            const resolved =
-              inspected && typeof inspected.nodeId === 'number'
-                ? await t.resolveTapTarget(inspected.nodeId)
-                : null
-            const node = (resolved && resolved.target) || inspected
-            const coordinateTarget = node
-              ? {
-                  nodeId: node.nodeId ?? null,
-                  id: node.id ?? null,
-                  testID: node.testID ?? null,
-                  text: node.text ?? node.accessibilityLabel ?? null,
-                  type: node.type ?? null,
-                }
-              : null
-            const coordinateAncestors = Array.isArray(inspected?.ancestors)
-              ? inspected.ancestors
-              : []
-            const ancestorMatchesRequested = coordinateAncestors.some(
-              (ancestor) =>
-                (typeof requestedTarget.nodeId === 'number' &&
-                  ancestor?.nodeId === requestedTarget.nodeId) ||
-                (requestedTarget.testID &&
-                  ancestor?.testID === requestedTarget.testID) ||
-                (requestedTarget.id && ancestor?.id === requestedTarget.id),
-            )
-            const requestedTargetMatched =
-              coordinateTarget === null
-                ? false
-                : typeof requestedTarget.nodeId === 'number' &&
-                    typeof coordinateTarget.nodeId === 'number'
-                  ? requestedTarget.nodeId === coordinateTarget.nodeId ||
-                    ancestorMatchesRequested
-                  : requestedTarget.testID && coordinateTarget.testID
-                    ? requestedTarget.testID === coordinateTarget.testID
-                      || ancestorMatchesRequested
-                    : requestedTarget.id && coordinateTarget.id
-                      ? requestedTarget.id === coordinateTarget.id
-                        || ancestorMatchesRequested
-                      : ancestorMatchesRequested
-            if (!requestedTargetMatched) {
-              return {
-                requestedTargetMatched,
-                requestedTarget,
-                coordinateTarget,
-              }
-            }
-            if (typeof t.activatePressAt !== 'function') {
-              return { error: 'tap-target-activation-unavailable' }
-            }
-            const activation = await t.activatePressAt(
-              ${payload.cx},
-              ${payload.cy},
-              requestedTarget.nodeId,
-            )
-            if (activation?.ok) {
-              window.dispatchEvent(
-                new CustomEvent('sootsim:agentAction', {
-                  detail: {
-                    type: 'tap',
-                    x: ${payload.cx},
-                    y: ${payload.cy},
-                    target: requestedTarget,
-                  },
-                }),
-              )
-            }
-            return {
-              activation,
-              requestedTargetMatched,
-              requestedTarget,
-              coordinateTarget,
-            }
-          })()`,
-        })
-        if (verification?.error) {
-          return {
-            payload,
-            result: {
-              hit: false,
-              reason: verification.error,
-              requestedTarget,
-            },
-            attempts,
-            failure: 'special',
-          }
-        }
-        if (verification?.requestedTargetMatched === false) {
-          return {
-            payload,
-            result: {
+        const { tree } = await inspectTree(bridge, 50)
+        if (Array.isArray(tree) && tree.length > 0) {
+          const hit = hitTestSemanticTree(tree, payload.cx, payload.cy)
+          const matched =
+            hit !== null &&
+            (nodeMatchesRequested(hit.node, requestedTarget) ||
+              hit.ancestors.some((ancestor) =>
+                nodeMatchesRequested(ancestor, requestedTarget),
+              ))
+          if (hit && !matched) {
+            lastResult = {
               hit: false,
               reason: 'target-covered',
-              ...verification,
-            },
-            attempts,
-            failure: 'missed',
+              requestedTarget,
+              coordinateTarget: coveringNodeSummary(hit.node),
+            }
+            return { payload, result: lastResult, attempts, failure: 'missed' }
           }
         }
-        const dispatched = verification?.activation?.tap ?? verification?.activation
-        const result =
-          dispatched && typeof dispatched === 'object'
-            ? { ...dispatched, ...verification?.activation, ...verification }
-            : { hit: !!dispatched, ...verification }
+        const result = await tapCoordinates(
+          bridge,
+          payload.cx,
+          payload.cy,
+          requestedTarget,
+        )
         lastResult = result
         if (isTapSuccess(result)) return { payload, result, attempts }
       }
@@ -293,6 +294,7 @@ export async function tapResolvedTarget(
 
     const remaining = deadline - Date.now()
     if (remaining <= 0) break
+    if (bridge.plane === 'cloud' || timing.retryWaitMs <= 0) break
     try {
       await waitForSootsimIdle({
         bridge,
@@ -327,66 +329,81 @@ export async function tapCoordinates(
   })
 }
 
+function payloadFromResolved(
+  resolved: {
+    x: number
+    y: number
+    testID?: string | null
+    text?: string | null
+    type?: string | null
+  },
+  match: {
+    nodeId?: number | null
+    testID?: string | null
+    text?: string | null
+    type?: string | null
+  },
+  extra: Record<string, unknown> = {},
+) {
+  return {
+    cx: resolved.x,
+    cy: resolved.y,
+    match: {
+      nodeId: match.nodeId ?? null,
+      id: match.testID ?? null,
+      testID: match.testID ?? null,
+      text: match.text ?? null,
+      type: match.type ?? null,
+    },
+    target: {
+      nodeId: match.nodeId ?? null,
+      id: resolved.testID ?? match.testID ?? null,
+      testID: resolved.testID ?? match.testID ?? null,
+      text: resolved.text ?? match.text ?? null,
+      type: resolved.type ?? match.type ?? null,
+    },
+    strategy: 'matched-node',
+    ...extra,
+  }
+}
+
+interface TextCandidate {
+  node: SimSemanticNode
+  ancestorTestIDs: string[]
+}
+
+function collectTextCandidates(
+  nodes: readonly SimSemanticNode[],
+  ancestors: string[] = [],
+): TextCandidate[] {
+  const out: TextCandidate[] = []
+  for (const node of nodes) {
+    out.push({ node, ancestorTestIDs: ancestors })
+    if (node.children?.length) {
+      const next = node.testID ? [...ancestors, node.testID] : ancestors
+      out.push(...collectTextCandidates(node.children, next))
+    }
+  }
+  return out
+}
+
+function nodeText(node: SimSemanticNode): string {
+  return node.text ?? node.label ?? ''
+}
+
 export async function tapById(
   bridge: InspectBridge,
   query: string,
   opts: { agent?: boolean; timing?: TapTimingOptions } = {},
 ): Promise<TapOutcome> {
-  const arg = JSON.stringify(query)
   return tapResolvedTarget(bridge, {
     agent: opts.agent,
     timing: opts.timing,
-    resolve: () =>
-      bridge.send({
-        type: 'evaluate',
-        code: `(async () => {
-          const t = window.__sootsimTest
-          if (!t) return null
-          const n = (await t.findByTestId(${arg})) || (await t.findById(${arg}))
-          if (!n || !n.absolutePosition || !n.layout) return { cx: null }
-          const resolved =
-            typeof n.nodeId === 'number' && typeof t.resolveTapTarget === 'function'
-              ? await t.resolveTapTarget(n.nodeId)
-              : null
-          const target = (resolved && resolved.target) || n
-          const cx =
-            resolved && typeof resolved.cx === 'number'
-              ? resolved.cx
-              : n.absolutePosition.x + (n.layout.width || 0) / 2
-          const cy =
-            resolved && typeof resolved.cy === 'number'
-              ? resolved.cy
-              : n.absolutePosition.y + (n.layout.height || 0) / 2
-          const scr = ${SCREEN_EVAL}
-          const offscreen =
-            !!scr && (cx < 0 || cy < 0 || cx > scr.width || cy > scr.height)
-          return {
-            cx,
-            cy,
-            ...(offscreen ? { offscreen: true, screen: scr } : {}),
-            match: {
-              nodeId: n.nodeId ?? null,
-              id: n.id,
-              testID: n.testID,
-              text: n.text ?? n.accessibilityLabel ?? null,
-              type: n.type,
-            },
-            target: {
-              nodeId: target.nodeId ?? null,
-              id: target.id,
-              testID: target.testID,
-              text:
-                target.text ??
-                target.accessibilityLabel ??
-                n.text ??
-                n.accessibilityLabel ??
-                null,
-              type: target.type,
-            },
-            strategy: (resolved && resolved.strategy) || 'matched-node',
-          }
-        })()`,
-      }),
+    resolve: async () => {
+      const resolved = await resolveTargetCoords(bridge, { mode: 'testid', value: query })
+      if (!resolved) return { cx: null }
+      return payloadFromResolved(resolved, resolved)
+    },
   })
 }
 
@@ -396,153 +413,96 @@ export async function tapByText(
   options: TapTextOptions = {},
   opts: { agent?: boolean; timing?: TapTimingOptions } = {},
 ): Promise<TapOutcome> {
-  const filterArg = JSON.stringify({
-    query,
-    exact: !!options.exact,
-    role: options.role ?? null,
-    within: options.within ?? null,
-    minX: options.minX ?? null,
-    maxX: options.maxX ?? null,
-    minY: options.minY ?? null,
-    maxY: options.maxY ?? null,
-    near: options.near ?? null,
-    nth: options.nth ?? null,
-    first: !!options.first,
-  })
-
   return tapResolvedTarget(bridge, {
     agent: opts.agent,
     timing: opts.timing,
     textFallback: query,
-    resolve: () =>
-      bridge.send({
-        type: 'evaluate',
-        code: `(async () => {
-          const t = window.__sootsimTest
-          if (!t) return { error: 'bridge-not-ready' }
-          const F = ${filterArg}
-
-          const res = await t.queryTextCandidates({
-            query: F.query,
-            exact: !!F.exact,
-            ...(F.role ? { role: F.role } : {}),
-          })
-
-          let candidates = res.candidates || []
-
-          candidates = candidates.filter((c) => {
-            const ax = c.info.absolutePosition && c.info.absolutePosition.x
-            const ay = c.info.absolutePosition && c.info.absolutePosition.y
-            if (F.minX !== null && !(ax >= F.minX)) return false
-            if (F.maxX !== null && !(ax <= F.maxX)) return false
-            if (F.minY !== null && !(ay >= F.minY)) return false
-            if (F.maxY !== null && !(ay <= F.maxY)) return false
-            if (F.within && !c.ancestorTestIDs.includes(F.within)) return false
-            return true
-          })
+    resolve: async () => {
+      const { tree } = await inspectTree(bridge, 50)
+      if (!Array.isArray(tree)) return { matched: 0, total: 0 }
+      const needle = options.exact ? query : query.toLowerCase()
+      let candidates = collectTextCandidates(tree).filter(({ node, ancestorTestIDs }) => {
+        const text = nodeText(node)
+        if (!text) return false
+        if (options.exact ? text !== needle : !text.toLowerCase().includes(needle)) {
+          return false
+        }
+        if (options.role && node.role !== options.role) return false
+        if (options.within && !ancestorTestIDs.includes(options.within)) return false
+        const ax = node.geometry.x
+        const ay = node.geometry.y
+        if (options.minX != null && !(ax >= options.minX)) return false
+        if (options.maxX != null && !(ax <= options.maxX)) return false
+        if (options.minY != null && !(ay >= options.minY)) return false
+        if (options.maxY != null && !(ay <= options.maxY)) return false
+        return true
+      })
 
-          if (F.near) {
-            candidates.sort((a, b) => {
-              const ax = (a.info.absolutePosition && a.info.absolutePosition.x) || 0
-              const ay = (a.info.absolutePosition && a.info.absolutePosition.y) || 0
-              const bx = (b.info.absolutePosition && b.info.absolutePosition.x) || 0
-              const by = (b.info.absolutePosition && b.info.absolutePosition.y) || 0
-              return (
-                Math.hypot(ax - F.near.x, ay - F.near.y) -
-                Math.hypot(bx - F.near.x, by - F.near.y)
-              )
-            })
-          } else {
-            candidates.sort((a, b) => {
-              const ay = (a.info.absolutePosition && a.info.absolutePosition.y) || 0
-              const by = (b.info.absolutePosition && b.info.absolutePosition.y) || 0
-              if (Math.abs(ay - by) > 2) return ay - by
-              const ax = (a.info.absolutePosition && a.info.absolutePosition.x) || 0
-              const bx = (b.info.absolutePosition && b.info.absolutePosition.x) || 0
-              return ax - bx
-            })
+      if (options.near) {
+        const { x, y } = options.near
+        candidates.sort(
+          (a, b) =>
+            Math.hypot(a.node.geometry.x - x, a.node.geometry.y - y) -
+            Math.hypot(b.node.geometry.x - x, b.node.geometry.y - y),
+        )
+      } else {
+        candidates.sort((a, b) => {
+          if (Math.abs(a.node.geometry.y - b.node.geometry.y) > 2) {
+            return a.node.geometry.y - b.node.geometry.y
           }
+          return a.node.geometry.x - b.node.geometry.x
+        })
+      }
 
-          const total = candidates.length
-          if (total === 0) return { matched: 0, total: 0 }
+      const total = candidates.length
+      if (total === 0) return { matched: 0, total: 0 }
 
-          let idx = 0
-          if (F.nth !== null) {
-            idx = F.nth < 0 ? total + F.nth : F.nth
-            if (idx < 0 || idx >= total) {
-              return { matched: 0, total, nthOutOfRange: true, nth: F.nth }
-            }
-          } else if (total > 1 && !F.first && !F.near) {
-            return {
-              ambiguous: true,
-              total,
-              candidates: candidates.slice(0, 10).map((c, i) => ({
-                idx: i,
-                nodeId: c.info.nodeId,
-                type: c.info.type,
-                testID: c.info.testID,
-                text: (c.info.text || '').slice(0, 60),
-                abs: c.info.absolutePosition,
-                layout: c.info.layout
-                  ? {
-                      width: Math.round(c.info.layout.width || 0),
-                      height: Math.round(c.info.layout.height || 0),
-                    }
-                  : null,
-                ancestorTestIDs: (c.ancestorTestIDs || []).slice(0, 5),
-              })),
-            }
-          }
-
-          const picked = candidates[idx]
-          const n = picked.info
-          if (!n.absolutePosition || !n.layout) return { matched: 0, total }
-
-          const resolved =
-            typeof n.nodeId === 'number' &&
-            typeof t.resolveTapTarget === 'function'
-              ? await t.resolveTapTarget(n.nodeId)
-              : null
-          const target = (resolved && resolved.target) || n
-          const cx =
-            resolved && typeof resolved.cx === 'number'
-              ? resolved.cx
-              : n.absolutePosition.x + (n.layout.width || 0) / 2
-          const cy =
-            resolved && typeof resolved.cy === 'number'
-              ? resolved.cy
-              : n.absolutePosition.y + (n.layout.height || 0) / 2
-          const scr = ${SCREEN_EVAL}
-          const offscreen =
-            !!scr && (cx < 0 || cy < 0 || cx > scr.width || cy > scr.height)
-          return {
-            cx,
-            cy,
-            ...(offscreen ? { offscreen: true, screen: scr } : {}),
-            match: {
-              nodeId: n.nodeId ?? null,
-              id: n.id,
-              testID: n.testID,
-              type: n.type,
-            },
-            target: {
-              nodeId: target.nodeId ?? null,
-              id: target.id,
-              testID: target.testID,
-              text:
-                target.text ??
-                target.accessibilityLabel ??
-                n.text ??
-                n.accessibilityLabel ??
-                null,
-              type: target.type,
+      let idx = 0
+      if (options.nth != null) {
+        idx = options.nth < 0 ? total + options.nth : options.nth
+        if (idx < 0 || idx >= total) {
+          return { matched: 0, total, nthOutOfRange: true, nth: options.nth }
+        }
+      } else if (total > 1 && !options.first && !options.near) {
+        return {
+          ambiguous: true,
+          total,
+          candidates: candidates.slice(0, 10).map((candidate, index) => ({
+            idx: index,
+            nodeId: candidate.node.nodeId,
+            type: candidate.node.type,
+            testID: candidate.node.testID ?? null,
+            text: nodeText(candidate.node).slice(0, 60),
+            abs: { x: candidate.node.geometry.x, y: candidate.node.geometry.y },
+            layout: {
+              width: Math.round(candidate.node.geometry.width),
+              height: Math.round(candidate.node.geometry.height),
             },
-            strategy: (resolved && resolved.strategy) || 'matched-node',
-            total,
-            idx,
-          }
-        })()`,
-      }),
+            ancestorTestIDs: candidate.ancestorTestIDs.slice(0, 5),
+          })),
+        }
+      }
+
+      const picked = candidates[idx]
+      if (!picked) return { matched: 0, total }
+      const node = picked.node
+      let resolved = node.testID
+        ? await resolveTargetCoords(bridge, { mode: 'testid', value: node.testID })
+        : await resolveTargetCoords(bridge, {
+            mode: 'text',
+            value: nodeText(node) || query,
+          })
+      if (!resolved) {
+        resolved = {
+          x: node.geometry.x + node.geometry.width / 2,
+          y: node.geometry.y + node.geometry.height / 2,
+          testID: node.testID ?? null,
+          text: nodeText(node) || query,
+          type: node.type,
+        }
+      }
+      return payloadFromResolved(resolved, node, { total, idx })
+    },
   })
 }
 
@@ -551,72 +511,40 @@ export async function tapBest(
   query: string,
   opts: { agent?: boolean; timing?: TapTimingOptions } = {},
 ): Promise<TapOutcome> {
-  const arg = JSON.stringify(query)
   return tapResolvedTarget(bridge, {
     agent: opts.agent,
     timing: opts.timing,
     textFallback: query,
     resolve: async () => {
-      const payload = await bridge.send({
-        type: 'evaluate',
-        code: `(async () => {
-          const t = window.__sootsimTest
-          if (!t) return { error: 'bridge-not-ready' }
-          const byTestId =
-            (await t.findByTestId(${arg})) || (await t.findById(${arg}))
-          if (byTestId && byTestId.absolutePosition && byTestId.layout) {
-            return {
-              strategy: 'testid',
-              node: {
-                nodeId: byTestId.nodeId ?? null,
-                id: byTestId.id,
-                testID: byTestId.testID,
-                type: byTestId.type,
-                text: byTestId.text,
-                absolutePosition: byTestId.absolutePosition,
-                layout: byTestId.layout,
-              },
-              screen: ${SCREEN_EVAL},
-            }
-          }
-          const byText = await t.findByText(${arg})
-          if (byText && byText.absolutePosition && byText.layout) {
-            return {
-              strategy: 'text',
-              node: {
-                nodeId: byText.nodeId ?? null,
-                id: byText.id,
-                testID: byText.testID,
-                type: byText.type,
-                text: byText.text,
-                absolutePosition: byText.absolutePosition,
-                layout: byText.layout,
-              },
-              screen: ${SCREEN_EVAL},
-            }
-          }
-          return { strategy: 'none' }
-        })()`,
-      })
-      if (!payload || !('node' in payload)) return payload
-      const node = payload.node
-      const cx = node.absolutePosition.x + node.layout.width / 2
-      const cy = node.absolutePosition.y + node.layout.height / 2
-      const screen = payload.screen ?? null
-      const offscreen =
-        !!screen && (cx < 0 || cy < 0 || cx > screen.width || cy > screen.height)
-      return {
-        ...payload,
-        cx,
-        cy,
-        ...(offscreen ? { offscreen: true, screen } : {}),
-        target: {
-          id: node.id,
-          testID: node.testID,
-          text: payload.strategy === 'text' ? query : node.text,
-          type: node.type,
-        },
+      const byId = await resolveTargetCoords(bridge, { mode: 'testid', value: query })
+      if (byId) {
+        return {
+          ...payloadFromResolved(byId, byId),
+          strategy: 'testid',
+          node: {
+            nodeId: null,
+            id: byId.testID ?? query,
+            testID: byId.testID ?? query,
+            type: byId.type ?? null,
+            text: byId.text ?? null,
+          },
+        }
+      }
+      const byText = await resolveTargetCoords(bridge, { mode: 'text', value: query })
+      if (byText) {
+        return {
+          ...payloadFromResolved(byText, byText),
+          strategy: 'text',
+          node: {
+            nodeId: null,
+            id: byText.testID ?? null,
+            testID: byText.testID ?? null,
+            type: byText.type ?? null,
+            text: byText.text ?? query,
+          },
+        }
       }
+      return { strategy: 'none' }
     },
   })
 }