react-native-ai-debugger 1.10.11 → 1.10.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -20
- package/build/__tests__/unit/screenState.test.js +204 -6
- package/build/__tests__/unit/screenState.test.js.map +1 -1
- package/build/__tests__/unit/tap.test.js +0 -2
- package/build/__tests__/unit/tap.test.js.map +1 -1
- package/build/core/connection.js +1 -1
- package/build/core/connection.js.map +1 -1
- package/build/core/debugGlobals.d.ts.map +1 -1
- package/build/core/debugGlobals.js +13 -8
- package/build/core/debugGlobals.js.map +1 -1
- package/build/core/guides.js +2 -2
- package/build/core/nativeOnlyHints.js +1 -1
- package/build/core/nativeOnlyHints.js.map +1 -1
- package/build/core/screenState.d.ts +78 -10
- package/build/core/screenState.d.ts.map +1 -1
- package/build/core/screenState.js +271 -39
- package/build/core/screenState.js.map +1 -1
- package/build/core/sdkBridge.js +5 -5
- package/build/core/sdkBridge.js.map +1 -1
- package/build/pro/tap.d.ts +0 -4
- package/build/pro/tap.d.ts.map +1 -1
- package/build/pro/tap.js +0 -98
- package/build/pro/tap.js.map +1 -1
- package/build/tools/componentTools.d.ts.map +1 -1
- package/build/tools/componentTools.js +11 -10
- package/build/tools/componentTools.js.map +1 -1
- package/build/tools/networkTools.js +5 -5
- package/build/tools/networkTools.js.map +1 -1
- package/package.json +4 -4
|
@@ -25,10 +25,41 @@ export interface ScreenStatePressable {
|
|
|
25
25
|
/** What pressing triggers: "onPress=handleSubmit()", "onBack=goBack()", or "onPress→onBack" (prop route when the fn is anonymous). Absent when nothing meaningful survives Hermes. */
|
|
26
26
|
onPressHint?: string | null;
|
|
27
27
|
}
|
|
28
|
+
export interface ScreenStateText {
|
|
29
|
+
text: string;
|
|
30
|
+
center: {
|
|
31
|
+
x: number;
|
|
32
|
+
y: number;
|
|
33
|
+
};
|
|
34
|
+
bounds: {
|
|
35
|
+
x: number;
|
|
36
|
+
y: number;
|
|
37
|
+
width: number;
|
|
38
|
+
height: number;
|
|
39
|
+
};
|
|
40
|
+
blockedByOverlay?: boolean;
|
|
41
|
+
}
|
|
42
|
+
export interface ScreenStateImage {
|
|
43
|
+
src?: string | null;
|
|
44
|
+
alt?: string | null;
|
|
45
|
+
center: {
|
|
46
|
+
x: number;
|
|
47
|
+
y: number;
|
|
48
|
+
};
|
|
49
|
+
bounds: {
|
|
50
|
+
x: number;
|
|
51
|
+
y: number;
|
|
52
|
+
width: number;
|
|
53
|
+
height: number;
|
|
54
|
+
};
|
|
55
|
+
blockedByOverlay?: boolean;
|
|
56
|
+
}
|
|
28
57
|
export interface ScreenStateOverlay {
|
|
29
58
|
type: "BottomSheet" | "Modal" | "Alert" | "ActionSheet" | "Unknown";
|
|
30
59
|
title: string | null;
|
|
31
60
|
pressables: ScreenStatePressable[];
|
|
61
|
+
texts?: ScreenStateText[];
|
|
62
|
+
images?: ScreenStateImage[];
|
|
32
63
|
}
|
|
33
64
|
export interface ScreenStateRoute {
|
|
34
65
|
name: string;
|
|
@@ -39,6 +70,8 @@ export interface ScreenState {
|
|
|
39
70
|
route: ScreenStateRoute | null;
|
|
40
71
|
overlays: ScreenStateOverlay[];
|
|
41
72
|
pressables: ScreenStatePressable[];
|
|
73
|
+
texts: ScreenStateText[];
|
|
74
|
+
images: ScreenStateImage[];
|
|
42
75
|
}
|
|
43
76
|
export declare function markPressablesCoveredByOverlay(pressables: ScreenStatePressable[], overlayBounds: {
|
|
44
77
|
x: number;
|
|
@@ -51,14 +84,30 @@ export declare function markPressablesCoveredByOverlay(pressables: ScreenStatePr
|
|
|
51
84
|
* pressable's icon child name carries recognizable semantics:
|
|
52
85
|
* { label: "[FloatingHeader]", icon: "SvgChevronBackward" }
|
|
53
86
|
* → label "[SvgChevronBackward — possibly back button]"
|
|
87
|
+
* An icon button whose only text is a count badge (cart with "1") keeps that
|
|
88
|
+
* count as nearby context while the label is upgraded to the icon's meaning:
|
|
89
|
+
* { label: "1", icon: "SvgCartNew" } → label "[SvgCartNew — possibly cart button]", nearbyText "1"
|
|
54
90
|
* Labels from text/a11y are never touched (icon is null for those).
|
|
55
91
|
*/
|
|
56
92
|
export declare function applyIconHintToLabel(p: ScreenStatePressable): ScreenStatePressable;
|
|
57
93
|
/**
|
|
58
|
-
*
|
|
59
|
-
* screenshot
|
|
94
|
+
* Coordinate converter for any positioned item (pressable, text, image) — lets
|
|
95
|
+
* screenshot tools map points/dp into screenshot pixels (including conditional
|
|
96
|
+
* shifts like the iOS safe-area band). Reads only center/bounds, so it applies
|
|
97
|
+
* uniformly across item types.
|
|
60
98
|
*/
|
|
61
|
-
export type
|
|
99
|
+
export type ItemCoordConverter = (item: {
|
|
100
|
+
center: {
|
|
101
|
+
x: number;
|
|
102
|
+
y: number;
|
|
103
|
+
};
|
|
104
|
+
bounds: {
|
|
105
|
+
x: number;
|
|
106
|
+
y: number;
|
|
107
|
+
width: number;
|
|
108
|
+
height: number;
|
|
109
|
+
};
|
|
110
|
+
}) => {
|
|
62
111
|
center: {
|
|
63
112
|
x: number;
|
|
64
113
|
y: number;
|
|
@@ -70,21 +119,40 @@ export type PressableCoordConverter = (p: ScreenStatePressable) => {
|
|
|
70
119
|
height: number;
|
|
71
120
|
};
|
|
72
121
|
};
|
|
122
|
+
/** Alias kept for existing call sites; prefer ItemCoordConverter. */
|
|
123
|
+
export type PressableCoordConverter = ItemCoordConverter;
|
|
124
|
+
/** One merged-list line for a static text node — coordinates match pressable lines. */
|
|
125
|
+
export declare function formatTextEntry(t: ScreenStateText, convert?: ItemCoordConverter, opts?: {
|
|
126
|
+
fullText?: boolean;
|
|
127
|
+
}): string;
|
|
128
|
+
/** One merged-list line for an image node — coordinates match pressable lines. */
|
|
129
|
+
export declare function formatImageEntry(img: ScreenStateImage, convert?: ItemCoordConverter): string;
|
|
73
130
|
/**
|
|
74
|
-
* Render a ScreenState as the
|
|
75
|
-
*
|
|
76
|
-
*
|
|
131
|
+
* Render a ScreenState as the orientation summary used by get_screen_state and the
|
|
132
|
+
* screenshot tools. By default it merges pressables, static text (📝), and images
|
|
133
|
+
* (🖼) into one spatially-ordered list per reachability group — enough to read and
|
|
134
|
+
* navigate the screen without a screenshot:
|
|
135
|
+
* 📍 Detail [Detail]
|
|
77
136
|
* 🎯 Pressables:
|
|
78
|
-
* (
|
|
137
|
+
* (210, 175) 🖼 Image 420x350 src="…"
|
|
138
|
+
* (146, 394) 📝 "Valya product" frame:(20,382 251x24)
|
|
139
|
+
* (210, 838) <Button /> "In cart" frame:(20,810 380x56)
|
|
140
|
+
* pressablesOnly restores the lean pressable-only snapshot; fullText disables the
|
|
141
|
+
* 80-char text truncation.
|
|
79
142
|
*/
|
|
80
|
-
export declare function formatScreenStateSummary(ss: ScreenState, convert?:
|
|
143
|
+
export declare function formatScreenStateSummary(ss: ScreenState, convert?: ItemCoordConverter, opts?: {
|
|
144
|
+
pressablesOnly?: boolean;
|
|
145
|
+
fullText?: boolean;
|
|
146
|
+
}): string;
|
|
81
147
|
export declare function describePressHandler(raw: unknown): string | null;
|
|
82
148
|
/**
|
|
83
149
|
* Fallback handler context from the custom component's on* props, for when the
|
|
84
150
|
* direct onPress is anonymous (Hermes discards source even in dev bundles, and
|
|
85
151
|
* names like navigation.goBack are lost to computed assignment):
|
|
86
|
-
* -
|
|
87
|
-
*
|
|
152
|
+
* - only trust the prop whose value IS the touchable's onPress (pass-through);
|
|
153
|
+
* a non-matched candidate is an unverifiable guess that mislabels every button
|
|
154
|
+
* in a multi-button container (FloatingHeader exposes only onBack, but its menu
|
|
155
|
+
* and cart buttons run internal handlers) — so it yields null
|
|
88
156
|
* - named fn → "onBack=goBack()"; nameless → "onPress→onBack" (the prop name
|
|
89
157
|
* alone is greppable context); a bare nameless "onPress" prop adds nothing → null
|
|
90
158
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"screenState.d.ts","sourceRoot":"","sources":["../../src/core/screenState.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAQlD,MAAM,WAAW,oBAAoB;IACjC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,mGAAmG;IACnG,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,iGAAiG;IACjG,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,yFAAyF;IACzF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kGAAkG;IAClG,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,sFAAsF;IACtF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,sLAAsL;IACtL,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,kBAAkB;IAC/B,IAAI,EAAE,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,aAAa,GAAG,SAAS,CAAC;IACpE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,UAAU,EAAE,oBAAoB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"screenState.d.ts","sourceRoot":"","sources":["../../src/core/screenState.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAQlD,MAAM,WAAW,oBAAoB;IACjC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,mGAAmG;IACnG,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,iGAAiG;IACjG,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,yFAAyF;IACzF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,kGAAkG;IAClG,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,sFAAsF;IACtF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,sLAAsL;IACtL,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC7B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,IAAI,EAAE,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,aAAa,GAAG,SAAS,CAAC;IACpE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,UAAU,EAAE,oBAAoB,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACvC,KAAK,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IACxB,KAAK,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC/B,QAAQ,EAAE,kBAAkB,EAAE,CAAC;IAC/B,UAAU,EAAE,oBAAoB,EAAE,CAAC;IACnC,KAAK,EAAE,eAAe,EAAE,CAAC;IACzB,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC9B;AAMD,wBAAgB,8BAA8B,CAC1C,UAAU,EAAE,oBAAoB,EAAE,EAClC,aAAa,EAAE;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACvE,oBAAoB,EAAE,CAWxB;AAKD;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,oBAAoB,GAAG,oBAAoB,CAYlF;AAED;;;;;GAKG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,IAAI,EAAE;IACpC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CACnE,KAAK;IACF,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjC,KAAK,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAClE,CAAC;AACF,qEAAqE;AACrE,MAAM,MAAM,uBAAuB,GAAG,kBAAkB,CAAC;AAWzD,uFAAuF;AACvF,wBAAgB,eAAe,CAC3B,CAAC,EAAE,eAAe,EAClB,OAAO,GAAE,kBAAmC,EAC5C,IAAI,GAAE;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAO,GAClC,MAAM,CAIR;AAED,kFAAkF;AAClF,wBAAgB,gBAAgB,CAC5B,GAAG,EAAE,gBAAgB,EACrB,OAAO,GAAE,kBAAmC,GAC7C,MAAM,CAKR;AAKD;;;;;;;;;;;;GAYG;AACH,wBAAgB,wBAAwB,CACpC,EAAE,EAAE,WAAW,EACf,OAAO,GAAE,kBAAmC,EAC5C,IAAI,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAO,GAC5D,MAAM,CAmFR;AAoBD,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAQhE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAWhE;AAED,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,OAAO,GAAG,WAAW,GAAG,IAAI,CAWzE;AAMD,wBAAsB,cAAc,CAChC,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,OAAO,CAAC,eAAe,GAAG;IAAE,WAAW,CAAC,EAAE,WAAW,CAAA;CAAE,CAAC,CA6lC1D"}
|
|
@@ -15,32 +15,68 @@ export function markPressablesCoveredByOverlay(pressables, overlayBounds) {
|
|
|
15
15
|
}
|
|
16
16
|
return pressables;
|
|
17
17
|
}
|
|
18
|
+
/** A count badge such as "1" or "99+" — the only "label" an icon button like a cart carries. */
|
|
19
|
+
const COUNT_BADGE_LABEL = /^\d{1,3}\+?$/;
|
|
18
20
|
/**
|
|
19
21
|
* Replace component-name fallback labels with semantic icon labels when the
|
|
20
22
|
* pressable's icon child name carries recognizable semantics:
|
|
21
23
|
* { label: "[FloatingHeader]", icon: "SvgChevronBackward" }
|
|
22
24
|
* → label "[SvgChevronBackward — possibly back button]"
|
|
25
|
+
* An icon button whose only text is a count badge (cart with "1") keeps that
|
|
26
|
+
* count as nearby context while the label is upgraded to the icon's meaning:
|
|
27
|
+
* { label: "1", icon: "SvgCartNew" } → label "[SvgCartNew — possibly cart button]", nearbyText "1"
|
|
23
28
|
* Labels from text/a11y are never touched (icon is null for those).
|
|
24
29
|
*/
|
|
25
30
|
export function applyIconHintToLabel(p) {
|
|
26
31
|
if (!p.icon)
|
|
27
32
|
return p;
|
|
28
33
|
const hint = iconSemanticHint(p.icon);
|
|
29
|
-
if (hint)
|
|
34
|
+
if (hint) {
|
|
35
|
+
if (p.label && COUNT_BADGE_LABEL.test(p.label.trim()) && !p.nearbyText) {
|
|
36
|
+
p.nearbyText = p.label;
|
|
37
|
+
}
|
|
30
38
|
p.label = `[${p.icon} — ${hint}]`;
|
|
31
|
-
|
|
39
|
+
}
|
|
40
|
+
else if (!p.label) {
|
|
32
41
|
p.label = `[${p.icon}]`;
|
|
42
|
+
}
|
|
33
43
|
return p;
|
|
34
44
|
}
|
|
35
|
-
const identityCoords = (
|
|
45
|
+
const identityCoords = (item) => ({ center: item.center, frame: item.bounds });
|
|
46
|
+
const TEXT_DISPLAY_MAX = 80;
|
|
47
|
+
const IMAGE_SRC_DISPLAY_MAX = 60;
|
|
48
|
+
function truncate(s, max) {
|
|
49
|
+
return s.length > max ? s.slice(0, max) + "…" : s;
|
|
50
|
+
}
|
|
51
|
+
/** One merged-list line for a static text node — coordinates match pressable lines. */
|
|
52
|
+
export function formatTextEntry(t, convert = identityCoords, opts = {}) {
|
|
53
|
+
const { center, frame } = convert(t);
|
|
54
|
+
const body = opts.fullText ? t.text : truncate(t.text, TEXT_DISPLAY_MAX);
|
|
55
|
+
return ` (${center.x}, ${center.y}) 📝 "${body}" frame:(${frame.x},${frame.y} ${frame.width}x${frame.height})`;
|
|
56
|
+
}
|
|
57
|
+
/** One merged-list line for an image node — coordinates match pressable lines. */
|
|
58
|
+
export function formatImageEntry(img, convert = identityCoords) {
|
|
59
|
+
const { center, frame } = convert(img);
|
|
60
|
+
const src = img.src ? ` src="${truncate(img.src, IMAGE_SRC_DISPLAY_MAX)}"` : "";
|
|
61
|
+
const alt = img.alt ? ` alt="${img.alt}"` : "";
|
|
62
|
+
return ` (${center.x}, ${center.y}) 🖼 Image ${frame.width}x${frame.height}${src}${alt} frame:(${frame.x},${frame.y} ${frame.width}x${frame.height})`;
|
|
63
|
+
}
|
|
64
|
+
const TEXT_CAP = 60;
|
|
65
|
+
const IMAGE_CAP = 40;
|
|
36
66
|
/**
|
|
37
|
-
* Render a ScreenState as the
|
|
38
|
-
*
|
|
39
|
-
*
|
|
67
|
+
* Render a ScreenState as the orientation summary used by get_screen_state and the
|
|
68
|
+
* screenshot tools. By default it merges pressables, static text (📝), and images
|
|
69
|
+
* (🖼) into one spatially-ordered list per reachability group — enough to read and
|
|
70
|
+
* navigate the screen without a screenshot:
|
|
71
|
+
* 📍 Detail [Detail]
|
|
40
72
|
* 🎯 Pressables:
|
|
41
|
-
* (
|
|
73
|
+
* (210, 175) 🖼 Image 420x350 src="…"
|
|
74
|
+
* (146, 394) 📝 "Valya product" frame:(20,382 251x24)
|
|
75
|
+
* (210, 838) <Button /> "In cart" frame:(20,810 380x56)
|
|
76
|
+
* pressablesOnly restores the lean pressable-only snapshot; fullText disables the
|
|
77
|
+
* 80-char text truncation.
|
|
42
78
|
*/
|
|
43
|
-
export function formatScreenStateSummary(ss, convert = identityCoords) {
|
|
79
|
+
export function formatScreenStateSummary(ss, convert = identityCoords, opts = {}) {
|
|
44
80
|
const lines = [];
|
|
45
81
|
if (ss.route) {
|
|
46
82
|
lines.push(`📍 Currently focused screen: "${ss.route.name}" [navigation stack: ${ss.route.stack.join(" > ")}]`);
|
|
@@ -54,38 +90,75 @@ export function formatScreenStateSummary(ss, convert = identityCoords) {
|
|
|
54
90
|
}
|
|
55
91
|
const formatPressable = (p) => {
|
|
56
92
|
const { center, frame } = convert(p);
|
|
57
|
-
|
|
93
|
+
// 🔘 marks tap targets in the enriched view (where 📝 text / 🖼 images also
|
|
94
|
+
// appear); omitted under pressablesOnly to keep the lean legacy lines.
|
|
95
|
+
const marker = opts.pressablesOnly ? "" : " 🔘";
|
|
96
|
+
return ` (${center.x}, ${center.y})${marker}${p.component ? ` <${p.component} />` : ""} ${p.label ? `"${p.label}"` : "(unlabeled)"}` +
|
|
58
97
|
`${p.nearbyText ? ` near "${p.nearbyText}"` : ""}${p.onPressHint ? ` ${p.onPressHint}` : ""}` +
|
|
59
98
|
`${p.testID ? ` testID="${p.testID}"` : ""}${p.isInput ? " [input]" : ""}` +
|
|
60
99
|
` frame:(${frame.x},${frame.y} ${frame.width}x${frame.height})`;
|
|
61
100
|
};
|
|
101
|
+
// Merge pressables + (unless pressablesOnly) texts + images for one reachability
|
|
102
|
+
// group, sorted spatially (top→bottom, then left→right). A text duplicating a
|
|
103
|
+
// pressable's nearbyText is dropped; texts/images cap with an explicit marker.
|
|
104
|
+
const renderGroup = (pressables, texts, images) => {
|
|
105
|
+
const out = [];
|
|
106
|
+
const nearby = new Set(pressables.map((p) => (p.nearbyText || "").trim()).filter(Boolean));
|
|
107
|
+
const freshTexts = opts.pressablesOnly ? [] : texts.filter((t) => !nearby.has(t.text.trim()));
|
|
108
|
+
const useTexts = freshTexts.slice(0, TEXT_CAP);
|
|
109
|
+
const useImages = opts.pressablesOnly ? [] : images.slice(0, IMAGE_CAP);
|
|
110
|
+
const rows = [
|
|
111
|
+
...pressables.map((p) => ({ y: p.center.y, x: p.center.x, line: formatPressable(p) })),
|
|
112
|
+
...useTexts.map((t) => ({ y: t.center.y, x: t.center.x, line: formatTextEntry(t, convert, opts) })),
|
|
113
|
+
...useImages.map((img) => ({ y: img.center.y, x: img.center.x, line: formatImageEntry(img, convert) })),
|
|
114
|
+
];
|
|
115
|
+
rows.sort((a, b) => (a.y !== b.y ? a.y - b.y : a.x - b.x));
|
|
116
|
+
rows.forEach((r) => out.push(r.line));
|
|
117
|
+
if (!opts.pressablesOnly) {
|
|
118
|
+
const droppedT = freshTexts.length - useTexts.length;
|
|
119
|
+
const droppedI = images.length - useImages.length;
|
|
120
|
+
if (droppedT > 0)
|
|
121
|
+
out.push(` … +${droppedT} more text`);
|
|
122
|
+
if (droppedI > 0)
|
|
123
|
+
out.push(` … +${droppedI} more images`);
|
|
124
|
+
}
|
|
125
|
+
return out;
|
|
126
|
+
};
|
|
62
127
|
if (ss.overlays.length > 0) {
|
|
63
128
|
for (const overlay of ss.overlays) {
|
|
64
129
|
lines.push(`\n🔲 ${overlay.type}${overlay.title ? ` — "${overlay.title}"` : ""}:`);
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
else
|
|
130
|
+
const body = renderGroup(overlay.pressables, overlay.texts ?? [], overlay.images ?? []);
|
|
131
|
+
if (body.length > 0)
|
|
132
|
+
body.forEach((l) => lines.push(l));
|
|
133
|
+
else
|
|
69
134
|
lines.push(" (no pressables)");
|
|
70
|
-
}
|
|
71
135
|
}
|
|
72
|
-
const
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
136
|
+
const reachableP = ss.pressables.filter((p) => !p.blockedByOverlay);
|
|
137
|
+
const blockedP = ss.pressables.filter((p) => p.blockedByOverlay);
|
|
138
|
+
const reachableT = ss.texts.filter((t) => !t.blockedByOverlay);
|
|
139
|
+
const blockedT = ss.texts.filter((t) => t.blockedByOverlay);
|
|
140
|
+
const reachableI = ss.images.filter((i) => !i.blockedByOverlay);
|
|
141
|
+
const blockedI = ss.images.filter((i) => i.blockedByOverlay);
|
|
142
|
+
const rootBody = renderGroup(reachableP, reachableT, reachableI);
|
|
143
|
+
if (opts.pressablesOnly) {
|
|
144
|
+
lines.push(`\n🎯 Root pressables: ${reachableP.length > 0 ? "" : "(none reachable)"}`);
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
lines.push(`\n🎯 Reachable (outside any overlay):${rootBody.length > 0 ? "" : " (nothing reachable)"}`);
|
|
148
|
+
}
|
|
149
|
+
rootBody.forEach((l) => lines.push(l));
|
|
150
|
+
if (blockedP.length > 0 || blockedT.length > 0 || blockedI.length > 0) {
|
|
77
151
|
lines.push(`\n🚫 Blocked by overlay (visible on the underlying screen but taps will NOT reach them until the overlay closes):`);
|
|
78
|
-
|
|
152
|
+
renderGroup(blockedP, blockedT, blockedI).forEach((l) => lines.push(l));
|
|
79
153
|
}
|
|
80
154
|
}
|
|
81
155
|
else {
|
|
82
|
-
lines.push("\n🎯 Pressables:");
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
else
|
|
156
|
+
lines.push(opts.pressablesOnly ? "\n🎯 Pressables:" : "\n🎯 On screen:");
|
|
157
|
+
const body = renderGroup(ss.pressables, ss.texts, ss.images);
|
|
158
|
+
if (body.length > 0)
|
|
159
|
+
body.forEach((l) => lines.push(l));
|
|
160
|
+
else
|
|
87
161
|
lines.push(" (none)");
|
|
88
|
-
}
|
|
89
162
|
}
|
|
90
163
|
return lines.join("\n");
|
|
91
164
|
}
|
|
@@ -124,8 +197,10 @@ export function describePressHandler(raw) {
|
|
|
124
197
|
* Fallback handler context from the custom component's on* props, for when the
|
|
125
198
|
* direct onPress is anonymous (Hermes discards source even in dev bundles, and
|
|
126
199
|
* names like navigation.goBack are lost to computed assignment):
|
|
127
|
-
* -
|
|
128
|
-
*
|
|
200
|
+
* - only trust the prop whose value IS the touchable's onPress (pass-through);
|
|
201
|
+
* a non-matched candidate is an unverifiable guess that mislabels every button
|
|
202
|
+
* in a multi-button container (FloatingHeader exposes only onBack, but its menu
|
|
203
|
+
* and cart buttons run internal handlers) — so it yields null
|
|
129
204
|
* - named fn → "onBack=goBack()"; nameless → "onPress→onBack" (the prop name
|
|
130
205
|
* alone is greppable context); a bare nameless "onPress" prop adds nothing → null
|
|
131
206
|
*/
|
|
@@ -133,7 +208,7 @@ export function describePropHandlers(raw) {
|
|
|
133
208
|
if (!Array.isArray(raw) || raw.length === 0)
|
|
134
209
|
return null;
|
|
135
210
|
const entries = raw.filter((e) => !!e && typeof e === "object" && typeof e.p === "string");
|
|
136
|
-
const pick = entries.find((e) => e.same) ??
|
|
211
|
+
const pick = entries.find((e) => e.same) ?? null;
|
|
137
212
|
if (!pick)
|
|
138
213
|
return null;
|
|
139
214
|
const name = meaningfulHandlerName(pick.n);
|
|
@@ -153,6 +228,8 @@ export function parseScreenStateResponse(raw) {
|
|
|
153
228
|
route: r.route ?? null,
|
|
154
229
|
overlays: r.overlays ?? [],
|
|
155
230
|
pressables: r.pressables ?? [],
|
|
231
|
+
texts: r.texts ?? [],
|
|
232
|
+
images: r.images ?? [],
|
|
156
233
|
};
|
|
157
234
|
}
|
|
158
235
|
// ============================================================================
|
|
@@ -273,7 +350,8 @@ export async function getScreenState(options = {}) {
|
|
|
273
350
|
|
|
274
351
|
// SDK navigation ref (highest priority — works even when NavigationContainer is wrapped)
|
|
275
352
|
try {
|
|
276
|
-
var
|
|
353
|
+
var sdk = globalThis.__EXECBRO__ || globalThis.__RN_AI_DEVTOOLS__;
|
|
354
|
+
var sdkNav = sdk && sdk.navigation;
|
|
277
355
|
if (sdkNav && typeof sdkNav.getCurrentRoute === 'function') {
|
|
278
356
|
var currentRoute = sdkNav.getCurrentRoute();
|
|
279
357
|
var rootState = typeof sdkNav.getRootState === 'function' ? sdkNav.getRootState() : null;
|
|
@@ -487,6 +565,9 @@ export async function getScreenState(options = {}) {
|
|
|
487
565
|
|
|
488
566
|
var hostFibers = [];
|
|
489
567
|
var fiberMeta = [];
|
|
568
|
+
// A count badge ("1", "99+") is the only text on icon buttons like a cart — treat it
|
|
569
|
+
// as no-own-label so the icon child still surfaces (the count is kept as nearby text).
|
|
570
|
+
var COUNT_BADGE = /^\\d{1,3}\\+?$/;
|
|
490
571
|
|
|
491
572
|
var RN_PRIMITIVES = /^(Animated\\(.*|withAnimated.*|AnimatedComponent.*|ForwardRef.*|memo\\(.*|Context\\.Consumer|Context\\.Provider|ScrollViewContext(Base)?|VirtualizedListContext(Resetter)?|TextInputContext|KeyboardAvoidingViewContext|RCT.*|RNS.*|RNC.*|ViewManagerAdapter_.*|VirtualizedList.*|CellRenderer.*|FrameSizeProvider.*|MaybeScreenContainer|MaybeScreen|Navigation.*|Screen$|ScreenStack|ScreenContainer|ScreenContentWrapper|SceneView|DelayedFreeze|Freeze|Suspender|DebugContainer|StaticContainer|SafeAreaProvider.*|SafeAreaFrameContext|SafeAreaInsetsContext|ExpoRoot|ExpoRootComponent|GestureHandler.*|NativeViewGestureHandler|GestureDetector|PanGestureHandler|Reanimated.*|BottomTabNavigator|TabLayout|RouteNode|Route$|KeyboardProvider|PortalProviderComponent|BottomSheetModalProviderWrapper|ThemeContext|ThemeProvider|TextAncestorContext|PressabilityDebugView|TouchableHighlightImpl|StatusBarOverlay|BottomSheetHostingContainerComponent|BottomSheetGestureHandlersProvider|BottomSheetBackdropContainerComponent|BottomSheetContainerComponent|BottomSheetDraggableViewComponent|BottomSheetHandleContainerComponent|BottomSheetBackgroundContainerComponent|DebuggingOverlay|InspectorDeferred|Inspector|InspectorOverlay|InspectorPanel|StyleInspector|BoxInspector|BoxContainer|ElementBox|BorderBox|InspectorPanelButton)$/;
|
|
492
573
|
var GENERIC_COMPONENT = /^(View|TouchableOpacity|TouchableHighlight|TouchableWithoutFeedback|Pressable|TouchableNativeFeedback|Text|RCTView|RCTText|Unknown)$/;
|
|
@@ -662,7 +743,8 @@ export async function getScreenState(options = {}) {
|
|
|
662
743
|
var a11y = hProps.accessibilityLabel || pProps.accessibilityLabel || null;
|
|
663
744
|
var baseLabel = a11y || (text && text.length > 0 ? text.slice(0, 80) : null) || null;
|
|
664
745
|
var label = resolveLabel(pressableFiber, hostFiber, baseLabel, testID);
|
|
665
|
-
var
|
|
746
|
+
var badgeOnly = baseLabel ? COUNT_BADGE.test(baseLabel.trim()) : false;
|
|
747
|
+
var icon = (baseLabel && !badgeOnly) ? null : findMeaningfulChildName(pressableFiber);
|
|
666
748
|
if (!baseLabel && !icon) {
|
|
667
749
|
var ownName = getComponentName(pressableFiber);
|
|
668
750
|
if (ownName && !GENERIC_COMPONENT.test(ownName) && !RN_PRIMITIVES.test(ownName)) icon = ownName;
|
|
@@ -686,7 +768,8 @@ export async function getScreenState(options = {}) {
|
|
|
686
768
|
var testID2 = p2.testID || p2.nativeID || null;
|
|
687
769
|
var baseLabel2 = a11y2 || (text2 && text2.length > 0 ? text2.slice(0, 80) : null) || null;
|
|
688
770
|
var label2 = resolveLabel(fiber, hosts3[0], baseLabel2, testID2);
|
|
689
|
-
var
|
|
771
|
+
var badgeOnly2 = baseLabel2 ? COUNT_BADGE.test(baseLabel2.trim()) : false;
|
|
772
|
+
var icon2 = (baseLabel2 && !badgeOnly2) ? null : findMeaningfulChildName(fiber);
|
|
690
773
|
if (!baseLabel2 && !icon2) {
|
|
691
774
|
var ownName2 = getComponentName(fiber);
|
|
692
775
|
if (ownName2 && !GENERIC_COMPONENT.test(ownName2) && !RN_PRIMITIVES.test(ownName2)) icon2 = ownName2;
|
|
@@ -732,6 +815,7 @@ export async function getScreenState(options = {}) {
|
|
|
732
815
|
|
|
733
816
|
var textFibers = [];
|
|
734
817
|
var textContents = [];
|
|
818
|
+
var textOverlayIdx = [];
|
|
735
819
|
|
|
736
820
|
function extractTextString(fiber) {
|
|
737
821
|
var p = fiber.memoizedProps;
|
|
@@ -750,12 +834,20 @@ export async function getScreenState(options = {}) {
|
|
|
750
834
|
return '';
|
|
751
835
|
}
|
|
752
836
|
|
|
753
|
-
function walkTexts(fiber, depth, insidePressable, inHidden) {
|
|
837
|
+
function walkTexts(fiber, depth, insidePressable, inHidden, ovIdx) {
|
|
754
838
|
if (!fiber || depth > 5000) return;
|
|
755
839
|
var name = getComponentName(fiber);
|
|
756
840
|
var props = fiber.memoizedProps;
|
|
757
841
|
var nextHidden = inHidden || isScreenHidden(name, props);
|
|
758
842
|
|
|
843
|
+
// Overlay membership by ancestry (same as the pressable walk) so a sheet's
|
|
844
|
+
// text is grouped with the sheet, not flagged as blocked behind it.
|
|
845
|
+
if (ovIdx == null) {
|
|
846
|
+
for (var ofiT = 0; ofiT < overlayFiberMeta.length; ofiT++) {
|
|
847
|
+
if (overlayFiberMeta[ofiT].fiber === fiber) { ovIdx = ofiT; break; }
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
|
|
759
851
|
var hasOnPress = props && typeof props.onPress === 'function';
|
|
760
852
|
var isInputHere = props && (typeof props.onChangeText === 'function' || typeof props.onFocus === 'function');
|
|
761
853
|
var nextInside = insidePressable || !!hasOnPress || !!isInputHere;
|
|
@@ -765,7 +857,7 @@ export async function getScreenState(options = {}) {
|
|
|
765
857
|
// for proxy bounds (Fabric RCTText has no publicInstance).
|
|
766
858
|
if (!insidePressable && !nextHidden && name !== 'RCTText' && typeof fiber.type !== 'string') {
|
|
767
859
|
var str = extractTextString(fiber);
|
|
768
|
-
if (str && str.length > 0 && str.length <=
|
|
860
|
+
if (str && str.length > 0 && str.length <= 300) {
|
|
769
861
|
var up = fiber;
|
|
770
862
|
var upDepth = 0;
|
|
771
863
|
var measurableT = null;
|
|
@@ -780,17 +872,73 @@ export async function getScreenState(options = {}) {
|
|
|
780
872
|
if (measurableT) {
|
|
781
873
|
textFibers.push(measurableT);
|
|
782
874
|
textContents.push(str);
|
|
875
|
+
textOverlayIdx.push(ovIdx);
|
|
783
876
|
}
|
|
784
877
|
}
|
|
785
878
|
}
|
|
786
879
|
|
|
787
880
|
var child = fiber.child;
|
|
788
881
|
while (child) {
|
|
789
|
-
walkTexts(child, depth + 1, nextInside, nextHidden);
|
|
882
|
+
walkTexts(child, depth + 1, nextInside, nextHidden, ovIdx);
|
|
790
883
|
child = child.sibling;
|
|
791
884
|
}
|
|
792
885
|
}
|
|
793
|
-
walkTexts(roots[0].current, 0, false, false);
|
|
886
|
+
walkTexts(roots[0].current, 0, false, false, null);
|
|
887
|
+
|
|
888
|
+
// ------------------------------------------------------------------
|
|
889
|
+
// 3c. Images — host image components, with source + a11y label. Climb to
|
|
890
|
+
// the nearest measurable host (same proxy-bounds trick as text).
|
|
891
|
+
// ------------------------------------------------------------------
|
|
892
|
+
|
|
893
|
+
var imageFibers = [];
|
|
894
|
+
var imageMeta = []; // { src, alt }
|
|
895
|
+
var IMG_NAME = /^(Image|RCTImageView|ExpoImage|FastImage|ImageBackground)$/;
|
|
896
|
+
|
|
897
|
+
function imageSource(props) {
|
|
898
|
+
if (!props) return null;
|
|
899
|
+
var s = props.source;
|
|
900
|
+
if (s == null) return null;
|
|
901
|
+
if (Array.isArray(s)) s = s[0];
|
|
902
|
+
if (typeof s === 'number') return 'asset#' + s;
|
|
903
|
+
if (s && typeof s === 'object' && typeof s.uri === 'string') {
|
|
904
|
+
return s.uri.length > 200 ? s.uri.slice(0, 200) : s.uri;
|
|
905
|
+
}
|
|
906
|
+
if (typeof s === 'string') return s.length > 200 ? s.slice(0, 200) : s;
|
|
907
|
+
return null;
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
function walkImages(fiber, depth, inHidden, ovIdx) {
|
|
911
|
+
if (!fiber || depth > 5000) return;
|
|
912
|
+
var name = getComponentName(fiber);
|
|
913
|
+
var props = fiber.memoizedProps;
|
|
914
|
+
var nextHidden = inHidden || isScreenHidden(name, props);
|
|
915
|
+
if (ovIdx == null) {
|
|
916
|
+
for (var ofiI = 0; ofiI < overlayFiberMeta.length; ofiI++) {
|
|
917
|
+
if (overlayFiberMeta[ofiI].fiber === fiber) { ovIdx = ofiI; break; }
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
if (!nextHidden && name && IMG_NAME.test(name) && typeof fiber.type !== 'string') {
|
|
921
|
+
var up = fiber;
|
|
922
|
+
var upD = 0;
|
|
923
|
+
var measurableI = null;
|
|
924
|
+
while (up && upD < 20) {
|
|
925
|
+
if (typeof up.type === 'string' && getMeasurable(up)) { measurableI = up; break; }
|
|
926
|
+
up = up.return;
|
|
927
|
+
upD++;
|
|
928
|
+
}
|
|
929
|
+
if (measurableI) {
|
|
930
|
+
imageFibers.push(measurableI);
|
|
931
|
+
imageMeta.push({
|
|
932
|
+
src: imageSource(props),
|
|
933
|
+
alt: (props && typeof props.accessibilityLabel === 'string') ? props.accessibilityLabel.slice(0, 80) : null,
|
|
934
|
+
overlayIdx: ovIdx
|
|
935
|
+
});
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
var child = fiber.child;
|
|
939
|
+
while (child) { walkImages(child, depth + 1, nextHidden, ovIdx); child = child.sibling; }
|
|
940
|
+
}
|
|
941
|
+
walkImages(roots[0].current, 0, false, null);
|
|
794
942
|
|
|
795
943
|
// ------------------------------------------------------------------
|
|
796
944
|
// 4. Store everything in globalThis for the resolve call
|
|
@@ -829,6 +977,9 @@ export async function getScreenState(options = {}) {
|
|
|
829
977
|
globalThis.__screenStateOverlayMeasurements = new Array(overlayHostFibers.length).fill(null);
|
|
830
978
|
globalThis.__screenStateTextContents = textContents;
|
|
831
979
|
globalThis.__screenStateTextMeasurements = new Array(textFibers.length).fill(null);
|
|
980
|
+
globalThis.__screenStateTextOverlayIdx = textOverlayIdx;
|
|
981
|
+
globalThis.__screenStateImageMeta = imageMeta;
|
|
982
|
+
globalThis.__screenStateImageMeasurements = new Array(imageFibers.length).fill(null);
|
|
832
983
|
|
|
833
984
|
// Dispatch all measureInWindow calls (pressables + root + overlay hosts)
|
|
834
985
|
for (var i = 0; i < hostFibers.length; i++) {
|
|
@@ -858,6 +1009,15 @@ export async function getScreenState(options = {}) {
|
|
|
858
1009
|
})(txi);
|
|
859
1010
|
} catch(e) {}
|
|
860
1011
|
}
|
|
1012
|
+
for (var imi = 0; imi < imageFibers.length; imi++) {
|
|
1013
|
+
try {
|
|
1014
|
+
(function(idx) {
|
|
1015
|
+
getMeasurable(imageFibers[idx]).measureInWindow(function(fx, fy, fw, fh) {
|
|
1016
|
+
globalThis.__screenStateImageMeasurements[idx] = { x: fx, y: fy, width: fw, height: fh };
|
|
1017
|
+
});
|
|
1018
|
+
})(imi);
|
|
1019
|
+
} catch(e) {}
|
|
1020
|
+
}
|
|
861
1021
|
|
|
862
1022
|
return { count: hostFibers.length, overlayCount: overlayFiberMeta.length };
|
|
863
1023
|
})()
|
|
@@ -883,8 +1043,14 @@ export async function getScreenState(options = {}) {
|
|
|
883
1043
|
var overlayMeasurements = globalThis.__screenStateOverlayMeasurements || [];
|
|
884
1044
|
var textContents = globalThis.__screenStateTextContents || [];
|
|
885
1045
|
var textMeasurements = globalThis.__screenStateTextMeasurements || [];
|
|
1046
|
+
var textOverlayIdx = globalThis.__screenStateTextOverlayIdx || [];
|
|
1047
|
+
var imageMeta = globalThis.__screenStateImageMeta || [];
|
|
1048
|
+
var imageMeasurements = globalThis.__screenStateImageMeasurements || [];
|
|
886
1049
|
globalThis.__screenStateTextContents = null;
|
|
887
1050
|
globalThis.__screenStateTextMeasurements = null;
|
|
1051
|
+
globalThis.__screenStateTextOverlayIdx = null;
|
|
1052
|
+
globalThis.__screenStateImageMeta = null;
|
|
1053
|
+
globalThis.__screenStateImageMeasurements = null;
|
|
888
1054
|
globalThis.__screenStateFibers = null;
|
|
889
1055
|
globalThis.__screenStateMeta = null;
|
|
890
1056
|
globalThis.__screenStateMeasurements = null;
|
|
@@ -1042,6 +1208,46 @@ export async function getScreenState(options = {}) {
|
|
|
1042
1208
|
delete allPressables[si].hasOwnLabel;
|
|
1043
1209
|
}
|
|
1044
1210
|
|
|
1211
|
+
// Build typed text/image lists (pair each measurement with its meta by index,
|
|
1212
|
+
// viewport-filter) and assign each to a reachability group via the same overlay
|
|
1213
|
+
// logic used for pressables (content containment vs blocking backdrop).
|
|
1214
|
+
var rootTexts = [], rootImages = [];
|
|
1215
|
+
for (var ov = 0; ov < overlays.length; ov++) { overlays[ov].texts = []; overlays[ov].images = []; }
|
|
1216
|
+
// Group an item: by fiber ancestry (overlayIdx) first — a sheet's own text/image
|
|
1217
|
+
// belongs to the sheet; else geometric containment in an overlay's content; else
|
|
1218
|
+
// root, flagged blocked when it sits under a blocking overlay (the screen behind).
|
|
1219
|
+
function pushClassified(entry, kind, ovIdx) {
|
|
1220
|
+
for (var a = 0; a < overlays.length; a++) {
|
|
1221
|
+
if (ovIdx != null && ovIdx === overlays[a].origIdx) { overlays[a][kind].push(entry); return; }
|
|
1222
|
+
}
|
|
1223
|
+
for (var c = 0; c < overlays.length; c++) {
|
|
1224
|
+
if (overlays[c].hasContent && inside(entry.bounds, overlays[c].contentBounds)) { overlays[c][kind].push(entry); return; }
|
|
1225
|
+
}
|
|
1226
|
+
var blk = false;
|
|
1227
|
+
for (var b = 0; b < overlays.length; b++) { if (inside(entry.bounds, overlays[b].blockBounds)) { blk = true; break; } }
|
|
1228
|
+
if (blk) entry.blockedByOverlay = true;
|
|
1229
|
+
(kind === 'texts' ? rootTexts : rootImages).push(entry);
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
for (var ti = 0; ti < textContents.length; ti++) {
|
|
1233
|
+
var tm = textMeasurements[ti];
|
|
1234
|
+
if (!tm || tm.width <= 0 || tm.height <= 0) continue;
|
|
1235
|
+
if (tm.x + tm.width < 0 || tm.y + tm.height < 0) continue;
|
|
1236
|
+
if (tm.x > viewportW || tm.y > viewportH) continue;
|
|
1237
|
+
pushClassified({ text: textContents[ti],
|
|
1238
|
+
center: { x: Math.round(tm.x + tm.width/2), y: Math.round(tm.y + tm.height/2) },
|
|
1239
|
+
bounds: { x: Math.round(tm.x), y: Math.round(tm.y), width: Math.round(tm.width), height: Math.round(tm.height) } }, 'texts', textOverlayIdx[ti]);
|
|
1240
|
+
}
|
|
1241
|
+
for (var ii = 0; ii < imageMeta.length; ii++) {
|
|
1242
|
+
var im = imageMeasurements[ii];
|
|
1243
|
+
if (!im || im.width <= 0 || im.height <= 0) continue;
|
|
1244
|
+
if (im.x + im.width < 0 || im.y + im.height < 0) continue;
|
|
1245
|
+
if (im.x > viewportW || im.y > viewportH) continue;
|
|
1246
|
+
pushClassified({ src: imageMeta[ii].src, alt: imageMeta[ii].alt,
|
|
1247
|
+
center: { x: Math.round(im.x + im.width/2), y: Math.round(im.y + im.height/2) },
|
|
1248
|
+
bounds: { x: Math.round(im.x), y: Math.round(im.y), width: Math.round(im.width), height: Math.round(im.height) } }, 'images', imageMeta[ii].overlayIdx);
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1045
1251
|
// Sort visually (top-to-bottom, left-to-right) — walk order is mount order,
|
|
1046
1252
|
// which puts late-mounted overlays like floating headers at the end.
|
|
1047
1253
|
function byPosition(a, b) {
|
|
@@ -1049,14 +1255,20 @@ export async function getScreenState(options = {}) {
|
|
|
1049
1255
|
return a.center.x - b.center.x;
|
|
1050
1256
|
}
|
|
1051
1257
|
rootPressables.sort(byPosition);
|
|
1052
|
-
|
|
1258
|
+
rootTexts.sort(byPosition);
|
|
1259
|
+
rootImages.sort(byPosition);
|
|
1260
|
+
for (var so = 0; so < overlays.length; so++) {
|
|
1261
|
+
overlays[so].pressables.sort(byPosition);
|
|
1262
|
+
overlays[so].texts.sort(byPosition);
|
|
1263
|
+
overlays[so].images.sort(byPosition);
|
|
1264
|
+
}
|
|
1053
1265
|
|
|
1054
1266
|
// Strip bounds from overlay objects (not in public interface)
|
|
1055
1267
|
var cleanOverlays = overlays.map(function(o) {
|
|
1056
|
-
return { type: o.type, title: o.title, pressables: o.pressables };
|
|
1268
|
+
return { type: o.type, title: o.title, pressables: o.pressables, texts: o.texts, images: o.images };
|
|
1057
1269
|
});
|
|
1058
1270
|
|
|
1059
|
-
return { route: route, overlays: cleanOverlays, pressables: rootPressables };
|
|
1271
|
+
return { route: route, overlays: cleanOverlays, pressables: rootPressables, texts: rootTexts, images: rootImages };
|
|
1060
1272
|
})()
|
|
1061
1273
|
`;
|
|
1062
1274
|
const resolveResult = await executeInApp(resolveExpression, false, { timeoutMs: 15000, originatingToolName: "get_screen_state" }, device);
|
|
@@ -1089,6 +1301,26 @@ export async function getScreenState(options = {}) {
|
|
|
1089
1301
|
for (const overlay of screenState.overlays) {
|
|
1090
1302
|
overlay.pressables = dedupPressables(overlay.pressables);
|
|
1091
1303
|
}
|
|
1304
|
+
// Dedup texts/images: a single visual element often maps to nested host nodes
|
|
1305
|
+
// (Image > ExpoImage) or repeated text spans that all measure to the same box.
|
|
1306
|
+
const dedupBy = (list, keyExtra) => {
|
|
1307
|
+
const seen = new Set();
|
|
1308
|
+
return list.filter((item) => {
|
|
1309
|
+
const key = `${item.center.x},${item.center.y}|${keyExtra(item)}`;
|
|
1310
|
+
if (seen.has(key))
|
|
1311
|
+
return false;
|
|
1312
|
+
seen.add(key);
|
|
1313
|
+
return true;
|
|
1314
|
+
});
|
|
1315
|
+
};
|
|
1316
|
+
screenState.texts = dedupBy(screenState.texts, (t) => t.text);
|
|
1317
|
+
screenState.images = dedupBy(screenState.images, (i) => i.src ?? "");
|
|
1318
|
+
for (const overlay of screenState.overlays) {
|
|
1319
|
+
if (overlay.texts)
|
|
1320
|
+
overlay.texts = dedupBy(overlay.texts, (t) => t.text);
|
|
1321
|
+
if (overlay.images)
|
|
1322
|
+
overlay.images = dedupBy(overlay.images, (i) => i.src ?? "");
|
|
1323
|
+
}
|
|
1092
1324
|
// Semantic icon hints + onPress handler hints
|
|
1093
1325
|
const decorate = (p) => {
|
|
1094
1326
|
applyIconHintToLabel(p);
|