testchimp-runner-core 0.0.34 → 0.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/execution-service.d.ts +1 -4
- package/dist/execution-service.d.ts.map +1 -1
- package/dist/execution-service.js +155 -468
- package/dist/execution-service.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -1
- package/dist/index.js.map +1 -1
- package/dist/orchestrator/decision-parser.d.ts +18 -0
- package/dist/orchestrator/decision-parser.d.ts.map +1 -0
- package/dist/orchestrator/decision-parser.js +127 -0
- package/dist/orchestrator/decision-parser.js.map +1 -0
- package/dist/orchestrator/index.d.ts +4 -2
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +14 -2
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/orchestrator-agent.d.ts +17 -14
- package/dist/orchestrator/orchestrator-agent.d.ts.map +1 -1
- package/dist/orchestrator/orchestrator-agent.js +534 -204
- package/dist/orchestrator/orchestrator-agent.js.map +1 -1
- package/dist/orchestrator/orchestrator-prompts.d.ts +14 -2
- package/dist/orchestrator/orchestrator-prompts.d.ts.map +1 -1
- package/dist/orchestrator/orchestrator-prompts.js +529 -247
- package/dist/orchestrator/orchestrator-prompts.js.map +1 -1
- package/dist/orchestrator/page-som-handler.d.ts +106 -0
- package/dist/orchestrator/page-som-handler.d.ts.map +1 -0
- package/dist/orchestrator/page-som-handler.js +1353 -0
- package/dist/orchestrator/page-som-handler.js.map +1 -0
- package/dist/orchestrator/som-types.d.ts +149 -0
- package/dist/orchestrator/som-types.d.ts.map +1 -0
- package/dist/orchestrator/som-types.js +87 -0
- package/dist/orchestrator/som-types.js.map +1 -0
- package/dist/orchestrator/tool-registry.d.ts +2 -0
- package/dist/orchestrator/tool-registry.d.ts.map +1 -1
- package/dist/orchestrator/tool-registry.js.map +1 -1
- package/dist/orchestrator/tools/index.d.ts +4 -1
- package/dist/orchestrator/tools/index.d.ts.map +1 -1
- package/dist/orchestrator/tools/index.js +7 -2
- package/dist/orchestrator/tools/index.js.map +1 -1
- package/dist/orchestrator/tools/refresh-som-markers.d.ts +12 -0
- package/dist/orchestrator/tools/refresh-som-markers.d.ts.map +1 -0
- package/dist/orchestrator/tools/refresh-som-markers.js +64 -0
- package/dist/orchestrator/tools/refresh-som-markers.js.map +1 -0
- package/dist/orchestrator/tools/view-previous-screenshot.d.ts +15 -0
- package/dist/orchestrator/tools/view-previous-screenshot.d.ts.map +1 -0
- package/dist/orchestrator/tools/view-previous-screenshot.js +92 -0
- package/dist/orchestrator/tools/view-previous-screenshot.js.map +1 -0
- package/dist/orchestrator/types.d.ts +23 -1
- package/dist/orchestrator/types.d.ts.map +1 -1
- package/dist/orchestrator/types.js +11 -1
- package/dist/orchestrator/types.js.map +1 -1
- package/dist/scenario-service.d.ts +5 -0
- package/dist/scenario-service.d.ts.map +1 -1
- package/dist/scenario-service.js +17 -0
- package/dist/scenario-service.js.map +1 -1
- package/dist/scenario-worker-class.d.ts +4 -0
- package/dist/scenario-worker-class.d.ts.map +1 -1
- package/dist/scenario-worker-class.js +18 -3
- package/dist/scenario-worker-class.js.map +1 -1
- package/dist/testing/agent-tester.d.ts +35 -0
- package/dist/testing/agent-tester.d.ts.map +1 -0
- package/dist/testing/agent-tester.js +84 -0
- package/dist/testing/agent-tester.js.map +1 -0
- package/dist/testing/ref-translator-tester.d.ts +44 -0
- package/dist/testing/ref-translator-tester.d.ts.map +1 -0
- package/dist/testing/ref-translator-tester.js +104 -0
- package/dist/testing/ref-translator-tester.js.map +1 -0
- package/dist/utils/hierarchical-selector.d.ts +47 -0
- package/dist/utils/hierarchical-selector.d.ts.map +1 -0
- package/dist/utils/hierarchical-selector.js +212 -0
- package/dist/utils/hierarchical-selector.js.map +1 -0
- package/dist/utils/page-info-retry.d.ts +14 -0
- package/dist/utils/page-info-retry.d.ts.map +1 -0
- package/dist/utils/page-info-retry.js +60 -0
- package/dist/utils/page-info-retry.js.map +1 -0
- package/dist/utils/page-info-utils.d.ts +1 -0
- package/dist/utils/page-info-utils.d.ts.map +1 -1
- package/dist/utils/page-info-utils.js +46 -18
- package/dist/utils/page-info-utils.js.map +1 -1
- package/dist/utils/ref-attacher.d.ts +21 -0
- package/dist/utils/ref-attacher.d.ts.map +1 -0
- package/dist/utils/ref-attacher.js +149 -0
- package/dist/utils/ref-attacher.js.map +1 -0
- package/dist/utils/ref-translator.d.ts +49 -0
- package/dist/utils/ref-translator.d.ts.map +1 -0
- package/dist/utils/ref-translator.js +276 -0
- package/dist/utils/ref-translator.js.map +1 -0
- package/package.json +6 -1
- package/RELEASE_0.0.26.md +0 -165
- package/RELEASE_0.0.27.md +0 -236
- package/RELEASE_0.0.28.md +0 -286
- package/plandocs/BEFORE_AFTER_VERIFICATION.md +0 -148
- package/plandocs/COORDINATE_MODE_DIAGNOSIS.md +0 -144
- package/plandocs/CREDIT_CALLBACK_ARCHITECTURE.md +0 -253
- package/plandocs/HUMAN_LIKE_IMPROVEMENTS.md +0 -642
- package/plandocs/IMPLEMENTATION_STATUS.md +0 -108
- package/plandocs/INTEGRATION_COMPLETE.md +0 -322
- package/plandocs/MULTI_AGENT_ARCHITECTURE_REVIEW.md +0 -844
- package/plandocs/ORCHESTRATOR_MVP_SUMMARY.md +0 -539
- package/plandocs/PHASE1_ABSTRACTION_COMPLETE.md +0 -241
- package/plandocs/PHASE1_FINAL_STATUS.md +0 -210
- package/plandocs/PHASE_1_COMPLETE.md +0 -165
- package/plandocs/PHASE_1_SUMMARY.md +0 -184
- package/plandocs/PLANNING_SESSION_SUMMARY.md +0 -372
- package/plandocs/PROMPT_OPTIMIZATION_ANALYSIS.md +0 -120
- package/plandocs/PROMPT_SANITY_CHECK.md +0 -120
- package/plandocs/SCRIPT_CLEANUP_FEATURE.md +0 -201
- package/plandocs/SCRIPT_GENERATION_ARCHITECTURE.md +0 -364
- package/plandocs/SELECTOR_IMPROVEMENTS.md +0 -139
- package/plandocs/SESSION_SUMMARY_v0.0.33.md +0 -151
- package/plandocs/TROUBLESHOOTING_SESSION.md +0 -72
- package/plandocs/VISION_DIAGNOSTICS_IMPROVEMENTS.md +0 -336
- package/plandocs/VISUAL_AGENT_EVOLUTION_PLAN.md +0 -396
- package/plandocs/WHATS_NEW_v0.0.33.md +0 -183
- package/src/auth-config.ts +0 -84
- package/src/credit-usage-service.ts +0 -188
- package/src/env-loader.ts +0 -103
- package/src/execution-service.ts +0 -1413
- package/src/file-handler.ts +0 -104
- package/src/index.ts +0 -422
- package/src/llm-facade.ts +0 -821
- package/src/llm-provider.ts +0 -53
- package/src/model-constants.ts +0 -35
- package/src/orchestrator/index.ts +0 -34
- package/src/orchestrator/orchestrator-agent.ts +0 -862
- package/src/orchestrator/orchestrator-agent.ts.backup +0 -1386
- package/src/orchestrator/orchestrator-prompts.ts +0 -474
- package/src/orchestrator/tool-registry.ts +0 -182
- package/src/orchestrator/tools/check-page-ready.ts +0 -75
- package/src/orchestrator/tools/extract-data.ts +0 -92
- package/src/orchestrator/tools/index.ts +0 -12
- package/src/orchestrator/tools/inspect-page.ts +0 -42
- package/src/orchestrator/tools/recall-history.ts +0 -72
- package/src/orchestrator/tools/take-screenshot.ts +0 -128
- package/src/orchestrator/tools/verify-action-result.ts +0 -159
- package/src/orchestrator/types.ts +0 -248
- package/src/playwright-mcp-service.ts +0 -224
- package/src/progress-reporter.ts +0 -144
- package/src/prompts.ts +0 -842
- package/src/providers/backend-proxy-llm-provider.ts +0 -91
- package/src/providers/local-llm-provider.ts +0 -38
- package/src/scenario-service.ts +0 -232
- package/src/scenario-worker-class.ts +0 -1089
- package/src/script-utils.ts +0 -203
- package/src/types.ts +0 -239
- package/src/utils/browser-utils.ts +0 -348
- package/src/utils/coordinate-converter.ts +0 -162
- package/src/utils/page-info-utils.ts +0 -250
- package/testchimp-runner-core-0.0.33.tgz +0 -0
- package/tsconfig.json +0 -19
|
@@ -13,7 +13,9 @@ async function getEnhancedPageInfo(input) {
|
|
|
13
13
|
if ('accessibility' in input) {
|
|
14
14
|
// Input is a Page object
|
|
15
15
|
hasPageObject = true;
|
|
16
|
-
|
|
16
|
+
// CRITICAL: Use interestingOnly: false to get FULL ARIA tree like Playwright MCP
|
|
17
|
+
// This includes all structural nodes (generic, paragraph, etc.) not just interactive ones
|
|
18
|
+
const snapshot = await input.accessibility.snapshot({ interestingOnly: false });
|
|
17
19
|
const url = input.url();
|
|
18
20
|
const title = await input.title();
|
|
19
21
|
domSnapshot = { url, title, accessibilityTree: snapshot };
|
|
@@ -34,17 +36,34 @@ async function getEnhancedPageInfo(input) {
|
|
|
34
36
|
'[role="textbox"]',
|
|
35
37
|
'[role="checkbox"]',
|
|
36
38
|
'[role="radio"]',
|
|
37
|
-
'[onclick]'
|
|
39
|
+
'[onclick]',
|
|
40
|
+
'[type="submit"]'
|
|
38
41
|
];
|
|
39
42
|
const allInteractive = new Set();
|
|
40
43
|
interactiveSelectors.forEach((selector) => {
|
|
41
44
|
doc.querySelectorAll(selector).forEach((el) => allInteractive.add(el));
|
|
42
45
|
});
|
|
46
|
+
// CRITICAL: Also detect elements that are styled/behave as interactive
|
|
47
|
+
// (e.g., divs/spans with cursor:pointer, click handlers)
|
|
48
|
+
// This catches modern web apps that use non-semantic HTML
|
|
49
|
+
const allElements = doc.querySelectorAll('div, span, p, li, td');
|
|
50
|
+
allElements.forEach((el) => {
|
|
51
|
+
const styles = globalThis.window.getComputedStyle(el);
|
|
52
|
+
const hasClickHandler = el.onclick ||
|
|
53
|
+
el.getAttribute('onclick') ||
|
|
54
|
+
el.hasAttribute('data-action') ||
|
|
55
|
+
el.hasAttribute('data-click');
|
|
56
|
+
// Include if: cursor is pointer, has click handler, or has tabindex (keyboard focusable)
|
|
57
|
+
if (styles.cursor === 'pointer' || hasClickHandler || el.tabIndex >= 0) {
|
|
58
|
+
allInteractive.add(el);
|
|
59
|
+
}
|
|
60
|
+
});
|
|
43
61
|
allInteractive.forEach((el) => {
|
|
44
62
|
const rect = el.getBoundingClientRect();
|
|
45
63
|
// Skip invisible elements
|
|
46
|
-
if (rect.width === 0 || rect.height === 0)
|
|
64
|
+
if (rect.width === 0 || rect.height === 0) {
|
|
47
65
|
return;
|
|
66
|
+
}
|
|
48
67
|
const id = el.id || null;
|
|
49
68
|
const tag = el.tagName.toLowerCase();
|
|
50
69
|
const text = el.textContent?.trim().substring(0, 40) || '';
|
|
@@ -65,6 +84,11 @@ async function getEnhancedPageInfo(input) {
|
|
|
65
84
|
if (text && text.length > 0 && text.length < 30) {
|
|
66
85
|
selectors.push(`getByRole('${role}', {name: '${text}'})`);
|
|
67
86
|
}
|
|
87
|
+
// 2b. BEST: getByRole for buttons with type attribute (more specific)
|
|
88
|
+
if (tag === 'button' && el.type) {
|
|
89
|
+
selectors.push(`getByRole('button', {name: '${text}'})`);
|
|
90
|
+
selectors.push(`button[type="${el.type}"]`);
|
|
91
|
+
}
|
|
68
92
|
// 3. BEST: getByPlaceholder for inputs with placeholders
|
|
69
93
|
if (placeholder && placeholder.length > 0 && placeholder.length < 50) {
|
|
70
94
|
selectors.push(`getByPlaceholder('${placeholder}')`);
|
|
@@ -78,7 +102,19 @@ async function getEnhancedPageInfo(input) {
|
|
|
78
102
|
el.getAttribute('data-id') || el.getAttribute('data-test');
|
|
79
103
|
if (dataTestId)
|
|
80
104
|
selectors.push(`getByTestId('${dataTestId}')`);
|
|
81
|
-
// 6.
|
|
105
|
+
// 6. GOOD: name attribute (especially for form inputs - very common and stable)
|
|
106
|
+
const nameAttr = el.getAttribute('name');
|
|
107
|
+
if (nameAttr && (tag === 'input' || tag === 'textarea' || tag === 'select')) {
|
|
108
|
+
selectors.push(`input[name="${nameAttr}"]`);
|
|
109
|
+
}
|
|
110
|
+
// 6b. GOOD: class-based selectors for buttons (common in modern frameworks)
|
|
111
|
+
if (tag === 'button' && el.className) {
|
|
112
|
+
const classes = el.className.split(' ').filter((c) => c.length > 0 && !c.includes('userfront-'));
|
|
113
|
+
if (classes.length > 0) {
|
|
114
|
+
selectors.push(`button.${classes[0]}`);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
// 7. LAST RESORT: ID selector (only if stable - avoid auto-generated IDs with unicode)
|
|
82
118
|
// Skip auto-generated Ant Design IDs like «r3»-form-item or rc_select_xxx
|
|
83
119
|
if (id && !id.includes('«') && !id.match(/^(rc_|:r[0-9]+:|__)/)) {
|
|
84
120
|
selectors.push(`#${id}`);
|
|
@@ -88,6 +124,7 @@ async function getEnhancedPageInfo(input) {
|
|
|
88
124
|
id,
|
|
89
125
|
text,
|
|
90
126
|
type,
|
|
127
|
+
name: nameAttr || '', // CRITICAL: Capture name attribute for form inputs
|
|
91
128
|
role,
|
|
92
129
|
ariaLabel,
|
|
93
130
|
placeholder,
|
|
@@ -144,22 +181,13 @@ async function getEnhancedPageInfo(input) {
|
|
|
144
181
|
}
|
|
145
182
|
}
|
|
146
183
|
formattedElements = selectedElements.map((e, i) => {
|
|
147
|
-
const idInfo = e.id ? `#${e.id}` : '
|
|
184
|
+
const idInfo = e.id ? `#${e.id}` : '';
|
|
148
185
|
const typeInfo = e.type ? ` type="${e.type}"` : '';
|
|
186
|
+
const nameInfo = e.name ? ` name="${e.name}"` : '';
|
|
149
187
|
const placeholderInfo = e.placeholder ? ` placeholder="${e.placeholder}"` : '';
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
selectorsInfo = e.selectors[0]; // Best option
|
|
154
|
-
if (e.selectors.length > 1) {
|
|
155
|
-
selectorsInfo += ` (or: ${e.selectors.slice(1, 3).join(', ')})`; // Show up to 2 alternatives
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
else {
|
|
159
|
-
selectorsInfo = 'No reliable selector available';
|
|
160
|
-
}
|
|
161
|
-
return `${i + 1}. [${e.bbox.x},${e.bbox.y} ${e.bbox.width}x${e.bbox.height}] <${e.tag}> ${idInfo}${typeInfo}${placeholderInfo} "${e.text}"
|
|
162
|
-
→ ${selectorsInfo}`;
|
|
188
|
+
const roleInfo = e.role ? ` role="${e.role}"` : '';
|
|
189
|
+
// Compact format: show all relevant attributes for LLM to construct selectors
|
|
190
|
+
return `${i + 1}. [${e.bbox.x},${e.bbox.y} ${e.bbox.width}x${e.bbox.height}] <${e.tag}>${idInfo}${typeInfo}${nameInfo}${placeholderInfo}${roleInfo} "${e.text}"`;
|
|
163
191
|
}).join('\n');
|
|
164
192
|
// Show stats for omitted elements
|
|
165
193
|
const omitted = interactiveElements.length - selectedElements.length;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"page-info-utils.js","sourceRoot":"","sources":["../../src/utils/page-info-utils.ts"],"names":[],"mappings":";AAAA;;;GAGG;;
|
|
1
|
+
{"version":3,"file":"page-info-utils.js","sourceRoot":"","sources":["../../src/utils/page-info-utils.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAyBH,kDAgQC;AAhQM,KAAK,UAAU,mBAAmB,CAAC,KAAmE;IAC3G,IAAI,WAAW,GAA2D,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC;IACxI,IAAI,mBAAmB,GAAyB,EAAE,CAAC;IACnD,IAAI,aAAa,GAAG,KAAK,CAAC;IAE1B,IAAI,CAAC;QACH,IAAI,eAAe,IAAI,KAAK,EAAE,CAAC;YAC7B,yBAAyB;YACzB,aAAa,GAAG,IAAI,CAAC;YACrB,iFAAiF;YACjF,0FAA0F;YAC1F,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YAChF,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YAClC,WAAW,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,CAAC;YAE1D,qEAAqE;YACrE,IAAI,CAAC;gBACH,mBAAmB,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE;oBAC9C,MAAM,GAAG,GAAI,UAAkB,CAAC,QAAQ,CAAC;oBACzC,MAAM,QAAQ,GAAU,EAAE,CAAC;oBAE3B,+BAA+B;oBAC/B,MAAM,oBAAoB,GAAG;wBAC3B,QAAQ;wBACR,OAAO;wBACP,UAAU;wBACV,QAAQ;wBACR,SAAS;wBACT,iBAAiB;wBACjB,eAAe;wBACf,kBAAkB;wBAClB,mBAAmB;wBACnB,gBAAgB;wBAChB,WAAW;wBACX,iBAAiB;qBAClB,CAAC;oBAEF,MAAM,cAAc,GAAG,IAAI,GAAG,EAAO,CAAC;oBACtC,oBAAoB,CAAC,OAAO,CAAC,CAAC,QAAgB,EAAE,EAAE;wBAChD,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;oBAC9E,CAAC,CAAC,CAAC;oBAEH,uEAAuE;oBACvE,yDAAyD;oBACzD,0DAA0D;oBAC1D,MAAM,WAAW,GAAG,GAAG,CAAC,gBAAgB,CAAC,sBAAsB,CAAC,CAAC;oBACjE,WAAW,CAAC,OAAO,CAAC,CAAC,EAAO,EAAE,EAAE;wBAC9B,MAAM,MAAM,GAAI,UAAkB,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;wBAC/D,MAAM,eAAe,GAAG,EAAE,CAAC,OAAO;4BACX,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC;4BAC1B,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC;4BAC9B,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;wBAErD,yFAAyF;wBACzF,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,eAAe,IAAI,EAAE,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC;4BACvE,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACzB,CAAC;oBACH,CAAC,CAAC,CAAC;oBAEH,cAAc,CAAC,OAAO,CAAC,CAAC,EAAO,EAAE,EAAE;wBACjC,MAAM,IAAI,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAC;wBAExC,0BAA0B;wBAC1B,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC1C,OAAO;wBACT,CAAC;wBAED,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,IAAI,CAAC;wBACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;wBACrC,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;wBAC3D,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;wBAC3B,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;wBACjE,MAAM,SAAS,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;wBACtD,MAAM,WAAW,GAAG,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;wBAEzC,uFAAuF;wBACvF,MAAM,SAAS,GAAa,EAAE,CAAC;wBAE/B,6DAA6D;wBAC7D,IAAI,CAAC,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACrG,MAAM,SAAS,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;4BACnD,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gCAC/D,SAAS,CAAC,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,CAAC;4BAC/C,CAAC;wBACH,CAAC;wBAED,yDAAyD;wBACzD,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;4BAChD,SAAS,CAAC,IAAI,CAAC,cAAc,IAAI,cAAc,IAAI,KAAK,CAAC,CAAC;wBAC5D,CAAC;wBAED,sEAAsE;wBACtE,IAAI,GAAG,KAAK,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;4BAChC,SAAS,CAAC,IAAI,CAAC,+BAA+B,IAAI,KAAK,CAAC,CAAC;4BACzD,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC;wBAC9C,CAAC;wBAED,yDAAyD;wBACzD,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;4BACrE,SAAS,CAAC,IAAI,CAAC,qBAAqB,WAAW,IAAI,CAAC,CAAC;wBACvD,CAAC;wBAED,qFAAqF;wBACrF,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;4BACjG,SAAS,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC;wBACzC,CAAC;wBAED,qDAAqD;wBACrD,MAAM,UAAU,GAAG,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC;4BAClE,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;wBAC7E,IAAI,UAAU;4BAAE,SAAS,CAAC,IAAI,CAAC,gBAAgB,UAAU,IAAI,CAAC,CAAC;wBAE/D,gFAAgF;wBAChF,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;wBACzC,IAAI,QAAQ,IAAI,CAAC,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC;4BAC5E,SAAS,CAAC,IAAI,CAAC,eAAe,QAAQ,IAAI,CAAC,CAAC;wBAC9C,CAAC;wBAED,4EAA4E;wBAC5E,IAAI,GAAG,KAAK,QAAQ,IAAI,EAAE,CAAC,SAAS,EAAE,CAAC;4BACrC,MAAM,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;4BACzG,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gCACvB,SAAS,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;4BACzC,CAAC;wBACH,CAAC;wBAED,uFAAuF;wBACvF,0EAA0E;wBAC1E,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;4BAChE,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;wBAC3B,CAAC;wBAED,QAAQ,CAAC,IAAI,CAAC;4BACZ,GAAG;4BACH,EAAE;4BACF,IAAI;4BACJ,IAAI;4BACJ,IAAI,EAAE,QAAQ,IAAI,EAAE,EAAG,mDAAmD;4BAC1E,IAAI;4BACJ,SAAS;4BACT,WAAW;4BACX,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gCACrB,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gCACrB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;gCAC7B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;6BAChC;4BACD,SAAS;yBACV,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;oBAEH,OAAO,QAAQ,CAAC;gBAClB,CAAC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,SAAc,EAAE,CAAC;gBACxB,oEAAoE;gBACpE,OAAO,CAAC,KAAK,CAAC,+DAA+D,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;gBAClG,mBAAmB,GAAG,EAAE,CAAC;YAC3B,CAAC;QACH,CAAC;aAAM,CAAC;YACN,iCAAiC;YACjC,WAAW,GAAG,KAAK,CAAC;QACtB,CAAC;QAED,uCAAuC;QACvC,IAAI,aAAa,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,mCAAmC,mBAAmB,CAAC,MAAM,8BAA8B,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5H,CAAC;QAED,6EAA6E;QAC7E,IAAI,iBAAiB,GAAG,EAAE,CAAC;QAE3B,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,2DAA2D;YAC3D,MAAM,WAAW,GAAyB,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAyB,EAAE,CAAC;YAE3C,iBAAiB;YACjB,KAAK,MAAM,EAAE,IAAI,mBAAmB,EAAE,CAAC;gBACrC,6DAA6D;gBAC7D,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC;oBAC1D,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC;oBAClE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACvB,CAAC;qBAAM,CAAC;oBACN,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,0BAA0B;YAC1B,MAAM,gBAAgB,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAElD,+EAA+E;YAC/E,IAAI,gBAAgB,CAAC,MAAM,GAAG,EAAE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,qBAAqB;gBAClF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvD,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;YAED,iBAAiB,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAChD,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtC,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnD,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnD,MAAM,eAAe,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/E,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAEnD,8EAA8E;gBAC9E,OAAO,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,CAAC,GAAG,IAAI,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,eAAe,GAAG,QAAQ,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC;YACnK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEd,kCAAkC;YAClC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;YACrE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,iBAAiB,IAAI,aAAa,OAAO,iBAAiB,CAAC;gBAE3D,wCAAwC;gBACxC,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;gBACxE,MAAM,gBAAgB,GAA2B,EAAE,CAAC;gBAEpD,KAAK,MAAM,EAAE,IAAI,mBAAmB,EAAE,CAAC;oBACrC,IAAI,EAAE,CAAC,GAAG,KAAK,GAAG,IAAI,EAAE,CAAC,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC,GAAG,KAAK,KAAK,EAAE,CAAC;wBAC1D,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;wBAC5D,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;oBACnE,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,iBAAiB,IAAI,qBAAqB,CAAC;oBAC3C,iBAAiB,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC;yBAClD,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC;yBACjC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,KAAK,OAAO,EAAE,CAAC;yBACjD,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,iBAAiB,GAAG,6EAA6E,CAAC;QACpG,CAAC;QAED,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,GAAG;YACpB,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,YAAY,EAAE,WAAW,CAAC,iBAAiB;YAC3C,mBAAmB;YACnB,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,+CAA+C;QAC/C,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACvE,OAAO;YACL,GAAG,EAAE,WAAW,EAAE,GAAG,IAAI,SAAS;YAClC,KAAK,EAAE,SAAS;YAChB,YAAY,EAAE,IAAI;YAClB,mBAAmB,EAAE,EAAE;YACvB,iBAAiB,EAAE,sBAAsB,KAAK,CAAC,OAAO,EAAE;SACzD,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ref Attacher - Injects stable refs as data-tc-id attributes into the DOM
|
|
3
|
+
* This makes refs stable across agent invocations and translations
|
|
4
|
+
*/
|
|
5
|
+
import { PageInfo } from './page-info-utils';
|
|
6
|
+
export interface RefAttacherResult {
|
|
7
|
+
success: boolean;
|
|
8
|
+
refsAttached: number;
|
|
9
|
+
pageInfo: PageInfo;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Attach refs from ARIA tree as data-tc-id attributes in the DOM
|
|
13
|
+
* This ensures refs are stable and can be used by both agent and translator
|
|
14
|
+
*/
|
|
15
|
+
export declare function attachRefsToDOM(page: any): Promise<RefAttacherResult>;
|
|
16
|
+
/**
|
|
17
|
+
* Get page info using existing data-tc-id attributes
|
|
18
|
+
* This allows using stable refs without regenerating them
|
|
19
|
+
*/
|
|
20
|
+
export declare function getPageInfoFromAttachedRefs(page: any): Promise<PageInfo>;
|
|
21
|
+
//# sourceMappingURL=ref-attacher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ref-attacher.d.ts","sourceRoot":"","sources":["../../src/utils/ref-attacher.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAuB,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAElE,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED;;;GAGG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkH3E;AAED;;;GAGG;AACH,wBAAsB,2BAA2B,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CA0C9E"}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Ref Attacher - Injects stable refs as data-tc-id attributes into the DOM
|
|
4
|
+
* This makes refs stable across agent invocations and translations
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.attachRefsToDOM = attachRefsToDOM;
|
|
8
|
+
exports.getPageInfoFromAttachedRefs = getPageInfoFromAttachedRefs;
|
|
9
|
+
const page_info_utils_1 = require("./page-info-utils");
|
|
10
|
+
/**
|
|
11
|
+
* Attach refs from ARIA tree as data-tc-id attributes in the DOM
|
|
12
|
+
* This ensures refs are stable and can be used by both agent and translator
|
|
13
|
+
*/
|
|
14
|
+
async function attachRefsToDOM(page) {
|
|
15
|
+
try {
|
|
16
|
+
// 1. Get page info with refs generated
|
|
17
|
+
const pageInfo = await (0, page_info_utils_1.getEnhancedPageInfo)(page);
|
|
18
|
+
if (!pageInfo.refMap || pageInfo.refMap.size === 0) {
|
|
19
|
+
return {
|
|
20
|
+
success: false,
|
|
21
|
+
refsAttached: 0,
|
|
22
|
+
pageInfo
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
// 2. Build a mapping of element characteristics to refs
|
|
26
|
+
const refMappings = [];
|
|
27
|
+
pageInfo.refMap.forEach((element, ref) => {
|
|
28
|
+
refMappings.push({
|
|
29
|
+
ref,
|
|
30
|
+
tag: element.tag,
|
|
31
|
+
role: element.role,
|
|
32
|
+
text: element.text || '',
|
|
33
|
+
name: element.name || '',
|
|
34
|
+
placeholder: element.placeholder || '',
|
|
35
|
+
type: element.type || '',
|
|
36
|
+
ariaLabel: element.ariaLabel || ''
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
// 3. Inject refs into DOM as data-tc-id attributes
|
|
40
|
+
const refsAttached = await page.evaluate((mappings) => {
|
|
41
|
+
let attached = 0;
|
|
42
|
+
const doc = globalThis.document;
|
|
43
|
+
// Get all elements
|
|
44
|
+
const allElements = Array.from(doc.querySelectorAll('*'));
|
|
45
|
+
for (const mapping of mappings) {
|
|
46
|
+
// Try to find matching element in DOM
|
|
47
|
+
let matchedElement = null;
|
|
48
|
+
// Strategy 1: Match by name attribute (most reliable for inputs)
|
|
49
|
+
if (mapping.name) {
|
|
50
|
+
matchedElement = doc.querySelector(`${mapping.tag}[name="${mapping.name}"]`);
|
|
51
|
+
}
|
|
52
|
+
// Strategy 2: Match by placeholder
|
|
53
|
+
if (!matchedElement && mapping.placeholder) {
|
|
54
|
+
matchedElement = doc.querySelector(`${mapping.tag}[placeholder="${mapping.placeholder}"]`);
|
|
55
|
+
}
|
|
56
|
+
// Strategy 3: Match by aria-label
|
|
57
|
+
if (!matchedElement && mapping.ariaLabel) {
|
|
58
|
+
matchedElement = doc.querySelector(`${mapping.tag}[aria-label="${mapping.ariaLabel}"]`);
|
|
59
|
+
}
|
|
60
|
+
// Strategy 4: Match by text content (for buttons, labels, etc.)
|
|
61
|
+
if (!matchedElement && mapping.text && mapping.text.length > 0 && mapping.text.length < 100) {
|
|
62
|
+
for (const el of allElements) {
|
|
63
|
+
const element = el;
|
|
64
|
+
if (element.tagName.toLowerCase() === mapping.tag &&
|
|
65
|
+
element.textContent?.trim() === mapping.text.trim()) {
|
|
66
|
+
matchedElement = el;
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// Strategy 5: Match by role and index (fallback)
|
|
72
|
+
if (!matchedElement && mapping.role) {
|
|
73
|
+
const roleElements = Array.from(doc.querySelectorAll(`[role="${mapping.role}"]`));
|
|
74
|
+
if (roleElements.length > 0) {
|
|
75
|
+
// Use first matching role element that doesn't have a ref yet
|
|
76
|
+
for (const el of roleElements) {
|
|
77
|
+
const element = el;
|
|
78
|
+
if (!element.hasAttribute('data-tc-id')) {
|
|
79
|
+
matchedElement = el;
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
// Attach ref as data-tc-id
|
|
86
|
+
if (matchedElement && !matchedElement.hasAttribute('data-tc-id')) {
|
|
87
|
+
matchedElement.setAttribute('data-tc-id', mapping.ref);
|
|
88
|
+
attached++;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return attached;
|
|
92
|
+
}, refMappings);
|
|
93
|
+
console.log(`[RefAttacher] ✓ Attached ${refsAttached} refs to DOM as data-tc-id`);
|
|
94
|
+
return {
|
|
95
|
+
success: true,
|
|
96
|
+
refsAttached,
|
|
97
|
+
pageInfo
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
console.error(`[RefAttacher] ✗ Failed to attach refs: ${error.message}`);
|
|
102
|
+
throw error;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Get page info using existing data-tc-id attributes
|
|
107
|
+
* This allows using stable refs without regenerating them
|
|
108
|
+
*/
|
|
109
|
+
async function getPageInfoFromAttachedRefs(page) {
|
|
110
|
+
// Get base page info
|
|
111
|
+
const pageInfo = await (0, page_info_utils_1.getEnhancedPageInfo)(page);
|
|
112
|
+
// Extract refs from data-tc-id attributes
|
|
113
|
+
const domRefs = await page.evaluate(() => {
|
|
114
|
+
const doc = globalThis.document;
|
|
115
|
+
const elementsWithRefs = Array.from(doc.querySelectorAll('[data-tc-id]'));
|
|
116
|
+
return elementsWithRefs.map((el) => ({
|
|
117
|
+
ref: el.getAttribute('data-tc-id'),
|
|
118
|
+
tag: el.tagName.toLowerCase(),
|
|
119
|
+
role: el.getAttribute('role') || '',
|
|
120
|
+
text: el.textContent?.trim().substring(0, 100) || '',
|
|
121
|
+
name: el.getAttribute('name') || '',
|
|
122
|
+
placeholder: el.getAttribute('placeholder') || '',
|
|
123
|
+
type: el.getAttribute('type') || '',
|
|
124
|
+
ariaLabel: el.getAttribute('aria-label') || ''
|
|
125
|
+
}));
|
|
126
|
+
});
|
|
127
|
+
// Merge DOM refs into refMap
|
|
128
|
+
const updatedRefMap = new Map(pageInfo.refMap);
|
|
129
|
+
domRefs.forEach((item) => {
|
|
130
|
+
if (item.ref && !updatedRefMap.has(item.ref)) {
|
|
131
|
+
updatedRefMap.set(item.ref, {
|
|
132
|
+
tag: item.tag,
|
|
133
|
+
role: item.role,
|
|
134
|
+
text: item.text,
|
|
135
|
+
name: item.name,
|
|
136
|
+
placeholder: item.placeholder,
|
|
137
|
+
type: item.type,
|
|
138
|
+
ariaLabel: item.ariaLabel,
|
|
139
|
+
id: null,
|
|
140
|
+
selectors: []
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
return {
|
|
145
|
+
...pageInfo,
|
|
146
|
+
refMap: updatedRefMap
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
//# sourceMappingURL=ref-attacher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ref-attacher.js","sourceRoot":"","sources":["../../src/utils/ref-attacher.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAcH,0CAkHC;AAMD,kEA0CC;AA9KD,uDAAkE;AAQlE;;;GAGG;AACI,KAAK,UAAU,eAAe,CAAC,IAAS;IAC7C,IAAI,CAAC;QACH,uCAAuC;QACvC,MAAM,QAAQ,GAAG,MAAM,IAAA,qCAAmB,EAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACnD,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,YAAY,EAAE,CAAC;gBACf,QAAQ;aACT,CAAC;QACJ,CAAC;QAED,wDAAwD;QACxD,MAAM,WAAW,GASZ,EAAE,CAAC;QAER,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;YACvC,WAAW,CAAC,IAAI,CAAC;gBACf,GAAG;gBACH,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE;gBACxB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE;gBACxB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;gBACtC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE;gBACxB,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE;aACnC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,mDAAmD;QACnD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAe,EAAE,EAAE;YAC3D,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,MAAM,GAAG,GAAI,UAAkB,CAAC,QAAQ,CAAC;YAEzC,mBAAmB;YACnB,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;YAE1D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,sCAAsC;gBACtC,IAAI,cAAc,GAAQ,IAAI,CAAC;gBAE/B,iEAAiE;gBACjE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;oBACjB,cAAc,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,GAAG,UAAU,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;gBAC/E,CAAC;gBAED,mCAAmC;gBACnC,IAAI,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;oBAC3C,cAAc,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,GAAG,iBAAiB,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;gBAC7F,CAAC;gBAED,kCAAkC;gBAClC,IAAI,CAAC,cAAc,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;oBACzC,cAAc,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,GAAG,gBAAgB,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;gBAC1F,CAAC;gBAED,gEAAgE;gBAChE,IAAI,CAAC,cAAc,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBAC5F,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,EAAS,CAAC;wBAC1B,IAAI,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,GAAG;4BAC7C,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;4BACxD,cAAc,GAAG,EAAE,CAAC;4BACpB,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,iDAAiD;gBACjD,IAAI,CAAC,cAAc,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;oBACpC,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,UAAU,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;oBAClF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,8DAA8D;wBAC9D,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;4BAC9B,MAAM,OAAO,GAAG,EAAS,CAAC;4BAC1B,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,EAAE,CAAC;gCACxC,cAAc,GAAG,EAAE,CAAC;gCACpB,MAAM;4BACR,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,2BAA2B;gBAC3B,IAAI,cAAc,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,EAAE,CAAC;oBACjE,cAAc,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;oBACvD,QAAQ,EAAE,CAAC;gBACb,CAAC;YACH,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,EAAE,WAAW,CAAC,CAAC;QAEhB,OAAO,CAAC,GAAG,CAAC,4BAA4B,YAAY,4BAA4B,CAAC,CAAC;QAElF,OAAO;YACL,OAAO,EAAE,IAAI;YACb,YAAY;YACZ,QAAQ;SACT,CAAC;IAEJ,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,0CAA0C,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,2BAA2B,CAAC,IAAS;IACzD,qBAAqB;IACrB,MAAM,QAAQ,GAAG,MAAM,IAAA,qCAAmB,EAAC,IAAI,CAAC,CAAC;IAEjD,0CAA0C;IAC1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACvC,MAAM,GAAG,GAAI,UAAkB,CAAC,QAAQ,CAAC;QACzC,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,CAAC;QAC1E,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,CAAC;YACxC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC;YAClC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE;YAC7B,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE;YACnC,IAAI,EAAE,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE;YACpD,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE;YACnC,WAAW,EAAE,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE;YACjD,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE;YACnC,SAAS,EAAE,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE;SAC/C,CAAC,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,6BAA6B;IAC7B,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAC,IAAS,EAAE,EAAE;QAC5B,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE;gBAC1B,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,EAAE,EAAE,IAAI;gBACR,SAAS,EAAE,EAAE;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,GAAG,QAAQ;QACX,MAAM,EAAE,aAAa;KACtB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ref Translation Utility
|
|
3
|
+
* Converts structured ref commands to Playwright commands with fallback alternatives
|
|
4
|
+
* Uses hierarchical ARIA tree context for robust selector generation
|
|
5
|
+
*/
|
|
6
|
+
import { RefCommand } from '../orchestrator/types';
|
|
7
|
+
export declare class RefTranslator {
|
|
8
|
+
private logger?;
|
|
9
|
+
private hierarchicalGenerator;
|
|
10
|
+
constructor(logger?: (message: string, level?: 'log' | 'error' | 'warn') => void);
|
|
11
|
+
/**
|
|
12
|
+
* LEGACY: Translate string commands (supports regex getByRef pattern)
|
|
13
|
+
* Use translateRefCommands() instead for new code
|
|
14
|
+
*/
|
|
15
|
+
translateStringCommands(commands: string[], page: any, refMap?: Map<string, any>, ariaTree?: any): Promise<Array<{
|
|
16
|
+
original: string;
|
|
17
|
+
alternatives: string[];
|
|
18
|
+
successful?: string;
|
|
19
|
+
}>>;
|
|
20
|
+
/**
|
|
21
|
+
* Translation layer: Convert ref commands to actual Playwright commands
|
|
22
|
+
* Tries multiple selector alternatives until one succeeds
|
|
23
|
+
*
|
|
24
|
+
* Agent outputs:
|
|
25
|
+
* { ref: 'e22', operation: 'fill', value: 'test@test.com' }
|
|
26
|
+
* This generates and tries multiple Playwright alternatives using hierarchical context
|
|
27
|
+
* Returns: The command that worked (for script storage)
|
|
28
|
+
*/
|
|
29
|
+
translateRefCommands(refCommands: RefCommand[], page: any, refMap?: Map<string, any>, ariaTree?: any): Promise<Array<{
|
|
30
|
+
original: RefCommand;
|
|
31
|
+
alternatives: string[];
|
|
32
|
+
successful?: string;
|
|
33
|
+
}>>;
|
|
34
|
+
/**
|
|
35
|
+
* Generate drag commands with coordinates
|
|
36
|
+
*/
|
|
37
|
+
private generateDragCommands;
|
|
38
|
+
/**
|
|
39
|
+
* Convert selectors to full Playwright commands with action
|
|
40
|
+
*/
|
|
41
|
+
private convertSelectorsToCommands;
|
|
42
|
+
/**
|
|
43
|
+
* Convert selector string to locator chain for building commands
|
|
44
|
+
*/
|
|
45
|
+
private selectorToLocatorChain;
|
|
46
|
+
private getActionCode;
|
|
47
|
+
private escape;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=ref-translator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ref-translator.d.ts","sourceRoot":"","sources":["../../src/utils/ref-translator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAC,CAA8D;IAC7E,OAAO,CAAC,qBAAqB,CAAgC;gBAEjD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,OAAO,GAAG,MAAM,KAAK,IAAI;IAKhF;;;OAGG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,EAAE,GAAG,EACT,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,QAAQ,CAAC,EAAE,GAAG,GACb,OAAO,CAAC,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAuCpF;;;;;;;;OAQG;IACG,oBAAoB,CACxB,WAAW,EAAE,UAAU,EAAE,EACzB,IAAI,EAAE,GAAG,EACT,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,QAAQ,CAAC,EAAE,GAAG,GACb,OAAO,CAAC,KAAK,CAAC;QAAE,QAAQ,EAAE,UAAU,CAAC;QAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAsFxF;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAuB5B;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAyClC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IA2B9B,OAAO,CAAC,aAAa;IAyCrB,OAAO,CAAC,MAAM;CAGf"}
|