yunti-browser-runtime 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -4
- package/bin/yunti-browser-runtime.js +6 -0
- package/docs/ACTION_RESULT_COVERAGE.md +41 -0
- package/docs/AGENT_WORKFLOW_CONTRACT.md +97 -0
- package/docs/EXECUTION_PLAN.md +1399 -5
- package/docs/EXTENSION_DISTRIBUTION.md +145 -0
- package/docs/EXTENSION_PERMISSION_STRATEGY.md +164 -0
- package/docs/EXTENSION_STORE_COPY.md +210 -0
- package/docs/INSTALL.md +2 -1
- package/docs/NEXT_MAJOR_PLAN.md +808 -0
- package/docs/PROJECT_STATUS.md +1012 -6
- package/docs/PUBLISHING_BLOCKERS.md +1 -1
- package/docs/RELEASE.md +6 -2
- package/docs/RELEASE_0_2_0_CHECKLIST.md +820 -0
- package/docs/ROADMAP.md +44 -0
- package/docs/SECURITY.md +34 -0
- package/docs/TOOL_GUIDE.md +282 -5
- package/extension/content.js +76 -7
- package/extension/dom-observer.js +588 -0
- package/extension/manifest.json +2 -2
- package/extension/session-manager.js +2 -0
- package/extension/tool-handlers.js +1120 -94
- package/mcp/bridge-hub.js +257 -3
- package/mcp/http-server.js +213 -0
- package/mcp/memory.js +5 -5
- package/mcp/redaction.js +52 -3
- package/mcp/server.js +2 -0
- package/mcp/tools.js +417 -38
- package/package.json +4 -2
- package/scripts/check-action-result-coverage.js +145 -0
- package/scripts/doctor.js +4 -0
- package/scripts/package-extension.js +1 -0
- package/scripts/release-check.js +3 -0
- package/skills/yunti-browser-runtime/SKILL.md +125 -2
|
@@ -73,12 +73,16 @@ export function createToolDispatcher({
|
|
|
73
73
|
result = await evaluateScript(tabId, session, event.arguments || {})
|
|
74
74
|
} else if (event.tool === "yunti_take_snapshot") {
|
|
75
75
|
result = await takeSnapshot(tabId, session, event.arguments || {})
|
|
76
|
+
} else if (event.tool === "yunti_observe_page") {
|
|
77
|
+
result = await observePage(tabId, session, event.arguments || {})
|
|
76
78
|
} else if (event.tool === "yunti_click") {
|
|
77
79
|
result = await clickByUid(tabId, session, event.arguments || {})
|
|
78
80
|
} else if (event.tool === "yunti_hover") {
|
|
79
81
|
result = await hoverByUid(tabId, session, event.arguments || {})
|
|
80
82
|
} else if (event.tool === "yunti_fill") {
|
|
81
83
|
result = await fillByUid(tabId, session, event.arguments || {})
|
|
84
|
+
} else if (event.tool === "yunti_select") {
|
|
85
|
+
result = await selectElement(tabId, session, event.arguments || {})
|
|
82
86
|
} else if (event.tool === "yunti_fill_form") {
|
|
83
87
|
result = await fillForm(tabId, session, event.arguments || {})
|
|
84
88
|
} else if (event.tool === "yunti_wait_for") {
|
|
@@ -105,6 +109,8 @@ export function createToolDispatcher({
|
|
|
105
109
|
result = await typeTextByUid(tabId, session, event.arguments || {})
|
|
106
110
|
} else if (event.tool === "yunti_press_key") {
|
|
107
111
|
result = await pressKeyByUid(tabId, session, event.arguments || {})
|
|
112
|
+
} else if (event.tool === "yunti_scroll") {
|
|
113
|
+
result = await scrollPage(tabId, session, event.arguments || {})
|
|
108
114
|
} else {
|
|
109
115
|
result = await chrome.tabs.sendMessage(tabId, {
|
|
110
116
|
type: "yunti_execute_tool",
|
|
@@ -343,8 +349,8 @@ export function createToolDispatcher({
|
|
|
343
349
|
}
|
|
344
350
|
}
|
|
345
351
|
|
|
346
|
-
//
|
|
347
|
-
const
|
|
352
|
+
// Latest page uid state from yunti_observe_page or yunti_take_snapshot.
|
|
353
|
+
const pageUidStore = new Map()
|
|
348
354
|
|
|
349
355
|
async function takeSnapshot(tabId, session, args = {}) {
|
|
350
356
|
const maxElements = Number.isFinite(Number(args.maxElements))
|
|
@@ -367,12 +373,7 @@ export function createToolDispatcher({
|
|
|
367
373
|
elements = await domFallbackSnapshot(tabId, maxElements)
|
|
368
374
|
}
|
|
369
375
|
|
|
370
|
-
|
|
371
|
-
const uidMap = {}
|
|
372
|
-
for (const el of elements) {
|
|
373
|
-
uidMap[el.uid] = { ...el }
|
|
374
|
-
}
|
|
375
|
-
snapshotStore.set(session.browserSessionId, uidMap)
|
|
376
|
+
storePageUidMap(session, "snapshot", elements)
|
|
376
377
|
|
|
377
378
|
return {
|
|
378
379
|
browserSessionId: session.browserSessionId,
|
|
@@ -383,6 +384,38 @@ export function createToolDispatcher({
|
|
|
383
384
|
snapshotId: `snap-${Date.now()}`,
|
|
384
385
|
}
|
|
385
386
|
}
|
|
387
|
+
|
|
388
|
+
async function observePage(tabId, session, args = {}) {
|
|
389
|
+
const observation = await chrome.tabs.sendMessage(tabId, {
|
|
390
|
+
type: "yunti_execute_tool",
|
|
391
|
+
tool: "yunti_observe_page",
|
|
392
|
+
arguments: args,
|
|
393
|
+
})
|
|
394
|
+
if (Array.isArray(observation?.elements)) {
|
|
395
|
+
storePageUidMap(session, "observe", [
|
|
396
|
+
...observation.elements,
|
|
397
|
+
...(Array.isArray(observation.scrollableContainers) ? observation.scrollableContainers : []),
|
|
398
|
+
], {
|
|
399
|
+
observationId: observation.observationId,
|
|
400
|
+
uidMapVersion: observation.uidMapVersion,
|
|
401
|
+
})
|
|
402
|
+
}
|
|
403
|
+
return observation
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
function storePageUidMap(session, source, elements, meta = {}) {
|
|
407
|
+
const uidMap = {}
|
|
408
|
+
for (const el of elements || []) {
|
|
409
|
+
if (!el?.uid) continue
|
|
410
|
+
uidMap[el.uid] = { ...el, source }
|
|
411
|
+
}
|
|
412
|
+
pageUidStore.set(session.browserSessionId, {
|
|
413
|
+
source,
|
|
414
|
+
uidMap,
|
|
415
|
+
storedAt: Date.now(),
|
|
416
|
+
...meta,
|
|
417
|
+
})
|
|
418
|
+
}
|
|
386
419
|
|
|
387
420
|
function flattenAXTree(nodes, maxElements, includeHidden) {
|
|
388
421
|
const elements = []
|
|
@@ -511,15 +544,30 @@ export function createToolDispatcher({
|
|
|
511
544
|
}
|
|
512
545
|
return clickAtCoordinate(tabId, session, x, y)
|
|
513
546
|
}
|
|
514
|
-
|
|
547
|
+
const selector = String(args.selector || "").trim()
|
|
548
|
+
if (!selector) {
|
|
515
549
|
throw new Error("yunti_click requires uid, selector, or both x and y. Call yunti_take_snapshot to get uid, pass a CSS selector, or use yunti_click_at for coordinate-only clicks.")
|
|
516
550
|
}
|
|
517
551
|
// Fall back to content script for selector-based click
|
|
518
|
-
|
|
552
|
+
const contentResult = await chrome.tabs.sendMessage(tabId, {
|
|
519
553
|
type: "yunti_execute_tool",
|
|
520
554
|
tool: "yunti_click",
|
|
521
555
|
arguments: args,
|
|
522
556
|
})
|
|
557
|
+
if (contentResult && typeof contentResult === "object" && contentResult.clicked === true) {
|
|
558
|
+
return {
|
|
559
|
+
...contentResult,
|
|
560
|
+
selector,
|
|
561
|
+
browserSessionId: session.browserSessionId,
|
|
562
|
+
method: "selector",
|
|
563
|
+
action: "click",
|
|
564
|
+
target: { selector, method: "selector" },
|
|
565
|
+
ok: true,
|
|
566
|
+
recoverable: false,
|
|
567
|
+
nextStepHint: "Selector click dispatched. Observe again, read page state, or use a fresh uid when possible to verify the intended change.",
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
return contentResult
|
|
523
571
|
}
|
|
524
572
|
|
|
525
573
|
async function hoverByUid(tabId, session, args = {}) {
|
|
@@ -534,7 +582,20 @@ export function createToolDispatcher({
|
|
|
534
582
|
throw new Error("yunti_hover coordinate mode requires both x and y. Call yunti_take_snapshot to get uid, pass selector, or provide both coordinates.")
|
|
535
583
|
}
|
|
536
584
|
await mouseMove(tabId, x, y)
|
|
537
|
-
|
|
585
|
+
const roundedX = Math.round(x)
|
|
586
|
+
const roundedY = Math.round(y)
|
|
587
|
+
return {
|
|
588
|
+
hovered: true,
|
|
589
|
+
x: roundedX,
|
|
590
|
+
y: roundedY,
|
|
591
|
+
browserSessionId: session.browserSessionId,
|
|
592
|
+
method: "coordinate",
|
|
593
|
+
action: "hover",
|
|
594
|
+
target: { method: "coordinate", x: roundedX, y: roundedY },
|
|
595
|
+
ok: true,
|
|
596
|
+
recoverable: false,
|
|
597
|
+
nextStepHint: "Coordinate hover dispatched. Observe again, read page state, or use a fresh uid when possible to verify menus, tooltips, or hover-only controls.",
|
|
598
|
+
}
|
|
538
599
|
}
|
|
539
600
|
const selector = String(args.selector || "").trim()
|
|
540
601
|
if (!selector) {
|
|
@@ -542,13 +603,20 @@ export function createToolDispatcher({
|
|
|
542
603
|
}
|
|
543
604
|
const point = await resolveSelectorCenter(tabId, selector)
|
|
544
605
|
await mouseMove(tabId, point.x, point.y)
|
|
606
|
+
const roundedX = Math.round(point.x)
|
|
607
|
+
const roundedY = Math.round(point.y)
|
|
545
608
|
return {
|
|
546
609
|
hovered: true,
|
|
547
610
|
selector,
|
|
548
|
-
x:
|
|
549
|
-
y:
|
|
611
|
+
x: roundedX,
|
|
612
|
+
y: roundedY,
|
|
550
613
|
browserSessionId: session.browserSessionId,
|
|
551
614
|
method: "selector",
|
|
615
|
+
action: "hover",
|
|
616
|
+
target: { selector, method: "selector", x: roundedX, y: roundedY },
|
|
617
|
+
ok: true,
|
|
618
|
+
recoverable: false,
|
|
619
|
+
nextStepHint: "Selector hover dispatched. Observe again, read page state, or use a fresh uid when possible to verify menus, tooltips, or hover-only controls.",
|
|
552
620
|
}
|
|
553
621
|
}
|
|
554
622
|
|
|
@@ -557,76 +625,803 @@ export function createToolDispatcher({
|
|
|
557
625
|
const uid = String(args.uid || "").trim()
|
|
558
626
|
const value = String(args.value)
|
|
559
627
|
if (uid) {
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
const
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
el.
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
628
|
+
try {
|
|
629
|
+
const { x, y } = await resolveUidCenter(tabId, session, uid)
|
|
630
|
+
await mouseClick(tabId, x, y, 1)
|
|
631
|
+
await delayCdp(100)
|
|
632
|
+
// Clear existing value
|
|
633
|
+
const elements = await chromeDebuggerSendCommand(
|
|
634
|
+
{ tabId },
|
|
635
|
+
"Runtime.evaluate",
|
|
636
|
+
{ expression: `(() => {
|
|
637
|
+
const el = document.elementFromPoint(${x}, ${y});
|
|
638
|
+
if (!el) return 'not found';
|
|
639
|
+
const tag = el.tagName.toLowerCase();
|
|
640
|
+
const rect = el.getBoundingClientRect();
|
|
641
|
+
const style = getComputedStyle(el);
|
|
642
|
+
const type = String(el.type || '').toLowerCase();
|
|
643
|
+
const disabled = Boolean(el.disabled || el.getAttribute('aria-disabled') === 'true');
|
|
644
|
+
const readOnly = Boolean(el.readOnly || el.getAttribute('aria-readonly') === 'true');
|
|
645
|
+
const hidden = Boolean(el.hidden || style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0 || rect.width <= 0 || rect.height <= 0);
|
|
646
|
+
const valueEditable = tag === 'input' || tag === 'textarea';
|
|
647
|
+
const blockedInputType = tag === 'input' && ['button', 'checkbox', 'color', 'file', 'hidden', 'image', 'radio', 'range', 'reset', 'submit'].includes(type);
|
|
648
|
+
const editable = el.isContentEditable || tag === 'select' || (valueEditable && !blockedInputType);
|
|
649
|
+
if (tag === 'select') {
|
|
650
|
+
const opts = Array.from(el.options);
|
|
651
|
+
return { tag: 'select', type, disabled, readOnly, hidden, editable, options: opts.map(o => ({ value: o.value, text: o.text.slice(0, 80) })), selectedIndex: el.selectedIndex };
|
|
652
|
+
}
|
|
653
|
+
const beforeText = el.isContentEditable ? String(el.textContent || '') : '';
|
|
654
|
+
if (el.isContentEditable) {
|
|
655
|
+
el.focus();
|
|
656
|
+
el.textContent = '';
|
|
657
|
+
} else if (valueEditable && !blockedInputType && !disabled && !readOnly && !hidden) {
|
|
658
|
+
el.focus();
|
|
659
|
+
el.select();
|
|
660
|
+
}
|
|
661
|
+
return {
|
|
662
|
+
tag,
|
|
663
|
+
type,
|
|
664
|
+
contentEditable: el.isContentEditable,
|
|
665
|
+
disabled,
|
|
666
|
+
readOnly,
|
|
667
|
+
hidden,
|
|
668
|
+
editable,
|
|
669
|
+
before: el.isContentEditable ? { textLength: beforeText.length } : undefined,
|
|
670
|
+
};
|
|
671
|
+
})()`,
|
|
672
|
+
returnByValue: true,
|
|
580
673
|
}
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
674
|
+
)
|
|
675
|
+
const elInfo = elements?.result?.value
|
|
676
|
+
|
|
677
|
+
if (elInfo === "not found") {
|
|
678
|
+
return buildUidFillFailureResult(session, uid, {
|
|
679
|
+
code: "ELEMENT_NOT_FOUND",
|
|
680
|
+
error: `No element found at uid ${uid} coordinates`,
|
|
681
|
+
})
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
if (elInfo?.hidden || elInfo?.disabled || elInfo?.readOnly || elInfo?.editable === false) {
|
|
685
|
+
return buildUidFillFailureResult(session, uid, {
|
|
686
|
+
code: "TARGET_NOT_EDITABLE",
|
|
687
|
+
error: buildFillEditabilityError(uid, elInfo),
|
|
688
|
+
element: summarizeFillTarget(elInfo),
|
|
689
|
+
})
|
|
584
690
|
}
|
|
585
|
-
)
|
|
586
|
-
const elInfo = elements?.result?.value
|
|
587
691
|
|
|
588
|
-
|
|
692
|
+
// Handle select element
|
|
693
|
+
if (elInfo?.tag === "select") {
|
|
694
|
+
const targetOption = elInfo.options?.find(
|
|
695
|
+
o => o.value === value || o.text === value
|
|
696
|
+
)
|
|
697
|
+
if (targetOption) {
|
|
698
|
+
await chromeDebuggerSendCommand(
|
|
699
|
+
{ tabId },
|
|
700
|
+
"Runtime.evaluate",
|
|
701
|
+
{
|
|
702
|
+
expression: `(() => {
|
|
703
|
+
const el = document.elementFromPoint(${x}, ${y});
|
|
704
|
+
if (el) el.value = ${JSON.stringify(targetOption.value)};
|
|
705
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
706
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
707
|
+
})()`,
|
|
708
|
+
}
|
|
709
|
+
)
|
|
710
|
+
return {
|
|
711
|
+
filled: true,
|
|
712
|
+
uid,
|
|
713
|
+
method: "select",
|
|
714
|
+
value: targetOption.value,
|
|
715
|
+
browserSessionId: session.browserSessionId,
|
|
716
|
+
action: "fill",
|
|
717
|
+
target: { uid, method: "select" },
|
|
718
|
+
ok: true,
|
|
719
|
+
recoverable: false,
|
|
720
|
+
nextStepHint: "Select value dispatched. Observe again, read page state, or evaluate the select value to verify the intended change.",
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
return buildUidFillFailureResult(session, uid, {
|
|
724
|
+
code: "OPTION_NOT_FOUND",
|
|
725
|
+
error: `Option '${value}' not found in select at uid ${uid}`,
|
|
726
|
+
availableValues: elInfo.options?.map((option) => option.value).slice(0, 50),
|
|
727
|
+
availableTexts: elInfo.options?.map((option) => option.text).slice(0, 50),
|
|
728
|
+
})
|
|
729
|
+
}
|
|
589
730
|
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
const
|
|
593
|
-
o => o.value === value || o.text === value
|
|
594
|
-
)
|
|
595
|
-
if (targetOption) {
|
|
731
|
+
// Type text via CDP Input.dispatchKeyEvent
|
|
732
|
+
const text = value
|
|
733
|
+
for (const char of text) {
|
|
596
734
|
await chromeDebuggerSendCommand(
|
|
597
735
|
{ tabId },
|
|
598
|
-
"
|
|
599
|
-
{
|
|
600
|
-
expression: `(() => {
|
|
601
|
-
const el = document.elementFromPoint(${x}, ${y});
|
|
602
|
-
if (el) el.value = ${JSON.stringify(targetOption.value)};
|
|
603
|
-
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
604
|
-
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
605
|
-
})()`,
|
|
606
|
-
}
|
|
736
|
+
"Input.dispatchKeyEvent",
|
|
737
|
+
{ type: "char", text: char, unmodifiedText: char }
|
|
607
738
|
)
|
|
608
|
-
return { filled: true, uid, method: "select", value: targetOption.value, browserSessionId: session.browserSessionId }
|
|
609
739
|
}
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
const text = value
|
|
615
|
-
for (const char of text) {
|
|
616
|
-
await chromeDebuggerSendCommand(
|
|
740
|
+
const method = elInfo?.contentEditable ? "contenteditable" : "keyboard"
|
|
741
|
+
const before = elInfo?.before
|
|
742
|
+
await delayCdp(50)
|
|
743
|
+
const verification = await chromeDebuggerSendCommand(
|
|
617
744
|
{ tabId },
|
|
618
|
-
"
|
|
619
|
-
{
|
|
745
|
+
"Runtime.evaluate",
|
|
746
|
+
{
|
|
747
|
+
expression: `(() => {
|
|
748
|
+
const el = document.elementFromPoint(${x}, ${y});
|
|
749
|
+
if (!el) return { found: false };
|
|
750
|
+
const tag = el.tagName.toLowerCase();
|
|
751
|
+
const type = String(el.type || '').toLowerCase();
|
|
752
|
+
const expected = ${JSON.stringify(text)};
|
|
753
|
+
const actual = el.isContentEditable
|
|
754
|
+
? String(el.textContent || '')
|
|
755
|
+
: (tag === 'input' || tag === 'textarea')
|
|
756
|
+
? String(el.value || '')
|
|
757
|
+
: undefined;
|
|
758
|
+
if (actual === undefined) {
|
|
759
|
+
return { found: true, tag, type, contentEditable: el.isContentEditable };
|
|
760
|
+
}
|
|
761
|
+
return {
|
|
762
|
+
found: true,
|
|
763
|
+
tag,
|
|
764
|
+
type,
|
|
765
|
+
contentEditable: el.isContentEditable,
|
|
766
|
+
valueMatches: actual === expected,
|
|
767
|
+
valueLength: actual.length,
|
|
768
|
+
expectedLength: expected.length,
|
|
769
|
+
};
|
|
770
|
+
})()`,
|
|
771
|
+
returnByValue: true,
|
|
772
|
+
}
|
|
620
773
|
)
|
|
621
|
-
|
|
622
|
-
|
|
774
|
+
const verificationInfo = verification?.result?.value
|
|
775
|
+
const verifiedAfter = verificationInfo?.found && Number.isFinite(Number(verificationInfo.valueLength))
|
|
776
|
+
? { textLength: Number(verificationInfo.valueLength) }
|
|
777
|
+
: undefined
|
|
778
|
+
const after = verifiedAfter || (elInfo?.contentEditable ? { textLength: text.length } : undefined)
|
|
779
|
+
if (verificationInfo?.found && verificationInfo.valueMatches === false) {
|
|
780
|
+
return buildUidFillFailureResult(session, uid, {
|
|
781
|
+
code: "VALUE_NOT_APPLIED",
|
|
782
|
+
error: `Filled value did not remain on uid ${uid}`,
|
|
783
|
+
method,
|
|
784
|
+
expectedValueLength: Number(verificationInfo.expectedLength),
|
|
785
|
+
actualValueLength: Number(verificationInfo.valueLength),
|
|
786
|
+
...(before ? { before } : {}),
|
|
787
|
+
after,
|
|
788
|
+
element: summarizeFillTarget({
|
|
789
|
+
tag: verificationInfo.tag,
|
|
790
|
+
type: verificationInfo.type,
|
|
791
|
+
contentEditable: verificationInfo.contentEditable,
|
|
792
|
+
}),
|
|
793
|
+
})
|
|
794
|
+
}
|
|
795
|
+
return {
|
|
796
|
+
filled: true,
|
|
797
|
+
uid,
|
|
798
|
+
method,
|
|
799
|
+
value: text,
|
|
800
|
+
...(before ? { before } : {}),
|
|
801
|
+
...(after ? { after } : {}),
|
|
802
|
+
browserSessionId: session.browserSessionId,
|
|
803
|
+
action: "fill",
|
|
804
|
+
target: { uid, method },
|
|
805
|
+
ok: true,
|
|
806
|
+
recoverable: false,
|
|
807
|
+
nextStepHint: elInfo?.contentEditable
|
|
808
|
+
? "Contenteditable fill dispatched. Observe again, read page text, or evaluate textContent to verify the intended change."
|
|
809
|
+
: "Fill dispatched. Observe again, read page state, or evaluate the field value to verify the intended change.",
|
|
810
|
+
}
|
|
811
|
+
} catch (error) {
|
|
812
|
+
return buildUidFillFailureResult(session, uid, {
|
|
813
|
+
error: error?.message || "Uid fill failed",
|
|
814
|
+
})
|
|
815
|
+
}
|
|
623
816
|
}
|
|
624
817
|
|
|
625
|
-
|
|
818
|
+
const selector = String(args.selector || "").trim()
|
|
819
|
+
let result
|
|
820
|
+
try {
|
|
821
|
+
result = await chrome.tabs.sendMessage(tabId, {
|
|
822
|
+
type: "yunti_execute_tool",
|
|
823
|
+
tool: "yunti_fill",
|
|
824
|
+
arguments: args,
|
|
825
|
+
})
|
|
826
|
+
} catch (error) {
|
|
827
|
+
return buildSelectorFillFailureResult(session, selector, {
|
|
828
|
+
error: error?.message || "Selector fill failed",
|
|
829
|
+
})
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
if (!result || typeof result !== "object" || result.filled !== true) {
|
|
833
|
+
return buildSelectorFillFailureResult(session, selector, result)
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
return {
|
|
837
|
+
...result,
|
|
838
|
+
selector,
|
|
839
|
+
method: result.method || "selector",
|
|
840
|
+
browserSessionId: session.browserSessionId,
|
|
841
|
+
action: "fill",
|
|
842
|
+
target: { selector, method: "selector" },
|
|
843
|
+
ok: true,
|
|
844
|
+
recoverable: false,
|
|
845
|
+
nextStepHint: "Selector fill dispatched. Observe again, read page state, or evaluate the field value to verify the intended change.",
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
function buildSelectorFillFailureResult(session, selector, fillResult = {}) {
|
|
850
|
+
const error = fillResult?.error || "Selector fill failed"
|
|
851
|
+
const code = fillResult?.code || inferFillFailureCode(error, "SELECTOR_FILL_FAILED")
|
|
852
|
+
const recoveryHint = {
|
|
853
|
+
reason: "selector-fill-failed",
|
|
854
|
+
recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_evaluate_script", "yunti_fill"],
|
|
855
|
+
nextAction: code === "ELEMENT_NOT_FOUND" ? "observe-again" : "inspect-target-element",
|
|
856
|
+
decision: code === "ELEMENT_NOT_FOUND" ? "refresh-observation-or-selector-before-retry" : "inspect-editability-before-retry",
|
|
857
|
+
selector,
|
|
858
|
+
message: "The selector fill could not be completed. Inspect whether the selector still matches an editable element, observe again for a fresh uid, or evaluate the field before retrying.",
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
return {
|
|
862
|
+
...(fillResult && typeof fillResult === "object" ? fillResult : {}),
|
|
863
|
+
filled: false,
|
|
864
|
+
selector,
|
|
865
|
+
browserSessionId: session.browserSessionId,
|
|
866
|
+
action: "fill",
|
|
867
|
+
target: { selector, method: "selector" },
|
|
868
|
+
ok: false,
|
|
869
|
+
recoverable: true,
|
|
870
|
+
code,
|
|
871
|
+
error,
|
|
872
|
+
recoveryHint,
|
|
873
|
+
nextStepHint: "Selector fill failed. Observe again for a fresh uid, inspect whether the target is editable, or retry with a stable selector before repeating the same fill.",
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
function buildUidFillFailureResult(session, uid, fillResult = {}) {
|
|
878
|
+
const availableValues = Array.isArray(fillResult?.availableValues) ? fillResult.availableValues : undefined
|
|
879
|
+
const availableTexts = Array.isArray(fillResult?.availableTexts) ? fillResult.availableTexts : undefined
|
|
880
|
+
const error = fillResult?.error || `Cannot fill uid ${uid}`
|
|
881
|
+
const code = fillResult?.code || inferFillFailureCode(error, "UID_FILL_FAILED")
|
|
882
|
+
const recoveryHint = {
|
|
883
|
+
reason: "uid-fill-failed",
|
|
884
|
+
recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_evaluate_script", "yunti_fill"],
|
|
885
|
+
nextAction: "inspect-target-element",
|
|
886
|
+
decision: "inspect-editability-before-retry",
|
|
887
|
+
uid,
|
|
888
|
+
...(availableValues ? { availableValues } : {}),
|
|
889
|
+
...(availableTexts ? { availableTexts } : {}),
|
|
890
|
+
message: "The uid fill could not be completed. Refresh observation if the uid may be stale, inspect whether the target is editable, or retry with selector fallback.",
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
if (code === "ELEMENT_NOT_FOUND") {
|
|
894
|
+
recoveryHint.nextAction = "observe-again"
|
|
895
|
+
recoveryHint.decision = "refresh-observation-before-retry"
|
|
896
|
+
} else if (code === "OPTION_NOT_FOUND") {
|
|
897
|
+
recoveryHint.nextAction = "inspect-available-options"
|
|
898
|
+
recoveryHint.decision = "inspect-options-before-retry"
|
|
899
|
+
} else if (code === "VALUE_NOT_APPLIED") {
|
|
900
|
+
recoveryHint.nextAction = "verify-field-state"
|
|
901
|
+
recoveryHint.decision = "inspect-controlled-or-masked-field-before-retry"
|
|
902
|
+
recoveryHint.message = "The fill dispatched, but the field value did not remain afterward. Inspect whether the target is framework-controlled, masked, or requires typing/press_key semantics before retrying."
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
return {
|
|
906
|
+
...(fillResult && typeof fillResult === "object" ? fillResult : {}),
|
|
907
|
+
filled: false,
|
|
908
|
+
uid,
|
|
909
|
+
browserSessionId: session.browserSessionId,
|
|
910
|
+
action: "fill",
|
|
911
|
+
target: { uid, method: "uid" },
|
|
912
|
+
ok: false,
|
|
913
|
+
recoverable: true,
|
|
914
|
+
code,
|
|
915
|
+
error,
|
|
916
|
+
...(availableValues ? { availableValues } : {}),
|
|
917
|
+
...(availableTexts ? { availableTexts } : {}),
|
|
918
|
+
recoveryHint,
|
|
919
|
+
nextStepHint: "Uid fill failed. Observe again for a fresh uid, inspect whether the target is editable or a select with available options, or retry with selector fallback before repeating the same fill.",
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
function inferFillFailureCode(error, fallbackCode) {
|
|
924
|
+
const message = String(error || "").toLowerCase()
|
|
925
|
+
if (message.includes("not found") || message.includes("cannot resolve coordinates")) return "ELEMENT_NOT_FOUND"
|
|
926
|
+
if (message.includes("did not remain") || message.includes("did not stick") || message.includes("not applied")) return "VALUE_NOT_APPLIED"
|
|
927
|
+
if (
|
|
928
|
+
message.includes("not editable") ||
|
|
929
|
+
message.includes("no editable target") ||
|
|
930
|
+
message.includes("cannot accept text") ||
|
|
931
|
+
message.includes("disabled") ||
|
|
932
|
+
message.includes("read only") ||
|
|
933
|
+
message.includes("readonly") ||
|
|
934
|
+
message.includes("hidden") ||
|
|
935
|
+
message.includes("no size")
|
|
936
|
+
) return "TARGET_NOT_EDITABLE"
|
|
937
|
+
return fallbackCode
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
function buildFillEditabilityError(uid, elInfo = {}) {
|
|
941
|
+
const tag = elInfo?.tag || "unknown"
|
|
942
|
+
const type = elInfo?.type ? ` type=${elInfo.type}` : ""
|
|
943
|
+
const reasons = []
|
|
944
|
+
if (elInfo?.hidden) reasons.push("hidden or has no size")
|
|
945
|
+
if (elInfo?.disabled) reasons.push("disabled")
|
|
946
|
+
if (elInfo?.readOnly) reasons.push("readonly")
|
|
947
|
+
if (elInfo?.editable === false) reasons.push("not editable")
|
|
948
|
+
const reason = reasons.length ? reasons.join(", ") : "not editable"
|
|
949
|
+
return `Target at uid ${uid} is not editable (${tag}${type}: ${reason})`
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
function summarizeFillTarget(elInfo = {}) {
|
|
953
|
+
return {
|
|
954
|
+
tag: elInfo?.tag,
|
|
955
|
+
...(elInfo?.type ? { type: elInfo.type } : {}),
|
|
956
|
+
...(elInfo?.contentEditable !== undefined ? { contentEditable: Boolean(elInfo.contentEditable) } : {}),
|
|
957
|
+
...(elInfo?.disabled !== undefined ? { disabled: Boolean(elInfo.disabled) } : {}),
|
|
958
|
+
...(elInfo?.readOnly !== undefined ? { readOnly: Boolean(elInfo.readOnly) } : {}),
|
|
959
|
+
...(elInfo?.hidden !== undefined ? { hidden: Boolean(elInfo.hidden) } : {}),
|
|
960
|
+
...(elInfo?.editable !== undefined ? { editable: Boolean(elInfo.editable) } : {}),
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
async function scrollPage(tabId, session, args = {}) {
|
|
965
|
+
const uid = String(args.uid || "").trim()
|
|
966
|
+
let scrollArgs = args
|
|
967
|
+
if (uid) {
|
|
968
|
+
let point
|
|
969
|
+
try {
|
|
970
|
+
point = await resolveUidCenter(tabId, session, uid)
|
|
971
|
+
} catch (error) {
|
|
972
|
+
return buildUidScrollFailureResult(session, uid, {
|
|
973
|
+
error: error?.message || "Uid scroll failed",
|
|
974
|
+
})
|
|
975
|
+
}
|
|
976
|
+
const { x, y } = point
|
|
977
|
+
scrollArgs = { ...args, x, y }
|
|
978
|
+
}
|
|
979
|
+
const result = await chrome.tabs.sendMessage(tabId, {
|
|
626
980
|
type: "yunti_execute_tool",
|
|
627
|
-
tool: "
|
|
628
|
-
arguments:
|
|
981
|
+
tool: "yunti_scroll",
|
|
982
|
+
arguments: scrollArgs,
|
|
629
983
|
})
|
|
984
|
+
|
|
985
|
+
if (!result || typeof result !== "object" || result.scrolled !== true) {
|
|
986
|
+
return result
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
const hasComparableScrollPosition =
|
|
990
|
+
Number.isFinite(Number(result.before?.left)) &&
|
|
991
|
+
Number.isFinite(Number(result.before?.top)) &&
|
|
992
|
+
Number.isFinite(Number(result.after?.left)) &&
|
|
993
|
+
Number.isFinite(Number(result.after?.top))
|
|
994
|
+
const moved = hasComparableScrollPosition
|
|
995
|
+
? Number(result.before.left) !== Number(result.after.left) ||
|
|
996
|
+
Number(result.before.top) !== Number(result.after.top)
|
|
997
|
+
: undefined
|
|
998
|
+
const noMovement = moved === false
|
|
999
|
+
const edgeHint = noMovement ? inferScrollEdgeHint(result.deltaX, result.deltaY) : undefined
|
|
1000
|
+
const recoveryHint = noMovement ? buildScrollRecoveryHint(session, uid, edgeHint, result) : undefined
|
|
1001
|
+
const partialMovement = moved === true ? buildScrollPartialMovementHint(result, scrollArgs) : undefined
|
|
1002
|
+
const coordinateFallbackHint = !uid ? buildCoordinateScrollFallbackHint(result) : undefined
|
|
1003
|
+
|
|
1004
|
+
return {
|
|
1005
|
+
...result,
|
|
1006
|
+
browserSessionId: session.browserSessionId,
|
|
1007
|
+
action: "scroll",
|
|
1008
|
+
...(uid ? { uid, method: "uid" } : {}),
|
|
1009
|
+
...(uid ? { scrollTarget: { uid, method: "uid" } } : {}),
|
|
1010
|
+
...(moved !== undefined ? { moved } : {}),
|
|
1011
|
+
...(noMovement ? { code: "NO_SCROLL_MOVEMENT" } : {}),
|
|
1012
|
+
...(edgeHint ? { edgeHint } : {}),
|
|
1013
|
+
...(recoveryHint ? { recoveryHint } : {}),
|
|
1014
|
+
...(partialMovement ? { partialMovement } : {}),
|
|
1015
|
+
...(coordinateFallbackHint ? { coordinateFallbackHint } : {}),
|
|
1016
|
+
ok: !noMovement,
|
|
1017
|
+
recoverable: noMovement,
|
|
1018
|
+
nextStepHint: noMovement
|
|
1019
|
+
? `Scroll dispatched but before/after positions did not change${edgeHint ? ` (${edgeHint})` : ""}. Observe again, inspect scroll boundaries, try the nearest scrollable container uid, or stop repeating the same scroll.`
|
|
1020
|
+
: partialMovement
|
|
1021
|
+
? `Scroll moved partially${partialMovement.edgeHint ? ` (${partialMovement.edgeHint})` : ""}. Observe again and compare scroll positions before repeating the same scroll.`
|
|
1022
|
+
: coordinateFallbackHint
|
|
1023
|
+
? "Coordinate scroll fell back to document scrolling. Observe again, inspect scrollableContainers[], and prefer a fresh scrollable container uid if a nested panel was intended."
|
|
1024
|
+
: uid
|
|
1025
|
+
? "Uid-targeted scroll dispatched. Observe again or read page state to verify the intended container position."
|
|
1026
|
+
: "Scroll dispatched. Observe again or read page state to verify the intended viewport or container position.",
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
function buildCoordinateScrollFallbackHint(result = {}) {
|
|
1031
|
+
if (result.coordinateScrollFallback !== "document") return undefined
|
|
1032
|
+
return {
|
|
1033
|
+
reason: "coordinate-scroll-document-fallback",
|
|
1034
|
+
nextAction: "observe-for-scrollable-container",
|
|
1035
|
+
decision: "prefer-fresh-scrollable-container-uid",
|
|
1036
|
+
recommendedTools: ["yunti_observe_page", "yunti_scroll"],
|
|
1037
|
+
...(result.coordinateTarget ? { coordinateTarget: result.coordinateTarget } : {}),
|
|
1038
|
+
message: "The coordinate scroll did not find a nested scrollable container and fell back to document scrolling. Observe again and choose a fresh scrollableContainers[] uid when a panel or sidebar was intended.",
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
function buildUidScrollFailureResult(session, uid, scrollResult = {}) {
|
|
1043
|
+
const error = scrollResult?.error || `Cannot scroll uid ${uid}`
|
|
1044
|
+
const code = scrollResult?.code || inferScrollFailureCode(error)
|
|
1045
|
+
const recoveryHint = {
|
|
1046
|
+
reason: "uid-scroll-failed",
|
|
1047
|
+
recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_scroll"],
|
|
1048
|
+
nextAction: "observe-again",
|
|
1049
|
+
decision: "refresh-scrollable-container-uid-before-retry",
|
|
1050
|
+
uid,
|
|
1051
|
+
message: "The uid scroll could not resolve a current target. Refresh observation, choose a fresh scrollable container uid from scrollableContainers[], or use document/coordinate scroll fallback before retrying.",
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
return {
|
|
1055
|
+
...(scrollResult && typeof scrollResult === "object" ? scrollResult : {}),
|
|
1056
|
+
scrolled: false,
|
|
1057
|
+
uid,
|
|
1058
|
+
browserSessionId: session.browserSessionId,
|
|
1059
|
+
action: "scroll",
|
|
1060
|
+
target: { uid, method: "uid" },
|
|
1061
|
+
ok: false,
|
|
1062
|
+
recoverable: true,
|
|
1063
|
+
code,
|
|
1064
|
+
error,
|
|
1065
|
+
recoveryHint,
|
|
1066
|
+
nextStepHint: "Uid scroll failed. Observe again for a fresh scrollable container uid, inspect scrollableContainers[], or retry with document/coordinate fallback before repeating the same uid scroll.",
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
function inferScrollFailureCode(error) {
|
|
1071
|
+
const message = String(error || "").toLowerCase()
|
|
1072
|
+
if (message.includes("not found") || message.includes("uid") && message.includes("latest page uid map")) {
|
|
1073
|
+
return "UID_NOT_FOUND"
|
|
1074
|
+
}
|
|
1075
|
+
if (message.includes("cannot resolve coordinates") || message.includes("off-screen") || message.includes("hidden") || message.includes("stale")) {
|
|
1076
|
+
return "UID_COORDINATES_UNAVAILABLE"
|
|
1077
|
+
}
|
|
1078
|
+
return "UID_SCROLL_FAILED"
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
function inferScrollEdgeHint(deltaX, deltaY) {
|
|
1082
|
+
const x = Number(deltaX)
|
|
1083
|
+
const y = Number(deltaY)
|
|
1084
|
+
if (Number.isFinite(y) && y > 0) return "possible-bottom-edge"
|
|
1085
|
+
if (Number.isFinite(y) && y < 0) return "possible-top-edge"
|
|
1086
|
+
if (Number.isFinite(x) && x > 0) return "possible-right-edge"
|
|
1087
|
+
if (Number.isFinite(x) && x < 0) return "possible-left-edge"
|
|
1088
|
+
return "no-delta"
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
function buildScrollPartialMovementHint(result = {}, args = {}) {
|
|
1092
|
+
const beforeLeft = Number(result.before?.left)
|
|
1093
|
+
const beforeTop = Number(result.before?.top)
|
|
1094
|
+
const afterLeft = Number(result.after?.left)
|
|
1095
|
+
const afterTop = Number(result.after?.top)
|
|
1096
|
+
if (
|
|
1097
|
+
!Number.isFinite(beforeLeft) ||
|
|
1098
|
+
!Number.isFinite(beforeTop) ||
|
|
1099
|
+
!Number.isFinite(afterLeft) ||
|
|
1100
|
+
!Number.isFinite(afterTop)
|
|
1101
|
+
) return undefined
|
|
1102
|
+
|
|
1103
|
+
const requestedDeltaX = Number(result.deltaX ?? args.deltaX ?? 0)
|
|
1104
|
+
const requestedDeltaY = Number(result.deltaY ?? args.deltaY ?? 0)
|
|
1105
|
+
const actualDeltaX = afterLeft - beforeLeft
|
|
1106
|
+
const actualDeltaY = afterTop - beforeTop
|
|
1107
|
+
const axes = []
|
|
1108
|
+
if (isPartialScrollMovement(requestedDeltaX, actualDeltaX)) axes.push("horizontal")
|
|
1109
|
+
if (isPartialScrollMovement(requestedDeltaY, actualDeltaY)) axes.push("vertical")
|
|
1110
|
+
if (!axes.length) return undefined
|
|
1111
|
+
|
|
1112
|
+
return {
|
|
1113
|
+
reason: "partial-scroll-movement",
|
|
1114
|
+
axes,
|
|
1115
|
+
requestedDeltaX,
|
|
1116
|
+
requestedDeltaY,
|
|
1117
|
+
actualDeltaX,
|
|
1118
|
+
actualDeltaY,
|
|
1119
|
+
edgeHint: inferScrollEdgeHint(requestedDeltaX, requestedDeltaY),
|
|
1120
|
+
nextAction: "observe-again",
|
|
1121
|
+
decision: "observe-before-continuing-scroll",
|
|
1122
|
+
message: "The scroll position changed, but less than the requested delta. Observe again before repeating the same scroll to verify whether the target container hit an edge or a different container should be used.",
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
function isPartialScrollMovement(requestedDelta, actualDelta) {
|
|
1127
|
+
if (!Number.isFinite(requestedDelta) || !Number.isFinite(actualDelta)) return false
|
|
1128
|
+
if (requestedDelta === 0 || actualDelta === 0) return false
|
|
1129
|
+
if (Math.sign(requestedDelta) !== Math.sign(actualDelta)) return false
|
|
1130
|
+
return Math.abs(actualDelta) < Math.abs(requestedDelta)
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
function buildScrollRecoveryHint(session, uid, edgeHint, result = {}) {
|
|
1134
|
+
const hint = {
|
|
1135
|
+
reason: "no-scroll-movement",
|
|
1136
|
+
recommendedTools: ["yunti_observe_page", "yunti_scroll"],
|
|
1137
|
+
nextAction: "observe-again",
|
|
1138
|
+
}
|
|
1139
|
+
if (edgeHint) hint.edgeHint = edgeHint
|
|
1140
|
+
if (uid) {
|
|
1141
|
+
hint.uid = uid
|
|
1142
|
+
hint.currentTarget = "scrollable-container"
|
|
1143
|
+
const target = pageUidStore.get(session.browserSessionId)?.uidMap?.[uid]
|
|
1144
|
+
if (target) {
|
|
1145
|
+
hint.lastObservedContainer = {
|
|
1146
|
+
uid,
|
|
1147
|
+
canScrollVertical: Boolean(target.canScrollVertical),
|
|
1148
|
+
canScrollHorizontal: Boolean(target.canScrollHorizontal),
|
|
1149
|
+
...(Number.isFinite(Number(target.pixelsAbove)) ? { pixelsAbove: Number(target.pixelsAbove) } : {}),
|
|
1150
|
+
...(Number.isFinite(Number(target.pixelsBelow)) ? { pixelsBelow: Number(target.pixelsBelow) } : {}),
|
|
1151
|
+
...(Number.isFinite(Number(target.pixelsLeft)) ? { pixelsLeft: Number(target.pixelsLeft) } : {}),
|
|
1152
|
+
...(Number.isFinite(Number(target.pixelsRight)) ? { pixelsRight: Number(target.pixelsRight) } : {}),
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
} else {
|
|
1156
|
+
hint.currentTarget = "document-or-coordinate-container"
|
|
1157
|
+
}
|
|
1158
|
+
if (edgeHint === "possible-bottom-edge" || edgeHint === "possible-top-edge") {
|
|
1159
|
+
hint.nextAction = uid ? "try-opposite-direction-or-nearest-container" : "observe-for-scrollable-container"
|
|
1160
|
+
} else if (edgeHint === "possible-right-edge" || edgeHint === "possible-left-edge") {
|
|
1161
|
+
hint.nextAction = uid ? "try-opposite-horizontal-direction-or-nearest-container" : "observe-for-horizontal-container"
|
|
1162
|
+
} else if (edgeHint === "no-delta") {
|
|
1163
|
+
hint.nextAction = "provide-nonzero-scroll-delta"
|
|
1164
|
+
}
|
|
1165
|
+
const decision = buildScrollRecoveryDecision(uid, edgeHint)
|
|
1166
|
+
if (decision) hint.decision = decision
|
|
1167
|
+
const suggestedRetry = buildScrollSuggestedRetry(uid, edgeHint, result)
|
|
1168
|
+
if (suggestedRetry) hint.suggestedRetry = suggestedRetry
|
|
1169
|
+
hint.message = "The scroll command dispatched, but the scroll position did not change. Refresh observation before retrying, inspect scroll boundaries, or target a different scrollable container uid."
|
|
1170
|
+
return hint
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
function buildScrollRecoveryDecision(uid, edgeHint) {
|
|
1174
|
+
if (edgeHint === "no-delta") return "provide-nonzero-delta"
|
|
1175
|
+
if (edgeHint === "possible-bottom-edge" || edgeHint === "possible-top-edge") {
|
|
1176
|
+
return uid ? "retry-opposite-vertical-on-same-container-once" : "observe-for-scrollable-container"
|
|
1177
|
+
}
|
|
1178
|
+
if (edgeHint === "possible-right-edge" || edgeHint === "possible-left-edge") {
|
|
1179
|
+
return uid ? "retry-opposite-horizontal-on-same-container-once" : "observe-for-horizontal-scrollable-container"
|
|
1180
|
+
}
|
|
1181
|
+
return undefined
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
function buildScrollSuggestedRetry(uid, edgeHint, result = {}) {
|
|
1185
|
+
const retry = {}
|
|
1186
|
+
const deltaX = Number(result.deltaX)
|
|
1187
|
+
const deltaY = Number(result.deltaY)
|
|
1188
|
+
if (edgeHint === "possible-bottom-edge" || edgeHint === "possible-top-edge") {
|
|
1189
|
+
if (!Number.isFinite(deltaY) || deltaY === 0) return undefined
|
|
1190
|
+
retry.deltaY = -deltaY
|
|
1191
|
+
} else if (edgeHint === "possible-right-edge" || edgeHint === "possible-left-edge") {
|
|
1192
|
+
if (!Number.isFinite(deltaX) || deltaX === 0) return undefined
|
|
1193
|
+
retry.deltaX = -deltaX
|
|
1194
|
+
} else {
|
|
1195
|
+
return undefined
|
|
1196
|
+
}
|
|
1197
|
+
if (uid) retry.uid = uid
|
|
1198
|
+
retry.note = uid
|
|
1199
|
+
? "Try the opposite direction once on the same observed container, then observe again or switch to a nearer scrollable container if it still does not move."
|
|
1200
|
+
: "Observe first for a scrollable container uid; only use this opposite delta if document scrolling is still the intended target."
|
|
1201
|
+
return retry
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
async function selectElement(tabId, session, args = {}) {
|
|
1205
|
+
const uid = String(args.uid || "").trim()
|
|
1206
|
+
const value = String(args.value ?? "")
|
|
1207
|
+
const text = String(args.text ?? "")
|
|
1208
|
+
if (uid) {
|
|
1209
|
+
const matchMode = text ? "text" : "value"
|
|
1210
|
+
const targetOption = text || value
|
|
1211
|
+
const { x, y } = await resolveUidCenter(tabId, session, uid)
|
|
1212
|
+
await ensureCdpAttached(tabId, "1.3")
|
|
1213
|
+
const result = await chromeDebuggerSendCommand(
|
|
1214
|
+
{ tabId },
|
|
1215
|
+
"Runtime.evaluate",
|
|
1216
|
+
{
|
|
1217
|
+
expression: `(() => {
|
|
1218
|
+
const el = document.elementFromPoint(${x}, ${y});
|
|
1219
|
+
if (!el) return { ok: false, code: "ELEMENT_NOT_FOUND", error: "No element found at uid coordinates" };
|
|
1220
|
+
if (el.tagName.toLowerCase() !== "select") {
|
|
1221
|
+
return { ok: false, code: "NOT_SELECT", error: "Element at uid is not a select element" };
|
|
1222
|
+
}
|
|
1223
|
+
const options = Array.from(el.options);
|
|
1224
|
+
const option = options.find((item) => {
|
|
1225
|
+
if (${JSON.stringify(matchMode)} === "text") return item.text.trim() === ${JSON.stringify(targetOption)};
|
|
1226
|
+
return item.value === ${JSON.stringify(targetOption)};
|
|
1227
|
+
});
|
|
1228
|
+
if (!option) {
|
|
1229
|
+
return {
|
|
1230
|
+
ok: false,
|
|
1231
|
+
code: "OPTION_NOT_FOUND",
|
|
1232
|
+
error: ${JSON.stringify(matchMode)} === "text" ? "Option text not found" : "Option value not found",
|
|
1233
|
+
availableValues: options.map((item) => item.value).slice(0, 50),
|
|
1234
|
+
availableTexts: options.map((item) => item.text.trim()).slice(0, 50),
|
|
1235
|
+
};
|
|
1236
|
+
}
|
|
1237
|
+
if (option.disabled) {
|
|
1238
|
+
return {
|
|
1239
|
+
ok: false,
|
|
1240
|
+
code: "OPTION_DISABLED",
|
|
1241
|
+
error: ${JSON.stringify(matchMode)} === "text" ? "Option text is disabled" : "Option value is disabled",
|
|
1242
|
+
disabledValue: option.value,
|
|
1243
|
+
disabledText: option.text.trim(),
|
|
1244
|
+
availableValues: options.map((item) => item.value).slice(0, 50),
|
|
1245
|
+
availableTexts: options.map((item) => item.text.trim()).slice(0, 50),
|
|
1246
|
+
};
|
|
1247
|
+
}
|
|
1248
|
+
el.value = option.value;
|
|
1249
|
+
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
1250
|
+
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
1251
|
+
return {
|
|
1252
|
+
ok: true,
|
|
1253
|
+
value: option.value,
|
|
1254
|
+
selectedIndex: el.selectedIndex,
|
|
1255
|
+
optionText: option.text.slice(0, 80),
|
|
1256
|
+
};
|
|
1257
|
+
})()`,
|
|
1258
|
+
returnByValue: true,
|
|
1259
|
+
}
|
|
1260
|
+
)
|
|
1261
|
+
const selectResult = result?.result?.value
|
|
1262
|
+
if (!selectResult?.ok) {
|
|
1263
|
+
return buildUidSelectFailureResult(session, uid, matchMode, targetOption, selectResult)
|
|
1264
|
+
}
|
|
1265
|
+
return {
|
|
1266
|
+
selected: true,
|
|
1267
|
+
uid,
|
|
1268
|
+
value: selectResult.value,
|
|
1269
|
+
text: selectResult.optionText,
|
|
1270
|
+
selectedIndex: selectResult.selectedIndex,
|
|
1271
|
+
browserSessionId: session.browserSessionId,
|
|
1272
|
+
action: "select",
|
|
1273
|
+
target: { uid, method: matchMode === "text" ? "uid.text" : "uid.value" },
|
|
1274
|
+
ok: true,
|
|
1275
|
+
recoverable: false,
|
|
1276
|
+
nextStepHint: matchMode === "text"
|
|
1277
|
+
? "Uid select dispatched by visible option text. Observe again, read page state, or evaluate the select value to verify the intended change."
|
|
1278
|
+
: "Uid select dispatched by option value. Observe again, read page state, or evaluate the select value to verify the intended change.",
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
const selector = String(args.selector || "").trim()
|
|
1283
|
+
let result
|
|
1284
|
+
try {
|
|
1285
|
+
result = await chrome.tabs.sendMessage(tabId, {
|
|
1286
|
+
type: "yunti_execute_tool",
|
|
1287
|
+
tool: "yunti_select",
|
|
1288
|
+
arguments: args,
|
|
1289
|
+
})
|
|
1290
|
+
} catch (error) {
|
|
1291
|
+
return buildSelectorSelectFailureResult(session, selector, args.value, {
|
|
1292
|
+
code: "SELECTOR_SELECT_FAILED",
|
|
1293
|
+
error: error?.message || "Selector select failed",
|
|
1294
|
+
})
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
if (!result || typeof result !== "object" || result.selected !== true) {
|
|
1298
|
+
return buildSelectorSelectFailureResult(session, selector, args.value, result)
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
const requestedValue = String(args.value ?? "")
|
|
1302
|
+
const disabledOption = Array.isArray(result.options)
|
|
1303
|
+
? result.options.find((option) => String(option?.value ?? "") === requestedValue && option.disabled === true)
|
|
1304
|
+
: undefined
|
|
1305
|
+
if (requestedValue && disabledOption) {
|
|
1306
|
+
return buildSelectorSelectFailureResult(session, selector, requestedValue, {
|
|
1307
|
+
...result,
|
|
1308
|
+
selected: false,
|
|
1309
|
+
code: "OPTION_DISABLED",
|
|
1310
|
+
error: "Option value is disabled",
|
|
1311
|
+
disabledValue: String(disabledOption.value ?? ""),
|
|
1312
|
+
disabledText: String(disabledOption.text ?? ""),
|
|
1313
|
+
})
|
|
1314
|
+
}
|
|
1315
|
+
if (requestedValue && String(result.value ?? "") !== requestedValue) {
|
|
1316
|
+
return buildSelectorSelectFailureResult(session, selector, requestedValue, {
|
|
1317
|
+
...result,
|
|
1318
|
+
code: "OPTION_NOT_FOUND",
|
|
1319
|
+
error: "Option value not found",
|
|
1320
|
+
actualValue: result.value,
|
|
1321
|
+
})
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
return {
|
|
1325
|
+
...result,
|
|
1326
|
+
selector,
|
|
1327
|
+
browserSessionId: session.browserSessionId,
|
|
1328
|
+
action: "select",
|
|
1329
|
+
target: { selector, method: "selector" },
|
|
1330
|
+
ok: true,
|
|
1331
|
+
recoverable: false,
|
|
1332
|
+
nextStepHint: "Select dispatched. Observe again, read page state, or evaluate the select value to verify the intended change.",
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
function buildSelectorSelectFailureResult(session, selector, targetOption, selectResult = {}) {
|
|
1337
|
+
const availableValues = Array.isArray(selectResult?.availableValues) ? selectResult.availableValues : undefined
|
|
1338
|
+
const availableTexts = Array.isArray(selectResult?.availableTexts) ? selectResult.availableTexts : undefined
|
|
1339
|
+
const disabledValue = selectResult?.disabledValue !== undefined ? String(selectResult.disabledValue) : undefined
|
|
1340
|
+
const disabledText = selectResult?.disabledText !== undefined ? String(selectResult.disabledText) : undefined
|
|
1341
|
+
const code = selectResult?.code || (selectResult?.selected === false ? "SELECTOR_SELECT_FAILED" : "SELECTOR_SELECT_FAILED")
|
|
1342
|
+
const error = selectResult?.error || "Selector select failed"
|
|
1343
|
+
const recoveryHint = {
|
|
1344
|
+
reason: "selector-select-failed",
|
|
1345
|
+
recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_evaluate_script", "yunti_select"],
|
|
1346
|
+
nextAction: code === "OPTION_NOT_FOUND" || code === "OPTION_DISABLED" ? "inspect-available-options" : "inspect-target-element",
|
|
1347
|
+
decision: code === "OPTION_DISABLED" ? "choose-enabled-option-or-unlock-field" : code === "OPTION_NOT_FOUND" ? "inspect-options-before-retry" : "use-select-element-or-uid-fallback",
|
|
1348
|
+
selector,
|
|
1349
|
+
matchMode: "value",
|
|
1350
|
+
targetOption: String(targetOption ?? ""),
|
|
1351
|
+
...(availableValues ? { availableValues } : {}),
|
|
1352
|
+
...(availableTexts ? { availableTexts } : {}),
|
|
1353
|
+
...(disabledValue ? { disabledValue } : {}),
|
|
1354
|
+
...(disabledText ? { disabledText } : {}),
|
|
1355
|
+
message: "The selector select could not be completed. Inspect the target select element and available options before retrying, or use a fresh uid/value fallback.",
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
return {
|
|
1359
|
+
...(selectResult && typeof selectResult === "object" ? selectResult : {}),
|
|
1360
|
+
selected: false,
|
|
1361
|
+
selector,
|
|
1362
|
+
browserSessionId: session.browserSessionId,
|
|
1363
|
+
action: "select",
|
|
1364
|
+
target: { selector, method: "selector" },
|
|
1365
|
+
ok: false,
|
|
1366
|
+
recoverable: true,
|
|
1367
|
+
code,
|
|
1368
|
+
error,
|
|
1369
|
+
matchMode: "value",
|
|
1370
|
+
targetOption: String(targetOption ?? ""),
|
|
1371
|
+
...(availableValues ? { availableValues } : {}),
|
|
1372
|
+
...(availableTexts ? { availableTexts } : {}),
|
|
1373
|
+
recoveryHint,
|
|
1374
|
+
nextStepHint: "Selector select failed. Inspect available options, observe again for a fresh uid, or retry with uid/value fallback before repeating the same selector select.",
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
function buildUidSelectFailureResult(session, uid, matchMode, targetOption, selectResult = {}) {
|
|
1379
|
+
const availableValues = Array.isArray(selectResult.availableValues) ? selectResult.availableValues : undefined
|
|
1380
|
+
const availableTexts = Array.isArray(selectResult.availableTexts) ? selectResult.availableTexts : undefined
|
|
1381
|
+
const disabledValue = selectResult?.disabledValue !== undefined ? String(selectResult.disabledValue) : undefined
|
|
1382
|
+
const disabledText = selectResult?.disabledText !== undefined ? String(selectResult.disabledText) : undefined
|
|
1383
|
+
const recoveryHint = {
|
|
1384
|
+
reason: "uid-select-failed",
|
|
1385
|
+
recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_evaluate_script", "yunti_select"],
|
|
1386
|
+
nextAction: "inspect-available-options",
|
|
1387
|
+
decision: selectResult.code === "OPTION_DISABLED" ? "choose-enabled-option-or-unlock-field" : "inspect-options-before-retry",
|
|
1388
|
+
uid,
|
|
1389
|
+
matchMode,
|
|
1390
|
+
targetOption,
|
|
1391
|
+
...(availableValues ? { availableValues } : {}),
|
|
1392
|
+
...(availableTexts ? { availableTexts } : {}),
|
|
1393
|
+
...(disabledValue ? { disabledValue } : {}),
|
|
1394
|
+
...(disabledText ? { disabledText } : {}),
|
|
1395
|
+
message: "The select option could not be matched. Inspect available options before retrying, refresh observation if the uid may be stale, or use selector/value fallback.",
|
|
1396
|
+
}
|
|
1397
|
+
|
|
1398
|
+
if (selectResult.code === "ELEMENT_NOT_FOUND") {
|
|
1399
|
+
recoveryHint.nextAction = "observe-again"
|
|
1400
|
+
recoveryHint.decision = "refresh-observation-before-retry"
|
|
1401
|
+
} else if (selectResult.code === "NOT_SELECT") {
|
|
1402
|
+
recoveryHint.nextAction = "inspect-target-element"
|
|
1403
|
+
recoveryHint.decision = "use-select-element-or-selector-fallback"
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
return {
|
|
1407
|
+
selected: false,
|
|
1408
|
+
uid,
|
|
1409
|
+
browserSessionId: session.browserSessionId,
|
|
1410
|
+
action: "select",
|
|
1411
|
+
target: { uid, method: matchMode === "text" ? "uid.text" : "uid.value" },
|
|
1412
|
+
ok: false,
|
|
1413
|
+
recoverable: true,
|
|
1414
|
+
code: selectResult.code || "UID_SELECT_FAILED",
|
|
1415
|
+
error: selectResult.error || `Cannot select option at uid ${uid}`,
|
|
1416
|
+
matchMode,
|
|
1417
|
+
targetOption,
|
|
1418
|
+
...(availableValues ? { availableValues } : {}),
|
|
1419
|
+
...(availableTexts ? { availableTexts } : {}),
|
|
1420
|
+
...(disabledValue ? { disabledValue } : {}),
|
|
1421
|
+
...(disabledText ? { disabledText } : {}),
|
|
1422
|
+
recoveryHint,
|
|
1423
|
+
nextStepHint: "Uid select failed. Inspect available options, observe again for a fresh uid, or retry with selector/value fallback before repeating the same select.",
|
|
1424
|
+
}
|
|
630
1425
|
}
|
|
631
1426
|
|
|
632
1427
|
function validateFillArgs(args = {}) {
|
|
@@ -648,18 +1443,57 @@ export function createToolDispatcher({
|
|
|
648
1443
|
async function clickViaSnapshotUid(tabId, session, uid) {
|
|
649
1444
|
const { x, y } = await resolveUidCenter(tabId, session, uid)
|
|
650
1445
|
await mouseClick(tabId, x, y, 1)
|
|
651
|
-
|
|
1446
|
+
const roundedX = Math.round(x)
|
|
1447
|
+
const roundedY = Math.round(y)
|
|
1448
|
+
return {
|
|
1449
|
+
clicked: true,
|
|
1450
|
+
uid,
|
|
1451
|
+
x: roundedX,
|
|
1452
|
+
y: roundedY,
|
|
1453
|
+
browserSessionId: session.browserSessionId,
|
|
1454
|
+
action: "click",
|
|
1455
|
+
target: { uid, x: roundedX, y: roundedY },
|
|
1456
|
+
ok: true,
|
|
1457
|
+
recoverable: false,
|
|
1458
|
+
nextStepHint: "Click dispatched. Observe again or read page state to verify the intended change.",
|
|
1459
|
+
}
|
|
652
1460
|
}
|
|
653
1461
|
|
|
654
1462
|
async function hoverViaSnapshotUid(tabId, session, uid) {
|
|
655
1463
|
const { x, y } = await resolveUidCenter(tabId, session, uid)
|
|
656
1464
|
await mouseMove(tabId, x, y)
|
|
657
|
-
|
|
1465
|
+
const roundedX = Math.round(x)
|
|
1466
|
+
const roundedY = Math.round(y)
|
|
1467
|
+
return {
|
|
1468
|
+
hovered: true,
|
|
1469
|
+
uid,
|
|
1470
|
+
x: roundedX,
|
|
1471
|
+
y: roundedY,
|
|
1472
|
+
browserSessionId: session.browserSessionId,
|
|
1473
|
+
action: "hover",
|
|
1474
|
+
target: { uid, x: roundedX, y: roundedY },
|
|
1475
|
+
ok: true,
|
|
1476
|
+
recoverable: false,
|
|
1477
|
+
nextStepHint: "Hover dispatched. Observe again or read page state to verify menus, tooltips, or hover-only controls.",
|
|
1478
|
+
}
|
|
658
1479
|
}
|
|
659
1480
|
|
|
660
1481
|
async function clickAtCoordinate(tabId, session, x, y) {
|
|
661
1482
|
await mouseClick(tabId, x, y, 1)
|
|
662
|
-
|
|
1483
|
+
const roundedX = Math.round(x)
|
|
1484
|
+
const roundedY = Math.round(y)
|
|
1485
|
+
return {
|
|
1486
|
+
clicked: true,
|
|
1487
|
+
x: roundedX,
|
|
1488
|
+
y: roundedY,
|
|
1489
|
+
browserSessionId: session.browserSessionId,
|
|
1490
|
+
method: "coordinate",
|
|
1491
|
+
action: "click",
|
|
1492
|
+
target: { method: "coordinate", x: roundedX, y: roundedY },
|
|
1493
|
+
ok: true,
|
|
1494
|
+
recoverable: false,
|
|
1495
|
+
nextStepHint: "Coordinate click dispatched. Observe again, read page state, or use a fresh uid when possible to verify the intended change.",
|
|
1496
|
+
}
|
|
663
1497
|
}
|
|
664
1498
|
|
|
665
1499
|
async function resolveSelectorCenter(tabId, selector) {
|
|
@@ -695,9 +1529,12 @@ export function createToolDispatcher({
|
|
|
695
1529
|
}
|
|
696
1530
|
|
|
697
1531
|
async function resolveUidCenter(tabId, session, uid) {
|
|
698
|
-
const
|
|
1532
|
+
const uidState = pageUidStore.get(session.browserSessionId)
|
|
1533
|
+
const uidMap = uidState?.uidMap
|
|
699
1534
|
if (!uidMap || !uidMap[uid]) {
|
|
700
|
-
throw new Error(
|
|
1535
|
+
throw new Error(
|
|
1536
|
+
`uid ${uid} not found in the latest page uid map. Run yunti_observe_page or yunti_take_snapshot again before retrying.`
|
|
1537
|
+
)
|
|
701
1538
|
}
|
|
702
1539
|
const el = uidMap[uid]
|
|
703
1540
|
|
|
@@ -724,7 +1561,7 @@ export function createToolDispatcher({
|
|
|
724
1561
|
}
|
|
725
1562
|
}
|
|
726
1563
|
|
|
727
|
-
throw new Error(`Cannot resolve coordinates for uid ${uid}. Element may be off-screen or
|
|
1564
|
+
throw new Error(`Cannot resolve coordinates for uid ${uid}. Element may be off-screen, hidden, or stale. Run yunti_observe_page again before retrying.`)
|
|
728
1565
|
}
|
|
729
1566
|
|
|
730
1567
|
async function mouseClick(tabId, x, y, clickCount = 1) {
|
|
@@ -756,24 +1593,60 @@ export function createToolDispatcher({
|
|
|
756
1593
|
for (const field of fields) {
|
|
757
1594
|
try {
|
|
758
1595
|
const uid = String(field.uid || "").trim()
|
|
1596
|
+
let fieldResult
|
|
759
1597
|
if (uid) {
|
|
760
|
-
await fillByUid(tabId, session, { uid, value: String(field.value || "") })
|
|
1598
|
+
fieldResult = await fillByUid(tabId, session, { uid, value: String(field.value || "") })
|
|
761
1599
|
} else {
|
|
762
|
-
await
|
|
763
|
-
type: "yunti_execute_tool",
|
|
764
|
-
tool: "yunti_fill",
|
|
765
|
-
arguments: { selector: field.selector, value: field.value },
|
|
766
|
-
})
|
|
1600
|
+
fieldResult = await fillByUid(tabId, session, { selector: field.selector, value: field.value })
|
|
767
1601
|
}
|
|
768
|
-
|
|
769
|
-
|
|
1602
|
+
const normalized = buildFillFormFieldResult(field, fieldResult)
|
|
1603
|
+
if (normalized.ok) filled++
|
|
1604
|
+
else failed++
|
|
1605
|
+
results.push(normalized)
|
|
770
1606
|
} catch (err) {
|
|
771
1607
|
failed++
|
|
772
1608
|
results.push({ uid: field.uid, selector: field.selector, ok: false, error: err?.message || String(err) })
|
|
773
1609
|
}
|
|
774
1610
|
}
|
|
775
1611
|
|
|
776
|
-
return {
|
|
1612
|
+
return {
|
|
1613
|
+
filled,
|
|
1614
|
+
failed,
|
|
1615
|
+
results,
|
|
1616
|
+
browserSessionId: session.browserSessionId,
|
|
1617
|
+
action: "fill_form",
|
|
1618
|
+
target: {
|
|
1619
|
+
fieldCount: fields.length,
|
|
1620
|
+
filled,
|
|
1621
|
+
failed,
|
|
1622
|
+
},
|
|
1623
|
+
ok: failed === 0,
|
|
1624
|
+
recoverable: failed > 0,
|
|
1625
|
+
nextStepHint: failed === 0
|
|
1626
|
+
? "Form fill dispatched. Observe again, read page state, or evaluate field values to verify the intended changes."
|
|
1627
|
+
: "Form fill partially failed. Inspect per-field results, observe again for fresh uids, or retry failed fields with selector fallback.",
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
function buildFillFormFieldResult(field, fieldResult = {}) {
|
|
1632
|
+
const ok = Boolean(fieldResult?.filled || fieldResult?.ok === true)
|
|
1633
|
+
if (ok) return { uid: field.uid, selector: field.selector, ok: true }
|
|
1634
|
+
|
|
1635
|
+
const diagnostics = fieldResult && typeof fieldResult === "object" ? fieldResult : {}
|
|
1636
|
+
const preservedKeys = [
|
|
1637
|
+
"code",
|
|
1638
|
+
"error",
|
|
1639
|
+
"recoveryHint",
|
|
1640
|
+
"availableValues",
|
|
1641
|
+
"availableTexts",
|
|
1642
|
+
"nextStepHint",
|
|
1643
|
+
]
|
|
1644
|
+
const result = { uid: field.uid, selector: field.selector, ok: false }
|
|
1645
|
+
for (const key of preservedKeys) {
|
|
1646
|
+
if (diagnostics[key] !== undefined) result[key] = diagnostics[key]
|
|
1647
|
+
}
|
|
1648
|
+
if (!result.error) result.error = "fill failed"
|
|
1649
|
+
return result
|
|
777
1650
|
}
|
|
778
1651
|
|
|
779
1652
|
async function waitForCondition(tabId, session, args = {}) {
|
|
@@ -795,7 +1668,12 @@ export function createToolDispatcher({
|
|
|
795
1668
|
if (urlContains) {
|
|
796
1669
|
const tab = await chrome.tabs.get(tabId)
|
|
797
1670
|
if (tab.url && tab.url.includes(urlContains)) {
|
|
798
|
-
return
|
|
1671
|
+
return buildWaitForSuccess(
|
|
1672
|
+
session,
|
|
1673
|
+
{ text, selector, urlContains, timeoutMs },
|
|
1674
|
+
{ found: true, condition: "urlContains", value: urlContains },
|
|
1675
|
+
Date.now() - startTime
|
|
1676
|
+
)
|
|
799
1677
|
}
|
|
800
1678
|
}
|
|
801
1679
|
|
|
@@ -811,16 +1689,72 @@ export function createToolDispatcher({
|
|
|
811
1689
|
"Runtime.evaluate",
|
|
812
1690
|
{ expression, returnByValue: true }
|
|
813
1691
|
)
|
|
814
|
-
|
|
1692
|
+
|
|
815
1693
|
if (result?.result?.value) {
|
|
816
|
-
return
|
|
1694
|
+
return buildWaitForSuccess(
|
|
1695
|
+
session,
|
|
1696
|
+
{ text, selector, urlContains, timeoutMs },
|
|
1697
|
+
result.result.value,
|
|
1698
|
+
Date.now() - startTime
|
|
1699
|
+
)
|
|
817
1700
|
}
|
|
818
1701
|
}
|
|
819
1702
|
|
|
820
1703
|
await delayCdp(200)
|
|
821
1704
|
}
|
|
822
1705
|
|
|
823
|
-
return {
|
|
1706
|
+
return buildWaitForTimeout(session, { text, selector, urlContains, timeoutMs }, timeoutMs)
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
function buildWaitForTarget({ text, selector, urlContains, timeoutMs } = {}) {
|
|
1710
|
+
const target = {}
|
|
1711
|
+
if (text) target.text = text
|
|
1712
|
+
if (selector) target.selector = selector
|
|
1713
|
+
if (urlContains) target.urlContains = urlContains
|
|
1714
|
+
if (timeoutMs !== undefined) target.timeoutMs = timeoutMs
|
|
1715
|
+
return target
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
function buildWaitForSuccess(session, args, match = {}, waitedMs = 0) {
|
|
1719
|
+
const condition = typeof match.condition === "string"
|
|
1720
|
+
? match.condition
|
|
1721
|
+
: match.found === "selector" || match.found === "text"
|
|
1722
|
+
? match.found
|
|
1723
|
+
: undefined
|
|
1724
|
+
const result = {
|
|
1725
|
+
found: true,
|
|
1726
|
+
...match,
|
|
1727
|
+
waitedMs,
|
|
1728
|
+
browserSessionId: session.browserSessionId,
|
|
1729
|
+
action: "wait_for",
|
|
1730
|
+
target: buildWaitForTarget(args),
|
|
1731
|
+
ok: true,
|
|
1732
|
+
recoverable: false,
|
|
1733
|
+
nextStepHint: "Wait condition matched. Call yunti_observe_page and continue with a fresh uid before acting on newly rendered content.",
|
|
1734
|
+
}
|
|
1735
|
+
if (condition && result.condition === undefined) result.condition = condition
|
|
1736
|
+
return result
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1739
|
+
function buildWaitForTimeout(session, args, waitedMs) {
|
|
1740
|
+
return {
|
|
1741
|
+
found: false,
|
|
1742
|
+
waitedMs,
|
|
1743
|
+
browserSessionId: session.browserSessionId,
|
|
1744
|
+
action: "wait_for",
|
|
1745
|
+
target: buildWaitForTarget(args),
|
|
1746
|
+
ok: false,
|
|
1747
|
+
recoverable: true,
|
|
1748
|
+
code: "WAIT_TIMEOUT",
|
|
1749
|
+
recoveryHint: {
|
|
1750
|
+
reason: "condition-not-met-before-timeout",
|
|
1751
|
+
recommendedTools: ["yunti_observe_page", "yunti_get_page_snapshot", "yunti_take_screenshot"],
|
|
1752
|
+
nextAction: "observe-or-adjust-condition",
|
|
1753
|
+
decision: "observe-before-retry",
|
|
1754
|
+
message: "The expected text, selector, or URL state did not appear before the timeout. Observe the page or adjust the wait condition before repeating the same wait.",
|
|
1755
|
+
},
|
|
1756
|
+
nextStepHint: "Wait timed out. Observe the current page, inspect whether the condition changed, or adjust the wait target before retrying.",
|
|
1757
|
+
}
|
|
824
1758
|
}
|
|
825
1759
|
|
|
826
1760
|
async function handleDialog(tabId, session, args = {}) {
|
|
@@ -1000,7 +1934,20 @@ export function createToolDispatcher({
|
|
|
1000
1934
|
type: "mouseReleased", x: toX, y: toY, button, clickCount: 1,
|
|
1001
1935
|
})
|
|
1002
1936
|
|
|
1003
|
-
|
|
1937
|
+
const from = { x: Math.round(fromX), y: Math.round(fromY) }
|
|
1938
|
+
const to = { x: Math.round(toX), y: Math.round(toY) }
|
|
1939
|
+
return {
|
|
1940
|
+
dragged: true,
|
|
1941
|
+
from,
|
|
1942
|
+
to,
|
|
1943
|
+
steps,
|
|
1944
|
+
browserSessionId: session.browserSessionId,
|
|
1945
|
+
action: "drag",
|
|
1946
|
+
target: { method: "coordinate", from, to },
|
|
1947
|
+
ok: true,
|
|
1948
|
+
recoverable: false,
|
|
1949
|
+
nextStepHint: "Drag dispatched. Observe again or read page state to verify the intended movement or drop result.",
|
|
1950
|
+
}
|
|
1004
1951
|
}
|
|
1005
1952
|
|
|
1006
1953
|
async function uploadFile(tabId, session, args = {}) {
|
|
@@ -1036,18 +1983,52 @@ export function createToolDispatcher({
|
|
|
1036
1983
|
backendNodeId,
|
|
1037
1984
|
})
|
|
1038
1985
|
|
|
1039
|
-
return {
|
|
1986
|
+
return {
|
|
1987
|
+
uploaded: true,
|
|
1988
|
+
fileCount: filePaths.length,
|
|
1989
|
+
filePaths,
|
|
1990
|
+
browserSessionId: session.browserSessionId,
|
|
1991
|
+
action: "upload_file",
|
|
1992
|
+
target: uid ? { uid, method: "uid" } : { selector, method: "selector" },
|
|
1993
|
+
ok: true,
|
|
1994
|
+
recoverable: false,
|
|
1995
|
+
nextStepHint: "File upload dispatched. Observe again, read page state, or verify the selected file input before submitting any form.",
|
|
1996
|
+
}
|
|
1040
1997
|
}
|
|
1041
1998
|
|
|
1042
1999
|
async function typeTextByUid(tabId, session, args = {}) {
|
|
1043
2000
|
const uid = String(args.uid || "").trim()
|
|
1044
2001
|
if (!uid) {
|
|
1045
2002
|
// Fall back to content script for selector/coordinate-based typing
|
|
1046
|
-
|
|
2003
|
+
const selector = String(args.selector || "").trim()
|
|
2004
|
+
const hasX = Number.isFinite(Number(args.x))
|
|
2005
|
+
const hasY = Number.isFinite(Number(args.y))
|
|
2006
|
+
const result = await chrome.tabs.sendMessage(tabId, {
|
|
1047
2007
|
type: "yunti_execute_tool",
|
|
1048
2008
|
tool: "yunti_type_text",
|
|
1049
2009
|
arguments: args,
|
|
1050
2010
|
})
|
|
2011
|
+
|
|
2012
|
+
if (!result || typeof result !== "object" || result.typed !== true) {
|
|
2013
|
+
return result
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
const target =
|
|
2017
|
+
selector
|
|
2018
|
+
? { selector, method: "selector" }
|
|
2019
|
+
: hasX && hasY
|
|
2020
|
+
? { method: "coordinate", x: Math.round(Number(args.x)), y: Math.round(Number(args.y)) }
|
|
2021
|
+
: { method: "focused" }
|
|
2022
|
+
|
|
2023
|
+
return {
|
|
2024
|
+
...result,
|
|
2025
|
+
browserSessionId: session.browserSessionId,
|
|
2026
|
+
action: "type_text",
|
|
2027
|
+
target,
|
|
2028
|
+
ok: true,
|
|
2029
|
+
recoverable: false,
|
|
2030
|
+
nextStepHint: "Type text dispatched. Observe again, read page state, or evaluate the active field value to verify the intended change.",
|
|
2031
|
+
}
|
|
1051
2032
|
}
|
|
1052
2033
|
|
|
1053
2034
|
const text = String(args.text || "")
|
|
@@ -1067,17 +2048,52 @@ export function createToolDispatcher({
|
|
|
1067
2048
|
)
|
|
1068
2049
|
}
|
|
1069
2050
|
|
|
1070
|
-
return {
|
|
2051
|
+
return {
|
|
2052
|
+
typed: true,
|
|
2053
|
+
uid,
|
|
2054
|
+
text,
|
|
2055
|
+
method: "cdp.keyboard",
|
|
2056
|
+
browserSessionId: session.browserSessionId,
|
|
2057
|
+
action: "type_text",
|
|
2058
|
+
target: { uid, method: "cdp.keyboard" },
|
|
2059
|
+
ok: true,
|
|
2060
|
+
recoverable: false,
|
|
2061
|
+
nextStepHint: "Type text dispatched. Observe again, read page state, or evaluate the field value to verify the intended change.",
|
|
2062
|
+
}
|
|
1071
2063
|
}
|
|
1072
2064
|
|
|
1073
2065
|
async function pressKeyByUid(tabId, session, args = {}) {
|
|
1074
2066
|
const uid = String(args.uid || "").trim()
|
|
1075
2067
|
if (!uid) {
|
|
1076
|
-
|
|
2068
|
+
const selector = String(args.selector || "").trim()
|
|
2069
|
+
const hasX = Number.isFinite(Number(args.x))
|
|
2070
|
+
const hasY = Number.isFinite(Number(args.y))
|
|
2071
|
+
const result = await chrome.tabs.sendMessage(tabId, {
|
|
1077
2072
|
type: "yunti_execute_tool",
|
|
1078
2073
|
tool: "yunti_press_key",
|
|
1079
2074
|
arguments: args,
|
|
1080
2075
|
})
|
|
2076
|
+
|
|
2077
|
+
if (!result || typeof result !== "object" || result.pressed !== true) {
|
|
2078
|
+
return result
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
const target =
|
|
2082
|
+
selector
|
|
2083
|
+
? { selector, method: "selector" }
|
|
2084
|
+
: hasX && hasY
|
|
2085
|
+
? { method: "coordinate", x: Math.round(Number(args.x)), y: Math.round(Number(args.y)) }
|
|
2086
|
+
: { method: "focused" }
|
|
2087
|
+
|
|
2088
|
+
return {
|
|
2089
|
+
...result,
|
|
2090
|
+
browserSessionId: session.browserSessionId,
|
|
2091
|
+
action: "press_key",
|
|
2092
|
+
target,
|
|
2093
|
+
ok: true,
|
|
2094
|
+
recoverable: false,
|
|
2095
|
+
nextStepHint: "Key press dispatched. Observe again, read page state, or evaluate the active field value to verify the intended effect.",
|
|
2096
|
+
}
|
|
1081
2097
|
}
|
|
1082
2098
|
|
|
1083
2099
|
const key = String(args.key || "").trim()
|
|
@@ -1106,7 +2122,17 @@ export function createToolDispatcher({
|
|
|
1106
2122
|
nativeVirtualKeyCode: keyDef.keyCode,
|
|
1107
2123
|
})
|
|
1108
2124
|
|
|
1109
|
-
return {
|
|
2125
|
+
return {
|
|
2126
|
+
pressed: true,
|
|
2127
|
+
uid,
|
|
2128
|
+
key,
|
|
2129
|
+
browserSessionId: session.browserSessionId,
|
|
2130
|
+
action: "press_key",
|
|
2131
|
+
target: { uid, method: "cdp.keyboard" },
|
|
2132
|
+
ok: true,
|
|
2133
|
+
recoverable: false,
|
|
2134
|
+
nextStepHint: "Key press dispatched. Observe again, read page state, or evaluate the field value to verify the intended effect.",
|
|
2135
|
+
}
|
|
1110
2136
|
}
|
|
1111
2137
|
|
|
1112
2138
|
const KEY_MAP = {
|