yunti-browser-runtime 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -73,12 +73,16 @@ export function createToolDispatcher({
73
73
  result = await evaluateScript(tabId, session, event.arguments || {})
74
74
  } else if (event.tool === "yunti_take_snapshot") {
75
75
  result = await takeSnapshot(tabId, session, event.arguments || {})
76
+ } else if (event.tool === "yunti_observe_page") {
77
+ result = await observePage(tabId, session, event.arguments || {})
76
78
  } else if (event.tool === "yunti_click") {
77
79
  result = await clickByUid(tabId, session, event.arguments || {})
78
80
  } else if (event.tool === "yunti_hover") {
79
81
  result = await hoverByUid(tabId, session, event.arguments || {})
80
82
  } else if (event.tool === "yunti_fill") {
81
83
  result = await fillByUid(tabId, session, event.arguments || {})
84
+ } else if (event.tool === "yunti_select") {
85
+ result = await selectElement(tabId, session, event.arguments || {})
82
86
  } else if (event.tool === "yunti_fill_form") {
83
87
  result = await fillForm(tabId, session, event.arguments || {})
84
88
  } else if (event.tool === "yunti_wait_for") {
@@ -105,6 +109,8 @@ export function createToolDispatcher({
105
109
  result = await typeTextByUid(tabId, session, event.arguments || {})
106
110
  } else if (event.tool === "yunti_press_key") {
107
111
  result = await pressKeyByUid(tabId, session, event.arguments || {})
112
+ } else if (event.tool === "yunti_scroll") {
113
+ result = await scrollPage(tabId, session, event.arguments || {})
108
114
  } else {
109
115
  result = await chrome.tabs.sendMessage(tabId, {
110
116
  type: "yunti_execute_tool",
@@ -343,8 +349,8 @@ export function createToolDispatcher({
343
349
  }
344
350
  }
345
351
 
346
- // Snapshot state — maps uid to element info for click/fill/hover
347
- const snapshotStore = new Map()
352
+ // Latest page uid state from yunti_observe_page or yunti_take_snapshot.
353
+ const pageUidStore = new Map()
348
354
 
349
355
  async function takeSnapshot(tabId, session, args = {}) {
350
356
  const maxElements = Number.isFinite(Number(args.maxElements))
@@ -367,12 +373,7 @@ export function createToolDispatcher({
367
373
  elements = await domFallbackSnapshot(tabId, maxElements)
368
374
  }
369
375
 
370
- // Build uid mapping
371
- const uidMap = {}
372
- for (const el of elements) {
373
- uidMap[el.uid] = { ...el }
374
- }
375
- snapshotStore.set(session.browserSessionId, uidMap)
376
+ storePageUidMap(session, "snapshot", elements)
376
377
 
377
378
  return {
378
379
  browserSessionId: session.browserSessionId,
@@ -383,6 +384,38 @@ export function createToolDispatcher({
383
384
  snapshotId: `snap-${Date.now()}`,
384
385
  }
385
386
  }
387
+
388
+ async function observePage(tabId, session, args = {}) {
389
+ const observation = await chrome.tabs.sendMessage(tabId, {
390
+ type: "yunti_execute_tool",
391
+ tool: "yunti_observe_page",
392
+ arguments: args,
393
+ })
394
+ if (Array.isArray(observation?.elements)) {
395
+ storePageUidMap(session, "observe", [
396
+ ...observation.elements,
397
+ ...(Array.isArray(observation.scrollableContainers) ? observation.scrollableContainers : []),
398
+ ], {
399
+ observationId: observation.observationId,
400
+ uidMapVersion: observation.uidMapVersion,
401
+ })
402
+ }
403
+ return observation
404
+ }
405
+
406
+ function storePageUidMap(session, source, elements, meta = {}) {
407
+ const uidMap = {}
408
+ for (const el of elements || []) {
409
+ if (!el?.uid) continue
410
+ uidMap[el.uid] = { ...el, source }
411
+ }
412
+ pageUidStore.set(session.browserSessionId, {
413
+ source,
414
+ uidMap,
415
+ storedAt: Date.now(),
416
+ ...meta,
417
+ })
418
+ }
386
419
 
387
420
  function flattenAXTree(nodes, maxElements, includeHidden) {
388
421
  const elements = []
@@ -511,15 +544,30 @@ export function createToolDispatcher({
511
544
  }
512
545
  return clickAtCoordinate(tabId, session, x, y)
513
546
  }
514
- if (!String(args.selector || "").trim()) {
547
+ const selector = String(args.selector || "").trim()
548
+ if (!selector) {
515
549
  throw new Error("yunti_click requires uid, selector, or both x and y. Call yunti_take_snapshot to get uid, pass a CSS selector, or use yunti_click_at for coordinate-only clicks.")
516
550
  }
517
551
  // Fall back to content script for selector-based click
518
- return chrome.tabs.sendMessage(tabId, {
552
+ const contentResult = await chrome.tabs.sendMessage(tabId, {
519
553
  type: "yunti_execute_tool",
520
554
  tool: "yunti_click",
521
555
  arguments: args,
522
556
  })
557
+ if (contentResult && typeof contentResult === "object" && contentResult.clicked === true) {
558
+ return {
559
+ ...contentResult,
560
+ selector,
561
+ browserSessionId: session.browserSessionId,
562
+ method: "selector",
563
+ action: "click",
564
+ target: { selector, method: "selector" },
565
+ ok: true,
566
+ recoverable: false,
567
+ nextStepHint: "Selector click dispatched. Observe again, read page state, or use a fresh uid when possible to verify the intended change.",
568
+ }
569
+ }
570
+ return contentResult
523
571
  }
524
572
 
525
573
  async function hoverByUid(tabId, session, args = {}) {
@@ -534,7 +582,20 @@ export function createToolDispatcher({
534
582
  throw new Error("yunti_hover coordinate mode requires both x and y. Call yunti_take_snapshot to get uid, pass selector, or provide both coordinates.")
535
583
  }
536
584
  await mouseMove(tabId, x, y)
537
- return { hovered: true, x: Math.round(x), y: Math.round(y), browserSessionId: session.browserSessionId, method: "coordinate" }
585
+ const roundedX = Math.round(x)
586
+ const roundedY = Math.round(y)
587
+ return {
588
+ hovered: true,
589
+ x: roundedX,
590
+ y: roundedY,
591
+ browserSessionId: session.browserSessionId,
592
+ method: "coordinate",
593
+ action: "hover",
594
+ target: { method: "coordinate", x: roundedX, y: roundedY },
595
+ ok: true,
596
+ recoverable: false,
597
+ nextStepHint: "Coordinate hover dispatched. Observe again, read page state, or use a fresh uid when possible to verify menus, tooltips, or hover-only controls.",
598
+ }
538
599
  }
539
600
  const selector = String(args.selector || "").trim()
540
601
  if (!selector) {
@@ -542,13 +603,20 @@ export function createToolDispatcher({
542
603
  }
543
604
  const point = await resolveSelectorCenter(tabId, selector)
544
605
  await mouseMove(tabId, point.x, point.y)
606
+ const roundedX = Math.round(point.x)
607
+ const roundedY = Math.round(point.y)
545
608
  return {
546
609
  hovered: true,
547
610
  selector,
548
- x: Math.round(point.x),
549
- y: Math.round(point.y),
611
+ x: roundedX,
612
+ y: roundedY,
550
613
  browserSessionId: session.browserSessionId,
551
614
  method: "selector",
615
+ action: "hover",
616
+ target: { selector, method: "selector", x: roundedX, y: roundedY },
617
+ ok: true,
618
+ recoverable: false,
619
+ nextStepHint: "Selector hover dispatched. Observe again, read page state, or use a fresh uid when possible to verify menus, tooltips, or hover-only controls.",
552
620
  }
553
621
  }
554
622
 
@@ -557,76 +625,803 @@ export function createToolDispatcher({
557
625
  const uid = String(args.uid || "").trim()
558
626
  const value = String(args.value)
559
627
  if (uid) {
560
- const { x, y } = await resolveUidCenter(tabId, session, uid)
561
- await mouseClick(tabId, x, y, 1)
562
- await delayCdp(100)
563
- // Clear existing value
564
- const elements = await chromeDebuggerSendCommand(
565
- { tabId },
566
- "Runtime.evaluate",
567
- { expression: `(() => {
568
- const el = document.elementFromPoint(${x}, ${y});
569
- if (!el) return 'not found';
570
- const tag = el.tagName.toLowerCase();
571
- if (tag === 'select') {
572
- const opts = Array.from(el.options);
573
- return { tag: 'select', options: opts.map(o => ({ value: o.value, text: o.text.slice(0, 80) })), selectedIndex: el.selectedIndex };
574
- }
575
- if (el.isContentEditable) {
576
- el.textContent = '';
577
- } else if (tag === 'input' || tag === 'textarea') {
578
- el.focus();
579
- el.select();
628
+ try {
629
+ const { x, y } = await resolveUidCenter(tabId, session, uid)
630
+ await mouseClick(tabId, x, y, 1)
631
+ await delayCdp(100)
632
+ // Clear existing value
633
+ const elements = await chromeDebuggerSendCommand(
634
+ { tabId },
635
+ "Runtime.evaluate",
636
+ { expression: `(() => {
637
+ const el = document.elementFromPoint(${x}, ${y});
638
+ if (!el) return 'not found';
639
+ const tag = el.tagName.toLowerCase();
640
+ const rect = el.getBoundingClientRect();
641
+ const style = getComputedStyle(el);
642
+ const type = String(el.type || '').toLowerCase();
643
+ const disabled = Boolean(el.disabled || el.getAttribute('aria-disabled') === 'true');
644
+ const readOnly = Boolean(el.readOnly || el.getAttribute('aria-readonly') === 'true');
645
+ const hidden = Boolean(el.hidden || style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0 || rect.width <= 0 || rect.height <= 0);
646
+ const valueEditable = tag === 'input' || tag === 'textarea';
647
+ const blockedInputType = tag === 'input' && ['button', 'checkbox', 'color', 'file', 'hidden', 'image', 'radio', 'range', 'reset', 'submit'].includes(type);
648
+ const editable = el.isContentEditable || tag === 'select' || (valueEditable && !blockedInputType);
649
+ if (tag === 'select') {
650
+ const opts = Array.from(el.options);
651
+ return { tag: 'select', type, disabled, readOnly, hidden, editable, options: opts.map(o => ({ value: o.value, text: o.text.slice(0, 80) })), selectedIndex: el.selectedIndex };
652
+ }
653
+ const beforeText = el.isContentEditable ? String(el.textContent || '') : '';
654
+ if (el.isContentEditable) {
655
+ el.focus();
656
+ el.textContent = '';
657
+ } else if (valueEditable && !blockedInputType && !disabled && !readOnly && !hidden) {
658
+ el.focus();
659
+ el.select();
660
+ }
661
+ return {
662
+ tag,
663
+ type,
664
+ contentEditable: el.isContentEditable,
665
+ disabled,
666
+ readOnly,
667
+ hidden,
668
+ editable,
669
+ before: el.isContentEditable ? { textLength: beforeText.length } : undefined,
670
+ };
671
+ })()`,
672
+ returnByValue: true,
580
673
  }
581
- return { tag, type: el.type, contentEditable: el.isContentEditable };
582
- })()`,
583
- returnByValue: true,
674
+ )
675
+ const elInfo = elements?.result?.value
676
+
677
+ if (elInfo === "not found") {
678
+ return buildUidFillFailureResult(session, uid, {
679
+ code: "ELEMENT_NOT_FOUND",
680
+ error: `No element found at uid ${uid} coordinates`,
681
+ })
682
+ }
683
+
684
+ if (elInfo?.hidden || elInfo?.disabled || elInfo?.readOnly || elInfo?.editable === false) {
685
+ return buildUidFillFailureResult(session, uid, {
686
+ code: "TARGET_NOT_EDITABLE",
687
+ error: buildFillEditabilityError(uid, elInfo),
688
+ element: summarizeFillTarget(elInfo),
689
+ })
584
690
  }
585
- )
586
- const elInfo = elements?.result?.value
587
691
 
588
- if (elInfo === "not found") throw new Error(`No element found at uid ${uid} coordinates`)
692
+ // Handle select element
693
+ if (elInfo?.tag === "select") {
694
+ const targetOption = elInfo.options?.find(
695
+ o => o.value === value || o.text === value
696
+ )
697
+ if (targetOption) {
698
+ await chromeDebuggerSendCommand(
699
+ { tabId },
700
+ "Runtime.evaluate",
701
+ {
702
+ expression: `(() => {
703
+ const el = document.elementFromPoint(${x}, ${y});
704
+ if (el) el.value = ${JSON.stringify(targetOption.value)};
705
+ el.dispatchEvent(new Event('change', { bubbles: true }));
706
+ el.dispatchEvent(new Event('input', { bubbles: true }));
707
+ })()`,
708
+ }
709
+ )
710
+ return {
711
+ filled: true,
712
+ uid,
713
+ method: "select",
714
+ value: targetOption.value,
715
+ browserSessionId: session.browserSessionId,
716
+ action: "fill",
717
+ target: { uid, method: "select" },
718
+ ok: true,
719
+ recoverable: false,
720
+ nextStepHint: "Select value dispatched. Observe again, read page state, or evaluate the select value to verify the intended change.",
721
+ }
722
+ }
723
+ return buildUidFillFailureResult(session, uid, {
724
+ code: "OPTION_NOT_FOUND",
725
+ error: `Option '${value}' not found in select at uid ${uid}`,
726
+ availableValues: elInfo.options?.map((option) => option.value).slice(0, 50),
727
+ availableTexts: elInfo.options?.map((option) => option.text).slice(0, 50),
728
+ })
729
+ }
589
730
 
590
- // Handle select element
591
- if (elInfo?.tag === "select") {
592
- const targetOption = elInfo.options?.find(
593
- o => o.value === value || o.text === value
594
- )
595
- if (targetOption) {
731
+ // Type text via CDP Input.dispatchKeyEvent
732
+ const text = value
733
+ for (const char of text) {
596
734
  await chromeDebuggerSendCommand(
597
735
  { tabId },
598
- "Runtime.evaluate",
599
- {
600
- expression: `(() => {
601
- const el = document.elementFromPoint(${x}, ${y});
602
- if (el) el.value = ${JSON.stringify(targetOption.value)};
603
- el.dispatchEvent(new Event('change', { bubbles: true }));
604
- el.dispatchEvent(new Event('input', { bubbles: true }));
605
- })()`,
606
- }
736
+ "Input.dispatchKeyEvent",
737
+ { type: "char", text: char, unmodifiedText: char }
607
738
  )
608
- return { filled: true, uid, method: "select", value: targetOption.value, browserSessionId: session.browserSessionId }
609
739
  }
610
- throw new Error(`Option '${value}' not found in select at uid ${uid}. Use yunti_take_snapshot to inspect the select element or pass an available option value/text.`)
611
- }
612
-
613
- // Type text via CDP Input.dispatchKeyEvent
614
- const text = value
615
- for (const char of text) {
616
- await chromeDebuggerSendCommand(
740
+ const method = elInfo?.contentEditable ? "contenteditable" : "keyboard"
741
+ const before = elInfo?.before
742
+ await delayCdp(50)
743
+ const verification = await chromeDebuggerSendCommand(
617
744
  { tabId },
618
- "Input.dispatchKeyEvent",
619
- { type: "char", text: char, unmodifiedText: char }
745
+ "Runtime.evaluate",
746
+ {
747
+ expression: `(() => {
748
+ const el = document.elementFromPoint(${x}, ${y});
749
+ if (!el) return { found: false };
750
+ const tag = el.tagName.toLowerCase();
751
+ const type = String(el.type || '').toLowerCase();
752
+ const expected = ${JSON.stringify(text)};
753
+ const actual = el.isContentEditable
754
+ ? String(el.textContent || '')
755
+ : (tag === 'input' || tag === 'textarea')
756
+ ? String(el.value || '')
757
+ : undefined;
758
+ if (actual === undefined) {
759
+ return { found: true, tag, type, contentEditable: el.isContentEditable };
760
+ }
761
+ return {
762
+ found: true,
763
+ tag,
764
+ type,
765
+ contentEditable: el.isContentEditable,
766
+ valueMatches: actual === expected,
767
+ valueLength: actual.length,
768
+ expectedLength: expected.length,
769
+ };
770
+ })()`,
771
+ returnByValue: true,
772
+ }
620
773
  )
621
- }
622
- return { filled: true, uid, method: "keyboard", value: text, browserSessionId: session.browserSessionId }
774
+ const verificationInfo = verification?.result?.value
775
+ const verifiedAfter = verificationInfo?.found && Number.isFinite(Number(verificationInfo.valueLength))
776
+ ? { textLength: Number(verificationInfo.valueLength) }
777
+ : undefined
778
+ const after = verifiedAfter || (elInfo?.contentEditable ? { textLength: text.length } : undefined)
779
+ if (verificationInfo?.found && verificationInfo.valueMatches === false) {
780
+ return buildUidFillFailureResult(session, uid, {
781
+ code: "VALUE_NOT_APPLIED",
782
+ error: `Filled value did not remain on uid ${uid}`,
783
+ method,
784
+ expectedValueLength: Number(verificationInfo.expectedLength),
785
+ actualValueLength: Number(verificationInfo.valueLength),
786
+ ...(before ? { before } : {}),
787
+ after,
788
+ element: summarizeFillTarget({
789
+ tag: verificationInfo.tag,
790
+ type: verificationInfo.type,
791
+ contentEditable: verificationInfo.contentEditable,
792
+ }),
793
+ })
794
+ }
795
+ return {
796
+ filled: true,
797
+ uid,
798
+ method,
799
+ value: text,
800
+ ...(before ? { before } : {}),
801
+ ...(after ? { after } : {}),
802
+ browserSessionId: session.browserSessionId,
803
+ action: "fill",
804
+ target: { uid, method },
805
+ ok: true,
806
+ recoverable: false,
807
+ nextStepHint: elInfo?.contentEditable
808
+ ? "Contenteditable fill dispatched. Observe again, read page text, or evaluate textContent to verify the intended change."
809
+ : "Fill dispatched. Observe again, read page state, or evaluate the field value to verify the intended change.",
810
+ }
811
+ } catch (error) {
812
+ return buildUidFillFailureResult(session, uid, {
813
+ error: error?.message || "Uid fill failed",
814
+ })
815
+ }
623
816
  }
624
817
 
625
- return chrome.tabs.sendMessage(tabId, {
818
+ const selector = String(args.selector || "").trim()
819
+ let result
820
+ try {
821
+ result = await chrome.tabs.sendMessage(tabId, {
822
+ type: "yunti_execute_tool",
823
+ tool: "yunti_fill",
824
+ arguments: args,
825
+ })
826
+ } catch (error) {
827
+ return buildSelectorFillFailureResult(session, selector, {
828
+ error: error?.message || "Selector fill failed",
829
+ })
830
+ }
831
+
832
+ if (!result || typeof result !== "object" || result.filled !== true) {
833
+ return buildSelectorFillFailureResult(session, selector, result)
834
+ }
835
+
836
+ return {
837
+ ...result,
838
+ selector,
839
+ method: result.method || "selector",
840
+ browserSessionId: session.browserSessionId,
841
+ action: "fill",
842
+ target: { selector, method: "selector" },
843
+ ok: true,
844
+ recoverable: false,
845
+ nextStepHint: "Selector fill dispatched. Observe again, read page state, or evaluate the field value to verify the intended change.",
846
+ }
847
+ }
848
+
849
+ function buildSelectorFillFailureResult(session, selector, fillResult = {}) {
850
+ const error = fillResult?.error || "Selector fill failed"
851
+ const code = fillResult?.code || inferFillFailureCode(error, "SELECTOR_FILL_FAILED")
852
+ const recoveryHint = {
853
+ reason: "selector-fill-failed",
854
+ recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_evaluate_script", "yunti_fill"],
855
+ nextAction: code === "ELEMENT_NOT_FOUND" ? "observe-again" : "inspect-target-element",
856
+ decision: code === "ELEMENT_NOT_FOUND" ? "refresh-observation-or-selector-before-retry" : "inspect-editability-before-retry",
857
+ selector,
858
+ message: "The selector fill could not be completed. Inspect whether the selector still matches an editable element, observe again for a fresh uid, or evaluate the field before retrying.",
859
+ }
860
+
861
+ return {
862
+ ...(fillResult && typeof fillResult === "object" ? fillResult : {}),
863
+ filled: false,
864
+ selector,
865
+ browserSessionId: session.browserSessionId,
866
+ action: "fill",
867
+ target: { selector, method: "selector" },
868
+ ok: false,
869
+ recoverable: true,
870
+ code,
871
+ error,
872
+ recoveryHint,
873
+ nextStepHint: "Selector fill failed. Observe again for a fresh uid, inspect whether the target is editable, or retry with a stable selector before repeating the same fill.",
874
+ }
875
+ }
876
+
877
+ function buildUidFillFailureResult(session, uid, fillResult = {}) {
878
+ const availableValues = Array.isArray(fillResult?.availableValues) ? fillResult.availableValues : undefined
879
+ const availableTexts = Array.isArray(fillResult?.availableTexts) ? fillResult.availableTexts : undefined
880
+ const error = fillResult?.error || `Cannot fill uid ${uid}`
881
+ const code = fillResult?.code || inferFillFailureCode(error, "UID_FILL_FAILED")
882
+ const recoveryHint = {
883
+ reason: "uid-fill-failed",
884
+ recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_evaluate_script", "yunti_fill"],
885
+ nextAction: "inspect-target-element",
886
+ decision: "inspect-editability-before-retry",
887
+ uid,
888
+ ...(availableValues ? { availableValues } : {}),
889
+ ...(availableTexts ? { availableTexts } : {}),
890
+ message: "The uid fill could not be completed. Refresh observation if the uid may be stale, inspect whether the target is editable, or retry with selector fallback.",
891
+ }
892
+
893
+ if (code === "ELEMENT_NOT_FOUND") {
894
+ recoveryHint.nextAction = "observe-again"
895
+ recoveryHint.decision = "refresh-observation-before-retry"
896
+ } else if (code === "OPTION_NOT_FOUND") {
897
+ recoveryHint.nextAction = "inspect-available-options"
898
+ recoveryHint.decision = "inspect-options-before-retry"
899
+ } else if (code === "VALUE_NOT_APPLIED") {
900
+ recoveryHint.nextAction = "verify-field-state"
901
+ recoveryHint.decision = "inspect-controlled-or-masked-field-before-retry"
902
+ recoveryHint.message = "The fill dispatched, but the field value did not remain afterward. Inspect whether the target is framework-controlled, masked, or requires typing/press_key semantics before retrying."
903
+ }
904
+
905
+ return {
906
+ ...(fillResult && typeof fillResult === "object" ? fillResult : {}),
907
+ filled: false,
908
+ uid,
909
+ browserSessionId: session.browserSessionId,
910
+ action: "fill",
911
+ target: { uid, method: "uid" },
912
+ ok: false,
913
+ recoverable: true,
914
+ code,
915
+ error,
916
+ ...(availableValues ? { availableValues } : {}),
917
+ ...(availableTexts ? { availableTexts } : {}),
918
+ recoveryHint,
919
+ nextStepHint: "Uid fill failed. Observe again for a fresh uid, inspect whether the target is editable or a select with available options, or retry with selector fallback before repeating the same fill.",
920
+ }
921
+ }
922
+
923
+ function inferFillFailureCode(error, fallbackCode) {
924
+ const message = String(error || "").toLowerCase()
925
+ if (message.includes("not found") || message.includes("cannot resolve coordinates")) return "ELEMENT_NOT_FOUND"
926
+ if (message.includes("did not remain") || message.includes("did not stick") || message.includes("not applied")) return "VALUE_NOT_APPLIED"
927
+ if (
928
+ message.includes("not editable") ||
929
+ message.includes("no editable target") ||
930
+ message.includes("cannot accept text") ||
931
+ message.includes("disabled") ||
932
+ message.includes("read only") ||
933
+ message.includes("readonly") ||
934
+ message.includes("hidden") ||
935
+ message.includes("no size")
936
+ ) return "TARGET_NOT_EDITABLE"
937
+ return fallbackCode
938
+ }
939
+
940
+ function buildFillEditabilityError(uid, elInfo = {}) {
941
+ const tag = elInfo?.tag || "unknown"
942
+ const type = elInfo?.type ? ` type=${elInfo.type}` : ""
943
+ const reasons = []
944
+ if (elInfo?.hidden) reasons.push("hidden or has no size")
945
+ if (elInfo?.disabled) reasons.push("disabled")
946
+ if (elInfo?.readOnly) reasons.push("readonly")
947
+ if (elInfo?.editable === false) reasons.push("not editable")
948
+ const reason = reasons.length ? reasons.join(", ") : "not editable"
949
+ return `Target at uid ${uid} is not editable (${tag}${type}: ${reason})`
950
+ }
951
+
952
+ function summarizeFillTarget(elInfo = {}) {
953
+ return {
954
+ tag: elInfo?.tag,
955
+ ...(elInfo?.type ? { type: elInfo.type } : {}),
956
+ ...(elInfo?.contentEditable !== undefined ? { contentEditable: Boolean(elInfo.contentEditable) } : {}),
957
+ ...(elInfo?.disabled !== undefined ? { disabled: Boolean(elInfo.disabled) } : {}),
958
+ ...(elInfo?.readOnly !== undefined ? { readOnly: Boolean(elInfo.readOnly) } : {}),
959
+ ...(elInfo?.hidden !== undefined ? { hidden: Boolean(elInfo.hidden) } : {}),
960
+ ...(elInfo?.editable !== undefined ? { editable: Boolean(elInfo.editable) } : {}),
961
+ }
962
+ }
963
+
964
+ async function scrollPage(tabId, session, args = {}) {
965
+ const uid = String(args.uid || "").trim()
966
+ let scrollArgs = args
967
+ if (uid) {
968
+ let point
969
+ try {
970
+ point = await resolveUidCenter(tabId, session, uid)
971
+ } catch (error) {
972
+ return buildUidScrollFailureResult(session, uid, {
973
+ error: error?.message || "Uid scroll failed",
974
+ })
975
+ }
976
+ const { x, y } = point
977
+ scrollArgs = { ...args, x, y }
978
+ }
979
+ const result = await chrome.tabs.sendMessage(tabId, {
626
980
  type: "yunti_execute_tool",
627
- tool: "yunti_fill",
628
- arguments: args,
981
+ tool: "yunti_scroll",
982
+ arguments: scrollArgs,
629
983
  })
984
+
985
+ if (!result || typeof result !== "object" || result.scrolled !== true) {
986
+ return result
987
+ }
988
+
989
+ const hasComparableScrollPosition =
990
+ Number.isFinite(Number(result.before?.left)) &&
991
+ Number.isFinite(Number(result.before?.top)) &&
992
+ Number.isFinite(Number(result.after?.left)) &&
993
+ Number.isFinite(Number(result.after?.top))
994
+ const moved = hasComparableScrollPosition
995
+ ? Number(result.before.left) !== Number(result.after.left) ||
996
+ Number(result.before.top) !== Number(result.after.top)
997
+ : undefined
998
+ const noMovement = moved === false
999
+ const edgeHint = noMovement ? inferScrollEdgeHint(result.deltaX, result.deltaY) : undefined
1000
+ const recoveryHint = noMovement ? buildScrollRecoveryHint(session, uid, edgeHint, result) : undefined
1001
+ const partialMovement = moved === true ? buildScrollPartialMovementHint(result, scrollArgs) : undefined
1002
+ const coordinateFallbackHint = !uid ? buildCoordinateScrollFallbackHint(result) : undefined
1003
+
1004
+ return {
1005
+ ...result,
1006
+ browserSessionId: session.browserSessionId,
1007
+ action: "scroll",
1008
+ ...(uid ? { uid, method: "uid" } : {}),
1009
+ ...(uid ? { scrollTarget: { uid, method: "uid" } } : {}),
1010
+ ...(moved !== undefined ? { moved } : {}),
1011
+ ...(noMovement ? { code: "NO_SCROLL_MOVEMENT" } : {}),
1012
+ ...(edgeHint ? { edgeHint } : {}),
1013
+ ...(recoveryHint ? { recoveryHint } : {}),
1014
+ ...(partialMovement ? { partialMovement } : {}),
1015
+ ...(coordinateFallbackHint ? { coordinateFallbackHint } : {}),
1016
+ ok: !noMovement,
1017
+ recoverable: noMovement,
1018
+ nextStepHint: noMovement
1019
+ ? `Scroll dispatched but before/after positions did not change${edgeHint ? ` (${edgeHint})` : ""}. Observe again, inspect scroll boundaries, try the nearest scrollable container uid, or stop repeating the same scroll.`
1020
+ : partialMovement
1021
+ ? `Scroll moved partially${partialMovement.edgeHint ? ` (${partialMovement.edgeHint})` : ""}. Observe again and compare scroll positions before repeating the same scroll.`
1022
+ : coordinateFallbackHint
1023
+ ? "Coordinate scroll fell back to document scrolling. Observe again, inspect scrollableContainers[], and prefer a fresh scrollable container uid if a nested panel was intended."
1024
+ : uid
1025
+ ? "Uid-targeted scroll dispatched. Observe again or read page state to verify the intended container position."
1026
+ : "Scroll dispatched. Observe again or read page state to verify the intended viewport or container position.",
1027
+ }
1028
+ }
1029
+
1030
+ function buildCoordinateScrollFallbackHint(result = {}) {
1031
+ if (result.coordinateScrollFallback !== "document") return undefined
1032
+ return {
1033
+ reason: "coordinate-scroll-document-fallback",
1034
+ nextAction: "observe-for-scrollable-container",
1035
+ decision: "prefer-fresh-scrollable-container-uid",
1036
+ recommendedTools: ["yunti_observe_page", "yunti_scroll"],
1037
+ ...(result.coordinateTarget ? { coordinateTarget: result.coordinateTarget } : {}),
1038
+ message: "The coordinate scroll did not find a nested scrollable container and fell back to document scrolling. Observe again and choose a fresh scrollableContainers[] uid when a panel or sidebar was intended.",
1039
+ }
1040
+ }
1041
+
1042
+ function buildUidScrollFailureResult(session, uid, scrollResult = {}) {
1043
+ const error = scrollResult?.error || `Cannot scroll uid ${uid}`
1044
+ const code = scrollResult?.code || inferScrollFailureCode(error)
1045
+ const recoveryHint = {
1046
+ reason: "uid-scroll-failed",
1047
+ recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_scroll"],
1048
+ nextAction: "observe-again",
1049
+ decision: "refresh-scrollable-container-uid-before-retry",
1050
+ uid,
1051
+ message: "The uid scroll could not resolve a current target. Refresh observation, choose a fresh scrollable container uid from scrollableContainers[], or use document/coordinate scroll fallback before retrying.",
1052
+ }
1053
+
1054
+ return {
1055
+ ...(scrollResult && typeof scrollResult === "object" ? scrollResult : {}),
1056
+ scrolled: false,
1057
+ uid,
1058
+ browserSessionId: session.browserSessionId,
1059
+ action: "scroll",
1060
+ target: { uid, method: "uid" },
1061
+ ok: false,
1062
+ recoverable: true,
1063
+ code,
1064
+ error,
1065
+ recoveryHint,
1066
+ nextStepHint: "Uid scroll failed. Observe again for a fresh scrollable container uid, inspect scrollableContainers[], or retry with document/coordinate fallback before repeating the same uid scroll.",
1067
+ }
1068
+ }
1069
+
1070
+ function inferScrollFailureCode(error) {
1071
+ const message = String(error || "").toLowerCase()
1072
+ if (message.includes("not found") || message.includes("uid") && message.includes("latest page uid map")) {
1073
+ return "UID_NOT_FOUND"
1074
+ }
1075
+ if (message.includes("cannot resolve coordinates") || message.includes("off-screen") || message.includes("hidden") || message.includes("stale")) {
1076
+ return "UID_COORDINATES_UNAVAILABLE"
1077
+ }
1078
+ return "UID_SCROLL_FAILED"
1079
+ }
1080
+
1081
+ function inferScrollEdgeHint(deltaX, deltaY) {
1082
+ const x = Number(deltaX)
1083
+ const y = Number(deltaY)
1084
+ if (Number.isFinite(y) && y > 0) return "possible-bottom-edge"
1085
+ if (Number.isFinite(y) && y < 0) return "possible-top-edge"
1086
+ if (Number.isFinite(x) && x > 0) return "possible-right-edge"
1087
+ if (Number.isFinite(x) && x < 0) return "possible-left-edge"
1088
+ return "no-delta"
1089
+ }
1090
+
1091
+ function buildScrollPartialMovementHint(result = {}, args = {}) {
1092
+ const beforeLeft = Number(result.before?.left)
1093
+ const beforeTop = Number(result.before?.top)
1094
+ const afterLeft = Number(result.after?.left)
1095
+ const afterTop = Number(result.after?.top)
1096
+ if (
1097
+ !Number.isFinite(beforeLeft) ||
1098
+ !Number.isFinite(beforeTop) ||
1099
+ !Number.isFinite(afterLeft) ||
1100
+ !Number.isFinite(afterTop)
1101
+ ) return undefined
1102
+
1103
+ const requestedDeltaX = Number(result.deltaX ?? args.deltaX ?? 0)
1104
+ const requestedDeltaY = Number(result.deltaY ?? args.deltaY ?? 0)
1105
+ const actualDeltaX = afterLeft - beforeLeft
1106
+ const actualDeltaY = afterTop - beforeTop
1107
+ const axes = []
1108
+ if (isPartialScrollMovement(requestedDeltaX, actualDeltaX)) axes.push("horizontal")
1109
+ if (isPartialScrollMovement(requestedDeltaY, actualDeltaY)) axes.push("vertical")
1110
+ if (!axes.length) return undefined
1111
+
1112
+ return {
1113
+ reason: "partial-scroll-movement",
1114
+ axes,
1115
+ requestedDeltaX,
1116
+ requestedDeltaY,
1117
+ actualDeltaX,
1118
+ actualDeltaY,
1119
+ edgeHint: inferScrollEdgeHint(requestedDeltaX, requestedDeltaY),
1120
+ nextAction: "observe-again",
1121
+ decision: "observe-before-continuing-scroll",
1122
+ message: "The scroll position changed, but less than the requested delta. Observe again before repeating the same scroll to verify whether the target container hit an edge or a different container should be used.",
1123
+ }
1124
+ }
1125
+
1126
+ function isPartialScrollMovement(requestedDelta, actualDelta) {
1127
+ if (!Number.isFinite(requestedDelta) || !Number.isFinite(actualDelta)) return false
1128
+ if (requestedDelta === 0 || actualDelta === 0) return false
1129
+ if (Math.sign(requestedDelta) !== Math.sign(actualDelta)) return false
1130
+ return Math.abs(actualDelta) < Math.abs(requestedDelta)
1131
+ }
1132
+
1133
+ function buildScrollRecoveryHint(session, uid, edgeHint, result = {}) {
1134
+ const hint = {
1135
+ reason: "no-scroll-movement",
1136
+ recommendedTools: ["yunti_observe_page", "yunti_scroll"],
1137
+ nextAction: "observe-again",
1138
+ }
1139
+ if (edgeHint) hint.edgeHint = edgeHint
1140
+ if (uid) {
1141
+ hint.uid = uid
1142
+ hint.currentTarget = "scrollable-container"
1143
+ const target = pageUidStore.get(session.browserSessionId)?.uidMap?.[uid]
1144
+ if (target) {
1145
+ hint.lastObservedContainer = {
1146
+ uid,
1147
+ canScrollVertical: Boolean(target.canScrollVertical),
1148
+ canScrollHorizontal: Boolean(target.canScrollHorizontal),
1149
+ ...(Number.isFinite(Number(target.pixelsAbove)) ? { pixelsAbove: Number(target.pixelsAbove) } : {}),
1150
+ ...(Number.isFinite(Number(target.pixelsBelow)) ? { pixelsBelow: Number(target.pixelsBelow) } : {}),
1151
+ ...(Number.isFinite(Number(target.pixelsLeft)) ? { pixelsLeft: Number(target.pixelsLeft) } : {}),
1152
+ ...(Number.isFinite(Number(target.pixelsRight)) ? { pixelsRight: Number(target.pixelsRight) } : {}),
1153
+ }
1154
+ }
1155
+ } else {
1156
+ hint.currentTarget = "document-or-coordinate-container"
1157
+ }
1158
+ if (edgeHint === "possible-bottom-edge" || edgeHint === "possible-top-edge") {
1159
+ hint.nextAction = uid ? "try-opposite-direction-or-nearest-container" : "observe-for-scrollable-container"
1160
+ } else if (edgeHint === "possible-right-edge" || edgeHint === "possible-left-edge") {
1161
+ hint.nextAction = uid ? "try-opposite-horizontal-direction-or-nearest-container" : "observe-for-horizontal-container"
1162
+ } else if (edgeHint === "no-delta") {
1163
+ hint.nextAction = "provide-nonzero-scroll-delta"
1164
+ }
1165
+ const decision = buildScrollRecoveryDecision(uid, edgeHint)
1166
+ if (decision) hint.decision = decision
1167
+ const suggestedRetry = buildScrollSuggestedRetry(uid, edgeHint, result)
1168
+ if (suggestedRetry) hint.suggestedRetry = suggestedRetry
1169
+ hint.message = "The scroll command dispatched, but the scroll position did not change. Refresh observation before retrying, inspect scroll boundaries, or target a different scrollable container uid."
1170
+ return hint
1171
+ }
1172
+
1173
+ function buildScrollRecoveryDecision(uid, edgeHint) {
1174
+ if (edgeHint === "no-delta") return "provide-nonzero-delta"
1175
+ if (edgeHint === "possible-bottom-edge" || edgeHint === "possible-top-edge") {
1176
+ return uid ? "retry-opposite-vertical-on-same-container-once" : "observe-for-scrollable-container"
1177
+ }
1178
+ if (edgeHint === "possible-right-edge" || edgeHint === "possible-left-edge") {
1179
+ return uid ? "retry-opposite-horizontal-on-same-container-once" : "observe-for-horizontal-scrollable-container"
1180
+ }
1181
+ return undefined
1182
+ }
1183
+
1184
+ function buildScrollSuggestedRetry(uid, edgeHint, result = {}) {
1185
+ const retry = {}
1186
+ const deltaX = Number(result.deltaX)
1187
+ const deltaY = Number(result.deltaY)
1188
+ if (edgeHint === "possible-bottom-edge" || edgeHint === "possible-top-edge") {
1189
+ if (!Number.isFinite(deltaY) || deltaY === 0) return undefined
1190
+ retry.deltaY = -deltaY
1191
+ } else if (edgeHint === "possible-right-edge" || edgeHint === "possible-left-edge") {
1192
+ if (!Number.isFinite(deltaX) || deltaX === 0) return undefined
1193
+ retry.deltaX = -deltaX
1194
+ } else {
1195
+ return undefined
1196
+ }
1197
+ if (uid) retry.uid = uid
1198
+ retry.note = uid
1199
+ ? "Try the opposite direction once on the same observed container, then observe again or switch to a nearer scrollable container if it still does not move."
1200
+ : "Observe first for a scrollable container uid; only use this opposite delta if document scrolling is still the intended target."
1201
+ return retry
1202
+ }
1203
+
1204
+ async function selectElement(tabId, session, args = {}) {
1205
+ const uid = String(args.uid || "").trim()
1206
+ const value = String(args.value ?? "")
1207
+ const text = String(args.text ?? "")
1208
+ if (uid) {
1209
+ const matchMode = text ? "text" : "value"
1210
+ const targetOption = text || value
1211
+ const { x, y } = await resolveUidCenter(tabId, session, uid)
1212
+ await ensureCdpAttached(tabId, "1.3")
1213
+ const result = await chromeDebuggerSendCommand(
1214
+ { tabId },
1215
+ "Runtime.evaluate",
1216
+ {
1217
+ expression: `(() => {
1218
+ const el = document.elementFromPoint(${x}, ${y});
1219
+ if (!el) return { ok: false, code: "ELEMENT_NOT_FOUND", error: "No element found at uid coordinates" };
1220
+ if (el.tagName.toLowerCase() !== "select") {
1221
+ return { ok: false, code: "NOT_SELECT", error: "Element at uid is not a select element" };
1222
+ }
1223
+ const options = Array.from(el.options);
1224
+ const option = options.find((item) => {
1225
+ if (${JSON.stringify(matchMode)} === "text") return item.text.trim() === ${JSON.stringify(targetOption)};
1226
+ return item.value === ${JSON.stringify(targetOption)};
1227
+ });
1228
+ if (!option) {
1229
+ return {
1230
+ ok: false,
1231
+ code: "OPTION_NOT_FOUND",
1232
+ error: ${JSON.stringify(matchMode)} === "text" ? "Option text not found" : "Option value not found",
1233
+ availableValues: options.map((item) => item.value).slice(0, 50),
1234
+ availableTexts: options.map((item) => item.text.trim()).slice(0, 50),
1235
+ };
1236
+ }
1237
+ if (option.disabled) {
1238
+ return {
1239
+ ok: false,
1240
+ code: "OPTION_DISABLED",
1241
+ error: ${JSON.stringify(matchMode)} === "text" ? "Option text is disabled" : "Option value is disabled",
1242
+ disabledValue: option.value,
1243
+ disabledText: option.text.trim(),
1244
+ availableValues: options.map((item) => item.value).slice(0, 50),
1245
+ availableTexts: options.map((item) => item.text.trim()).slice(0, 50),
1246
+ };
1247
+ }
1248
+ el.value = option.value;
1249
+ el.dispatchEvent(new Event("change", { bubbles: true }));
1250
+ el.dispatchEvent(new Event("input", { bubbles: true }));
1251
+ return {
1252
+ ok: true,
1253
+ value: option.value,
1254
+ selectedIndex: el.selectedIndex,
1255
+ optionText: option.text.slice(0, 80),
1256
+ };
1257
+ })()`,
1258
+ returnByValue: true,
1259
+ }
1260
+ )
1261
+ const selectResult = result?.result?.value
1262
+ if (!selectResult?.ok) {
1263
+ return buildUidSelectFailureResult(session, uid, matchMode, targetOption, selectResult)
1264
+ }
1265
+ return {
1266
+ selected: true,
1267
+ uid,
1268
+ value: selectResult.value,
1269
+ text: selectResult.optionText,
1270
+ selectedIndex: selectResult.selectedIndex,
1271
+ browserSessionId: session.browserSessionId,
1272
+ action: "select",
1273
+ target: { uid, method: matchMode === "text" ? "uid.text" : "uid.value" },
1274
+ ok: true,
1275
+ recoverable: false,
1276
+ nextStepHint: matchMode === "text"
1277
+ ? "Uid select dispatched by visible option text. Observe again, read page state, or evaluate the select value to verify the intended change."
1278
+ : "Uid select dispatched by option value. Observe again, read page state, or evaluate the select value to verify the intended change.",
1279
+ }
1280
+ }
1281
+
1282
+ const selector = String(args.selector || "").trim()
1283
+ let result
1284
+ try {
1285
+ result = await chrome.tabs.sendMessage(tabId, {
1286
+ type: "yunti_execute_tool",
1287
+ tool: "yunti_select",
1288
+ arguments: args,
1289
+ })
1290
+ } catch (error) {
1291
+ return buildSelectorSelectFailureResult(session, selector, args.value, {
1292
+ code: "SELECTOR_SELECT_FAILED",
1293
+ error: error?.message || "Selector select failed",
1294
+ })
1295
+ }
1296
+
1297
+ if (!result || typeof result !== "object" || result.selected !== true) {
1298
+ return buildSelectorSelectFailureResult(session, selector, args.value, result)
1299
+ }
1300
+
1301
+ const requestedValue = String(args.value ?? "")
1302
+ const disabledOption = Array.isArray(result.options)
1303
+ ? result.options.find((option) => String(option?.value ?? "") === requestedValue && option.disabled === true)
1304
+ : undefined
1305
+ if (requestedValue && disabledOption) {
1306
+ return buildSelectorSelectFailureResult(session, selector, requestedValue, {
1307
+ ...result,
1308
+ selected: false,
1309
+ code: "OPTION_DISABLED",
1310
+ error: "Option value is disabled",
1311
+ disabledValue: String(disabledOption.value ?? ""),
1312
+ disabledText: String(disabledOption.text ?? ""),
1313
+ })
1314
+ }
1315
+ if (requestedValue && String(result.value ?? "") !== requestedValue) {
1316
+ return buildSelectorSelectFailureResult(session, selector, requestedValue, {
1317
+ ...result,
1318
+ code: "OPTION_NOT_FOUND",
1319
+ error: "Option value not found",
1320
+ actualValue: result.value,
1321
+ })
1322
+ }
1323
+
1324
+ return {
1325
+ ...result,
1326
+ selector,
1327
+ browserSessionId: session.browserSessionId,
1328
+ action: "select",
1329
+ target: { selector, method: "selector" },
1330
+ ok: true,
1331
+ recoverable: false,
1332
+ nextStepHint: "Select dispatched. Observe again, read page state, or evaluate the select value to verify the intended change.",
1333
+ }
1334
+ }
1335
+
1336
+ function buildSelectorSelectFailureResult(session, selector, targetOption, selectResult = {}) {
1337
+ const availableValues = Array.isArray(selectResult?.availableValues) ? selectResult.availableValues : undefined
1338
+ const availableTexts = Array.isArray(selectResult?.availableTexts) ? selectResult.availableTexts : undefined
1339
+ const disabledValue = selectResult?.disabledValue !== undefined ? String(selectResult.disabledValue) : undefined
1340
+ const disabledText = selectResult?.disabledText !== undefined ? String(selectResult.disabledText) : undefined
1341
+ const code = selectResult?.code || (selectResult?.selected === false ? "SELECTOR_SELECT_FAILED" : "SELECTOR_SELECT_FAILED")
1342
+ const error = selectResult?.error || "Selector select failed"
1343
+ const recoveryHint = {
1344
+ reason: "selector-select-failed",
1345
+ recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_evaluate_script", "yunti_select"],
1346
+ nextAction: code === "OPTION_NOT_FOUND" || code === "OPTION_DISABLED" ? "inspect-available-options" : "inspect-target-element",
1347
+ decision: code === "OPTION_DISABLED" ? "choose-enabled-option-or-unlock-field" : code === "OPTION_NOT_FOUND" ? "inspect-options-before-retry" : "use-select-element-or-uid-fallback",
1348
+ selector,
1349
+ matchMode: "value",
1350
+ targetOption: String(targetOption ?? ""),
1351
+ ...(availableValues ? { availableValues } : {}),
1352
+ ...(availableTexts ? { availableTexts } : {}),
1353
+ ...(disabledValue ? { disabledValue } : {}),
1354
+ ...(disabledText ? { disabledText } : {}),
1355
+ message: "The selector select could not be completed. Inspect the target select element and available options before retrying, or use a fresh uid/value fallback.",
1356
+ }
1357
+
1358
+ return {
1359
+ ...(selectResult && typeof selectResult === "object" ? selectResult : {}),
1360
+ selected: false,
1361
+ selector,
1362
+ browserSessionId: session.browserSessionId,
1363
+ action: "select",
1364
+ target: { selector, method: "selector" },
1365
+ ok: false,
1366
+ recoverable: true,
1367
+ code,
1368
+ error,
1369
+ matchMode: "value",
1370
+ targetOption: String(targetOption ?? ""),
1371
+ ...(availableValues ? { availableValues } : {}),
1372
+ ...(availableTexts ? { availableTexts } : {}),
1373
+ recoveryHint,
1374
+ nextStepHint: "Selector select failed. Inspect available options, observe again for a fresh uid, or retry with uid/value fallback before repeating the same selector select.",
1375
+ }
1376
+ }
1377
+
1378
+ function buildUidSelectFailureResult(session, uid, matchMode, targetOption, selectResult = {}) {
1379
+ const availableValues = Array.isArray(selectResult.availableValues) ? selectResult.availableValues : undefined
1380
+ const availableTexts = Array.isArray(selectResult.availableTexts) ? selectResult.availableTexts : undefined
1381
+ const disabledValue = selectResult?.disabledValue !== undefined ? String(selectResult.disabledValue) : undefined
1382
+ const disabledText = selectResult?.disabledText !== undefined ? String(selectResult.disabledText) : undefined
1383
+ const recoveryHint = {
1384
+ reason: "uid-select-failed",
1385
+ recommendedTools: ["yunti_observe_page", "yunti_take_snapshot", "yunti_evaluate_script", "yunti_select"],
1386
+ nextAction: "inspect-available-options",
1387
+ decision: selectResult.code === "OPTION_DISABLED" ? "choose-enabled-option-or-unlock-field" : "inspect-options-before-retry",
1388
+ uid,
1389
+ matchMode,
1390
+ targetOption,
1391
+ ...(availableValues ? { availableValues } : {}),
1392
+ ...(availableTexts ? { availableTexts } : {}),
1393
+ ...(disabledValue ? { disabledValue } : {}),
1394
+ ...(disabledText ? { disabledText } : {}),
1395
+ message: "The select option could not be matched. Inspect available options before retrying, refresh observation if the uid may be stale, or use selector/value fallback.",
1396
+ }
1397
+
1398
+ if (selectResult.code === "ELEMENT_NOT_FOUND") {
1399
+ recoveryHint.nextAction = "observe-again"
1400
+ recoveryHint.decision = "refresh-observation-before-retry"
1401
+ } else if (selectResult.code === "NOT_SELECT") {
1402
+ recoveryHint.nextAction = "inspect-target-element"
1403
+ recoveryHint.decision = "use-select-element-or-selector-fallback"
1404
+ }
1405
+
1406
+ return {
1407
+ selected: false,
1408
+ uid,
1409
+ browserSessionId: session.browserSessionId,
1410
+ action: "select",
1411
+ target: { uid, method: matchMode === "text" ? "uid.text" : "uid.value" },
1412
+ ok: false,
1413
+ recoverable: true,
1414
+ code: selectResult.code || "UID_SELECT_FAILED",
1415
+ error: selectResult.error || `Cannot select option at uid ${uid}`,
1416
+ matchMode,
1417
+ targetOption,
1418
+ ...(availableValues ? { availableValues } : {}),
1419
+ ...(availableTexts ? { availableTexts } : {}),
1420
+ ...(disabledValue ? { disabledValue } : {}),
1421
+ ...(disabledText ? { disabledText } : {}),
1422
+ recoveryHint,
1423
+ nextStepHint: "Uid select failed. Inspect available options, observe again for a fresh uid, or retry with selector/value fallback before repeating the same select.",
1424
+ }
630
1425
  }
631
1426
 
632
1427
  function validateFillArgs(args = {}) {
@@ -648,18 +1443,57 @@ export function createToolDispatcher({
648
1443
  async function clickViaSnapshotUid(tabId, session, uid) {
649
1444
  const { x, y } = await resolveUidCenter(tabId, session, uid)
650
1445
  await mouseClick(tabId, x, y, 1)
651
- return { clicked: true, uid, x: Math.round(x), y: Math.round(y), browserSessionId: session.browserSessionId }
1446
+ const roundedX = Math.round(x)
1447
+ const roundedY = Math.round(y)
1448
+ return {
1449
+ clicked: true,
1450
+ uid,
1451
+ x: roundedX,
1452
+ y: roundedY,
1453
+ browserSessionId: session.browserSessionId,
1454
+ action: "click",
1455
+ target: { uid, x: roundedX, y: roundedY },
1456
+ ok: true,
1457
+ recoverable: false,
1458
+ nextStepHint: "Click dispatched. Observe again or read page state to verify the intended change.",
1459
+ }
652
1460
  }
653
1461
 
654
1462
  async function hoverViaSnapshotUid(tabId, session, uid) {
655
1463
  const { x, y } = await resolveUidCenter(tabId, session, uid)
656
1464
  await mouseMove(tabId, x, y)
657
- return { hovered: true, uid, x: Math.round(x), y: Math.round(y), browserSessionId: session.browserSessionId }
1465
+ const roundedX = Math.round(x)
1466
+ const roundedY = Math.round(y)
1467
+ return {
1468
+ hovered: true,
1469
+ uid,
1470
+ x: roundedX,
1471
+ y: roundedY,
1472
+ browserSessionId: session.browserSessionId,
1473
+ action: "hover",
1474
+ target: { uid, x: roundedX, y: roundedY },
1475
+ ok: true,
1476
+ recoverable: false,
1477
+ nextStepHint: "Hover dispatched. Observe again or read page state to verify menus, tooltips, or hover-only controls.",
1478
+ }
658
1479
  }
659
1480
 
660
1481
  async function clickAtCoordinate(tabId, session, x, y) {
661
1482
  await mouseClick(tabId, x, y, 1)
662
- return { clicked: true, x: Math.round(x), y: Math.round(y), browserSessionId: session.browserSessionId, method: "coordinate" }
1483
+ const roundedX = Math.round(x)
1484
+ const roundedY = Math.round(y)
1485
+ return {
1486
+ clicked: true,
1487
+ x: roundedX,
1488
+ y: roundedY,
1489
+ browserSessionId: session.browserSessionId,
1490
+ method: "coordinate",
1491
+ action: "click",
1492
+ target: { method: "coordinate", x: roundedX, y: roundedY },
1493
+ ok: true,
1494
+ recoverable: false,
1495
+ nextStepHint: "Coordinate click dispatched. Observe again, read page state, or use a fresh uid when possible to verify the intended change.",
1496
+ }
663
1497
  }
664
1498
 
665
1499
  async function resolveSelectorCenter(tabId, selector) {
@@ -695,9 +1529,12 @@ export function createToolDispatcher({
695
1529
  }
696
1530
 
697
1531
  async function resolveUidCenter(tabId, session, uid) {
698
- const uidMap = snapshotStore.get(session.browserSessionId)
1532
+ const uidState = pageUidStore.get(session.browserSessionId)
1533
+ const uidMap = uidState?.uidMap
699
1534
  if (!uidMap || !uidMap[uid]) {
700
- throw new Error(`uid ${uid} not found. Run yunti_take_snapshot first.`)
1535
+ throw new Error(
1536
+ `uid ${uid} not found in the latest page uid map. Run yunti_observe_page or yunti_take_snapshot again before retrying.`
1537
+ )
701
1538
  }
702
1539
  const el = uidMap[uid]
703
1540
 
@@ -724,7 +1561,7 @@ export function createToolDispatcher({
724
1561
  }
725
1562
  }
726
1563
 
727
- throw new Error(`Cannot resolve coordinates for uid ${uid}. Element may be off-screen or hidden.`)
1564
+ throw new Error(`Cannot resolve coordinates for uid ${uid}. Element may be off-screen, hidden, or stale. Run yunti_observe_page again before retrying.`)
728
1565
  }
729
1566
 
730
1567
  async function mouseClick(tabId, x, y, clickCount = 1) {
@@ -756,24 +1593,60 @@ export function createToolDispatcher({
756
1593
  for (const field of fields) {
757
1594
  try {
758
1595
  const uid = String(field.uid || "").trim()
1596
+ let fieldResult
759
1597
  if (uid) {
760
- await fillByUid(tabId, session, { uid, value: String(field.value || "") })
1598
+ fieldResult = await fillByUid(tabId, session, { uid, value: String(field.value || "") })
761
1599
  } else {
762
- await chrome.tabs.sendMessage(tabId, {
763
- type: "yunti_execute_tool",
764
- tool: "yunti_fill",
765
- arguments: { selector: field.selector, value: field.value },
766
- })
1600
+ fieldResult = await fillByUid(tabId, session, { selector: field.selector, value: field.value })
767
1601
  }
768
- filled++
769
- results.push({ uid: field.uid, selector: field.selector, ok: true })
1602
+ const normalized = buildFillFormFieldResult(field, fieldResult)
1603
+ if (normalized.ok) filled++
1604
+ else failed++
1605
+ results.push(normalized)
770
1606
  } catch (err) {
771
1607
  failed++
772
1608
  results.push({ uid: field.uid, selector: field.selector, ok: false, error: err?.message || String(err) })
773
1609
  }
774
1610
  }
775
1611
 
776
- return { filled, failed, results, browserSessionId: session.browserSessionId }
1612
+ return {
1613
+ filled,
1614
+ failed,
1615
+ results,
1616
+ browserSessionId: session.browserSessionId,
1617
+ action: "fill_form",
1618
+ target: {
1619
+ fieldCount: fields.length,
1620
+ filled,
1621
+ failed,
1622
+ },
1623
+ ok: failed === 0,
1624
+ recoverable: failed > 0,
1625
+ nextStepHint: failed === 0
1626
+ ? "Form fill dispatched. Observe again, read page state, or evaluate field values to verify the intended changes."
1627
+ : "Form fill partially failed. Inspect per-field results, observe again for fresh uids, or retry failed fields with selector fallback.",
1628
+ }
1629
+ }
1630
+
1631
+ function buildFillFormFieldResult(field, fieldResult = {}) {
1632
+ const ok = Boolean(fieldResult?.filled || fieldResult?.ok === true)
1633
+ if (ok) return { uid: field.uid, selector: field.selector, ok: true }
1634
+
1635
+ const diagnostics = fieldResult && typeof fieldResult === "object" ? fieldResult : {}
1636
+ const preservedKeys = [
1637
+ "code",
1638
+ "error",
1639
+ "recoveryHint",
1640
+ "availableValues",
1641
+ "availableTexts",
1642
+ "nextStepHint",
1643
+ ]
1644
+ const result = { uid: field.uid, selector: field.selector, ok: false }
1645
+ for (const key of preservedKeys) {
1646
+ if (diagnostics[key] !== undefined) result[key] = diagnostics[key]
1647
+ }
1648
+ if (!result.error) result.error = "fill failed"
1649
+ return result
777
1650
  }
778
1651
 
779
1652
  async function waitForCondition(tabId, session, args = {}) {
@@ -795,7 +1668,12 @@ export function createToolDispatcher({
795
1668
  if (urlContains) {
796
1669
  const tab = await chrome.tabs.get(tabId)
797
1670
  if (tab.url && tab.url.includes(urlContains)) {
798
- return { found: true, condition: "urlContains", value: urlContains, waitedMs: Date.now() - startTime, browserSessionId: session.browserSessionId }
1671
+ return buildWaitForSuccess(
1672
+ session,
1673
+ { text, selector, urlContains, timeoutMs },
1674
+ { found: true, condition: "urlContains", value: urlContains },
1675
+ Date.now() - startTime
1676
+ )
799
1677
  }
800
1678
  }
801
1679
 
@@ -811,16 +1689,72 @@ export function createToolDispatcher({
811
1689
  "Runtime.evaluate",
812
1690
  { expression, returnByValue: true }
813
1691
  )
814
-
1692
+
815
1693
  if (result?.result?.value) {
816
- return { found: true, ...result.result.value, waitedMs: Date.now() - startTime, browserSessionId: session.browserSessionId }
1694
+ return buildWaitForSuccess(
1695
+ session,
1696
+ { text, selector, urlContains, timeoutMs },
1697
+ result.result.value,
1698
+ Date.now() - startTime
1699
+ )
817
1700
  }
818
1701
  }
819
1702
 
820
1703
  await delayCdp(200)
821
1704
  }
822
1705
 
823
- return { found: false, waitedMs: timeoutMs, browserSessionId: session.browserSessionId }
1706
+ return buildWaitForTimeout(session, { text, selector, urlContains, timeoutMs }, timeoutMs)
1707
+ }
1708
+
1709
+ function buildWaitForTarget({ text, selector, urlContains, timeoutMs } = {}) {
1710
+ const target = {}
1711
+ if (text) target.text = text
1712
+ if (selector) target.selector = selector
1713
+ if (urlContains) target.urlContains = urlContains
1714
+ if (timeoutMs !== undefined) target.timeoutMs = timeoutMs
1715
+ return target
1716
+ }
1717
+
1718
+ function buildWaitForSuccess(session, args, match = {}, waitedMs = 0) {
1719
+ const condition = typeof match.condition === "string"
1720
+ ? match.condition
1721
+ : match.found === "selector" || match.found === "text"
1722
+ ? match.found
1723
+ : undefined
1724
+ const result = {
1725
+ found: true,
1726
+ ...match,
1727
+ waitedMs,
1728
+ browserSessionId: session.browserSessionId,
1729
+ action: "wait_for",
1730
+ target: buildWaitForTarget(args),
1731
+ ok: true,
1732
+ recoverable: false,
1733
+ nextStepHint: "Wait condition matched. Call yunti_observe_page and continue with a fresh uid before acting on newly rendered content.",
1734
+ }
1735
+ if (condition && result.condition === undefined) result.condition = condition
1736
+ return result
1737
+ }
1738
+
1739
+ function buildWaitForTimeout(session, args, waitedMs) {
1740
+ return {
1741
+ found: false,
1742
+ waitedMs,
1743
+ browserSessionId: session.browserSessionId,
1744
+ action: "wait_for",
1745
+ target: buildWaitForTarget(args),
1746
+ ok: false,
1747
+ recoverable: true,
1748
+ code: "WAIT_TIMEOUT",
1749
+ recoveryHint: {
1750
+ reason: "condition-not-met-before-timeout",
1751
+ recommendedTools: ["yunti_observe_page", "yunti_get_page_snapshot", "yunti_take_screenshot"],
1752
+ nextAction: "observe-or-adjust-condition",
1753
+ decision: "observe-before-retry",
1754
+ message: "The expected text, selector, or URL state did not appear before the timeout. Observe the page or adjust the wait condition before repeating the same wait.",
1755
+ },
1756
+ nextStepHint: "Wait timed out. Observe the current page, inspect whether the condition changed, or adjust the wait target before retrying.",
1757
+ }
824
1758
  }
825
1759
 
826
1760
  async function handleDialog(tabId, session, args = {}) {
@@ -1000,7 +1934,20 @@ export function createToolDispatcher({
1000
1934
  type: "mouseReleased", x: toX, y: toY, button, clickCount: 1,
1001
1935
  })
1002
1936
 
1003
- return { dragged: true, from: { x: Math.round(fromX), y: Math.round(fromY) }, to: { x: Math.round(toX), y: Math.round(toY) }, steps, browserSessionId: session.browserSessionId }
1937
+ const from = { x: Math.round(fromX), y: Math.round(fromY) }
1938
+ const to = { x: Math.round(toX), y: Math.round(toY) }
1939
+ return {
1940
+ dragged: true,
1941
+ from,
1942
+ to,
1943
+ steps,
1944
+ browserSessionId: session.browserSessionId,
1945
+ action: "drag",
1946
+ target: { method: "coordinate", from, to },
1947
+ ok: true,
1948
+ recoverable: false,
1949
+ nextStepHint: "Drag dispatched. Observe again or read page state to verify the intended movement or drop result.",
1950
+ }
1004
1951
  }
1005
1952
 
1006
1953
  async function uploadFile(tabId, session, args = {}) {
@@ -1036,18 +1983,52 @@ export function createToolDispatcher({
1036
1983
  backendNodeId,
1037
1984
  })
1038
1985
 
1039
- return { uploaded: true, fileCount: filePaths.length, filePaths, browserSessionId: session.browserSessionId }
1986
+ return {
1987
+ uploaded: true,
1988
+ fileCount: filePaths.length,
1989
+ filePaths,
1990
+ browserSessionId: session.browserSessionId,
1991
+ action: "upload_file",
1992
+ target: uid ? { uid, method: "uid" } : { selector, method: "selector" },
1993
+ ok: true,
1994
+ recoverable: false,
1995
+ nextStepHint: "File upload dispatched. Observe again, read page state, or verify the selected file input before submitting any form.",
1996
+ }
1040
1997
  }
1041
1998
 
1042
1999
  async function typeTextByUid(tabId, session, args = {}) {
1043
2000
  const uid = String(args.uid || "").trim()
1044
2001
  if (!uid) {
1045
2002
  // Fall back to content script for selector/coordinate-based typing
1046
- return chrome.tabs.sendMessage(tabId, {
2003
+ const selector = String(args.selector || "").trim()
2004
+ const hasX = Number.isFinite(Number(args.x))
2005
+ const hasY = Number.isFinite(Number(args.y))
2006
+ const result = await chrome.tabs.sendMessage(tabId, {
1047
2007
  type: "yunti_execute_tool",
1048
2008
  tool: "yunti_type_text",
1049
2009
  arguments: args,
1050
2010
  })
2011
+
2012
+ if (!result || typeof result !== "object" || result.typed !== true) {
2013
+ return result
2014
+ }
2015
+
2016
+ const target =
2017
+ selector
2018
+ ? { selector, method: "selector" }
2019
+ : hasX && hasY
2020
+ ? { method: "coordinate", x: Math.round(Number(args.x)), y: Math.round(Number(args.y)) }
2021
+ : { method: "focused" }
2022
+
2023
+ return {
2024
+ ...result,
2025
+ browserSessionId: session.browserSessionId,
2026
+ action: "type_text",
2027
+ target,
2028
+ ok: true,
2029
+ recoverable: false,
2030
+ nextStepHint: "Type text dispatched. Observe again, read page state, or evaluate the active field value to verify the intended change.",
2031
+ }
1051
2032
  }
1052
2033
 
1053
2034
  const text = String(args.text || "")
@@ -1067,17 +2048,52 @@ export function createToolDispatcher({
1067
2048
  )
1068
2049
  }
1069
2050
 
1070
- return { typed: true, uid, text, method: "cdp.keyboard", browserSessionId: session.browserSessionId }
2051
+ return {
2052
+ typed: true,
2053
+ uid,
2054
+ text,
2055
+ method: "cdp.keyboard",
2056
+ browserSessionId: session.browserSessionId,
2057
+ action: "type_text",
2058
+ target: { uid, method: "cdp.keyboard" },
2059
+ ok: true,
2060
+ recoverable: false,
2061
+ nextStepHint: "Type text dispatched. Observe again, read page state, or evaluate the field value to verify the intended change.",
2062
+ }
1071
2063
  }
1072
2064
 
1073
2065
  async function pressKeyByUid(tabId, session, args = {}) {
1074
2066
  const uid = String(args.uid || "").trim()
1075
2067
  if (!uid) {
1076
- return chrome.tabs.sendMessage(tabId, {
2068
+ const selector = String(args.selector || "").trim()
2069
+ const hasX = Number.isFinite(Number(args.x))
2070
+ const hasY = Number.isFinite(Number(args.y))
2071
+ const result = await chrome.tabs.sendMessage(tabId, {
1077
2072
  type: "yunti_execute_tool",
1078
2073
  tool: "yunti_press_key",
1079
2074
  arguments: args,
1080
2075
  })
2076
+
2077
+ if (!result || typeof result !== "object" || result.pressed !== true) {
2078
+ return result
2079
+ }
2080
+
2081
+ const target =
2082
+ selector
2083
+ ? { selector, method: "selector" }
2084
+ : hasX && hasY
2085
+ ? { method: "coordinate", x: Math.round(Number(args.x)), y: Math.round(Number(args.y)) }
2086
+ : { method: "focused" }
2087
+
2088
+ return {
2089
+ ...result,
2090
+ browserSessionId: session.browserSessionId,
2091
+ action: "press_key",
2092
+ target,
2093
+ ok: true,
2094
+ recoverable: false,
2095
+ nextStepHint: "Key press dispatched. Observe again, read page state, or evaluate the active field value to verify the intended effect.",
2096
+ }
1081
2097
  }
1082
2098
 
1083
2099
  const key = String(args.key || "").trim()
@@ -1106,7 +2122,17 @@ export function createToolDispatcher({
1106
2122
  nativeVirtualKeyCode: keyDef.keyCode,
1107
2123
  })
1108
2124
 
1109
- return { pressed: true, uid, key, browserSessionId: session.browserSessionId }
2125
+ return {
2126
+ pressed: true,
2127
+ uid,
2128
+ key,
2129
+ browserSessionId: session.browserSessionId,
2130
+ action: "press_key",
2131
+ target: { uid, method: "cdp.keyboard" },
2132
+ ok: true,
2133
+ recoverable: false,
2134
+ nextStepHint: "Key press dispatched. Observe again, read page state, or evaluate the field value to verify the intended effect.",
2135
+ }
1110
2136
  }
1111
2137
 
1112
2138
  const KEY_MAP = {