yunti-browser-runtime 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -15
- package/bin/yunti-browser-runtime.js +6 -0
- package/docs/ACTION_RESULT_COVERAGE.md +41 -0
- package/docs/AGENT_WORKFLOW_CONTRACT.md +97 -0
- package/docs/EXECUTION_PLAN.md +1433 -4
- package/docs/EXTENSION_DISTRIBUTION.md +145 -0
- package/docs/EXTENSION_PERMISSION_STRATEGY.md +164 -0
- package/docs/EXTENSION_STORE_COPY.md +210 -0
- package/docs/INSTALL.md +18 -8
- package/docs/NEXT_MAJOR_PLAN.md +808 -0
- package/docs/PROJECT_STATUS.md +1019 -8
- package/docs/PUBLISHING_BLOCKERS.md +2 -2
- package/docs/RELEASE.md +6 -2
- package/docs/RELEASE_0_2_0_CHECKLIST.md +820 -0
- package/docs/ROADMAP.md +44 -0
- package/docs/SECURITY.md +34 -0
- package/docs/TOOL_GUIDE.md +282 -5
- package/extension/content.js +76 -7
- package/extension/dom-observer.js +588 -0
- package/extension/manifest.json +2 -2
- package/extension/popup.css +6 -1
- package/extension/popup.html +10 -10
- package/extension/popup.js +21 -10
- package/extension/session-manager.js +2 -0
- package/extension/tool-handlers.js +1120 -94
- package/mcp/bridge-hub.js +257 -3
- package/mcp/http-server.js +213 -0
- package/mcp/memory.js +5 -5
- package/mcp/redaction.js +52 -3
- package/mcp/server.js +2 -0
- package/mcp/tools.js +417 -38
- package/package.json +4 -2
- package/scripts/check-action-result-coverage.js +145 -0
- package/scripts/doctor.js +4 -0
- package/scripts/package-extension.js +1 -0
- package/scripts/print-config.js +1 -1
- package/scripts/release-check.js +3 -0
- package/skills/yunti-browser-runtime/SKILL.md +125 -2
package/docs/ROADMAP.md
CHANGED
|
@@ -35,6 +35,50 @@
|
|
|
35
35
|
- Publish an npm package.
|
|
36
36
|
- Prepare public docs and examples.
|
|
37
37
|
|
|
38
|
+
## Phase 5: Local Install Simplification
|
|
39
|
+
|
|
40
|
+
- Default local loopback bridge usage does not require a token.
|
|
41
|
+
- MCP server starts the local bridge automatically.
|
|
42
|
+
- Extension popup stays zero-config on first run.
|
|
43
|
+
- Token, bridge URL, and page match customization stay in advanced settings.
|
|
44
|
+
|
|
45
|
+
## Phase 6: Best Browser Automation Runtime
|
|
46
|
+
|
|
47
|
+
See [NEXT_MAJOR_PLAN.md](NEXT_MAJOR_PLAN.md) for the durable `0.2.0` execution
|
|
48
|
+
plan.
|
|
49
|
+
|
|
50
|
+
- Keep the release Yunti-first: build the best local browser automation
|
|
51
|
+
operation layer for agents instead of pivoting to any single reference
|
|
52
|
+
project's shape.
|
|
53
|
+
- Absorb practical strengths from Playwright, Puppeteer, Selenium, CDP,
|
|
54
|
+
browser-use, Page Agent, BrowserGym, and extension runtimes.
|
|
55
|
+
- Sequence the work: first implement the Page Agent / browser-use inspired
|
|
56
|
+
observe/action spine, then absorb the other ecosystem lessons step by step.
|
|
57
|
+
- Treat Page Agent/browser-use as the first DOM operation reference only; Yunti's
|
|
58
|
+
full product boundary still includes MCP setup, real browser sessions, tabs,
|
|
59
|
+
CDP, network/console diagnostics, screenshots, uploads, and zero-config local
|
|
60
|
+
install.
|
|
61
|
+
- Define success in practical terms: fewer mis-clicks, fewer blind retries,
|
|
62
|
+
clearer recovery, default redaction, and deterministic browser-action
|
|
63
|
+
fixtures.
|
|
64
|
+
- Keep planning ahead of code for each slice: confirm the user-facing capability,
|
|
65
|
+
compatibility rule, non-goals, and validation path before implementation.
|
|
66
|
+
- Add agent-friendly page observation with text DOM, stable uids, and scroll
|
|
67
|
+
hints.
|
|
68
|
+
- Improve DOM actions for click, fill, select, contenteditable, and scrollable
|
|
69
|
+
containers.
|
|
70
|
+
- Document an observe-act-verify workflow for external agents.
|
|
71
|
+
- Promote DOM redaction and page content policy to first-class behavior.
|
|
72
|
+
- Add optional local runtime console without changing the zero-config install
|
|
73
|
+
path.
|
|
74
|
+
- Prepare extension distribution readiness for browser stores.
|
|
75
|
+
- Track store-facing permission/privacy readiness in
|
|
76
|
+
[EXTENSION_DISTRIBUTION.md](EXTENSION_DISTRIBUTION.md) before changing
|
|
77
|
+
manifest behavior.
|
|
78
|
+
- Keep the npm/unpacked path stable while evaluating the browser-store
|
|
79
|
+
permission strategy in
|
|
80
|
+
[EXTENSION_PERMISSION_STRATEGY.md](EXTENSION_PERMISSION_STRATEGY.md).
|
|
81
|
+
|
|
38
82
|
## Future: Remote Mode
|
|
39
83
|
|
|
40
84
|
Remote multi-user operation is intentionally out of the first release. It should
|
package/docs/SECURITY.md
CHANGED
|
@@ -17,7 +17,19 @@ The first release is local single-user software:
|
|
|
17
17
|
|
|
18
18
|
- Cookies and raw authorization headers are not returned by network tools.
|
|
19
19
|
- Network URLs, headers, request bodies, and console messages are sanitized.
|
|
20
|
+
- Learning memory sanitizes likely secrets and PII-like text before writing
|
|
21
|
+
title, detail, tags, or source fields to disk. Agents should still avoid
|
|
22
|
+
intentionally submitting raw secrets or personal data to memory.
|
|
23
|
+
- DOM observation defaults to `balanced` redaction for credential-like values.
|
|
24
|
+
`strict` redaction additionally hides likely email, phone, Luhn-valid
|
|
25
|
+
payment-card-like values, address-like text, page titles, labels, names,
|
|
26
|
+
visible text, placeholders, values, and select option text.
|
|
20
27
|
- Learning memory is stored under `~/.yunti_agent/users/{userId}/memory`.
|
|
28
|
+
- Raw CDP event diagnostics are an explicit low-level exception: payloads are
|
|
29
|
+
not redacted, are cached only in memory, and should be cleared after debugging
|
|
30
|
+
with `yunti_clear_cdp_events`.
|
|
31
|
+
- Screenshot tools return real visible pixels. They do not inherit DOM
|
|
32
|
+
redaction and should be used only when visual evidence is needed.
|
|
21
33
|
- The content script does not call product-specific login APIs.
|
|
22
34
|
|
|
23
35
|
## Browser Permissions
|
|
@@ -37,6 +49,18 @@ allowlist can be configured in the extension popup with page match patterns, and
|
|
|
37
49
|
store-distributed releases should re-review whether broad host permissions are
|
|
38
50
|
still appropriate for the intended audience.
|
|
39
51
|
|
|
52
|
+
The current browser-store readiness audit and submission checklist are tracked
|
|
53
|
+
in [EXTENSION_DISTRIBUTION.md](EXTENSION_DISTRIBUTION.md). That document is the
|
|
54
|
+
source for store-facing permission rationale, privacy copy, and any future
|
|
55
|
+
decision to keep or narrow broad host permissions.
|
|
56
|
+
Draft store-facing text is tracked in
|
|
57
|
+
[EXTENSION_STORE_COPY.md](EXTENSION_STORE_COPY.md) and must be reviewed before
|
|
58
|
+
any browser-store submission.
|
|
59
|
+
The pre-store permission strategy is tracked in
|
|
60
|
+
[EXTENSION_PERMISSION_STRATEGY.md](EXTENSION_PERMISSION_STRATEGY.md); the
|
|
61
|
+
current npm/unpacked path keeps broad local automation permissions, while a
|
|
62
|
+
store-candidate path should be designed separately before submission.
|
|
63
|
+
|
|
40
64
|
Users should only load the extension from a trusted local checkout.
|
|
41
65
|
|
|
42
66
|
## Privacy Notes
|
|
@@ -45,7 +69,17 @@ Users should only load the extension from a trusted local checkout.
|
|
|
45
69
|
hosted service.
|
|
46
70
|
- Raw cookies, authorization headers, passwords, and token-like fields are not
|
|
47
71
|
returned by the documented tools.
|
|
72
|
+
- Console diagnostics and learning memory use shared sanitization for likely
|
|
73
|
+
Bearer tokens, JWTs, private keys, long token-like values, emails, phones,
|
|
74
|
+
Luhn-valid payment-card-like values, and address-like text.
|
|
75
|
+
- DOM redaction applies to structured observation text, not to screenshots;
|
|
76
|
+
screenshots are visible page pixels and may contain sensitive content.
|
|
48
77
|
- Learning memory is local filesystem data and should not contain secrets.
|
|
78
|
+
- Raw CDP diagnostics may contain sensitive payloads; use them only when needed
|
|
79
|
+
for low-level debugging, filter by method/limit where possible, and clear them
|
|
80
|
+
after use.
|
|
81
|
+
- Screenshot artifacts may contain all visible sensitive page content; prefer
|
|
82
|
+
structured redacted observations when text/state is enough.
|
|
49
83
|
- Agents should summarize sensitive-looking output instead of repeating it.
|
|
50
84
|
|
|
51
85
|
## Not Yet Implemented
|
package/docs/TOOL_GUIDE.md
CHANGED
|
@@ -5,26 +5,128 @@
|
|
|
5
5
|
1. Call `yunti_get_tool_usage_hints` when unsure about parameters or routing.
|
|
6
6
|
2. Call `yunti_list_browser_targets` to understand the live browser state.
|
|
7
7
|
3. Keep the returned `browserSessionId` for follow-up page/CDP tools.
|
|
8
|
-
4.
|
|
8
|
+
4. Once `yunti_observe_page` is available in the connected runtime, prefer
|
|
9
|
+
`observe -> act by fresh uid -> observe/verify` for page operations.
|
|
10
|
+
5. If the session becomes stale, call `yunti_list_browser_targets` again and
|
|
9
11
|
retry with the latest route.
|
|
10
|
-
|
|
12
|
+
6. For multi-step page work, prefer one stable `browserSessionId` throughout the
|
|
11
13
|
task.
|
|
12
14
|
|
|
13
15
|
Sessions expire when the extension stops polling the local bridge. Stale-session
|
|
14
16
|
errors include a reason and recovery hint; do not keep retrying an expired id.
|
|
15
17
|
|
|
18
|
+
For the full P6.3.1 workflow contract and copyable prompt, see
|
|
19
|
+
[Agent Workflow Contract](AGENT_WORKFLOW_CONTRACT.md).
|
|
20
|
+
|
|
21
|
+
## Default Page Operation Contract
|
|
22
|
+
|
|
23
|
+
1. Call `yunti_get_tool_usage_hints` when tool usage is uncertain.
|
|
24
|
+
2. Call `yunti_list_browser_targets` and choose the intended
|
|
25
|
+
`browserSessionId`.
|
|
26
|
+
3. Call `yunti_observe_page` before page actions.
|
|
27
|
+
4. Prefer fresh uids for click, hover, fill, select, scroll, type, press,
|
|
28
|
+
upload, and drag operations.
|
|
29
|
+
5. After each action, verify by observing again or using snapshot, evaluate,
|
|
30
|
+
screenshot, network, or console tools.
|
|
31
|
+
6. For async rendering, validation, navigation, option loading, or infinite
|
|
32
|
+
scroll, call `yunti_wait_for`, then `yunti_observe_page`, then continue with
|
|
33
|
+
a fresh uid.
|
|
34
|
+
7. If a result has `ok: false`, `code`, `recoveryHint`, or `nextStepHint`,
|
|
35
|
+
follow that guidance before retrying.
|
|
36
|
+
8. Use selector or coordinate fallback only when fresh uids are unavailable or
|
|
37
|
+
as an explicit recovery/debugging path.
|
|
38
|
+
|
|
39
|
+
Ask the user before submitting, deleting, approving, purchasing, publishing,
|
|
40
|
+
uploading sensitive files, changing production data, exposing secrets, or
|
|
41
|
+
taking an action whose effect cannot be verified from page state.
|
|
42
|
+
|
|
43
|
+
## Minimal Use Cases
|
|
44
|
+
|
|
45
|
+
### Click
|
|
46
|
+
|
|
47
|
+
1. Call `yunti_list_browser_targets` and select the intended
|
|
48
|
+
`browserSessionId`.
|
|
49
|
+
2. Call `yunti_observe_page` and choose the target element uid from the fresh
|
|
50
|
+
observation.
|
|
51
|
+
3. Call `yunti_click` with `browserSessionId` and `uid`.
|
|
52
|
+
4. Read `ok`, `code`, `recoverable`, `recoveryHint`, and `nextStepHint`; if
|
|
53
|
+
recoverable, follow the hint before retrying.
|
|
54
|
+
5. Call `yunti_observe_page` again or use screenshot/evaluate to verify the
|
|
55
|
+
expected page change.
|
|
56
|
+
|
|
57
|
+
### Fill Form
|
|
58
|
+
|
|
59
|
+
1. Call `yunti_observe_page` and inspect field state: `fillable`, `readOnly`,
|
|
60
|
+
`disabled`, `fillBlockReason`, `selectedValue`, `selectedText`, and
|
|
61
|
+
`options[]`.
|
|
62
|
+
2. Fill editable fields with `yunti_fill` using fresh uids; select options with
|
|
63
|
+
`yunti_select` using `uid` / `value` or `uid` / `text`.
|
|
64
|
+
3. For multiple fields, use `yunti_fill_form` when fields are known and inspect
|
|
65
|
+
per-field `results[]`.
|
|
66
|
+
4. If async validation or option loading occurs, call `yunti_wait_for`, then
|
|
67
|
+
`yunti_observe_page`, then continue with fresh uids.
|
|
68
|
+
5. Before submitting or changing production data, ask the user for
|
|
69
|
+
confirmation.
|
|
70
|
+
|
|
71
|
+
### Scroll To Find
|
|
72
|
+
|
|
73
|
+
1. Call `yunti_observe_page` and inspect `scrollableContainers[]`.
|
|
74
|
+
2. Prefer `yunti_scroll` with a fresh scrollable container uid instead of
|
|
75
|
+
document scroll for nested panels.
|
|
76
|
+
3. After scrolling, call `yunti_observe_page` and search the new
|
|
77
|
+
`textTree` / `elements` for the target.
|
|
78
|
+
4. If `moved: false`, `partialMovement`, `edgeHint`, or `recoveryHint` appears,
|
|
79
|
+
follow that guidance before repeating the same scroll.
|
|
80
|
+
5. When async content loads after scrolling, call `yunti_wait_for`, then
|
|
81
|
+
`yunti_observe_page`, then continue with a fresh scroll container uid.
|
|
82
|
+
|
|
83
|
+
### Switch Tab
|
|
84
|
+
|
|
85
|
+
1. Call `yunti_list_browser_targets` to inspect current tabs and browser
|
|
86
|
+
targets.
|
|
87
|
+
2. Choose the intended `browserSessionId` for Yunti page tools.
|
|
88
|
+
3. Use `yunti_select_page` when switching to a registered Yunti page route.
|
|
89
|
+
4. Use `yunti_cdp_send_command` with `Target.activateTarget` only for raw
|
|
90
|
+
`targetId` / `tabId` browser target activation.
|
|
91
|
+
5. After switching, call `yunti_observe_page` or `yunti_get_page_snapshot` to
|
|
92
|
+
verify the active page before acting.
|
|
93
|
+
|
|
94
|
+
### Wait For Async Result
|
|
95
|
+
|
|
96
|
+
1. Call `yunti_wait_for` with expected `text`, `selector`, or `urlContains`.
|
|
97
|
+
2. If `ok: true`, call `yunti_observe_page` and use fresh uids for follow-up
|
|
98
|
+
actions.
|
|
99
|
+
3. If `code: "WAIT_TIMEOUT"`, observe or inspect current page state before
|
|
100
|
+
adjusting the condition or retrying.
|
|
101
|
+
4. Do not reuse pre-wait uids for newly rendered content.
|
|
102
|
+
5. Verify the final result with observe, snapshot, evaluate, screenshot,
|
|
103
|
+
network, or console tools.
|
|
104
|
+
|
|
16
105
|
## Core Tools
|
|
17
106
|
|
|
18
107
|
- `yunti_list_browser_targets`: canonical live inventory for tabs and targets.
|
|
19
108
|
- `yunti_list_pages`: compatibility alias for the same live inventory.
|
|
109
|
+
- `yunti_observe_page`: P6.1 observe-first page operation contract with fresh
|
|
110
|
+
uids, compact text tree, scroll metadata, and DOM redaction metadata.
|
|
111
|
+
Element entries may include `editable`, `fillable`, `readOnly`,
|
|
112
|
+
`fillBlockReason`, select `selectedIndex` / `selectedValue` /
|
|
113
|
+
`selectedText`, and `options[]` summaries for action planning.
|
|
114
|
+
Default `balanced` redaction hides credential-like values; `strict` also
|
|
115
|
+
hides likely email, phone, Luhn-valid payment-card-like values, address-like
|
|
116
|
+
text, page titles, labels, names, visible text, placeholders, values, and
|
|
117
|
+
select option text.
|
|
20
118
|
- `yunti_get_page_snapshot`: lightweight page state and visible context.
|
|
21
119
|
- `yunti_take_snapshot`: element-oriented snapshot for uid-based actions.
|
|
120
|
+
- `yunti_wait_for`: wait for async text, selector, or URL state before
|
|
121
|
+
observing again and continuing with fresh uids.
|
|
22
122
|
- `yunti_click`, `yunti_fill`, `yunti_hover`: common DOM actions.
|
|
23
123
|
- `yunti_cdp_send_command`: low-level CDP access routed through the extension.
|
|
24
124
|
- `yunti_get_network_log`, `yunti_list_network_requests`: sanitized network
|
|
25
125
|
observations.
|
|
26
|
-
- `yunti_list_console_messages`: console diagnostics.
|
|
27
|
-
- `
|
|
126
|
+
- `yunti_list_console_messages`: sanitized console diagnostics.
|
|
127
|
+
- `yunti_get_cdp_events`: raw low-level CDP diagnostics; clear after debugging.
|
|
128
|
+
- `yunti_remember_learning`, `yunti_get_learning_memory`: sanitized local agent
|
|
129
|
+
memory.
|
|
28
130
|
|
|
29
131
|
## Routing Rules
|
|
30
132
|
|
|
@@ -52,13 +154,171 @@ errors include a reason and recovery hint; do not keep retrying an expired id.
|
|
|
52
154
|
## Parameter Rules
|
|
53
155
|
|
|
54
156
|
- `yunti_click` and `yunti_hover` require a `uid`, a `selector`, or both `x`
|
|
55
|
-
and `y`;
|
|
157
|
+
and `y`; prefer a fresh uid from `yunti_observe_page` once available, or use
|
|
158
|
+
`yunti_take_snapshot` for the current compatibility path.
|
|
56
159
|
- `yunti_fill` requires `value` and either `uid` or `selector`; it does not
|
|
57
160
|
support coordinate-only targeting.
|
|
58
161
|
- `yunti_close_page` closes by `browserSessionId`; to close by raw `tabId` or
|
|
59
162
|
`targetId`, use `yunti_cdp_send_command` with `Target.closeTarget`.
|
|
60
163
|
- `yunti_forget_learning_memory` requires `id`, or `all=true` plus
|
|
61
164
|
`confirmed=true` when deleting every memory.
|
|
165
|
+
- `yunti_remember_learning` is for durable patterns and gotchas, not transcripts
|
|
166
|
+
or raw payloads. It sanitizes likely secrets and PII-like text before storage,
|
|
167
|
+
but agents should avoid submitting secrets, cookies, auth headers, private
|
|
168
|
+
keys, payment data, or personal contact details in the first place.
|
|
169
|
+
|
|
170
|
+
## Action Recovery Rules
|
|
171
|
+
|
|
172
|
+
- After a click, fill, or hover, observe again or read page state before treating
|
|
173
|
+
the action as successful.
|
|
174
|
+
- If a uid is stale or missing, call `yunti_observe_page` again for a fresh uid
|
|
175
|
+
before retrying the same action.
|
|
176
|
+
- If an element may be outside the viewport, use observe scroll hints and
|
|
177
|
+
`yunti_scroll` before falling back to coordinates.
|
|
178
|
+
- If the page is loading or changing, wait or observe again instead of blindly
|
|
179
|
+
repeating the same action.
|
|
180
|
+
- For async UI transitions, use `yunti_wait_for` for expected text, selector, or
|
|
181
|
+
page state, then call `yunti_observe_page` and continue with a fresh uid
|
|
182
|
+
instead of reusing an old target.
|
|
183
|
+
- Read `yunti_wait_for` structured fields when present: `ok: true` means the
|
|
184
|
+
wait condition matched; `code: "WAIT_TIMEOUT"` means inspect or adjust the
|
|
185
|
+
condition before repeating the same wait.
|
|
186
|
+
- If the target tab is uncertain, call `yunti_list_browser_targets` and route
|
|
187
|
+
follow-up work through the intended `browserSessionId`.
|
|
188
|
+
|
|
189
|
+
## Action Result Rules
|
|
190
|
+
|
|
191
|
+
- Current action results remain compatibility-shaped and may include fields such
|
|
192
|
+
as `clicked`, `hovered`, `filled`, `selected`, `scrolled`, `typed`, `pressed`,
|
|
193
|
+
`uploaded`, `dragged`, aggregate counts like `failed`, per-field `results`,
|
|
194
|
+
wait fields like `found` / `waitedMs`, `uid`, `selector`, `x`, `y`, `method`,
|
|
195
|
+
`valueLength`, `before`, `after`, and `browserSessionId`.
|
|
196
|
+
- P6.2 structured action result fields are being introduced additively. Agents
|
|
197
|
+
should prefer `action`, `target`, `ok`, `recoverable`, and `nextStepHint` when
|
|
198
|
+
present, while still preserving and reading compatibility fields. `code` and
|
|
199
|
+
richer before/after summaries remain follow-on fields where useful.
|
|
200
|
+
- Do not treat a dispatched action as final proof of success; verify page state
|
|
201
|
+
with observe, snapshot, evaluate, screenshot, or CDP when the workflow needs
|
|
202
|
+
proof.
|
|
203
|
+
- Compatibility fields must remain available while structured result fields are
|
|
204
|
+
introduced.
|
|
205
|
+
|
|
206
|
+
## Wait Guidance
|
|
207
|
+
|
|
208
|
+
- Use `yunti_wait_for` when async rendering, navigation, validation, dynamic
|
|
209
|
+
select options, or infinite-scroll content needs time to appear.
|
|
210
|
+
- Provide at least one of `text`, `selector`, or `urlContains`; set
|
|
211
|
+
`timeoutMs` only when the default is too short or too long.
|
|
212
|
+
- Successful waits preserve compatibility fields such as `found`, `text`,
|
|
213
|
+
`selector`, `condition`, `value`, `waitedMs`, and `browserSessionId`, and may
|
|
214
|
+
include `action: "wait_for"`, `target`, `ok: true`, `recoverable: false`, and
|
|
215
|
+
`nextStepHint`.
|
|
216
|
+
- Timeout waits return `found: false`, `ok: false`, `recoverable: true`,
|
|
217
|
+
`code: "WAIT_TIMEOUT"`, `recoveryHint`, and `nextStepHint`.
|
|
218
|
+
- After a successful wait, call `yunti_observe_page` again and use fresh uids
|
|
219
|
+
for newly rendered elements. After a timeout, observe or inspect current page
|
|
220
|
+
state before changing the wait condition or retrying.
|
|
221
|
+
|
|
222
|
+
## Fill Guidance
|
|
223
|
+
|
|
224
|
+
- Prefer a fresh `uid` from `yunti_observe_page` or `yunti_take_snapshot` when
|
|
225
|
+
filling inputs, textareas, selects, or contenteditable targets.
|
|
226
|
+
- Before filling, inspect observation fields such as `fillable`, `readOnly`,
|
|
227
|
+
`disabled`, `fillBlockReason`, and select `selectedValue` / `selectedText`
|
|
228
|
+
/ `options[]` when present.
|
|
229
|
+
- Uid-targeted contenteditable fills now return `method: "contenteditable"` and
|
|
230
|
+
include `before` / `after` text length summaries when available.
|
|
231
|
+
- Treat contenteditable results as dispatch evidence, not final proof. Verify
|
|
232
|
+
with observe, snapshot, evaluate `textContent`, or a page-specific assertion
|
|
233
|
+
when exact editor state matters.
|
|
234
|
+
- Selector-based fill remains compatible and may return content-script-shaped
|
|
235
|
+
fields such as `element` or `valueLength`.
|
|
236
|
+
- Uid and selector fill failures can return structured `filled: false`
|
|
237
|
+
diagnostics with `code`, `ok: false`, `recoverable: true`, `recoveryHint`,
|
|
238
|
+
and `nextStepHint`; follow `recoveryHint.nextAction` / `decision` before
|
|
239
|
+
repeating the same fill.
|
|
240
|
+
- If a field appears after async rendering or validation, wait for the expected
|
|
241
|
+
text/selector/state with `yunti_wait_for`, observe again, and fill with a
|
|
242
|
+
fresh editable uid.
|
|
243
|
+
- Non-editable, hidden, disabled, or readonly fill targets return
|
|
244
|
+
`code: "TARGET_NOT_EDITABLE"` diagnostics instead of being treated as
|
|
245
|
+
successful fills; inspect the target or wait/unlock the field before retrying.
|
|
246
|
+
- Uid keyboard/contenteditable fills verify the post-fill value when possible.
|
|
247
|
+
If the value does not remain, `yunti_fill` returns
|
|
248
|
+
`code: "VALUE_NOT_APPLIED"` with `expectedValueLength`, `actualValueLength`,
|
|
249
|
+
`recoveryHint`, and `nextStepHint` instead of exposing the raw field value.
|
|
250
|
+
Inspect whether the target is framework-controlled, masked, or needs
|
|
251
|
+
`yunti_type_text` / `yunti_press_key` semantics before retrying.
|
|
252
|
+
- Select-option fill misses include `availableValues` / `availableTexts` when
|
|
253
|
+
available, so inspect those options before retrying with `yunti_fill` or
|
|
254
|
+
`yunti_select`.
|
|
255
|
+
- `yunti_fill_form` preserves per-field compatibility results and can carry the
|
|
256
|
+
same structured failure details on failed `results[]` items, including
|
|
257
|
+
`code`, `recoveryHint`, `availableValues` / `availableTexts`, and
|
|
258
|
+
`nextStepHint`.
|
|
259
|
+
|
|
260
|
+
## Scroll Guidance
|
|
261
|
+
|
|
262
|
+
- Prefer a fresh `scrollableContainers[]` uid from `yunti_observe_page` when
|
|
263
|
+
scrolling nested app panels or sidebars.
|
|
264
|
+
- Uid-targeted scroll resolves the observed container center and reuses the
|
|
265
|
+
existing coordinate/container scroll path, so coordinate recovery remains
|
|
266
|
+
compatible.
|
|
267
|
+
- Uid scroll preserves the existing `target` compatibility field and adds
|
|
268
|
+
`uid`, `method: "uid"`, and `scrollTarget` for structured interpretation.
|
|
269
|
+
- Coordinate scroll results may include `coordinateTarget`,
|
|
270
|
+
`scrollContainerFound`, `coordinateScrollFallback: "document"`, and
|
|
271
|
+
`coordinateFallbackHint`. Use these fields to tell whether the coordinate
|
|
272
|
+
actually hit a nested scrollable container or fell back to document scrolling;
|
|
273
|
+
when fallback appears, observe again and prefer a fresh
|
|
274
|
+
`scrollableContainers[]` uid for nested panels.
|
|
275
|
+
- When `before` / `after` positions are comparable, scroll results include
|
|
276
|
+
`moved`; if positions do not change, the result reports
|
|
277
|
+
`code: "NO_SCROLL_MOVEMENT"`, `ok: false`, and `recoverable: true`.
|
|
278
|
+
When direction can be inferred, it also includes `edgeHint` such as
|
|
279
|
+
`possible-bottom-edge`, `possible-top-edge`, `possible-right-edge`, or
|
|
280
|
+
`possible-left-edge`.
|
|
281
|
+
- `NO_SCROLL_MOVEMENT` results include `recoveryHint` with a structured
|
|
282
|
+
`nextAction`, machine-readable `decision`, `recommendedTools`, current target
|
|
283
|
+
type, and last observed container metadata when a fresh container uid was
|
|
284
|
+
used. When the attempted direction is clear, `recoveryHint.suggestedRetry`
|
|
285
|
+
also provides a one-shot opposite `deltaX` / `deltaY` to try before switching
|
|
286
|
+
container or stopping.
|
|
287
|
+
- Partial movement results keep `ok: true` and add `partialMovement` with
|
|
288
|
+
requested/actual delta summaries, affected axes, `edgeHint`,
|
|
289
|
+
`decision: "observe-before-continuing-scroll"`, and `nextAction:
|
|
290
|
+
"observe-again"`. Observe again before repeating the same scroll.
|
|
291
|
+
- Uid scroll failures can return structured `scrolled: false` diagnostics such
|
|
292
|
+
as `code: "UID_NOT_FOUND"` or `code: "UID_COORDINATES_UNAVAILABLE"` with
|
|
293
|
+
`recoveryHint.decision: "refresh-scrollable-container-uid-before-retry"`.
|
|
294
|
+
Refresh observation and choose a fresh `scrollableContainers[]` uid before
|
|
295
|
+
repeating the same uid scroll.
|
|
296
|
+
- After scrolling, observe again and compare document or container `before` /
|
|
297
|
+
`after` positions before assuming the needed element is visible. Stop
|
|
298
|
+
repeating the same scroll when `moved: false` appears; use `recoveryHint`,
|
|
299
|
+
`decision`, `edgeHint`, and `suggestedRetry` to choose a different container,
|
|
300
|
+
direction, or recovery path.
|
|
301
|
+
- If scrolling depends on newly loaded content, wait for the expected
|
|
302
|
+
text/selector/state with `yunti_wait_for`, observe again, and choose a fresh
|
|
303
|
+
`scrollableContainers[]` uid before continuing.
|
|
304
|
+
|
|
305
|
+
## Select Guidance
|
|
306
|
+
|
|
307
|
+
- Current `yunti_select` runtime behavior supports selector/value, uid/value,
|
|
308
|
+
and uid/visible text.
|
|
309
|
+
- Use `text` with a fresh uid when the user-facing option label is clearer than
|
|
310
|
+
the option value; selector path remains selector/value compatible.
|
|
311
|
+
- Uid and selector option misses, disabled options, plus non-select targets return structured
|
|
312
|
+
`selected: false` diagnostics with `code`, `matchMode`, `targetOption`, and
|
|
313
|
+
`recoveryHint`; option misses include `availableValues` / `availableTexts`
|
|
314
|
+
when the page exposes them, while disabled-option failures can include
|
|
315
|
+
`disabledValue` / `disabledText`.
|
|
316
|
+
- Before retrying a failed select, inspect available options with observe,
|
|
317
|
+
snapshot, evaluate, a stable selector, or `recoveryHint.decision` instead of
|
|
318
|
+
blindly repeating it.
|
|
319
|
+
- If select options are populated asynchronously, use `yunti_wait_for` for the
|
|
320
|
+
expected option text or form state, then observe again and select with a fresh
|
|
321
|
+
uid/value or uid/text.
|
|
62
322
|
|
|
63
323
|
## Safety Rules
|
|
64
324
|
|
|
@@ -67,3 +327,20 @@ errors include a reason and recovery hint; do not keep retrying an expired id.
|
|
|
67
327
|
explicit user confirmation in the agent workflow.
|
|
68
328
|
- Tool outputs redact likely cookies, authorization headers, passwords, and
|
|
69
329
|
token-like values.
|
|
330
|
+
- Learning memory and console diagnostics sanitize likely secrets and PII-like
|
|
331
|
+
text before local storage. Raw CDP events are the explicit low-level exception
|
|
332
|
+
and should be cleared after debugging with `yunti_clear_cdp_events`.
|
|
333
|
+
- Use `yunti_observe_page` `redaction: "strict"` when the page may contain
|
|
334
|
+
personal information. Keep `redaction: "off"` only for explicit local
|
|
335
|
+
debugging.
|
|
336
|
+
- DOM observation redaction does not imply screenshot redaction; screenshots
|
|
337
|
+
represent visible page pixels and may include sensitive content.
|
|
338
|
+
- Prefer sanitized tools for diagnostics: `yunti_list_network_requests`,
|
|
339
|
+
`yunti_get_network_request`, `yunti_list_console_messages`, and
|
|
340
|
+
`yunti_get_console_message`.
|
|
341
|
+
- Use `yunti_get_cdp_events` only for low-level protocol debugging. Filter by
|
|
342
|
+
`method`, keep `limit` small, do not copy raw `params` into chat or learning
|
|
343
|
+
memory, and clear with `yunti_clear_cdp_events` when done.
|
|
344
|
+
- Use screenshots only when visual proof is needed. Prefer viewport screenshots
|
|
345
|
+
over `fullPage` when sufficient, and summarize safe findings instead of
|
|
346
|
+
storing raw image data in learning memory.
|
package/extension/content.js
CHANGED
|
@@ -366,6 +366,8 @@ function updateWidgetStatus(text, auth = null) {
|
|
|
366
366
|
|
|
367
367
|
async function executeTool(tool, args) {
|
|
368
368
|
switch (tool) {
|
|
369
|
+
case "yunti_observe_page":
|
|
370
|
+
return observePage(args)
|
|
369
371
|
case "yunti_get_page_snapshot":
|
|
370
372
|
return getPageSnapshot(args)
|
|
371
373
|
case "yunti_get_selected_context":
|
|
@@ -397,6 +399,14 @@ async function executeTool(tool, args) {
|
|
|
397
399
|
}
|
|
398
400
|
}
|
|
399
401
|
|
|
402
|
+
function observePage(args = {}) {
|
|
403
|
+
if (!window.YuntiBrowserRuntimeObserver?.observePage) {
|
|
404
|
+
throw new Error("yunti_observe_page is unavailable because the DOM observer module was not loaded. Refresh the page and try again.")
|
|
405
|
+
}
|
|
406
|
+
window.__YUNTI_BROWSER_SESSION_ID__ = browserSessionId
|
|
407
|
+
return window.YuntiBrowserRuntimeObserver.observePage(args)
|
|
408
|
+
}
|
|
409
|
+
|
|
400
410
|
async function getPageSnapshot(args = {}) {
|
|
401
411
|
const mode = args.mode === "detailed" ? "detailed" : "light"
|
|
402
412
|
const includeElements = args.includeElements === undefined ? mode === "detailed" : Boolean(args.includeElements)
|
|
@@ -604,7 +614,27 @@ function pressKey(args) {
|
|
|
604
614
|
function selectElement(args) {
|
|
605
615
|
const element = mustFind(args.selector)
|
|
606
616
|
if (!(element instanceof HTMLSelectElement)) throw new Error("Target is not a select element")
|
|
607
|
-
|
|
617
|
+
const value = String(args.value ?? "")
|
|
618
|
+
const options = Array.from(element.options)
|
|
619
|
+
const option = options.find((item) => item.value === value)
|
|
620
|
+
if (option?.disabled) {
|
|
621
|
+
return {
|
|
622
|
+
selected: false,
|
|
623
|
+
element: describeElement(element),
|
|
624
|
+
value: element.value,
|
|
625
|
+
code: "OPTION_DISABLED",
|
|
626
|
+
error: "Option value is disabled",
|
|
627
|
+
disabledValue: option.value,
|
|
628
|
+
disabledText: option.text.trim(),
|
|
629
|
+
options: options.slice(0, 50).map((item) => ({
|
|
630
|
+
value: item.value,
|
|
631
|
+
text: item.text.trim(),
|
|
632
|
+
disabled: Boolean(item.disabled),
|
|
633
|
+
selected: Boolean(item.selected),
|
|
634
|
+
})),
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
element.value = value
|
|
608
638
|
element.dispatchEvent(new Event("input", { bubbles: true }))
|
|
609
639
|
element.dispatchEvent(new Event("change", { bubbles: true }))
|
|
610
640
|
return { selected: true, element: describeElement(element), value: element.value }
|
|
@@ -613,14 +643,15 @@ function selectElement(args) {
|
|
|
613
643
|
function scrollPage(args) {
|
|
614
644
|
const deltaX = Number.isFinite(Number(args.deltaX)) ? Number(args.deltaX) : 0
|
|
615
645
|
const deltaY = Number.isFinite(Number(args.deltaY)) ? Number(args.deltaY) : 600
|
|
646
|
+
const hasCoordinateTarget = Number.isFinite(Number(args.x)) && Number.isFinite(Number(args.y))
|
|
647
|
+
const coordinateX = hasCoordinateTarget ? clamp(Number(args.x), 0, Math.max(0, window.innerWidth - 1)) : undefined
|
|
648
|
+
const coordinateY = hasCoordinateTarget ? clamp(Number(args.y), 0, Math.max(0, window.innerHeight - 1)) : undefined
|
|
616
649
|
const coordinateTarget =
|
|
617
|
-
|
|
618
|
-
? document.elementFromPoint(
|
|
619
|
-
clamp(Number(args.x), 0, Math.max(0, window.innerWidth - 1)),
|
|
620
|
-
clamp(Number(args.y), 0, Math.max(0, window.innerHeight - 1))
|
|
621
|
-
)
|
|
650
|
+
hasCoordinateTarget
|
|
651
|
+
? document.elementFromPoint(coordinateX, coordinateY)
|
|
622
652
|
: null
|
|
623
|
-
const
|
|
653
|
+
const scrollableAncestor = findScrollableAncestor(coordinateTarget)
|
|
654
|
+
const target = scrollableAncestor || document.scrollingElement || document.documentElement
|
|
624
655
|
const before = getScrollPosition(target)
|
|
625
656
|
target.scrollBy({ left: deltaX, top: deltaY, behavior: "auto" })
|
|
626
657
|
const after = getScrollPosition(target)
|
|
@@ -629,6 +660,18 @@ function scrollPage(args) {
|
|
|
629
660
|
deltaX,
|
|
630
661
|
deltaY,
|
|
631
662
|
target: target === document.scrollingElement ? "document" : describeElement(target),
|
|
663
|
+
...(hasCoordinateTarget
|
|
664
|
+
? {
|
|
665
|
+
coordinateTarget: {
|
|
666
|
+
x: Math.round(coordinateX),
|
|
667
|
+
y: Math.round(coordinateY),
|
|
668
|
+
found: Boolean(coordinateTarget),
|
|
669
|
+
...(coordinateTarget ? { element: describeElement(coordinateTarget) } : {}),
|
|
670
|
+
},
|
|
671
|
+
scrollContainerFound: Boolean(scrollableAncestor),
|
|
672
|
+
...(scrollableAncestor ? {} : { coordinateScrollFallback: "document" }),
|
|
673
|
+
}
|
|
674
|
+
: {}),
|
|
632
675
|
before,
|
|
633
676
|
after,
|
|
634
677
|
}
|
|
@@ -678,6 +721,7 @@ function dispatchPointerMouseSequence(element, x, y) {
|
|
|
678
721
|
}
|
|
679
722
|
|
|
680
723
|
function setEditableText(element, text, options = {}) {
|
|
724
|
+
assertEditableTarget(element)
|
|
681
725
|
if (element.isContentEditable) {
|
|
682
726
|
const next = options.replace ? text : `${element.textContent || ""}${text}`
|
|
683
727
|
element.textContent = next
|
|
@@ -692,6 +736,31 @@ function setEditableText(element, text, options = {}) {
|
|
|
692
736
|
element.dispatchEvent(new Event("change", { bubbles: true }))
|
|
693
737
|
}
|
|
694
738
|
|
|
739
|
+
function assertEditableTarget(element) {
|
|
740
|
+
if (!element) throw new Error("No editable target is focused")
|
|
741
|
+
const style = getComputedStyle(element)
|
|
742
|
+
const rect = element.getBoundingClientRect()
|
|
743
|
+
const tag = element.tagName.toLowerCase()
|
|
744
|
+
const type = String(element.type || "").toLowerCase()
|
|
745
|
+
if (element.hidden || style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0 || rect.width <= 0 || rect.height <= 0) {
|
|
746
|
+
throw new Error("Target element is hidden or has no size")
|
|
747
|
+
}
|
|
748
|
+
if (element.disabled || element.getAttribute("aria-disabled") === "true") {
|
|
749
|
+
throw new Error("Target element is disabled")
|
|
750
|
+
}
|
|
751
|
+
if (element.readOnly || element.getAttribute("aria-readonly") === "true") {
|
|
752
|
+
throw new Error("Target element is readonly")
|
|
753
|
+
}
|
|
754
|
+
if (element.isContentEditable) return
|
|
755
|
+
if ("value" in element) {
|
|
756
|
+
if (tag === "input" && ["button", "checkbox", "color", "file", "hidden", "image", "radio", "range", "reset", "submit"].includes(type)) {
|
|
757
|
+
throw new Error(`Target input type ${type} is not editable`)
|
|
758
|
+
}
|
|
759
|
+
return
|
|
760
|
+
}
|
|
761
|
+
throw new Error("Target element is not editable")
|
|
762
|
+
}
|
|
763
|
+
|
|
695
764
|
function setNativeValue(element, value) {
|
|
696
765
|
const prototype = Object.getPrototypeOf(element)
|
|
697
766
|
const descriptor = Object.getOwnPropertyDescriptor(prototype, "value")
|