yunti-browser-runtime 0.1.3 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -9
- package/bin/yunti-browser-runtime.js +6 -0
- package/docs/ACTION_RESULT_COVERAGE.md +41 -0
- package/docs/AGENT_WORKFLOW_CONTRACT.md +108 -0
- package/docs/EXECUTION_PLAN.md +1433 -5
- package/docs/EXTENSION_DISTRIBUTION.md +146 -0
- package/docs/EXTENSION_PERMISSION_STRATEGY.md +167 -0
- package/docs/EXTENSION_STORE_COPY.md +213 -0
- package/docs/INSTALL.md +13 -7
- package/docs/NEXT_MAJOR_PLAN.md +808 -0
- package/docs/PROJECT_STATUS.md +1037 -7
- package/docs/PUBLISHING_BLOCKERS.md +1 -1
- package/docs/RELEASE.md +6 -2
- package/docs/RELEASE_0_2_0_CHECKLIST.md +827 -0
- package/docs/ROADMAP.md +44 -0
- package/docs/SECURITY.md +34 -0
- package/docs/TOOL_GUIDE.md +290 -5
- package/extension/background.js +20 -0
- package/extension/cdp.js +4 -0
- package/extension/content.js +167 -16
- package/extension/dom-observer.js +588 -0
- package/extension/manifest.json +3 -2
- package/extension/popup.js +2 -2
- package/extension/session-manager.js +106 -4
- package/extension/tool-handlers.js +1090 -156
- package/mcp/bridge-hub.js +257 -3
- package/mcp/http-server.js +213 -0
- package/mcp/memory.js +5 -5
- package/mcp/redaction.js +52 -3
- package/mcp/server.js +2 -0
- package/mcp/tools.js +421 -38
- package/package.json +4 -2
- package/scripts/check-action-result-coverage.js +145 -0
- package/scripts/doctor.js +6 -1
- package/scripts/package-extension.js +1 -0
- package/scripts/release-check.js +3 -0
- package/skills/yunti-browser-runtime/SKILL.md +128 -3
package/docs/ROADMAP.md
CHANGED
|
@@ -35,6 +35,50 @@
|
|
|
35
35
|
- Publish an npm package.
|
|
36
36
|
- Prepare public docs and examples.
|
|
37
37
|
|
|
38
|
+
## Phase 5: Local Install Simplification
|
|
39
|
+
|
|
40
|
+
- Default local loopback bridge usage does not require a token.
|
|
41
|
+
- MCP server starts the local bridge automatically.
|
|
42
|
+
- Extension popup stays zero-config on first run.
|
|
43
|
+
- Token, bridge URL, and page match customization stay in advanced settings.
|
|
44
|
+
|
|
45
|
+
## Phase 6: Best Browser Automation Runtime
|
|
46
|
+
|
|
47
|
+
See [NEXT_MAJOR_PLAN.md](NEXT_MAJOR_PLAN.md) for the durable `0.2.0` execution
|
|
48
|
+
plan.
|
|
49
|
+
|
|
50
|
+
- Keep the release Yunti-first: build the best local browser automation
|
|
51
|
+
operation layer for agents instead of pivoting to any single reference
|
|
52
|
+
project's shape.
|
|
53
|
+
- Absorb practical strengths from Playwright, Puppeteer, Selenium, CDP,
|
|
54
|
+
browser-use, Page Agent, BrowserGym, and extension runtimes.
|
|
55
|
+
- Sequence the work: first implement the Page Agent / browser-use inspired
|
|
56
|
+
observe/action spine, then absorb the other ecosystem lessons step by step.
|
|
57
|
+
- Treat Page Agent/browser-use as the first DOM operation reference only; Yunti's
|
|
58
|
+
full product boundary still includes MCP setup, real browser sessions, tabs,
|
|
59
|
+
CDP, network/console diagnostics, screenshots, uploads, and zero-config local
|
|
60
|
+
install.
|
|
61
|
+
- Define success in practical terms: fewer mis-clicks, fewer blind retries,
|
|
62
|
+
clearer recovery, default redaction, and deterministic browser-action
|
|
63
|
+
fixtures.
|
|
64
|
+
- Keep planning ahead of code for each slice: confirm the user-facing capability,
|
|
65
|
+
compatibility rule, non-goals, and validation path before implementation.
|
|
66
|
+
- Add agent-friendly page observation with text DOM, stable uids, and scroll
|
|
67
|
+
hints.
|
|
68
|
+
- Improve DOM actions for click, fill, select, contenteditable, and scrollable
|
|
69
|
+
containers.
|
|
70
|
+
- Document an observe-act-verify workflow for external agents.
|
|
71
|
+
- Promote DOM redaction and page content policy to first-class behavior.
|
|
72
|
+
- Add optional local runtime console without changing the zero-config install
|
|
73
|
+
path.
|
|
74
|
+
- Prepare extension distribution readiness for browser stores.
|
|
75
|
+
- Track store-facing permission/privacy readiness in
|
|
76
|
+
[EXTENSION_DISTRIBUTION.md](EXTENSION_DISTRIBUTION.md) before changing
|
|
77
|
+
manifest behavior.
|
|
78
|
+
- Keep the npm/unpacked path stable while evaluating the browser-store
|
|
79
|
+
permission strategy in
|
|
80
|
+
[EXTENSION_PERMISSION_STRATEGY.md](EXTENSION_PERMISSION_STRATEGY.md).
|
|
81
|
+
|
|
38
82
|
## Future: Remote Mode
|
|
39
83
|
|
|
40
84
|
Remote multi-user operation is intentionally out of the first release. It should
|
package/docs/SECURITY.md
CHANGED
|
@@ -17,7 +17,19 @@ The first release is local single-user software:
|
|
|
17
17
|
|
|
18
18
|
- Cookies and raw authorization headers are not returned by network tools.
|
|
19
19
|
- Network URLs, headers, request bodies, and console messages are sanitized.
|
|
20
|
+
- Learning memory sanitizes likely secrets and PII-like text before writing
|
|
21
|
+
title, detail, tags, or source fields to disk. Agents should still avoid
|
|
22
|
+
intentionally submitting raw secrets or personal data to memory.
|
|
23
|
+
- DOM observation defaults to `balanced` redaction for credential-like values.
|
|
24
|
+
`strict` redaction additionally hides likely email, phone, Luhn-valid
|
|
25
|
+
payment-card-like values, address-like text, page titles, labels, names,
|
|
26
|
+
visible text, placeholders, values, and select option text.
|
|
20
27
|
- Learning memory is stored under `~/.yunti_agent/users/{userId}/memory`.
|
|
28
|
+
- Raw CDP event diagnostics are an explicit low-level exception: payloads are
|
|
29
|
+
not redacted, are cached only in memory, and should be cleared after debugging
|
|
30
|
+
with `yunti_clear_cdp_events`.
|
|
31
|
+
- Screenshot tools return real visible pixels. They do not inherit DOM
|
|
32
|
+
redaction and should be used only when visual evidence is needed.
|
|
21
33
|
- The content script does not call product-specific login APIs.
|
|
22
34
|
|
|
23
35
|
## Browser Permissions
|
|
@@ -37,6 +49,18 @@ allowlist can be configured in the extension popup with page match patterns, and
|
|
|
37
49
|
store-distributed releases should re-review whether broad host permissions are
|
|
38
50
|
still appropriate for the intended audience.
|
|
39
51
|
|
|
52
|
+
The current browser-store readiness audit and submission checklist are tracked
|
|
53
|
+
in [EXTENSION_DISTRIBUTION.md](EXTENSION_DISTRIBUTION.md). That document is the
|
|
54
|
+
source for store-facing permission rationale, privacy copy, and any future
|
|
55
|
+
decision to keep or narrow broad host permissions.
|
|
56
|
+
Draft store-facing text is tracked in
|
|
57
|
+
[EXTENSION_STORE_COPY.md](EXTENSION_STORE_COPY.md) and must be reviewed before
|
|
58
|
+
any browser-store submission.
|
|
59
|
+
The pre-store permission strategy is tracked in
|
|
60
|
+
[EXTENSION_PERMISSION_STRATEGY.md](EXTENSION_PERMISSION_STRATEGY.md); the
|
|
61
|
+
current npm/unpacked path keeps broad local automation permissions, while a
|
|
62
|
+
store-candidate path should be designed separately before submission.
|
|
63
|
+
|
|
40
64
|
Users should only load the extension from a trusted local checkout.
|
|
41
65
|
|
|
42
66
|
## Privacy Notes
|
|
@@ -45,7 +69,17 @@ Users should only load the extension from a trusted local checkout.
|
|
|
45
69
|
hosted service.
|
|
46
70
|
- Raw cookies, authorization headers, passwords, and token-like fields are not
|
|
47
71
|
returned by the documented tools.
|
|
72
|
+
- Console diagnostics and learning memory use shared sanitization for likely
|
|
73
|
+
Bearer tokens, JWTs, private keys, long token-like values, emails, phones,
|
|
74
|
+
Luhn-valid payment-card-like values, and address-like text.
|
|
75
|
+
- DOM redaction applies to structured observation text, not to screenshots;
|
|
76
|
+
screenshots are visible page pixels and may contain sensitive content.
|
|
48
77
|
- Learning memory is local filesystem data and should not contain secrets.
|
|
78
|
+
- Raw CDP diagnostics may contain sensitive payloads; use them only when needed
|
|
79
|
+
for low-level debugging, filter by method/limit where possible, and clear them
|
|
80
|
+
after use.
|
|
81
|
+
- Screenshot artifacts may contain all visible sensitive page content; prefer
|
|
82
|
+
structured redacted observations when text/state is enough.
|
|
49
83
|
- Agents should summarize sensitive-looking output instead of repeating it.
|
|
50
84
|
|
|
51
85
|
## Not Yet Implemented
|
package/docs/TOOL_GUIDE.md
CHANGED
|
@@ -5,26 +5,128 @@
|
|
|
5
5
|
1. Call `yunti_get_tool_usage_hints` when unsure about parameters or routing.
|
|
6
6
|
2. Call `yunti_list_browser_targets` to understand the live browser state.
|
|
7
7
|
3. Keep the returned `browserSessionId` for follow-up page/CDP tools.
|
|
8
|
-
4.
|
|
8
|
+
4. Once `yunti_observe_page` is available in the connected runtime, prefer
|
|
9
|
+
`observe -> act by fresh uid -> observe/verify` for page operations.
|
|
10
|
+
5. If the session becomes stale, call `yunti_list_browser_targets` again and
|
|
9
11
|
retry with the latest route.
|
|
10
|
-
|
|
12
|
+
6. For multi-step page work, prefer one stable `browserSessionId` throughout the
|
|
11
13
|
task.
|
|
12
14
|
|
|
13
15
|
Sessions expire when the extension stops polling the local bridge. Stale-session
|
|
14
16
|
errors include a reason and recovery hint; do not keep retrying an expired id.
|
|
15
17
|
|
|
18
|
+
For the full P6.3.1 workflow contract and copyable prompt, see
|
|
19
|
+
[Agent Workflow Contract](AGENT_WORKFLOW_CONTRACT.md).
|
|
20
|
+
|
|
21
|
+
## Default Page Operation Contract
|
|
22
|
+
|
|
23
|
+
1. Call `yunti_get_tool_usage_hints` when tool usage is uncertain.
|
|
24
|
+
2. Call `yunti_list_browser_targets` and choose the intended
|
|
25
|
+
`browserSessionId`.
|
|
26
|
+
3. Call `yunti_observe_page` before page actions.
|
|
27
|
+
4. Prefer fresh uids for click, hover, fill, select, scroll, type, press,
|
|
28
|
+
upload, and drag operations.
|
|
29
|
+
5. After each action, verify by observing again or using snapshot, evaluate,
|
|
30
|
+
screenshot, network, or console tools.
|
|
31
|
+
6. For async rendering, validation, navigation, option loading, or infinite
|
|
32
|
+
scroll, call `yunti_wait_for`, then `yunti_observe_page`, then continue with
|
|
33
|
+
a fresh uid.
|
|
34
|
+
7. If a result has `ok: false`, `code`, `recoveryHint`, or `nextStepHint`,
|
|
35
|
+
follow that guidance before retrying.
|
|
36
|
+
8. Use selector or coordinate fallback only when fresh uids are unavailable or
|
|
37
|
+
as an explicit recovery/debugging path.
|
|
38
|
+
|
|
39
|
+
Ask the user before submitting, deleting, approving, purchasing, publishing,
|
|
40
|
+
uploading sensitive files, changing production data, exposing secrets, or
|
|
41
|
+
taking an action whose effect cannot be verified from page state.
|
|
42
|
+
|
|
43
|
+
## Minimal Use Cases
|
|
44
|
+
|
|
45
|
+
### Click
|
|
46
|
+
|
|
47
|
+
1. Call `yunti_list_browser_targets` and select the intended
|
|
48
|
+
`browserSessionId`.
|
|
49
|
+
2. Call `yunti_observe_page` and choose the target element uid from the fresh
|
|
50
|
+
observation.
|
|
51
|
+
3. Call `yunti_click` with `browserSessionId` and `uid`.
|
|
52
|
+
4. Read `ok`, `code`, `recoverable`, `recoveryHint`, and `nextStepHint`; if
|
|
53
|
+
recoverable, follow the hint before retrying.
|
|
54
|
+
5. Call `yunti_observe_page` again or use screenshot/evaluate to verify the
|
|
55
|
+
expected page change.
|
|
56
|
+
|
|
57
|
+
### Fill Form
|
|
58
|
+
|
|
59
|
+
1. Call `yunti_observe_page` and inspect field state: `fillable`, `readOnly`,
|
|
60
|
+
`disabled`, `fillBlockReason`, `selectedValue`, `selectedText`, and
|
|
61
|
+
`options[]`.
|
|
62
|
+
2. Fill editable fields with `yunti_fill` using fresh uids; select options with
|
|
63
|
+
`yunti_select` using `uid` / `value` or `uid` / `text`.
|
|
64
|
+
3. For multiple fields, use `yunti_fill_form` when fields are known and inspect
|
|
65
|
+
per-field `results[]`.
|
|
66
|
+
4. If async validation or option loading occurs, call `yunti_wait_for`, then
|
|
67
|
+
`yunti_observe_page`, then continue with fresh uids.
|
|
68
|
+
5. Before submitting or changing production data, ask the user for
|
|
69
|
+
confirmation.
|
|
70
|
+
|
|
71
|
+
### Scroll To Find
|
|
72
|
+
|
|
73
|
+
1. Call `yunti_observe_page` and inspect `scrollableContainers[]`.
|
|
74
|
+
2. Prefer `yunti_scroll` with a fresh scrollable container uid instead of
|
|
75
|
+
document scroll for nested panels.
|
|
76
|
+
3. After scrolling, call `yunti_observe_page` and search the new
|
|
77
|
+
`textTree` / `elements` for the target.
|
|
78
|
+
4. If `moved: false`, `partialMovement`, `edgeHint`, or `recoveryHint` appears,
|
|
79
|
+
follow that guidance before repeating the same scroll.
|
|
80
|
+
5. When async content loads after scrolling, call `yunti_wait_for`, then
|
|
81
|
+
`yunti_observe_page`, then continue with a fresh scroll container uid.
|
|
82
|
+
|
|
83
|
+
### Switch Tab
|
|
84
|
+
|
|
85
|
+
1. Call `yunti_list_browser_targets` to inspect current tabs and browser
|
|
86
|
+
targets.
|
|
87
|
+
2. Choose the intended `browserSessionId` for Yunti page tools.
|
|
88
|
+
3. Use `yunti_select_page` when switching to a registered Yunti page route.
|
|
89
|
+
4. Use `yunti_cdp_send_command` with `Target.activateTarget` only for raw
|
|
90
|
+
`targetId` / `tabId` browser target activation.
|
|
91
|
+
5. After switching, call `yunti_observe_page` or `yunti_get_page_snapshot` to
|
|
92
|
+
verify the active page before acting.
|
|
93
|
+
|
|
94
|
+
### Wait For Async Result
|
|
95
|
+
|
|
96
|
+
1. Call `yunti_wait_for` with expected `text`, `selector`, or `urlContains`.
|
|
97
|
+
2. If `ok: true`, call `yunti_observe_page` and use fresh uids for follow-up
|
|
98
|
+
actions.
|
|
99
|
+
3. If `code: "WAIT_TIMEOUT"`, observe or inspect current page state before
|
|
100
|
+
adjusting the condition or retrying.
|
|
101
|
+
4. Do not reuse pre-wait uids for newly rendered content.
|
|
102
|
+
5. Verify the final result with observe, snapshot, evaluate, screenshot,
|
|
103
|
+
network, or console tools.
|
|
104
|
+
|
|
16
105
|
## Core Tools
|
|
17
106
|
|
|
18
107
|
- `yunti_list_browser_targets`: canonical live inventory for tabs and targets.
|
|
19
108
|
- `yunti_list_pages`: compatibility alias for the same live inventory.
|
|
109
|
+
- `yunti_observe_page`: P6.1 observe-first page operation contract with fresh
|
|
110
|
+
uids, compact text tree, scroll metadata, and DOM redaction metadata.
|
|
111
|
+
Element entries may include `editable`, `fillable`, `readOnly`,
|
|
112
|
+
`fillBlockReason`, select `selectedIndex` / `selectedValue` /
|
|
113
|
+
`selectedText`, and `options[]` summaries for action planning.
|
|
114
|
+
Default `balanced` redaction hides credential-like values; `strict` also
|
|
115
|
+
hides likely email, phone, Luhn-valid payment-card-like values, address-like
|
|
116
|
+
text, page titles, labels, names, visible text, placeholders, values, and
|
|
117
|
+
select option text.
|
|
20
118
|
- `yunti_get_page_snapshot`: lightweight page state and visible context.
|
|
21
119
|
- `yunti_take_snapshot`: element-oriented snapshot for uid-based actions.
|
|
120
|
+
- `yunti_wait_for`: wait for async text, selector, or URL state before
|
|
121
|
+
observing again and continuing with fresh uids.
|
|
22
122
|
- `yunti_click`, `yunti_fill`, `yunti_hover`: common DOM actions.
|
|
23
123
|
- `yunti_cdp_send_command`: low-level CDP access routed through the extension.
|
|
24
124
|
- `yunti_get_network_log`, `yunti_list_network_requests`: sanitized network
|
|
25
125
|
observations.
|
|
26
|
-
- `yunti_list_console_messages`: console diagnostics.
|
|
27
|
-
- `
|
|
126
|
+
- `yunti_list_console_messages`: sanitized console diagnostics.
|
|
127
|
+
- `yunti_get_cdp_events`: raw low-level CDP diagnostics; clear after debugging.
|
|
128
|
+
- `yunti_remember_learning`, `yunti_get_learning_memory`: sanitized local agent
|
|
129
|
+
memory.
|
|
28
130
|
|
|
29
131
|
## Routing Rules
|
|
30
132
|
|
|
@@ -41,6 +143,14 @@ errors include a reason and recovery hint; do not keep retrying an expired id.
|
|
|
41
143
|
|
|
42
144
|
## CDP Rules
|
|
43
145
|
|
|
146
|
+
- In `0.2.1+`, normal observe-first page actions should not automatically
|
|
147
|
+
attach Chrome debugger. Prefer `yunti_observe_page` plus fresh uid
|
|
148
|
+
`yunti_click`, `yunti_hover`, `yunti_fill`, `yunti_select`, `yunti_scroll`,
|
|
149
|
+
`yunti_type_text`, and `yunti_press_key` on anti-debug-sensitive pages.
|
|
150
|
+
- Chrome debugger banners may still appear for explicit low-level or advanced
|
|
151
|
+
tools such as `yunti_cdp_send_command`, raw CDP diagnostics, performance
|
|
152
|
+
tracing, screenshot fallback paths, drag, upload, emulation, resize, or
|
|
153
|
+
legacy snapshot compatibility.
|
|
44
154
|
- Do not run CDP method names in a shell.
|
|
45
155
|
- Use `yunti_cdp_send_command` with `method` and optional `params`.
|
|
46
156
|
- `params` must be an object when provided.
|
|
@@ -52,13 +162,171 @@ errors include a reason and recovery hint; do not keep retrying an expired id.
|
|
|
52
162
|
## Parameter Rules
|
|
53
163
|
|
|
54
164
|
- `yunti_click` and `yunti_hover` require a `uid`, a `selector`, or both `x`
|
|
55
|
-
and `y`;
|
|
165
|
+
and `y`; prefer a fresh uid from `yunti_observe_page` once available, or use
|
|
166
|
+
`yunti_take_snapshot` for the current compatibility path.
|
|
56
167
|
- `yunti_fill` requires `value` and either `uid` or `selector`; it does not
|
|
57
168
|
support coordinate-only targeting.
|
|
58
169
|
- `yunti_close_page` closes by `browserSessionId`; to close by raw `tabId` or
|
|
59
170
|
`targetId`, use `yunti_cdp_send_command` with `Target.closeTarget`.
|
|
60
171
|
- `yunti_forget_learning_memory` requires `id`, or `all=true` plus
|
|
61
172
|
`confirmed=true` when deleting every memory.
|
|
173
|
+
- `yunti_remember_learning` is for durable patterns and gotchas, not transcripts
|
|
174
|
+
or raw payloads. It sanitizes likely secrets and PII-like text before storage,
|
|
175
|
+
but agents should avoid submitting secrets, cookies, auth headers, private
|
|
176
|
+
keys, payment data, or personal contact details in the first place.
|
|
177
|
+
|
|
178
|
+
## Action Recovery Rules
|
|
179
|
+
|
|
180
|
+
- After a click, fill, or hover, observe again or read page state before treating
|
|
181
|
+
the action as successful.
|
|
182
|
+
- If a uid is stale or missing, call `yunti_observe_page` again for a fresh uid
|
|
183
|
+
before retrying the same action.
|
|
184
|
+
- If an element may be outside the viewport, use observe scroll hints and
|
|
185
|
+
`yunti_scroll` before falling back to coordinates.
|
|
186
|
+
- If the page is loading or changing, wait or observe again instead of blindly
|
|
187
|
+
repeating the same action.
|
|
188
|
+
- For async UI transitions, use `yunti_wait_for` for expected text, selector, or
|
|
189
|
+
page state, then call `yunti_observe_page` and continue with a fresh uid
|
|
190
|
+
instead of reusing an old target.
|
|
191
|
+
- Read `yunti_wait_for` structured fields when present: `ok: true` means the
|
|
192
|
+
wait condition matched; `code: "WAIT_TIMEOUT"` means inspect or adjust the
|
|
193
|
+
condition before repeating the same wait.
|
|
194
|
+
- If the target tab is uncertain, call `yunti_list_browser_targets` and route
|
|
195
|
+
follow-up work through the intended `browserSessionId`.
|
|
196
|
+
|
|
197
|
+
## Action Result Rules
|
|
198
|
+
|
|
199
|
+
- Current action results remain compatibility-shaped and may include fields such
|
|
200
|
+
as `clicked`, `hovered`, `filled`, `selected`, `scrolled`, `typed`, `pressed`,
|
|
201
|
+
`uploaded`, `dragged`, aggregate counts like `failed`, per-field `results`,
|
|
202
|
+
wait fields like `found` / `waitedMs`, `uid`, `selector`, `x`, `y`, `method`,
|
|
203
|
+
`valueLength`, `before`, `after`, and `browserSessionId`.
|
|
204
|
+
- P6.2 structured action result fields are being introduced additively. Agents
|
|
205
|
+
should prefer `action`, `target`, `ok`, `recoverable`, and `nextStepHint` when
|
|
206
|
+
present, while still preserving and reading compatibility fields. `code` and
|
|
207
|
+
richer before/after summaries remain follow-on fields where useful.
|
|
208
|
+
- Do not treat a dispatched action as final proof of success; verify page state
|
|
209
|
+
with observe, snapshot, evaluate, screenshot, or CDP when the workflow needs
|
|
210
|
+
proof.
|
|
211
|
+
- Compatibility fields must remain available while structured result fields are
|
|
212
|
+
introduced.
|
|
213
|
+
|
|
214
|
+
## Wait Guidance
|
|
215
|
+
|
|
216
|
+
- Use `yunti_wait_for` when async rendering, navigation, validation, dynamic
|
|
217
|
+
select options, or infinite-scroll content needs time to appear.
|
|
218
|
+
- Provide at least one of `text`, `selector`, or `urlContains`; set
|
|
219
|
+
`timeoutMs` only when the default is too short or too long.
|
|
220
|
+
- Successful waits preserve compatibility fields such as `found`, `text`,
|
|
221
|
+
`selector`, `condition`, `value`, `waitedMs`, and `browserSessionId`, and may
|
|
222
|
+
include `action: "wait_for"`, `target`, `ok: true`, `recoverable: false`, and
|
|
223
|
+
`nextStepHint`.
|
|
224
|
+
- Timeout waits return `found: false`, `ok: false`, `recoverable: true`,
|
|
225
|
+
`code: "WAIT_TIMEOUT"`, `recoveryHint`, and `nextStepHint`.
|
|
226
|
+
- After a successful wait, call `yunti_observe_page` again and use fresh uids
|
|
227
|
+
for newly rendered elements. After a timeout, observe or inspect current page
|
|
228
|
+
state before changing the wait condition or retrying.
|
|
229
|
+
|
|
230
|
+
## Fill Guidance
|
|
231
|
+
|
|
232
|
+
- Prefer a fresh `uid` from `yunti_observe_page` or `yunti_take_snapshot` when
|
|
233
|
+
filling inputs, textareas, selects, or contenteditable targets.
|
|
234
|
+
- Before filling, inspect observation fields such as `fillable`, `readOnly`,
|
|
235
|
+
`disabled`, `fillBlockReason`, and select `selectedValue` / `selectedText`
|
|
236
|
+
/ `options[]` when present.
|
|
237
|
+
- Uid-targeted contenteditable fills now return `method: "contenteditable"` and
|
|
238
|
+
include `before` / `after` text length summaries when available.
|
|
239
|
+
- Treat contenteditable results as dispatch evidence, not final proof. Verify
|
|
240
|
+
with observe, snapshot, evaluate `textContent`, or a page-specific assertion
|
|
241
|
+
when exact editor state matters.
|
|
242
|
+
- Selector-based fill remains compatible and may return content-script-shaped
|
|
243
|
+
fields such as `element` or `valueLength`.
|
|
244
|
+
- Uid and selector fill failures can return structured `filled: false`
|
|
245
|
+
diagnostics with `code`, `ok: false`, `recoverable: true`, `recoveryHint`,
|
|
246
|
+
and `nextStepHint`; follow `recoveryHint.nextAction` / `decision` before
|
|
247
|
+
repeating the same fill.
|
|
248
|
+
- If a field appears after async rendering or validation, wait for the expected
|
|
249
|
+
text/selector/state with `yunti_wait_for`, observe again, and fill with a
|
|
250
|
+
fresh editable uid.
|
|
251
|
+
- Non-editable, hidden, disabled, or readonly fill targets return
|
|
252
|
+
`code: "TARGET_NOT_EDITABLE"` diagnostics instead of being treated as
|
|
253
|
+
successful fills; inspect the target or wait/unlock the field before retrying.
|
|
254
|
+
- Uid keyboard/contenteditable fills verify the post-fill value when possible.
|
|
255
|
+
If the value does not remain, `yunti_fill` returns
|
|
256
|
+
`code: "VALUE_NOT_APPLIED"` with `expectedValueLength`, `actualValueLength`,
|
|
257
|
+
`recoveryHint`, and `nextStepHint` instead of exposing the raw field value.
|
|
258
|
+
Inspect whether the target is framework-controlled, masked, or needs
|
|
259
|
+
`yunti_type_text` / `yunti_press_key` semantics before retrying.
|
|
260
|
+
- Select-option fill misses include `availableValues` / `availableTexts` when
|
|
261
|
+
available, so inspect those options before retrying with `yunti_fill` or
|
|
262
|
+
`yunti_select`.
|
|
263
|
+
- `yunti_fill_form` preserves per-field compatibility results and can carry the
|
|
264
|
+
same structured failure details on failed `results[]` items, including
|
|
265
|
+
`code`, `recoveryHint`, `availableValues` / `availableTexts`, and
|
|
266
|
+
`nextStepHint`.
|
|
267
|
+
|
|
268
|
+
## Scroll Guidance
|
|
269
|
+
|
|
270
|
+
- Prefer a fresh `scrollableContainers[]` uid from `yunti_observe_page` when
|
|
271
|
+
scrolling nested app panels or sidebars.
|
|
272
|
+
- Uid-targeted scroll resolves the observed container center and reuses the
|
|
273
|
+
existing coordinate/container scroll path, so coordinate recovery remains
|
|
274
|
+
compatible.
|
|
275
|
+
- Uid scroll preserves the existing `target` compatibility field and adds
|
|
276
|
+
`uid`, `method: "uid"`, and `scrollTarget` for structured interpretation.
|
|
277
|
+
- Coordinate scroll results may include `coordinateTarget`,
|
|
278
|
+
`scrollContainerFound`, `coordinateScrollFallback: "document"`, and
|
|
279
|
+
`coordinateFallbackHint`. Use these fields to tell whether the coordinate
|
|
280
|
+
actually hit a nested scrollable container or fell back to document scrolling;
|
|
281
|
+
when fallback appears, observe again and prefer a fresh
|
|
282
|
+
`scrollableContainers[]` uid for nested panels.
|
|
283
|
+
- When `before` / `after` positions are comparable, scroll results include
|
|
284
|
+
`moved`; if positions do not change, the result reports
|
|
285
|
+
`code: "NO_SCROLL_MOVEMENT"`, `ok: false`, and `recoverable: true`.
|
|
286
|
+
When direction can be inferred, it also includes `edgeHint` such as
|
|
287
|
+
`possible-bottom-edge`, `possible-top-edge`, `possible-right-edge`, or
|
|
288
|
+
`possible-left-edge`.
|
|
289
|
+
- `NO_SCROLL_MOVEMENT` results include `recoveryHint` with a structured
|
|
290
|
+
`nextAction`, machine-readable `decision`, `recommendedTools`, current target
|
|
291
|
+
type, and last observed container metadata when a fresh container uid was
|
|
292
|
+
used. When the attempted direction is clear, `recoveryHint.suggestedRetry`
|
|
293
|
+
also provides a one-shot opposite `deltaX` / `deltaY` to try before switching
|
|
294
|
+
container or stopping.
|
|
295
|
+
- Partial movement results keep `ok: true` and add `partialMovement` with
|
|
296
|
+
requested/actual delta summaries, affected axes, `edgeHint`,
|
|
297
|
+
`decision: "observe-before-continuing-scroll"`, and `nextAction:
|
|
298
|
+
"observe-again"`. Observe again before repeating the same scroll.
|
|
299
|
+
- Uid scroll failures can return structured `scrolled: false` diagnostics such
|
|
300
|
+
as `code: "UID_NOT_FOUND"` or `code: "UID_COORDINATES_UNAVAILABLE"` with
|
|
301
|
+
`recoveryHint.decision: "refresh-scrollable-container-uid-before-retry"`.
|
|
302
|
+
Refresh observation and choose a fresh `scrollableContainers[]` uid before
|
|
303
|
+
repeating the same uid scroll.
|
|
304
|
+
- After scrolling, observe again and compare document or container `before` /
|
|
305
|
+
`after` positions before assuming the needed element is visible. Stop
|
|
306
|
+
repeating the same scroll when `moved: false` appears; use `recoveryHint`,
|
|
307
|
+
`decision`, `edgeHint`, and `suggestedRetry` to choose a different container,
|
|
308
|
+
direction, or recovery path.
|
|
309
|
+
- If scrolling depends on newly loaded content, wait for the expected
|
|
310
|
+
text/selector/state with `yunti_wait_for`, observe again, and choose a fresh
|
|
311
|
+
`scrollableContainers[]` uid before continuing.
|
|
312
|
+
|
|
313
|
+
## Select Guidance
|
|
314
|
+
|
|
315
|
+
- Current `yunti_select` runtime behavior supports selector/value, uid/value,
|
|
316
|
+
and uid/visible text.
|
|
317
|
+
- Use `text` with a fresh uid when the user-facing option label is clearer than
|
|
318
|
+
the option value; selector path remains selector/value compatible.
|
|
319
|
+
- Uid and selector option misses, disabled options, plus non-select targets return structured
|
|
320
|
+
`selected: false` diagnostics with `code`, `matchMode`, `targetOption`, and
|
|
321
|
+
`recoveryHint`; option misses include `availableValues` / `availableTexts`
|
|
322
|
+
when the page exposes them, while disabled-option failures can include
|
|
323
|
+
`disabledValue` / `disabledText`.
|
|
324
|
+
- Before retrying a failed select, inspect available options with observe,
|
|
325
|
+
snapshot, evaluate, a stable selector, or `recoveryHint.decision` instead of
|
|
326
|
+
blindly repeating it.
|
|
327
|
+
- If select options are populated asynchronously, use `yunti_wait_for` for the
|
|
328
|
+
expected option text or form state, then observe again and select with a fresh
|
|
329
|
+
uid/value or uid/text.
|
|
62
330
|
|
|
63
331
|
## Safety Rules
|
|
64
332
|
|
|
@@ -67,3 +335,20 @@ errors include a reason and recovery hint; do not keep retrying an expired id.
|
|
|
67
335
|
explicit user confirmation in the agent workflow.
|
|
68
336
|
- Tool outputs redact likely cookies, authorization headers, passwords, and
|
|
69
337
|
token-like values.
|
|
338
|
+
- Learning memory and console diagnostics sanitize likely secrets and PII-like
|
|
339
|
+
text before local storage. Raw CDP events are the explicit low-level exception
|
|
340
|
+
and should be cleared after debugging with `yunti_clear_cdp_events`.
|
|
341
|
+
- Use `yunti_observe_page` `redaction: "strict"` when the page may contain
|
|
342
|
+
personal information. Keep `redaction: "off"` only for explicit local
|
|
343
|
+
debugging.
|
|
344
|
+
- DOM observation redaction does not imply screenshot redaction; screenshots
|
|
345
|
+
represent visible page pixels and may include sensitive content.
|
|
346
|
+
- Prefer sanitized tools for diagnostics: `yunti_list_network_requests`,
|
|
347
|
+
`yunti_get_network_request`, `yunti_list_console_messages`, and
|
|
348
|
+
`yunti_get_console_message`.
|
|
349
|
+
- Use `yunti_get_cdp_events` only for low-level protocol debugging. Filter by
|
|
350
|
+
`method`, keep `limit` small, do not copy raw `params` into chat or learning
|
|
351
|
+
memory, and clear with `yunti_clear_cdp_events` when done.
|
|
352
|
+
- Use screenshots only when visual proof is needed. Prefer viewport screenshots
|
|
353
|
+
over `fullPage` when sufficient, and summarize safe findings instead of
|
|
354
|
+
storing raw image data in learning memory.
|
package/extension/background.js
CHANGED
|
@@ -18,6 +18,7 @@ const cdp = createCdpController({
|
|
|
18
18
|
postBridge,
|
|
19
19
|
startPolling,
|
|
20
20
|
forwardConsoleEvent,
|
|
21
|
+
ensureAllTabsRegistered: sessionManager.ensureAllTabsRegistered,
|
|
21
22
|
})
|
|
22
23
|
|
|
23
24
|
const { detachCdpTab, installCdpEventForwarder } = cdp
|
|
@@ -45,6 +46,23 @@ chrome.tabs.onRemoved.addListener((tabId) => {
|
|
|
45
46
|
|
|
46
47
|
chrome.tabs.onActivated.addListener(({ tabId }) => {
|
|
47
48
|
void sessionManager.activateTab(tabId)
|
|
49
|
+
void sessionManager.ensureTabRegistered(tabId, { reason: "tab_activated" }).catch(() => {})
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => {
|
|
53
|
+
if (changeInfo.status !== "complete") return
|
|
54
|
+
void sessionManager.ensureTabRegistered(tabId, {
|
|
55
|
+
tab,
|
|
56
|
+
reason: "tab_updated_complete",
|
|
57
|
+
}).catch(() => {})
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
chrome.runtime.onInstalled.addListener(() => {
|
|
61
|
+
void sessionManager.ensureAllTabsRegistered({ reason: "extension_installed" }).catch(() => {})
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
chrome.runtime.onStartup.addListener(() => {
|
|
65
|
+
void sessionManager.ensureAllTabsRegistered({ reason: "browser_startup" }).catch(() => {})
|
|
48
66
|
})
|
|
49
67
|
|
|
50
68
|
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
|
@@ -53,3 +71,5 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
|
|
53
71
|
)
|
|
54
72
|
return true
|
|
55
73
|
})
|
|
74
|
+
|
|
75
|
+
void sessionManager.ensureAllTabsRegistered({ reason: "background_started" }).catch(() => {})
|
package/extension/cdp.js
CHANGED
|
@@ -4,6 +4,7 @@ export function createCdpController({
|
|
|
4
4
|
postBridge,
|
|
5
5
|
startPolling,
|
|
6
6
|
forwardConsoleEvent,
|
|
7
|
+
ensureAllTabsRegistered,
|
|
7
8
|
}) {
|
|
8
9
|
const cdpAttachedTabs = new Set()
|
|
9
10
|
const cdpEnabledDomains = new Map()
|
|
@@ -159,6 +160,9 @@ export function createCdpController({
|
|
|
159
160
|
}
|
|
160
161
|
|
|
161
162
|
async function listBrowserTargets(session) {
|
|
163
|
+
if (typeof ensureAllTabsRegistered === "function") {
|
|
164
|
+
await ensureAllTabsRegistered({ reason: "list_browser_targets" }).catch(() => null)
|
|
165
|
+
}
|
|
162
166
|
const cdpResult = await interceptTargetGetTargets(session)
|
|
163
167
|
const targetInfos = Array.isArray(cdpResult?.result?.targetInfos)
|
|
164
168
|
? cdpResult.result.targetInfos
|