yunti-browser-runtime 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +256 -0
  3. package/bin/yunti-browser-runtime.js +86 -0
  4. package/docs/EXECUTION_PLAN.md +1278 -0
  5. package/docs/INSTALL.md +205 -0
  6. package/docs/PROJECT_INTENT.md +44 -0
  7. package/docs/PROJECT_STATUS.md +263 -0
  8. package/docs/PUBLISHING_BLOCKERS.md +110 -0
  9. package/docs/RELEASE.md +148 -0
  10. package/docs/ROADMAP.md +42 -0
  11. package/docs/SECURITY.md +56 -0
  12. package/docs/TOOL_GUIDE.md +69 -0
  13. package/extension/background.js +55 -0
  14. package/extension/cdp.js +582 -0
  15. package/extension/content.css +9 -0
  16. package/extension/content.js +946 -0
  17. package/extension/manifest.json +35 -0
  18. package/extension/network-monitor.js +140 -0
  19. package/extension/popup.css +66 -0
  20. package/extension/popup.html +30 -0
  21. package/extension/popup.js +55 -0
  22. package/extension/session-manager.js +332 -0
  23. package/extension/settings.js +94 -0
  24. package/extension/tool-handlers.js +1158 -0
  25. package/lib/runtime-paths.js +39 -0
  26. package/mcp/bridge-hub.js +604 -0
  27. package/mcp/http-server.js +326 -0
  28. package/mcp/json-rpc.js +35 -0
  29. package/mcp/memory.js +126 -0
  30. package/mcp/redaction.js +94 -0
  31. package/mcp/server.js +269 -0
  32. package/mcp/tools.js +1092 -0
  33. package/package.json +60 -0
  34. package/scripts/check-package-metadata.js +131 -0
  35. package/scripts/check-published-package.js +113 -0
  36. package/scripts/doctor.js +163 -0
  37. package/scripts/package-extension.js +137 -0
  38. package/scripts/print-config.js +116 -0
  39. package/scripts/release-check.js +472 -0
  40. package/skills/yunti-browser-runtime/SKILL.md +77 -0
@@ -0,0 +1,946 @@
1
+ const PATCH_ATTR = "data-yunti-browser-runtime-patch"
2
+ const DANGEROUS_RE = /提交|保存|删除|作废|关闭|下架|审核|确认|同意|拒绝|submit|save|delete|remove|approve|reject/i
3
+ const AUTH_CACHE_TTL_MS = 5000
4
+
5
+ const patchStore = new Map()
6
+ let browserSessionId = null
7
+ let widgetRoot = null
8
+ let registerTimer = null
9
+ let authStateCache = null
10
+ let extensionContextInvalidated = false
11
+ let pageWatchersInstalled = false
12
+
13
+ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
14
+ if (message?.type === "yunti_execute_tool") {
15
+ void executeTool(message.tool, message.arguments || {})
16
+ .then(sendResponse)
17
+ .catch((error) => sendResponse({ ok: false, error: error.message || String(error) }))
18
+ return true
19
+ }
20
+ if (message?.type === "yunti_refresh_registration") {
21
+ void register({ forceAuth: true }).then(sendResponse, (error) =>
22
+ sendResponse({ ok: false, error: error?.message || String(error) })
23
+ )
24
+ return true
25
+ }
26
+ return false
27
+ })
28
+
29
+ void bootstrapContent()
30
+
31
+ async function bootstrapContent() {
32
+ await register().catch((error) => {
33
+ updateWidgetStatus(error?.message || String(error))
34
+ return null
35
+ })
36
+ }
37
+
38
+ async function register(options = {}) {
39
+ if (extensionContextInvalidated) return contextInvalidatedResponse()
40
+ const support = await sendRuntimeMessage({
41
+ type: "yunti_is_supported_page",
42
+ page: { url: location.href, title: document.title },
43
+ })
44
+ if (!support?.supported) {
45
+ return { ok: false, error: support?.error || "not a browser page" }
46
+ }
47
+
48
+ const page = await getPageState({ forceAuth: Boolean(options.forceAuth) })
49
+ const response = await sendRuntimeMessage({
50
+ type: "yunti_content_ready",
51
+ page,
52
+ })
53
+ if (response?.ok) {
54
+ browserSessionId = response.session.browserSessionId
55
+ installPageWatchers()
56
+ installWidget()
57
+ }
58
+ const auth = response?.session?.auth || page.auth
59
+ updateWidgetStatus(
60
+ response?.ok
61
+ ? auth.loggedIn
62
+ ? auth.userName
63
+ ? `Connected: ${auth.userName}`
64
+ : "Connected to current page"
65
+ : "Connected to current page"
66
+ : response?.error || "未连接",
67
+ auth
68
+ )
69
+ return response
70
+ }
71
+
72
+ async function getPageState(options = {}) {
73
+ return {
74
+ url: location.href,
75
+ title: document.title,
76
+ client: getClientInfo(),
77
+ auth: await getAuthState(options),
78
+ }
79
+ }
80
+
81
+ function getClientInfo() {
82
+ const userAgent = String(navigator.userAgent || "")
83
+ return {
84
+ userAgent,
85
+ family: detectBrowserFamily(userAgent),
86
+ platform: String(navigator.userAgentData?.platform || navigator.platform || ""),
87
+ language: String(navigator.language || ""),
88
+ }
89
+ }
90
+
91
+ function detectBrowserFamily(userAgent) {
92
+ if (/Edg\//i.test(userAgent)) return "edge"
93
+ if (/OPR\//i.test(userAgent)) return "opera"
94
+ if (/Brave\//i.test(userAgent)) return "brave"
95
+ if (/Chrome\//i.test(userAgent) || /Chromium\//i.test(userAgent)) return "chrome"
96
+ if (/Firefox\//i.test(userAgent)) return "firefox"
97
+ if (/Safari\//i.test(userAgent)) return "safari"
98
+ return "unknown"
99
+ }
100
+
101
+ async function getAuthState(options = {}) {
102
+ if (!options.forceAuth) {
103
+ const cached = getCachedAuthState()
104
+ if (cached) return cached
105
+ }
106
+
107
+ const state = getDomAuthState()
108
+ cacheAuthState(state)
109
+ return state
110
+ }
111
+
112
+ function getCachedAuthState() {
113
+ if (!authStateCache) return null
114
+ if (authStateCache.href !== location.href) return null
115
+ if (Date.now() - authStateCache.at > AUTH_CACHE_TTL_MS) return null
116
+ return authStateCache.value
117
+ }
118
+
119
+ function cacheAuthState(value) {
120
+ authStateCache = {
121
+ href: location.href,
122
+ at: Date.now(),
123
+ value,
124
+ }
125
+ }
126
+
127
+ function getDomAuthState() {
128
+ const visibleText = collectVisibleText(5000)
129
+ const hashPath = location.hash.replace(/^#/, "")
130
+ const urlLooksLogin = /(^|[/?#])(?:login|signin|auth)(?:$|[/?#&=])/i.test(location.href)
131
+ const businessRoute = /^\/(?!login|signin|auth)(?:[a-z0-9_-]+\/?)+/i.test(hashPath)
132
+ const passwordInputs = [...document.querySelectorAll("input[type='password']")].filter(isVisible)
133
+ const loginText = /登录|登陆|用户名|账号|密码|验证码|sign in|login/i.test(visibleText)
134
+ const appShell = document.querySelector(
135
+ ".ant-layout,.ant-menu,.el-menu,.el-container,[class*='layout'],[class*='sidebar'],[class*='menu'],nav,aside"
136
+ )
137
+ const userMenu = getUserMenuState()
138
+ const businessText =
139
+ /工作台|仪表盘|首页|菜单|系统|管理|查询|搜索|筛选|新增|编辑|导入|导出|返回|列表|详情|设置|帮助中心/i.test(
140
+ visibleText
141
+ )
142
+
143
+ if (userMenu.loggedIn) {
144
+ return {
145
+ state: "authenticated",
146
+ loggedIn: true,
147
+ reason: userMenu.userName
148
+ ? `Detected user menu: ${userMenu.userName}`
149
+ : "Detected user menu and logout entry",
150
+ userName: userMenu.userName,
151
+ checkedAt: new Date().toISOString(),
152
+ }
153
+ }
154
+
155
+ if (!urlLooksLogin && businessRoute && (appShell || businessText)) {
156
+ return {
157
+ state: "authenticated",
158
+ loggedIn: true,
159
+ reason: "Detected application route and page shell",
160
+ checkedAt: new Date().toISOString(),
161
+ }
162
+ }
163
+
164
+ if (!urlLooksLogin && appShell && businessText) {
165
+ return {
166
+ state: "authenticated",
167
+ loggedIn: true,
168
+ reason: "Detected application navigation or content",
169
+ checkedAt: new Date().toISOString(),
170
+ }
171
+ }
172
+
173
+ if (urlLooksLogin || (passwordInputs.length > 0 && loginText) || (loginText && !appShell && !businessText)) {
174
+ return {
175
+ state: "unauthenticated",
176
+ loggedIn: false,
177
+ reason: "检测到登录页或登录表单",
178
+ checkedAt: new Date().toISOString(),
179
+ }
180
+ }
181
+
182
+ if (document.readyState === "loading") {
183
+ return {
184
+ state: "checking",
185
+ loggedIn: false,
186
+ reason: "页面仍在加载",
187
+ checkedAt: new Date().toISOString(),
188
+ }
189
+ }
190
+
191
+ return {
192
+ state: "unknown",
193
+ loggedIn: false,
194
+ reason: "Current page is connected; the agent should infer auth state from page context when needed",
195
+ checkedAt: new Date().toISOString(),
196
+ }
197
+ }
198
+
199
+ function getUserMenuState() {
200
+ const logoutByXPath = textFromXPath("//*[starts-with(@id, 'dropdown-menu-')]/li[2]/span")
201
+ const userByXPath = textFromXPath("//*[starts-with(@id, 'dropdown-menu-')]/li[1]")
202
+ const menuItems = [
203
+ ...document.querySelectorAll("[id^='dropdown-menu-'] li, [id^='dropdown-menu-'] span"),
204
+ ].filter(isVisible)
205
+ const logoutByCss = menuItems.some((item) => /退出登录|退出|注销|登出/i.test(item.innerText || ""))
206
+ const userByCss =
207
+ menuItems.map((item) => compactText(item.innerText || "")).find((text) => text && !/退出|注销|登出/i.test(text)) ||
208
+ ""
209
+
210
+ return {
211
+ loggedIn: /退出登录|退出|注销|登出/i.test(logoutByXPath || "") || logoutByCss,
212
+ userName: compactText(userByXPath || userByCss || "") || "",
213
+ }
214
+ }
215
+
216
+ function textFromXPath(xpath) {
217
+ try {
218
+ const node = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
219
+ .singleNodeValue
220
+ return node && isVisible(node.parentElement || node) ? node.textContent || "" : ""
221
+ } catch {
222
+ return ""
223
+ }
224
+ }
225
+
226
+ function installPageWatchers() {
227
+ if (pageWatchersInstalled) return
228
+ pageWatchersInstalled = true
229
+ window.addEventListener("popstate", scheduleRegister)
230
+ window.addEventListener("hashchange", scheduleRegister)
231
+ document.addEventListener("visibilitychange", () => {
232
+ if (!document.hidden) scheduleRegister()
233
+ })
234
+
235
+ for (const name of ["pushState", "replaceState"]) {
236
+ const original = history[name]
237
+ history[name] = function patchedHistoryState(...args) {
238
+ const result = original.apply(this, args)
239
+ scheduleRegister()
240
+ return result
241
+ }
242
+ }
243
+
244
+ const observer = new MutationObserver(scheduleRegister)
245
+ observer.observe(document.documentElement, {
246
+ childList: true,
247
+ subtree: true,
248
+ })
249
+ }
250
+
251
+ function scheduleRegister() {
252
+ if (extensionContextInvalidated) return
253
+ clearTimeout(registerTimer)
254
+ registerTimer = setTimeout(() => {
255
+ void register().catch((error) => updateWidgetStatus(error?.message || String(error)))
256
+ }, 500)
257
+ }
258
+
259
+ function installWidget() {
260
+ if (widgetRoot || document.getElementById("yunti-browser-runtime-widget")) return
261
+ const host = document.createElement("div")
262
+ host.id = "yunti-browser-runtime-widget"
263
+ host.style.position = "fixed"
264
+ host.style.right = "18px"
265
+ host.style.bottom = "18px"
266
+ host.style.zIndex = "2147483647"
267
+ document.documentElement.appendChild(host)
268
+
269
+ const shadow = host.attachShadow({ mode: "open" })
270
+ shadow.innerHTML = `
271
+ <style>
272
+ :host { all: initial; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
273
+ .fab {
274
+ width: 44px; height: 44px; border-radius: 999px; border: 0; cursor: pointer;
275
+ background: #0f766e; color: white; box-shadow: 0 10px 24px rgba(15, 23, 42, .24);
276
+ font: 700 15px/1 system-ui; display: grid; place-items: center;
277
+ }
278
+ .panel {
279
+ width: 292px; margin-bottom: 10px; border: 1px solid rgba(15, 23, 42, .14);
280
+ border-radius: 10px; background: white; color: #111827;
281
+ box-shadow: 0 18px 42px rgba(15, 23, 42, .24); overflow: hidden;
282
+ }
283
+ .hidden { display: none; }
284
+ header { padding: 12px 13px; background: #0f766e; color: white; }
285
+ h2 { margin: 0; font-size: 14px; letter-spacing: 0; }
286
+ p { margin: 5px 0 0; font-size: 12px; color: rgba(255,255,255,.86); }
287
+ .body { display: grid; gap: 9px; padding: 12px; }
288
+ .status { font-size: 12px; line-height: 1.4; color: #475569; word-break: break-word; }
289
+ button.action {
290
+ min-height: 32px; border-radius: 7px; border: 1px solid #cbd5e1; background: #fff;
291
+ color: #0f172a; cursor: pointer; font: 500 13px/1 system-ui;
292
+ }
293
+ button.action:disabled { opacity: .48; cursor: not-allowed; }
294
+ button.primary { border-color: #0f766e; background: #0f766e; color: white; }
295
+ .row { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; }
296
+ </style>
297
+ <div class="panel hidden" data-panel>
298
+ <header>
299
+ <h2>Yunti Browser Runtime</h2>
300
+ <p>当前页面临时增强,刷新后恢复。</p>
301
+ </header>
302
+ <div class="body">
303
+ <div class="status" data-status>正在连接...</div>
304
+ <button class="action" data-refresh>刷新连接</button>
305
+ </div>
306
+ </div>
307
+ <button class="fab" data-toggle title="Yunti Browser Runtime">AI</button>
308
+ `
309
+
310
+ widgetRoot = shadow
311
+ const panel = shadow.querySelector("[data-panel]")
312
+ shadow.querySelector("[data-toggle]").addEventListener("click", () => {
313
+ panel.classList.toggle("hidden")
314
+ })
315
+ shadow.querySelector("[data-refresh]").addEventListener("click", () => {
316
+ void register({ forceAuth: true }).catch((error) => updateWidgetStatus(error?.message || String(error)))
317
+ })
318
+ }
319
+
320
+ async function sendWidgetMessage(message) {
321
+ return sendRuntimeMessage(message)
322
+ }
323
+
324
+ async function sendRuntimeMessage(message) {
325
+ if (extensionContextInvalidated) return contextInvalidatedResponse()
326
+ try {
327
+ return await chrome.runtime.sendMessage(message)
328
+ } catch (error) {
329
+ if (isExtensionContextInvalidated(error)) {
330
+ return markExtensionContextInvalidated()
331
+ }
332
+ return { ok: false, error: error?.message || String(error) }
333
+ }
334
+ }
335
+
336
+ function isExtensionContextInvalidated(error) {
337
+ return /extension context invalidated/i.test(error?.message || String(error || ""))
338
+ }
339
+
340
+ function contextInvalidatedResponse() {
341
+ return { ok: false, error: "插件已重新加载,请刷新 browser page恢复连接。" }
342
+ }
343
+
344
+ function markExtensionContextInvalidated() {
345
+ extensionContextInvalidated = true
346
+ clearTimeout(registerTimer)
347
+ updateWidgetStatus(contextInvalidatedResponse().error)
348
+ return contextInvalidatedResponse()
349
+ }
350
+
351
+ function updateWidgetStatus(text, auth = null) {
352
+ if (!widgetRoot && browserSessionId) installWidget()
353
+ const status = widgetRoot?.querySelector("[data-status]")
354
+ if (status) {
355
+ status.textContent = `${text}${browserSessionId ? `\n${browserSessionId}` : ""}`
356
+ }
357
+ if (auth) {
358
+ for (const button of widgetRoot?.querySelectorAll("[data-refresh]") || []) {
359
+ button.disabled = false
360
+ button.title = auth.loggedIn
361
+ ? ""
362
+ : "Yunti页面已连接,登录态由 agent 结合页面和接口结果判断"
363
+ }
364
+ }
365
+ }
366
+
367
+ async function executeTool(tool, args) {
368
+ switch (tool) {
369
+ case "yunti_get_page_snapshot":
370
+ return getPageSnapshot(args)
371
+ case "yunti_get_selected_context":
372
+ return getSelectedContext()
373
+ case "yunti_fetch_with_cookie":
374
+ return fetchWithCookie(args)
375
+ case "yunti_apply_preview_patch":
376
+ return applyPreviewPatch(args)
377
+ case "yunti_rollback_preview_patch":
378
+ return rollbackPreviewPatch(args.patchId)
379
+ case "yunti_click":
380
+ return clickElement(args)
381
+ case "yunti_click_at":
382
+ return clickAt(args)
383
+ case "yunti_fill":
384
+ return fillElement(args)
385
+ case "yunti_type_text":
386
+ return typeText(args)
387
+ case "yunti_press_key":
388
+ return pressKey(args)
389
+ case "yunti_select":
390
+ return selectElement(args)
391
+ case "yunti_scroll":
392
+ return scrollPage(args)
393
+ case "yunti_request_user_confirmation":
394
+ return requestUserConfirmation(args)
395
+ default:
396
+ throw new Error(`Unknown Yunti browser tool: ${tool}`)
397
+ }
398
+ }
399
+
400
+ async function getPageSnapshot(args = {}) {
401
+ const mode = args.mode === "detailed" ? "detailed" : "light"
402
+ const includeElements = args.includeElements === undefined ? mode === "detailed" : Boolean(args.includeElements)
403
+ const defaultTextLength = includeElements ? 12000 : 4000
404
+ const maxTextLength = clamp(Number(args.maxTextLength ?? defaultTextLength), 0, 60000)
405
+ const snapshot = {
406
+ browserSessionId,
407
+ url: location.href,
408
+ origin: location.origin,
409
+ title: document.title,
410
+ auth: await getAuthState(),
411
+ mode,
412
+ selection: String(getSelection()?.toString() || "").trim().slice(0, 4000),
413
+ visibleText: collectVisibleText(maxTextLength),
414
+ overview: getPageOverview(),
415
+ capturedAt: new Date().toISOString(),
416
+ }
417
+ if (includeElements) {
418
+ snapshot.forms = [...document.forms].slice(0, 20).map(describeForm)
419
+ snapshot.buttons = [...document.querySelectorAll("button,input[type=button],input[type=submit],a")]
420
+ .filter(isVisible)
421
+ .slice(0, 100)
422
+ .map(describeClickable)
423
+ snapshot.inputs = [...document.querySelectorAll("input,textarea,select,[contenteditable=true]")]
424
+ .filter(isVisible)
425
+ .slice(0, 120)
426
+ .map(describeInput)
427
+ snapshot.links = [...document.querySelectorAll("a[href]")]
428
+ .filter(isVisible)
429
+ .slice(0, 80)
430
+ .map(describeClickable)
431
+ snapshot.tables = [...document.querySelectorAll("table")]
432
+ .filter(isVisible)
433
+ .slice(0, 20)
434
+ .map(describeTable)
435
+ }
436
+ return snapshot
437
+ }
438
+
439
+ function getSelectedContext() {
440
+ const selection = getSelection()
441
+ const text = String(selection?.toString() || "").trim()
442
+ let element = null
443
+ if (selection && selection.rangeCount > 0) {
444
+ const node = selection.getRangeAt(0).commonAncestorContainer
445
+ element = node.nodeType === Node.ELEMENT_NODE ? node : node.parentElement
446
+ }
447
+ return {
448
+ browserSessionId,
449
+ url: location.href,
450
+ title: document.title,
451
+ text,
452
+ element: element ? describeElement(element) : null,
453
+ }
454
+ }
455
+
456
+ async function fetchWithCookie(args) {
457
+ const url = new URL(args.url, location.href)
458
+ if (url.origin !== location.origin) {
459
+ throw new Error("yunti_fetch_with_cookie only allows current platform origin")
460
+ }
461
+
462
+ const method = String(args.method || "GET").toUpperCase()
463
+ const response = await fetch(url.toString(), {
464
+ method,
465
+ headers: args.headers || {},
466
+ body: args.body ?? undefined,
467
+ credentials: "include",
468
+ })
469
+ const maxBytes = clamp(Number(args.maxBytes || 200000), 1024, 1000000)
470
+ const text = await response.text()
471
+ return {
472
+ url: url.toString(),
473
+ status: response.status,
474
+ ok: response.ok,
475
+ redirected: response.redirected,
476
+ contentType: response.headers.get("content-type"),
477
+ body: text.slice(0, maxBytes),
478
+ truncated: text.length > maxBytes,
479
+ }
480
+ }
481
+
482
+ function applyPreviewPatch(args) {
483
+ const patchId = String(args.patchId || crypto.randomUUID())
484
+ const applied = []
485
+
486
+ for (const patch of args.patches || []) {
487
+ const target = mustFind(patch.selector)
488
+ const record = snapshotElement(target)
489
+ record.inserted = []
490
+
491
+ target.setAttribute(PATCH_ATTR, patchId)
492
+ target.classList.add("yunti-browser-runtime-highlight")
493
+
494
+ if (patch.text !== undefined) target.textContent = String(patch.text)
495
+ if (patch.html !== undefined) target.innerHTML = String(patch.html)
496
+ if (patch.value !== undefined && "value" in target) {
497
+ target.value = String(patch.value)
498
+ target.dispatchEvent(new Event("input", { bubbles: true }))
499
+ target.dispatchEvent(new Event("change", { bubbles: true }))
500
+ }
501
+ if (patch.style && typeof patch.style === "object") {
502
+ for (const [key, value] of Object.entries(patch.style)) target.style[key] = String(value)
503
+ }
504
+ if (patch.attributes && typeof patch.attributes === "object") {
505
+ for (const [key, value] of Object.entries(patch.attributes)) {
506
+ target.setAttribute(key, String(value))
507
+ }
508
+ }
509
+ if (patch.className) target.classList.add(...String(patch.className).split(/\s+/).filter(Boolean))
510
+ for (const [position, html] of [
511
+ ["beforeend", patch.appendHtml],
512
+ ["beforebegin", patch.beforeHtml],
513
+ ["afterend", patch.afterHtml],
514
+ ]) {
515
+ if (!html) continue
516
+ record.inserted.push(...insertTemporaryHtml(target, position, String(html), patchId))
517
+ }
518
+
519
+ applied.push({ selector: patch.selector, element: describeElement(target), record })
520
+ }
521
+
522
+ patchStore.set(patchId, applied)
523
+ return { patchId, applied: applied.length }
524
+ }
525
+
526
+ function rollbackPreviewPatch(patchId) {
527
+ const ids = patchId ? [patchId] : [...patchStore.keys()]
528
+ let restored = 0
529
+ for (const id of ids) {
530
+ const records = patchStore.get(id) || []
531
+ for (const item of records.reverse()) {
532
+ restoreElement(item.record)
533
+ restored += 1
534
+ }
535
+ patchStore.delete(id)
536
+ }
537
+ return { restored, remainingPatchIds: [...patchStore.keys()] }
538
+ }
539
+
540
+ function clickElement(args) {
541
+ const element = mustFind(args.selector)
542
+ element.scrollIntoView({ block: "center", inline: "center" })
543
+ element.click()
544
+ return { clicked: true, element: describeElement(element) }
545
+ }
546
+
547
+ function clickAt(args) {
548
+ const x = clamp(Number(args.x), 0, Math.max(0, window.innerWidth - 1))
549
+ const y = clamp(Number(args.y), 0, Math.max(0, window.innerHeight - 1))
550
+ const element = document.elementFromPoint(x, y)
551
+ if (!element) throw new Error(`No element found at viewport coordinate ${x},${y}`)
552
+ focusElement(element)
553
+ dispatchPointerMouseSequence(element, x, y)
554
+ return {
555
+ clicked: true,
556
+ x,
557
+ y,
558
+ element: describeElement(element),
559
+ }
560
+ }
561
+
562
+ function fillElement(args) {
563
+ const element = mustFind(args.selector)
564
+ const value = String(args.value ?? "")
565
+ element.scrollIntoView({ block: "center", inline: "center" })
566
+ setEditableText(element, value, { replace: true })
567
+ return { filled: true, element: describeElement(element), valueLength: value.length }
568
+ }
569
+
570
+ function typeText(args) {
571
+ const text = String(args.text ?? "")
572
+ const element = resolveTargetElement(args) || document.activeElement
573
+ if (!element || element === document.body || element === document.documentElement) {
574
+ throw new Error("No editable target is focused; pass selector or x/y")
575
+ }
576
+ focusElement(element)
577
+ setEditableText(element, text, { replace: Boolean(args.clear) })
578
+ return {
579
+ typed: true,
580
+ element: describeElement(element),
581
+ textLength: text.length,
582
+ mode: args.clear ? "replace" : "append",
583
+ }
584
+ }
585
+
586
+ function pressKey(args) {
587
+ const key = String(args.key || "")
588
+ if (!key) throw new Error("key is required")
589
+ const element = resolveTargetElement(args) || document.activeElement || document.body
590
+ focusElement(element)
591
+ const beforeValue = getEditableText(element)
592
+ dispatchKeyboardEvent(element, "keydown", key)
593
+ applySimpleKeyEdit(element, key)
594
+ dispatchKeyboardEvent(element, "keyup", key)
595
+ const afterValue = getEditableText(element)
596
+ return {
597
+ pressed: true,
598
+ key,
599
+ element: describeElement(element),
600
+ valueChanged: beforeValue !== afterValue,
601
+ }
602
+ }
603
+
604
+ function selectElement(args) {
605
+ const element = mustFind(args.selector)
606
+ if (!(element instanceof HTMLSelectElement)) throw new Error("Target is not a select element")
607
+ element.value = String(args.value ?? "")
608
+ element.dispatchEvent(new Event("input", { bubbles: true }))
609
+ element.dispatchEvent(new Event("change", { bubbles: true }))
610
+ return { selected: true, element: describeElement(element), value: element.value }
611
+ }
612
+
613
+ function scrollPage(args) {
614
+ const deltaX = Number.isFinite(Number(args.deltaX)) ? Number(args.deltaX) : 0
615
+ const deltaY = Number.isFinite(Number(args.deltaY)) ? Number(args.deltaY) : 600
616
+ const coordinateTarget =
617
+ Number.isFinite(Number(args.x)) && Number.isFinite(Number(args.y))
618
+ ? document.elementFromPoint(
619
+ clamp(Number(args.x), 0, Math.max(0, window.innerWidth - 1)),
620
+ clamp(Number(args.y), 0, Math.max(0, window.innerHeight - 1))
621
+ )
622
+ : null
623
+ const target = findScrollableAncestor(coordinateTarget) || document.scrollingElement || document.documentElement
624
+ const before = getScrollPosition(target)
625
+ target.scrollBy({ left: deltaX, top: deltaY, behavior: "auto" })
626
+ const after = getScrollPosition(target)
627
+ return {
628
+ scrolled: true,
629
+ deltaX,
630
+ deltaY,
631
+ target: target === document.scrollingElement ? "document" : describeElement(target),
632
+ before,
633
+ after,
634
+ }
635
+ }
636
+
637
+ function requestUserConfirmation(args) {
638
+ const approved = window.confirm(`${args.message || "Confirm agent action"}\n\n${args.detail || ""}`)
639
+ return { approved }
640
+ }
641
+
642
+ function resolveTargetElement(args = {}) {
643
+ if (args.selector) return mustFind(args.selector)
644
+ if (Number.isFinite(Number(args.x)) && Number.isFinite(Number(args.y))) {
645
+ return document.elementFromPoint(
646
+ clamp(Number(args.x), 0, Math.max(0, window.innerWidth - 1)),
647
+ clamp(Number(args.y), 0, Math.max(0, window.innerHeight - 1))
648
+ )
649
+ }
650
+ return null
651
+ }
652
+
653
+ function focusElement(element) {
654
+ if (element && typeof element.focus === "function") {
655
+ element.focus({ preventScroll: true })
656
+ }
657
+ }
658
+
659
+ function dispatchPointerMouseSequence(element, x, y) {
660
+ const common = {
661
+ bubbles: true,
662
+ cancelable: true,
663
+ composed: true,
664
+ clientX: x,
665
+ clientY: y,
666
+ view: window,
667
+ }
668
+ for (const type of ["pointerover", "pointermove", "pointerdown", "pointerup"]) {
669
+ try {
670
+ element.dispatchEvent(new PointerEvent(type, { ...common, pointerType: "mouse", isPrimary: true }))
671
+ } catch {
672
+ break
673
+ }
674
+ }
675
+ for (const type of ["mouseover", "mousemove", "mousedown", "mouseup", "click"]) {
676
+ element.dispatchEvent(new MouseEvent(type, common))
677
+ }
678
+ }
679
+
680
+ function setEditableText(element, text, options = {}) {
681
+ if (element.isContentEditable) {
682
+ const next = options.replace ? text : `${element.textContent || ""}${text}`
683
+ element.textContent = next
684
+ } else if ("value" in element) {
685
+ const previous = String(element.value || "")
686
+ const next = options.replace ? text : `${previous}${text}`
687
+ setNativeValue(element, next)
688
+ } else {
689
+ throw new Error("Target element cannot accept text")
690
+ }
691
+ element.dispatchEvent(new Event("input", { bubbles: true }))
692
+ element.dispatchEvent(new Event("change", { bubbles: true }))
693
+ }
694
+
695
+ function setNativeValue(element, value) {
696
+ const prototype = Object.getPrototypeOf(element)
697
+ const descriptor = Object.getOwnPropertyDescriptor(prototype, "value")
698
+ if (descriptor?.set) descriptor.set.call(element, value)
699
+ else element.value = value
700
+ }
701
+
702
+ function getEditableText(element) {
703
+ if (!element) return ""
704
+ if (element.isContentEditable) return element.textContent || ""
705
+ if ("value" in element) return String(element.value || "")
706
+ return ""
707
+ }
708
+
709
+ function dispatchKeyboardEvent(element, type, key) {
710
+ element.dispatchEvent(
711
+ new KeyboardEvent(type, {
712
+ key,
713
+ code: key.length === 1 ? `Key${key.toUpperCase()}` : key,
714
+ bubbles: true,
715
+ cancelable: true,
716
+ composed: true,
717
+ })
718
+ )
719
+ }
720
+
721
+ function applySimpleKeyEdit(element, key) {
722
+ if (key === "Backspace") {
723
+ const current = getEditableText(element)
724
+ if (current) setEditableText(element, current.slice(0, -1), { replace: true })
725
+ }
726
+ if (key === "Enter" && element instanceof HTMLTextAreaElement) {
727
+ setEditableText(element, "\n")
728
+ }
729
+ }
730
+
731
+ function findScrollableAncestor(element) {
732
+ let node = element
733
+ while (node && node !== document.body && node !== document.documentElement) {
734
+ const style = getComputedStyle(node)
735
+ const scrollableY = /(auto|scroll)/.test(style.overflowY) && node.scrollHeight > node.clientHeight
736
+ const scrollableX = /(auto|scroll)/.test(style.overflowX) && node.scrollWidth > node.clientWidth
737
+ if (scrollableY || scrollableX) return node
738
+ node = node.parentElement
739
+ }
740
+ return null
741
+ }
742
+
743
+ function getScrollPosition(element) {
744
+ return {
745
+ left: Math.round(element.scrollLeft || 0),
746
+ top: Math.round(element.scrollTop || 0),
747
+ }
748
+ }
749
+
750
+ function insertTemporaryHtml(target, position, html, patchId) {
751
+ const template = document.createElement("template")
752
+ template.innerHTML = html
753
+ const nodes = [...template.content.childNodes].map((node) => {
754
+ if (node.nodeType === Node.ELEMENT_NODE) {
755
+ node.setAttribute(PATCH_ATTR, patchId)
756
+ node.classList.add("yunti-browser-runtime-highlight")
757
+ return node
758
+ }
759
+ const span = document.createElement("span")
760
+ span.setAttribute(PATCH_ATTR, patchId)
761
+ span.className = "yunti-browser-runtime-highlight"
762
+ span.textContent = node.textContent || ""
763
+ return span
764
+ })
765
+
766
+ if (position === "beforeend") target.append(...nodes)
767
+ else if (position === "beforebegin") target.before(...nodes)
768
+ else if (position === "afterend") target.after(...nodes)
769
+ else throw new Error(`Unsupported insert position: ${position}`)
770
+
771
+ return nodes
772
+ }
773
+
774
+ function collectVisibleText(maxTextLength) {
775
+ if (maxTextLength <= 0) return ""
776
+ const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, {
777
+ acceptNode(node) {
778
+ const text = node.nodeValue?.replace(/\s+/g, " ").trim()
779
+ if (!text) return NodeFilter.FILTER_REJECT
780
+ return isVisible(node.parentElement) ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT
781
+ },
782
+ })
783
+ let out = ""
784
+ while (walker.nextNode() && out.length < maxTextLength) {
785
+ out += `${walker.currentNode.nodeValue.replace(/\s+/g, " ").trim()}\n`
786
+ }
787
+ return out.slice(0, maxTextLength)
788
+ }
789
+
790
+ function getPageOverview() {
791
+ const visibleButtons = [...document.querySelectorAll("button,input[type=button],input[type=submit]")]
792
+ .filter(isVisible)
793
+ .slice(0, 20)
794
+ .map((element) => compactText(element.innerText || element.value || element.getAttribute("aria-label") || ""))
795
+ .filter(Boolean)
796
+ const visibleInputs = [...document.querySelectorAll("input,textarea,select,[contenteditable=true]")]
797
+ .filter(isVisible)
798
+ .slice(0, 20)
799
+ .map((element) => ({
800
+ type: element.type || element.tagName.toLowerCase(),
801
+ name: element.getAttribute("name"),
802
+ placeholder: element.getAttribute("placeholder"),
803
+ label: findLabel(element),
804
+ }))
805
+ return {
806
+ heading: compactText(document.querySelector("h1,h2,h3,[role=heading]")?.innerText || ""),
807
+ path: location.hash || location.pathname,
808
+ counts: {
809
+ forms: document.forms.length,
810
+ buttons: [...document.querySelectorAll("button,input[type=button],input[type=submit]")].filter(isVisible)
811
+ .length,
812
+ links: [...document.querySelectorAll("a[href]")].filter(isVisible).length,
813
+ inputs: [...document.querySelectorAll("input,textarea,select,[contenteditable=true]")].filter(isVisible)
814
+ .length,
815
+ tables: [...document.querySelectorAll("table")].filter(isVisible).length,
816
+ },
817
+ sampleButtons: visibleButtons,
818
+ sampleInputs: visibleInputs,
819
+ }
820
+ }
821
+
822
+ function describeForm(form) {
823
+ return {
824
+ selector: cssPath(form),
825
+ id: form.id || null,
826
+ name: form.getAttribute("name"),
827
+ action: form.action || null,
828
+ method: form.method || null,
829
+ fields: [...form.querySelectorAll("input,textarea,select")]
830
+ .filter(isVisible)
831
+ .slice(0, 80)
832
+ .map(describeInput),
833
+ }
834
+ }
835
+
836
+ function describeClickable(element) {
837
+ return {
838
+ ...describeElement(element),
839
+ text: compactText(element.innerText || element.value || element.getAttribute("title") || ""),
840
+ href: element.href || null,
841
+ }
842
+ }
843
+
844
+ function describeInput(element) {
845
+ return {
846
+ ...describeElement(element),
847
+ type: element.type || element.tagName.toLowerCase(),
848
+ name: element.getAttribute("name"),
849
+ placeholder: element.getAttribute("placeholder"),
850
+ label: findLabel(element),
851
+ valuePreview: "value" in element ? String(element.value || "").slice(0, 80) : null,
852
+ }
853
+ }
854
+
855
+ function describeTable(table) {
856
+ const headers = [...table.querySelectorAll("th")].slice(0, 30).map((th) => compactText(th.innerText))
857
+ const rows = [...table.querySelectorAll("tr")]
858
+ .slice(0, 5)
859
+ .map((tr) => [...tr.children].slice(0, 10).map((td) => compactText(td.innerText)))
860
+ return { selector: cssPath(table), headers, rows }
861
+ }
862
+
863
+ function describeElement(element) {
864
+ return {
865
+ selector: cssPath(element),
866
+ tag: element.tagName.toLowerCase(),
867
+ id: element.id || null,
868
+ className: element.className || null,
869
+ text: compactText(element.innerText || element.textContent || ""),
870
+ }
871
+ }
872
+
873
+ function findLabel(element) {
874
+ if (element.id) {
875
+ const label = document.querySelector(`label[for="${CSS.escape(element.id)}"]`)
876
+ if (label) return compactText(label.innerText)
877
+ }
878
+ return compactText(element.closest("label")?.innerText || "")
879
+ }
880
+
881
+ function compactText(text) {
882
+ return String(text || "").replace(/\s+/g, " ").trim().slice(0, 160) || null
883
+ }
884
+
885
+ function mustFind(selector) {
886
+ const element = document.querySelector(selector)
887
+ if (!element) throw new Error(`Element not found: ${selector}`)
888
+ return element
889
+ }
890
+
891
+ function snapshotElement(element) {
892
+ return {
893
+ element,
894
+ html: element.innerHTML,
895
+ text: element.textContent,
896
+ value: "value" in element ? element.value : undefined,
897
+ className: element.className,
898
+ style: element.getAttribute("style"),
899
+ attrs: [...element.attributes].map((attr) => [attr.name, attr.value]),
900
+ }
901
+ }
902
+
903
+ function restoreElement(record) {
904
+ for (const node of record.inserted || []) node.remove()
905
+ const element = record.element
906
+ if (!element?.isConnected) return
907
+ while (element.attributes.length) element.removeAttribute(element.attributes[0].name)
908
+ for (const [name, value] of record.attrs) element.setAttribute(name, value)
909
+ element.innerHTML = record.html
910
+ if (record.value !== undefined && "value" in element) element.value = record.value
911
+ if (record.style === null) element.removeAttribute("style")
912
+ else element.setAttribute("style", record.style)
913
+ element.className = record.className
914
+ }
915
+
916
+ function isVisible(element) {
917
+ if (!element) return false
918
+ const style = getComputedStyle(element)
919
+ if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0) {
920
+ return false
921
+ }
922
+ const rect = element.getBoundingClientRect()
923
+ return rect.width > 0 && rect.height > 0
924
+ }
925
+
926
+ function cssPath(element) {
927
+ if (element.id) return `#${CSS.escape(element.id)}`
928
+ const parts = []
929
+ let node = element
930
+ while (node && node.nodeType === Node.ELEMENT_NODE && node !== document.body) {
931
+ let part = node.tagName.toLowerCase()
932
+ const parent = node.parentElement
933
+ if (parent) {
934
+ const same = [...parent.children].filter((child) => child.tagName === node.tagName)
935
+ if (same.length > 1) part += `:nth-of-type(${same.indexOf(node) + 1})`
936
+ }
937
+ parts.unshift(part)
938
+ node = parent
939
+ }
940
+ return parts.length ? parts.join(" > ") : "body"
941
+ }
942
+
943
+ function clamp(value, min, max) {
944
+ if (!Number.isFinite(value)) return min
945
+ return Math.max(min, Math.min(max, value))
946
+ }