sentienceapi 0.98.5 → 0.99.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -1099
- package/dist/actions.d.ts +37 -0
- package/dist/actions.d.ts.map +1 -1
- package/dist/actions.js +592 -0
- package/dist/actions.js.map +1 -1
- package/dist/agent-runtime.d.ts +56 -3
- package/dist/agent-runtime.d.ts.map +1 -1
- package/dist/agent-runtime.js +389 -1
- package/dist/agent-runtime.js.map +1 -1
- package/dist/agent.d.ts +3 -1
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +59 -5
- package/dist/agent.js.map +1 -1
- package/dist/browser.d.ts +24 -1
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +88 -1
- package/dist/browser.js.map +1 -1
- package/dist/cli.js +240 -0
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +8 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +21 -4
- package/dist/index.js.map +1 -1
- package/dist/read.d.ts +7 -0
- package/dist/read.d.ts.map +1 -1
- package/dist/read.js +42 -0
- package/dist/read.js.map +1 -1
- package/dist/runtime-agent.d.ts +72 -0
- package/dist/runtime-agent.d.ts.map +1 -0
- package/dist/runtime-agent.js +357 -0
- package/dist/runtime-agent.js.map +1 -0
- package/dist/snapshot.d.ts.map +1 -1
- package/dist/snapshot.js +8 -3
- package/dist/snapshot.js.map +1 -1
- package/dist/tools/context.d.ts +18 -0
- package/dist/tools/context.d.ts.map +1 -0
- package/dist/tools/context.js +40 -0
- package/dist/tools/context.js.map +1 -0
- package/dist/tools/defaults.d.ts +5 -0
- package/dist/tools/defaults.d.ts.map +1 -0
- package/dist/tools/defaults.js +368 -0
- package/dist/tools/defaults.js.map +1 -0
- package/dist/tools/filesystem.d.ts +12 -0
- package/dist/tools/filesystem.d.ts.map +1 -0
- package/dist/tools/filesystem.js +137 -0
- package/dist/tools/filesystem.js.map +1 -0
- package/dist/tools/index.d.ts +5 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +15 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/registry.d.ts +38 -0
- package/dist/tools/registry.d.ts.map +1 -0
- package/dist/tools/registry.js +100 -0
- package/dist/tools/registry.js.map +1 -0
- package/dist/types.d.ts +67 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/trace-event-builder.d.ts +20 -0
- package/dist/utils/trace-event-builder.d.ts.map +1 -1
- package/dist/utils/trace-event-builder.js +41 -2
- package/dist/utils/trace-event-builder.js.map +1 -1
- package/dist/utils/zod.d.ts +5 -0
- package/dist/utils/zod.d.ts.map +1 -0
- package/dist/utils/zod.js +80 -0
- package/dist/utils/zod.js.map +1 -0
- package/dist/verification.d.ts +9 -0
- package/dist/verification.d.ts.map +1 -1
- package/dist/verification.js +27 -0
- package/dist/verification.js.map +1 -1
- package/dist/vision-executor.d.ts +18 -0
- package/dist/vision-executor.d.ts.map +1 -0
- package/dist/vision-executor.js +60 -0
- package/dist/vision-executor.js.map +1 -0
- package/dist/visual-agent.d.ts.map +1 -1
- package/dist/visual-agent.js +15 -0
- package/dist/visual-agent.js.map +1 -1
- package/package.json +1 -1
- package/src/extension/background.js +1 -1
- package/src/extension/content.js +4 -3
- package/src/extension/injected_api.js +125 -57
- package/src/extension/manifest.json +1 -1
- package/src/extension/pkg/sentience_core_bg.wasm +0 -0
- package/src/extension/release.json +30 -30
package/dist/actions.d.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { IBrowser } from './protocols/browser-protocol';
|
|
5
5
|
import { ActionResult, BBox } from './types';
|
|
6
|
+
import { SnapshotOptions } from './snapshot';
|
|
6
7
|
import { CursorPolicy } from './cursor-policy';
|
|
7
8
|
export interface ClickRect {
|
|
8
9
|
x: number;
|
|
@@ -62,6 +63,34 @@ export declare function click(browser: IBrowser, elementId: number, useMouse?: b
|
|
|
62
63
|
* ```
|
|
63
64
|
*/
|
|
64
65
|
export declare function typeText(browser: IBrowser, elementId: number, text: string, takeSnapshot?: boolean, delayMs?: number): Promise<ActionResult>;
|
|
66
|
+
/**
|
|
67
|
+
* Clear the value of an input/textarea element (best-effort).
|
|
68
|
+
*/
|
|
69
|
+
export declare function clear(browser: IBrowser, elementId: number, takeSnapshot?: boolean): Promise<ActionResult>;
|
|
70
|
+
/**
|
|
71
|
+
* Ensure a checkbox/radio is checked (best-effort).
|
|
72
|
+
*/
|
|
73
|
+
export declare function check(browser: IBrowser, elementId: number, takeSnapshot?: boolean): Promise<ActionResult>;
|
|
74
|
+
/**
|
|
75
|
+
* Ensure a checkbox/radio is unchecked (best-effort).
|
|
76
|
+
*/
|
|
77
|
+
export declare function uncheck(browser: IBrowser, elementId: number, takeSnapshot?: boolean): Promise<ActionResult>;
|
|
78
|
+
/**
|
|
79
|
+
* Select an option in a <select> element by matching option value or label (best-effort).
|
|
80
|
+
*/
|
|
81
|
+
export declare function selectOption(browser: IBrowser, elementId: number, option: string, takeSnapshot?: boolean): Promise<ActionResult>;
|
|
82
|
+
/**
|
|
83
|
+
* Upload a local file via an <input type="file"> element (best-effort).
|
|
84
|
+
*/
|
|
85
|
+
export declare function uploadFile(browser: IBrowser, elementId: number, filePath: string, takeSnapshot?: boolean): Promise<ActionResult>;
|
|
86
|
+
/**
|
|
87
|
+
* Submit a form (best-effort) by clicking a submit control or calling requestSubmit().
|
|
88
|
+
*/
|
|
89
|
+
export declare function submit(browser: IBrowser, elementId: number, takeSnapshot?: boolean): Promise<ActionResult>;
|
|
90
|
+
/**
|
|
91
|
+
* Navigate back in history (best-effort).
|
|
92
|
+
*/
|
|
93
|
+
export declare function back(browser: IBrowser, takeSnapshot?: boolean): Promise<ActionResult>;
|
|
65
94
|
/**
|
|
66
95
|
* Scroll an element into view
|
|
67
96
|
*
|
|
@@ -108,6 +137,14 @@ export declare function scrollTo(browser: IBrowser, elementId: number, behavior?
|
|
|
108
137
|
* ```
|
|
109
138
|
*/
|
|
110
139
|
export declare function press(browser: IBrowser, key: string, takeSnapshot?: boolean): Promise<ActionResult>;
|
|
140
|
+
/**
|
|
141
|
+
* Send a sequence of key presses (e.g., "CMD+H", "CTRL+SHIFT+P").
|
|
142
|
+
*/
|
|
143
|
+
export declare function sendKeys(browser: IBrowser, sequence: string, takeSnapshot?: boolean, delayMs?: number): Promise<ActionResult>;
|
|
144
|
+
/**
|
|
145
|
+
* Navigate to a search results page for the given query.
|
|
146
|
+
*/
|
|
147
|
+
export declare function search(browser: IBrowser, query: string, engine?: string, takeSnapshot?: boolean, snapshotOptions?: SnapshotOptions | undefined): Promise<ActionResult>;
|
|
111
148
|
/**
|
|
112
149
|
* Click at the center of a rectangle using Playwright's native mouse simulation.
|
|
113
150
|
* This uses a hybrid approach: calculates center coordinates and uses mouse.click()
|
package/dist/actions.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAY,IAAI,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAY,IAAI,EAAE,MAAM,SAAS,CAAC;AACvD,OAAO,EAAY,eAAe,EAAE,MAAM,YAAY,CAAC;AAEvD,OAAO,EAAE,YAAY,EAAwB,MAAM,iBAAiB,CAAC;AAoCrE,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAoED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,KAAK,CACzB,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,QAAQ,GAAE,OAAc,EACxB,YAAY,GAAE,OAAe,EAC7B,YAAY,CAAC,EAAE,YAAY,GAC1B,OAAO,CAAC,YAAY,CAAC,CA2GvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,YAAY,GAAE,OAAe,EAC7B,OAAO,GAAE,MAAU,GAClB,OAAO,CAAC,YAAY,CAAC,CAoDvB;AAED;;GAEG;AACH,wBAAsB,KAAK,CACzB,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CA4DvB;AAED;;GAEG;AACH,wBAAsB,KAAK,CACzB,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CA2DvB;AAED;;GAEG;AACH,wBAAsB,OAAO,CAC3B,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CA2DvB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAwEvB;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CA8EvB;AAED;;GAEG;AACH,wBAAsB,MAAM,CAC1B,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAiFvB;AAED;;GAEG;AACH,wBAAsB,IAAI,CACxB,OAAO,EAAE,QAAQ,EACjB,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAiDvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,QAAQ,GAAE,QAAQ,GAAG,SAAS,GAAG,MAAiB,EAClD,KAAK,GAAE,OAAO,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAoB,EACxD,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAwDvB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,KAAK,CACzB,OAAO,EAAE,QAAQ,EACjB,GAAG,EAAE,MAAM,EACX,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAgCvB;AA4CD;;GAEG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,QAAQ,EACjB,QAAQ,EAAE,MAAM,EAChB,YAAY,GAAE,OAAe,EAC7B,OAAO,GAAE,MAAW,GACnB,OAAO,CAAC,YAAY,CAAC,CAqCvB;AAiBD;;GAEG;AACH,wBAAsB,MAAM,CAC1B,OAAO,EAAE,QAAQ,EACjB,KAAK,EAAE,MAAM,EACb,MAAM,GAAE,MAAqB,EAC7B,YAAY,GAAE,OAAe,EAC7B,eAAe,GAAE,eAAe,GAAG,SAAqB,GACvD,OAAO,CAAC,YAAY,CAAC,CAgCvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,QAAQ,EACjB,IAAI,EAAE,SAAS,GAAG,IAAI,EACtB,SAAS,GAAE,OAAc,EACzB,iBAAiB,GAAE,MAAY,EAC/B,YAAY,GAAE,OAAe,EAC7B,YAAY,CAAC,EAAE,YAAY,GAC1B,OAAO,CAAC,YAAY,CAAC,CAgGvB"}
|
package/dist/actions.js
CHANGED
|
@@ -5,8 +5,17 @@
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.click = click;
|
|
7
7
|
exports.typeText = typeText;
|
|
8
|
+
exports.clear = clear;
|
|
9
|
+
exports.check = check;
|
|
10
|
+
exports.uncheck = uncheck;
|
|
11
|
+
exports.selectOption = selectOption;
|
|
12
|
+
exports.uploadFile = uploadFile;
|
|
13
|
+
exports.submit = submit;
|
|
14
|
+
exports.back = back;
|
|
8
15
|
exports.scrollTo = scrollTo;
|
|
9
16
|
exports.press = press;
|
|
17
|
+
exports.sendKeys = sendKeys;
|
|
18
|
+
exports.search = search;
|
|
10
19
|
exports.clickRect = clickRect;
|
|
11
20
|
const snapshot_1 = require("./snapshot");
|
|
12
21
|
const browser_evaluator_1 = require("./utils/browser-evaluator");
|
|
@@ -273,6 +282,465 @@ async function typeText(browser, elementId, text, takeSnapshot = false, delayMs
|
|
|
273
282
|
snapshot_after: snapshotAfter,
|
|
274
283
|
};
|
|
275
284
|
}
|
|
285
|
+
/**
|
|
286
|
+
* Clear the value of an input/textarea element (best-effort).
|
|
287
|
+
*/
|
|
288
|
+
async function clear(browser, elementId, takeSnapshot = false) {
|
|
289
|
+
const page = browser.getPage();
|
|
290
|
+
if (!page)
|
|
291
|
+
throw new Error('Browser not started. Call start() first.');
|
|
292
|
+
const startTime = Date.now();
|
|
293
|
+
const urlBefore = page.url();
|
|
294
|
+
const ok = await browser_evaluator_1.BrowserEvaluator.evaluate(page, id => {
|
|
295
|
+
const el = window.sentience_registry?.[id];
|
|
296
|
+
if (!el)
|
|
297
|
+
return false;
|
|
298
|
+
try {
|
|
299
|
+
el.focus?.();
|
|
300
|
+
}
|
|
301
|
+
catch {
|
|
302
|
+
/* ignore */
|
|
303
|
+
}
|
|
304
|
+
if ('value' in el) {
|
|
305
|
+
el.value = '';
|
|
306
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
307
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
308
|
+
return true;
|
|
309
|
+
}
|
|
310
|
+
return false;
|
|
311
|
+
}, elementId);
|
|
312
|
+
if (!ok) {
|
|
313
|
+
return {
|
|
314
|
+
success: false,
|
|
315
|
+
duration_ms: Date.now() - startTime,
|
|
316
|
+
outcome: 'error',
|
|
317
|
+
error: { code: 'clear_failed', reason: 'Element not found or not clearable' },
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
try {
|
|
321
|
+
await page.waitForTimeout(250);
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
/* ignore */
|
|
325
|
+
}
|
|
326
|
+
const durationMs = Date.now() - startTime;
|
|
327
|
+
const urlAfter = page.url();
|
|
328
|
+
const urlChanged = urlBefore !== urlAfter;
|
|
329
|
+
const outcome = urlChanged ? 'navigated' : 'dom_updated';
|
|
330
|
+
let snapshotAfter;
|
|
331
|
+
if (takeSnapshot) {
|
|
332
|
+
snapshotAfter = await (0, snapshot_1.snapshot)(browser);
|
|
333
|
+
}
|
|
334
|
+
return {
|
|
335
|
+
success: true,
|
|
336
|
+
duration_ms: durationMs,
|
|
337
|
+
outcome,
|
|
338
|
+
url_changed: urlChanged,
|
|
339
|
+
snapshot_after: snapshotAfter,
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Ensure a checkbox/radio is checked (best-effort).
|
|
344
|
+
*/
|
|
345
|
+
async function check(browser, elementId, takeSnapshot = false) {
|
|
346
|
+
const page = browser.getPage();
|
|
347
|
+
if (!page)
|
|
348
|
+
throw new Error('Browser not started. Call start() first.');
|
|
349
|
+
const startTime = Date.now();
|
|
350
|
+
const urlBefore = page.url();
|
|
351
|
+
const ok = await browser_evaluator_1.BrowserEvaluator.evaluate(page, id => {
|
|
352
|
+
const el = window.sentience_registry?.[id];
|
|
353
|
+
if (!el)
|
|
354
|
+
return false;
|
|
355
|
+
try {
|
|
356
|
+
el.focus?.();
|
|
357
|
+
}
|
|
358
|
+
catch {
|
|
359
|
+
/* ignore */
|
|
360
|
+
}
|
|
361
|
+
if (!('checked' in el))
|
|
362
|
+
return false;
|
|
363
|
+
if (el.checked === true)
|
|
364
|
+
return true;
|
|
365
|
+
try {
|
|
366
|
+
el.click();
|
|
367
|
+
}
|
|
368
|
+
catch {
|
|
369
|
+
return false;
|
|
370
|
+
}
|
|
371
|
+
return true;
|
|
372
|
+
}, elementId);
|
|
373
|
+
if (!ok) {
|
|
374
|
+
return {
|
|
375
|
+
success: false,
|
|
376
|
+
duration_ms: Date.now() - startTime,
|
|
377
|
+
outcome: 'error',
|
|
378
|
+
error: { code: 'check_failed', reason: 'Element not found or not checkable' },
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
try {
|
|
382
|
+
await page.waitForTimeout(250);
|
|
383
|
+
}
|
|
384
|
+
catch {
|
|
385
|
+
/* ignore */
|
|
386
|
+
}
|
|
387
|
+
const durationMs = Date.now() - startTime;
|
|
388
|
+
const urlAfter = page.url();
|
|
389
|
+
const urlChanged = urlBefore !== urlAfter;
|
|
390
|
+
const outcome = urlChanged ? 'navigated' : 'dom_updated';
|
|
391
|
+
let snapshotAfter;
|
|
392
|
+
if (takeSnapshot)
|
|
393
|
+
snapshotAfter = await (0, snapshot_1.snapshot)(browser);
|
|
394
|
+
return {
|
|
395
|
+
success: true,
|
|
396
|
+
duration_ms: durationMs,
|
|
397
|
+
outcome,
|
|
398
|
+
url_changed: urlChanged,
|
|
399
|
+
snapshot_after: snapshotAfter,
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* Ensure a checkbox/radio is unchecked (best-effort).
|
|
404
|
+
*/
|
|
405
|
+
async function uncheck(browser, elementId, takeSnapshot = false) {
|
|
406
|
+
const page = browser.getPage();
|
|
407
|
+
if (!page)
|
|
408
|
+
throw new Error('Browser not started. Call start() first.');
|
|
409
|
+
const startTime = Date.now();
|
|
410
|
+
const urlBefore = page.url();
|
|
411
|
+
const ok = await browser_evaluator_1.BrowserEvaluator.evaluate(page, id => {
|
|
412
|
+
const el = window.sentience_registry?.[id];
|
|
413
|
+
if (!el)
|
|
414
|
+
return false;
|
|
415
|
+
try {
|
|
416
|
+
el.focus?.();
|
|
417
|
+
}
|
|
418
|
+
catch {
|
|
419
|
+
/* ignore */
|
|
420
|
+
}
|
|
421
|
+
if (!('checked' in el))
|
|
422
|
+
return false;
|
|
423
|
+
if (el.checked === false)
|
|
424
|
+
return true;
|
|
425
|
+
try {
|
|
426
|
+
el.click();
|
|
427
|
+
}
|
|
428
|
+
catch {
|
|
429
|
+
return false;
|
|
430
|
+
}
|
|
431
|
+
return true;
|
|
432
|
+
}, elementId);
|
|
433
|
+
if (!ok) {
|
|
434
|
+
return {
|
|
435
|
+
success: false,
|
|
436
|
+
duration_ms: Date.now() - startTime,
|
|
437
|
+
outcome: 'error',
|
|
438
|
+
error: { code: 'uncheck_failed', reason: 'Element not found or not uncheckable' },
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
try {
|
|
442
|
+
await page.waitForTimeout(250);
|
|
443
|
+
}
|
|
444
|
+
catch {
|
|
445
|
+
/* ignore */
|
|
446
|
+
}
|
|
447
|
+
const durationMs = Date.now() - startTime;
|
|
448
|
+
const urlAfter = page.url();
|
|
449
|
+
const urlChanged = urlBefore !== urlAfter;
|
|
450
|
+
const outcome = urlChanged ? 'navigated' : 'dom_updated';
|
|
451
|
+
let snapshotAfter;
|
|
452
|
+
if (takeSnapshot)
|
|
453
|
+
snapshotAfter = await (0, snapshot_1.snapshot)(browser);
|
|
454
|
+
return {
|
|
455
|
+
success: true,
|
|
456
|
+
duration_ms: durationMs,
|
|
457
|
+
outcome,
|
|
458
|
+
url_changed: urlChanged,
|
|
459
|
+
snapshot_after: snapshotAfter,
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* Select an option in a <select> element by matching option value or label (best-effort).
|
|
464
|
+
*/
|
|
465
|
+
async function selectOption(browser, elementId, option, takeSnapshot = false) {
|
|
466
|
+
const page = browser.getPage();
|
|
467
|
+
if (!page)
|
|
468
|
+
throw new Error('Browser not started. Call start() first.');
|
|
469
|
+
const startTime = Date.now();
|
|
470
|
+
const urlBefore = page.url();
|
|
471
|
+
const ok = await browser_evaluator_1.BrowserEvaluator.evaluate(page, (args) => {
|
|
472
|
+
const el = window.sentience_registry?.[args.id];
|
|
473
|
+
if (!el)
|
|
474
|
+
return false;
|
|
475
|
+
const tag = String(el.tagName || '').toUpperCase();
|
|
476
|
+
if (tag !== 'SELECT')
|
|
477
|
+
return false;
|
|
478
|
+
const needle = String(args.option ?? '');
|
|
479
|
+
const opts = Array.from(el.options || []);
|
|
480
|
+
let chosen = null;
|
|
481
|
+
for (const o of opts) {
|
|
482
|
+
const oo = o;
|
|
483
|
+
if (String(oo.value) === needle || String(oo.text) === needle) {
|
|
484
|
+
chosen = o;
|
|
485
|
+
break;
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
if (!chosen) {
|
|
489
|
+
for (const o of opts) {
|
|
490
|
+
const oo = o;
|
|
491
|
+
if (String(oo.text || '').includes(needle)) {
|
|
492
|
+
chosen = o;
|
|
493
|
+
break;
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
if (!chosen)
|
|
498
|
+
return false;
|
|
499
|
+
el.value = chosen.value;
|
|
500
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
501
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
502
|
+
return true;
|
|
503
|
+
}, { id: elementId, option });
|
|
504
|
+
if (!ok) {
|
|
505
|
+
return {
|
|
506
|
+
success: false,
|
|
507
|
+
duration_ms: Date.now() - startTime,
|
|
508
|
+
outcome: 'error',
|
|
509
|
+
error: { code: 'select_failed', reason: 'Element not found or option not found' },
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
try {
|
|
513
|
+
await page.waitForTimeout(250);
|
|
514
|
+
}
|
|
515
|
+
catch {
|
|
516
|
+
/* ignore */
|
|
517
|
+
}
|
|
518
|
+
const durationMs = Date.now() - startTime;
|
|
519
|
+
const urlAfter = page.url();
|
|
520
|
+
const urlChanged = urlBefore !== urlAfter;
|
|
521
|
+
const outcome = urlChanged ? 'navigated' : 'dom_updated';
|
|
522
|
+
let snapshotAfter;
|
|
523
|
+
if (takeSnapshot)
|
|
524
|
+
snapshotAfter = await (0, snapshot_1.snapshot)(browser);
|
|
525
|
+
return {
|
|
526
|
+
success: true,
|
|
527
|
+
duration_ms: durationMs,
|
|
528
|
+
outcome,
|
|
529
|
+
url_changed: urlChanged,
|
|
530
|
+
snapshot_after: snapshotAfter,
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
/**
|
|
534
|
+
* Upload a local file via an <input type="file"> element (best-effort).
|
|
535
|
+
*/
|
|
536
|
+
async function uploadFile(browser, elementId, filePath, takeSnapshot = false) {
|
|
537
|
+
const page = browser.getPage();
|
|
538
|
+
if (!page)
|
|
539
|
+
throw new Error('Browser not started. Call start() first.');
|
|
540
|
+
const startTime = Date.now();
|
|
541
|
+
const urlBefore = page.url();
|
|
542
|
+
let success = false;
|
|
543
|
+
let errorMsg;
|
|
544
|
+
try {
|
|
545
|
+
// First try: grab the exact element handle from the sentience registry.
|
|
546
|
+
try {
|
|
547
|
+
const handle = await page.evaluateHandle('(id) => (window.sentience_registry && window.sentience_registry[id]) || null', elementId);
|
|
548
|
+
const el = handle.asElement?.() ?? null;
|
|
549
|
+
if (!el)
|
|
550
|
+
throw new Error('Element not found');
|
|
551
|
+
await el.setInputFiles(filePath);
|
|
552
|
+
success = true;
|
|
553
|
+
}
|
|
554
|
+
catch {
|
|
555
|
+
// Fallback: resolve a selector from the element's attributes and use page.setInputFiles().
|
|
556
|
+
const attrs = await browser_evaluator_1.BrowserEvaluator.evaluate(page, id => {
|
|
557
|
+
const el = window.sentience_registry?.[id];
|
|
558
|
+
if (!el)
|
|
559
|
+
return null;
|
|
560
|
+
const tag = String(el.tagName || '').toUpperCase();
|
|
561
|
+
const type = String(el.type || '').toLowerCase();
|
|
562
|
+
const idAttr = el.id ? String(el.id) : null;
|
|
563
|
+
const nameAttr = el.name ? String(el.name) : null;
|
|
564
|
+
return { tag, type, id: idAttr, name: nameAttr };
|
|
565
|
+
}, elementId);
|
|
566
|
+
let selector = null;
|
|
567
|
+
if (attrs && attrs.tag === 'INPUT' && attrs.type === 'file') {
|
|
568
|
+
if (attrs.id)
|
|
569
|
+
selector = `input#${attrs.id}`;
|
|
570
|
+
else if (attrs.name)
|
|
571
|
+
selector = `input[name="${String(attrs.name).replace(/"/g, '\\"')}"]`;
|
|
572
|
+
}
|
|
573
|
+
if (!selector)
|
|
574
|
+
throw new Error('Element not found');
|
|
575
|
+
await page.setInputFiles(selector, filePath);
|
|
576
|
+
success = true;
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
catch (e) {
|
|
580
|
+
success = false;
|
|
581
|
+
errorMsg = String(e?.message ?? e);
|
|
582
|
+
}
|
|
583
|
+
try {
|
|
584
|
+
await page.waitForTimeout(250);
|
|
585
|
+
}
|
|
586
|
+
catch {
|
|
587
|
+
/* ignore */
|
|
588
|
+
}
|
|
589
|
+
const durationMs = Date.now() - startTime;
|
|
590
|
+
const urlAfter = page.url();
|
|
591
|
+
const urlChanged = urlBefore !== urlAfter;
|
|
592
|
+
const outcome = urlChanged ? 'navigated' : success ? 'dom_updated' : 'error';
|
|
593
|
+
let snapshotAfter;
|
|
594
|
+
if (takeSnapshot) {
|
|
595
|
+
try {
|
|
596
|
+
snapshotAfter = await (0, snapshot_1.snapshot)(browser);
|
|
597
|
+
}
|
|
598
|
+
catch {
|
|
599
|
+
/* ignore */
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
return {
|
|
603
|
+
success,
|
|
604
|
+
duration_ms: durationMs,
|
|
605
|
+
outcome,
|
|
606
|
+
url_changed: urlChanged,
|
|
607
|
+
snapshot_after: snapshotAfter,
|
|
608
|
+
error: success ? undefined : { code: 'upload_failed', reason: errorMsg ?? 'upload failed' },
|
|
609
|
+
};
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Submit a form (best-effort) by clicking a submit control or calling requestSubmit().
|
|
613
|
+
*/
|
|
614
|
+
async function submit(browser, elementId, takeSnapshot = false) {
|
|
615
|
+
const page = browser.getPage();
|
|
616
|
+
if (!page)
|
|
617
|
+
throw new Error('Browser not started. Call start() first.');
|
|
618
|
+
const startTime = Date.now();
|
|
619
|
+
const urlBefore = page.url();
|
|
620
|
+
const ok = await browser_evaluator_1.BrowserEvaluator.evaluate(page, id => {
|
|
621
|
+
const el = window.sentience_registry?.[id];
|
|
622
|
+
if (!el)
|
|
623
|
+
return false;
|
|
624
|
+
try {
|
|
625
|
+
el.focus?.();
|
|
626
|
+
}
|
|
627
|
+
catch {
|
|
628
|
+
/* ignore */
|
|
629
|
+
}
|
|
630
|
+
const tag = String(el.tagName || '').toUpperCase();
|
|
631
|
+
if (tag === 'FORM') {
|
|
632
|
+
if (typeof el.requestSubmit === 'function') {
|
|
633
|
+
el.requestSubmit();
|
|
634
|
+
return true;
|
|
635
|
+
}
|
|
636
|
+
try {
|
|
637
|
+
el.submit();
|
|
638
|
+
return true;
|
|
639
|
+
}
|
|
640
|
+
catch {
|
|
641
|
+
return false;
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
const form = el.form;
|
|
645
|
+
if (form && typeof form.requestSubmit === 'function') {
|
|
646
|
+
form.requestSubmit();
|
|
647
|
+
return true;
|
|
648
|
+
}
|
|
649
|
+
try {
|
|
650
|
+
el.click();
|
|
651
|
+
return true;
|
|
652
|
+
}
|
|
653
|
+
catch {
|
|
654
|
+
return false;
|
|
655
|
+
}
|
|
656
|
+
}, elementId);
|
|
657
|
+
if (!ok) {
|
|
658
|
+
return {
|
|
659
|
+
success: false,
|
|
660
|
+
duration_ms: Date.now() - startTime,
|
|
661
|
+
outcome: 'error',
|
|
662
|
+
error: { code: 'submit_failed', reason: 'Element not found or not submittable' },
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
try {
|
|
666
|
+
await page.waitForTimeout(500);
|
|
667
|
+
}
|
|
668
|
+
catch {
|
|
669
|
+
/* ignore */
|
|
670
|
+
}
|
|
671
|
+
const durationMs = Date.now() - startTime;
|
|
672
|
+
const urlAfter = page.url();
|
|
673
|
+
const urlChanged = urlBefore !== urlAfter;
|
|
674
|
+
const outcome = urlChanged ? 'navigated' : 'dom_updated';
|
|
675
|
+
let snapshotAfter;
|
|
676
|
+
if (takeSnapshot) {
|
|
677
|
+
try {
|
|
678
|
+
snapshotAfter = await (0, snapshot_1.snapshot)(browser);
|
|
679
|
+
}
|
|
680
|
+
catch {
|
|
681
|
+
/* ignore */
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
return {
|
|
685
|
+
success: true,
|
|
686
|
+
duration_ms: durationMs,
|
|
687
|
+
outcome,
|
|
688
|
+
url_changed: urlChanged,
|
|
689
|
+
snapshot_after: snapshotAfter,
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
/**
|
|
693
|
+
* Navigate back in history (best-effort).
|
|
694
|
+
*/
|
|
695
|
+
async function back(browser, takeSnapshot = false) {
|
|
696
|
+
const page = browser.getPage();
|
|
697
|
+
if (!page)
|
|
698
|
+
throw new Error('Browser not started. Call start() first.');
|
|
699
|
+
const startTime = Date.now();
|
|
700
|
+
const urlBefore = page.url();
|
|
701
|
+
let success = false;
|
|
702
|
+
let errorMsg;
|
|
703
|
+
try {
|
|
704
|
+
await page.goBack();
|
|
705
|
+
success = true;
|
|
706
|
+
}
|
|
707
|
+
catch (e) {
|
|
708
|
+
success = false;
|
|
709
|
+
errorMsg = String(e?.message ?? e);
|
|
710
|
+
}
|
|
711
|
+
try {
|
|
712
|
+
await page.waitForTimeout(500);
|
|
713
|
+
}
|
|
714
|
+
catch {
|
|
715
|
+
/* ignore */
|
|
716
|
+
}
|
|
717
|
+
const durationMs = Date.now() - startTime;
|
|
718
|
+
let urlChanged = false;
|
|
719
|
+
try {
|
|
720
|
+
urlChanged = urlBefore !== page.url();
|
|
721
|
+
}
|
|
722
|
+
catch {
|
|
723
|
+
urlChanged = true;
|
|
724
|
+
}
|
|
725
|
+
const outcome = urlChanged ? 'navigated' : success ? 'dom_updated' : 'error';
|
|
726
|
+
let snapshotAfter;
|
|
727
|
+
if (takeSnapshot) {
|
|
728
|
+
try {
|
|
729
|
+
snapshotAfter = await (0, snapshot_1.snapshot)(browser);
|
|
730
|
+
}
|
|
731
|
+
catch {
|
|
732
|
+
/* ignore */
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
return {
|
|
736
|
+
success,
|
|
737
|
+
duration_ms: durationMs,
|
|
738
|
+
outcome,
|
|
739
|
+
url_changed: urlChanged,
|
|
740
|
+
snapshot_after: snapshotAfter,
|
|
741
|
+
error: success ? undefined : { code: 'back_failed', reason: errorMsg ?? 'back failed' },
|
|
742
|
+
};
|
|
743
|
+
}
|
|
276
744
|
/**
|
|
277
745
|
* Scroll an element into view
|
|
278
746
|
*
|
|
@@ -390,6 +858,130 @@ async function press(browser, key, takeSnapshot = false) {
|
|
|
390
858
|
snapshot_after: snapshotAfter,
|
|
391
859
|
};
|
|
392
860
|
}
|
|
861
|
+
function normalizeKeyToken(token) {
|
|
862
|
+
const lookup = {
|
|
863
|
+
CMD: 'Meta',
|
|
864
|
+
COMMAND: 'Meta',
|
|
865
|
+
CTRL: 'Control',
|
|
866
|
+
CONTROL: 'Control',
|
|
867
|
+
ALT: 'Alt',
|
|
868
|
+
OPTION: 'Alt',
|
|
869
|
+
SHIFT: 'Shift',
|
|
870
|
+
ESC: 'Escape',
|
|
871
|
+
ESCAPE: 'Escape',
|
|
872
|
+
ENTER: 'Enter',
|
|
873
|
+
RETURN: 'Enter',
|
|
874
|
+
TAB: 'Tab',
|
|
875
|
+
SPACE: 'Space',
|
|
876
|
+
};
|
|
877
|
+
const upper = token.trim().toUpperCase();
|
|
878
|
+
return lookup[upper] ?? token.trim();
|
|
879
|
+
}
|
|
880
|
+
function parseKeySequence(sequence) {
|
|
881
|
+
const parts = [];
|
|
882
|
+
for (const rawPart of sequence.replace(/,/g, ' ').split(/\s+/)) {
|
|
883
|
+
let raw = rawPart.trim();
|
|
884
|
+
if (!raw)
|
|
885
|
+
continue;
|
|
886
|
+
if (raw.startsWith('{') && raw.endsWith('}')) {
|
|
887
|
+
raw = raw.slice(1, -1);
|
|
888
|
+
}
|
|
889
|
+
if (raw.includes('+')) {
|
|
890
|
+
const combo = raw
|
|
891
|
+
.split('+')
|
|
892
|
+
.filter(Boolean)
|
|
893
|
+
.map(token => normalizeKeyToken(token))
|
|
894
|
+
.join('+');
|
|
895
|
+
parts.push(combo);
|
|
896
|
+
}
|
|
897
|
+
else {
|
|
898
|
+
parts.push(normalizeKeyToken(raw));
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
return parts;
|
|
902
|
+
}
|
|
903
|
+
/**
|
|
904
|
+
* Send a sequence of key presses (e.g., "CMD+H", "CTRL+SHIFT+P").
|
|
905
|
+
*/
|
|
906
|
+
async function sendKeys(browser, sequence, takeSnapshot = false, delayMs = 50) {
|
|
907
|
+
const page = browser.getPage();
|
|
908
|
+
if (!page) {
|
|
909
|
+
throw new Error('Browser not started. Call start() first.');
|
|
910
|
+
}
|
|
911
|
+
const startTime = Date.now();
|
|
912
|
+
const urlBefore = page.url();
|
|
913
|
+
const keys = parseKeySequence(sequence);
|
|
914
|
+
if (keys.length === 0) {
|
|
915
|
+
throw new Error('send_keys sequence is empty');
|
|
916
|
+
}
|
|
917
|
+
for (const key of keys) {
|
|
918
|
+
await page.keyboard.press(key);
|
|
919
|
+
if (delayMs > 0) {
|
|
920
|
+
await page.waitForTimeout(delayMs);
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
const durationMs = Date.now() - startTime;
|
|
924
|
+
const urlAfter = page.url();
|
|
925
|
+
const urlChanged = urlBefore !== urlAfter;
|
|
926
|
+
const outcome = urlChanged ? 'navigated' : 'dom_updated';
|
|
927
|
+
let snapshotAfter;
|
|
928
|
+
if (takeSnapshot) {
|
|
929
|
+
snapshotAfter = await (0, snapshot_1.snapshot)(browser);
|
|
930
|
+
}
|
|
931
|
+
return {
|
|
932
|
+
success: true,
|
|
933
|
+
duration_ms: durationMs,
|
|
934
|
+
outcome,
|
|
935
|
+
url_changed: urlChanged,
|
|
936
|
+
snapshot_after: snapshotAfter,
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
function buildSearchUrl(query, engine) {
|
|
940
|
+
const q = encodeURIComponent(query).replace(/%20/g, '+');
|
|
941
|
+
const key = engine.trim().toLowerCase();
|
|
942
|
+
if (key === 'duckduckgo' || key === 'ddg') {
|
|
943
|
+
return `https://duckduckgo.com/?q=${q}`;
|
|
944
|
+
}
|
|
945
|
+
if (key === 'google.com' || key === 'google') {
|
|
946
|
+
return `https://www.google.com/search?q=${q}`;
|
|
947
|
+
}
|
|
948
|
+
if (key === 'bing') {
|
|
949
|
+
return `https://www.bing.com/search?q=${q}`;
|
|
950
|
+
}
|
|
951
|
+
throw new Error(`unsupported search engine: ${engine}`);
|
|
952
|
+
}
|
|
953
|
+
/**
|
|
954
|
+
* Navigate to a search results page for the given query.
|
|
955
|
+
*/
|
|
956
|
+
async function search(browser, query, engine = 'duckduckgo', takeSnapshot = false, snapshotOptions = undefined) {
|
|
957
|
+
const page = browser.getPage();
|
|
958
|
+
if (!page) {
|
|
959
|
+
throw new Error('Browser not started. Call start() first.');
|
|
960
|
+
}
|
|
961
|
+
if (!query.trim()) {
|
|
962
|
+
throw new Error('search query is empty');
|
|
963
|
+
}
|
|
964
|
+
const startTime = Date.now();
|
|
965
|
+
const urlBefore = page.url();
|
|
966
|
+
const url = buildSearchUrl(query, engine);
|
|
967
|
+
await browser.goto(url);
|
|
968
|
+
await page.waitForLoadState('networkidle');
|
|
969
|
+
const durationMs = Date.now() - startTime;
|
|
970
|
+
const urlAfter = page.url();
|
|
971
|
+
const urlChanged = urlBefore !== urlAfter;
|
|
972
|
+
const outcome = urlChanged ? 'navigated' : 'dom_updated';
|
|
973
|
+
let snapshotAfter;
|
|
974
|
+
if (takeSnapshot) {
|
|
975
|
+
snapshotAfter = await (0, snapshot_1.snapshot)(browser, snapshotOptions);
|
|
976
|
+
}
|
|
977
|
+
return {
|
|
978
|
+
success: true,
|
|
979
|
+
duration_ms: durationMs,
|
|
980
|
+
outcome,
|
|
981
|
+
url_changed: urlChanged,
|
|
982
|
+
snapshot_after: snapshotAfter,
|
|
983
|
+
};
|
|
984
|
+
}
|
|
393
985
|
/**
|
|
394
986
|
* Click at the center of a rectangle using Playwright's native mouse simulation.
|
|
395
987
|
* This uses a hybrid approach: calculates center coordinates and uses mouse.click()
|