sentienceapi 0.92.2 → 0.93.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -54
- package/dist/actions.d.ts +33 -1
- package/dist/actions.d.ts.map +1 -1
- package/dist/actions.js +81 -3
- package/dist/actions.js.map +1 -1
- package/dist/agent-runtime.d.ts +190 -0
- package/dist/agent-runtime.d.ts.map +1 -0
- package/dist/agent-runtime.js +257 -0
- package/dist/agent-runtime.js.map +1 -0
- package/dist/canonicalization.d.ts +126 -0
- package/dist/canonicalization.d.ts.map +1 -0
- package/dist/canonicalization.js +161 -0
- package/dist/canonicalization.js.map +1 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +19 -1
- package/dist/index.js.map +1 -1
- package/dist/ordinal.d.ts +90 -0
- package/dist/ordinal.d.ts.map +1 -0
- package/dist/ordinal.js +249 -0
- package/dist/ordinal.js.map +1 -0
- package/dist/snapshot-diff.d.ts +8 -15
- package/dist/snapshot-diff.d.ts.map +1 -1
- package/dist/snapshot-diff.js +38 -43
- package/dist/snapshot-diff.js.map +1 -1
- package/dist/snapshot.js +2 -0
- package/dist/snapshot.js.map +1 -1
- package/dist/tracing/indexer.d.ts.map +1 -1
- package/dist/tracing/indexer.js +3 -46
- package/dist/tracing/indexer.js.map +1 -1
- package/dist/tracing/types.d.ts +19 -0
- package/dist/tracing/types.d.ts.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/verification.d.ts +177 -0
- package/dist/verification.d.ts.map +1 -0
- package/dist/verification.js +315 -0
- package/dist/verification.js.map +1 -0
- package/package.json +1 -1
- package/src/extension/injected_api.js +9 -2
- package/src/extension/manifest.json +1 -1
- package/src/extension/pkg/sentience_core_bg.wasm +0 -0
- package/src/extension/release.json +47 -46
package/README.md
CHANGED
|
@@ -13,6 +13,7 @@ npx playwright install chromium
|
|
|
13
13
|
```
|
|
14
14
|
|
|
15
15
|
**For local development:**
|
|
16
|
+
|
|
16
17
|
```bash
|
|
17
18
|
npm install
|
|
18
19
|
npm run build
|
|
@@ -44,7 +45,7 @@ const response = await agent.execute(
|
|
|
44
45
|
console.log(response); // "I found the top result for wireless mouse on Amazon. It's priced at $24.99..."
|
|
45
46
|
|
|
46
47
|
// Follow-up questions maintain context
|
|
47
|
-
const followUp = await agent.chat(
|
|
48
|
+
const followUp = await agent.chat('Add it to cart');
|
|
48
49
|
console.log(followUp);
|
|
49
50
|
|
|
50
51
|
await browser.close();
|
|
@@ -141,6 +142,37 @@ await browser.close();
|
|
|
141
142
|
|
|
142
143
|
---
|
|
143
144
|
|
|
145
|
+
## 🆕 What's New (2026-01-06)
|
|
146
|
+
|
|
147
|
+
### Human-like Typing
|
|
148
|
+
|
|
149
|
+
Add realistic delays between keystrokes to mimic human typing:
|
|
150
|
+
|
|
151
|
+
```typescript
|
|
152
|
+
// Type instantly (default)
|
|
153
|
+
await typeText(browser, elementId, 'Hello World');
|
|
154
|
+
|
|
155
|
+
// Type with human-like delay (~10ms between keystrokes)
|
|
156
|
+
await typeText(browser, elementId, 'Hello World', false, 10);
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### Scroll to Element
|
|
160
|
+
|
|
161
|
+
Scroll elements into view with smooth animation:
|
|
162
|
+
|
|
163
|
+
```typescript
|
|
164
|
+
const snap = await snapshot(browser);
|
|
165
|
+
const button = find(snap, 'role=button text~"Submit"');
|
|
166
|
+
|
|
167
|
+
// Scroll element into view with smooth animation
|
|
168
|
+
await scrollTo(browser, button.id);
|
|
169
|
+
|
|
170
|
+
// Scroll instantly to top of viewport
|
|
171
|
+
await scrollTo(browser, button.id, 'instant', 'start');
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
144
176
|
<details>
|
|
145
177
|
<summary><h2>📊 Agent Execution Tracing (NEW in v0.3.1)</h2></summary>
|
|
146
178
|
|
|
@@ -154,7 +186,7 @@ import {
|
|
|
154
186
|
SentienceAgent,
|
|
155
187
|
OpenAIProvider,
|
|
156
188
|
Tracer,
|
|
157
|
-
JsonlTraceSink
|
|
189
|
+
JsonlTraceSink,
|
|
158
190
|
} from 'sentienceapi';
|
|
159
191
|
import { randomUUID } from 'crypto';
|
|
160
192
|
|
|
@@ -208,6 +240,38 @@ Traces are **100% compatible** with Python SDK traces - use the same tools to an
|
|
|
208
240
|
|
|
209
241
|
</details>
|
|
210
242
|
|
|
243
|
+
<details>
|
|
244
|
+
<summary><h2>🔍 Agent Runtime Verification</h2></summary>
|
|
245
|
+
|
|
246
|
+
`AgentRuntime` provides assertion predicates for runtime verification in agent loops, enabling programmatic verification of browser state during execution.
|
|
247
|
+
|
|
248
|
+
```typescript
|
|
249
|
+
import { SentienceBrowser } from 'sentienceapi';
|
|
250
|
+
import { AgentRuntime, urlContains, exists, allOf } from 'sentienceapi';
|
|
251
|
+
import { createTracer } from 'sentienceapi';
|
|
252
|
+
|
|
253
|
+
const browser = new SentienceBrowser();
|
|
254
|
+
await browser.start();
|
|
255
|
+
const tracer = await createTracer({ runId: 'my-run', uploadTrace: false });
|
|
256
|
+
const runtime = new AgentRuntime(browser, browser.getPage(), tracer);
|
|
257
|
+
|
|
258
|
+
// Navigate and take snapshot
|
|
259
|
+
await browser.getPage().goto('https://example.com');
|
|
260
|
+
runtime.beginStep('Verify page');
|
|
261
|
+
await runtime.snapshot();
|
|
262
|
+
|
|
263
|
+
// Run assertions
|
|
264
|
+
runtime.assert(urlContains('example.com'), 'on_correct_domain');
|
|
265
|
+
runtime.assert(exists('role=heading'), 'has_heading');
|
|
266
|
+
runtime.assertDone(exists("text~'Example'"), 'task_complete');
|
|
267
|
+
|
|
268
|
+
console.log(`Task done: ${runtime.isTaskDone}`);
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
**See example:** [examples/agent-runtime-verification.ts](examples/agent-runtime-verification.ts)
|
|
272
|
+
|
|
273
|
+
</details>
|
|
274
|
+
|
|
211
275
|
---
|
|
212
276
|
|
|
213
277
|
<details>
|
|
@@ -234,14 +298,14 @@ async function main() {
|
|
|
234
298
|
console.log(`Found ${snap.elements.length} elements`);
|
|
235
299
|
|
|
236
300
|
// Find first product in viewport using spatial filtering
|
|
237
|
-
const products = snap.elements
|
|
238
|
-
|
|
301
|
+
const products = snap.elements.filter(
|
|
302
|
+
el =>
|
|
239
303
|
el.role === 'link' &&
|
|
240
304
|
el.visual_cues.is_clickable &&
|
|
241
305
|
el.in_viewport &&
|
|
242
306
|
!el.is_occluded &&
|
|
243
|
-
el.bbox.y < 600
|
|
244
|
-
|
|
307
|
+
el.bbox.y < 600 // First row
|
|
308
|
+
);
|
|
245
309
|
|
|
246
310
|
if (products.length > 0) {
|
|
247
311
|
// Sort by position (left to right, top to bottom)
|
|
@@ -296,12 +360,14 @@ main();
|
|
|
296
360
|
**`snapshot(browser, options?)`** - Capture page state with AI-ranked elements
|
|
297
361
|
|
|
298
362
|
Features:
|
|
363
|
+
|
|
299
364
|
- Returns semantic elements with roles, text, importance scores, and bounding boxes
|
|
300
365
|
- Optional screenshot capture (PNG/JPEG)
|
|
301
366
|
- Optional visual overlay to see what elements are detected
|
|
302
367
|
- TypeScript types for type safety
|
|
303
368
|
|
|
304
369
|
**Example:**
|
|
370
|
+
|
|
305
371
|
```typescript
|
|
306
372
|
const snap = await snapshot(browser, { screenshot: true, show_overlay: true });
|
|
307
373
|
|
|
@@ -326,6 +392,7 @@ for (const element of snap.elements) {
|
|
|
326
392
|
- Powerful query DSL with multiple operators
|
|
327
393
|
|
|
328
394
|
**Query Examples:**
|
|
395
|
+
|
|
329
396
|
```typescript
|
|
330
397
|
// Find by role and text
|
|
331
398
|
const button = find(snap, 'role=button text="Sign in"');
|
|
@@ -366,12 +433,13 @@ All actions return `ActionResult` with success status, timing, and outcome:
|
|
|
366
433
|
const result = await click(browser, element.id);
|
|
367
434
|
|
|
368
435
|
console.log(`Success: ${result.success}`);
|
|
369
|
-
console.log(`Outcome: ${result.outcome}`);
|
|
436
|
+
console.log(`Outcome: ${result.outcome}`); // "navigated", "dom_updated", "error"
|
|
370
437
|
console.log(`Duration: ${result.duration_ms}ms`);
|
|
371
438
|
console.log(`URL changed: ${result.url_changed}`);
|
|
372
439
|
```
|
|
373
440
|
|
|
374
441
|
**Coordinate-based clicking:**
|
|
442
|
+
|
|
375
443
|
```typescript
|
|
376
444
|
import { clickRect } from './src';
|
|
377
445
|
|
|
@@ -389,7 +457,7 @@ if (element) {
|
|
|
389
457
|
x: element.bbox.x,
|
|
390
458
|
y: element.bbox.y,
|
|
391
459
|
w: element.bbox.width,
|
|
392
|
-
h: element.bbox.height
|
|
460
|
+
h: element.bbox.height,
|
|
393
461
|
});
|
|
394
462
|
}
|
|
395
463
|
```
|
|
@@ -403,6 +471,7 @@ if (element) {
|
|
|
403
471
|
- **`expect(browser, selector)`** - Assertion helper with fluent API
|
|
404
472
|
|
|
405
473
|
**Examples:**
|
|
474
|
+
|
|
406
475
|
```typescript
|
|
407
476
|
// Wait for element (auto-detects optimal interval based on API usage)
|
|
408
477
|
const result = await waitFor(browser, 'role=button text="Submit"', 10000);
|
|
@@ -420,9 +489,9 @@ const result = await waitFor(browser, 'role=button', 5000, undefined, true);
|
|
|
420
489
|
const result = await waitFor(browser, 'role=button', 5000, 500, false);
|
|
421
490
|
|
|
422
491
|
// Semantic wait conditions
|
|
423
|
-
await waitFor(browser, 'clickable=true', 5000);
|
|
424
|
-
await waitFor(browser, 'importance>100', 5000);
|
|
425
|
-
await waitFor(browser, 'role=link visible=true', 5000);
|
|
492
|
+
await waitFor(browser, 'clickable=true', 5000); // Wait for clickable element
|
|
493
|
+
await waitFor(browser, 'importance>100', 5000); // Wait for important element
|
|
494
|
+
await waitFor(browser, 'role=link visible=true', 5000); // Wait for visible link
|
|
426
495
|
|
|
427
496
|
// Assertions
|
|
428
497
|
await expect(browser, 'role=button text="Submit"').toExist(5000);
|
|
@@ -448,7 +517,7 @@ import { showOverlay, clearOverlay } from 'sentienceapi';
|
|
|
448
517
|
const snap = await snapshot(browser);
|
|
449
518
|
|
|
450
519
|
// Show overlay anytime without re-snapshotting
|
|
451
|
-
await showOverlay(browser, snap);
|
|
520
|
+
await showOverlay(browser, snap); // Auto-clears after 5 seconds
|
|
452
521
|
|
|
453
522
|
// Highlight specific target element in red
|
|
454
523
|
const button = find(snap, 'role=button text~"Submit"');
|
|
@@ -460,11 +529,13 @@ await clearOverlay(browser);
|
|
|
460
529
|
```
|
|
461
530
|
|
|
462
531
|
**Color Coding:**
|
|
532
|
+
|
|
463
533
|
- 🔴 Red: Target element
|
|
464
534
|
- 🔵 Blue: Primary elements (`is_primary=true`)
|
|
465
535
|
- 🟢 Green: Regular interactive elements
|
|
466
536
|
|
|
467
537
|
**Visual Indicators:**
|
|
538
|
+
|
|
468
539
|
- Border thickness/opacity scales with importance
|
|
469
540
|
- Semi-transparent fill
|
|
470
541
|
- Importance badges
|
|
@@ -477,21 +548,23 @@ await clearOverlay(browser);
|
|
|
477
548
|
<summary><h3>📄 Content Reading</h3></summary>
|
|
478
549
|
|
|
479
550
|
**`read(browser, options?)`** - Extract page content
|
|
551
|
+
|
|
480
552
|
- `format: "text"` - Plain text extraction
|
|
481
553
|
- `format: "markdown"` - High-quality markdown conversion (uses Turndown)
|
|
482
554
|
- `format: "raw"` - Cleaned HTML (default)
|
|
483
555
|
|
|
484
556
|
**Example:**
|
|
557
|
+
|
|
485
558
|
```typescript
|
|
486
559
|
import { read } from './src';
|
|
487
560
|
|
|
488
561
|
// Get markdown content
|
|
489
562
|
const result = await read(browser, { format: 'markdown' });
|
|
490
|
-
console.log(result.content);
|
|
563
|
+
console.log(result.content); // Markdown text
|
|
491
564
|
|
|
492
565
|
// Get plain text
|
|
493
566
|
const result = await read(browser, { format: 'text' });
|
|
494
|
-
console.log(result.content);
|
|
567
|
+
console.log(result.content); // Plain text
|
|
495
568
|
```
|
|
496
569
|
|
|
497
570
|
</details>
|
|
@@ -500,11 +573,13 @@ console.log(result.content); // Plain text
|
|
|
500
573
|
<summary><h3>📷 Screenshots</h3></summary>
|
|
501
574
|
|
|
502
575
|
**`screenshot(browser, options?)`** - Standalone screenshot capture
|
|
576
|
+
|
|
503
577
|
- Returns base64-encoded data URL
|
|
504
578
|
- PNG or JPEG format
|
|
505
579
|
- Quality control for JPEG (1-100)
|
|
506
580
|
|
|
507
581
|
**Example:**
|
|
582
|
+
|
|
508
583
|
```typescript
|
|
509
584
|
import { screenshot } from './src';
|
|
510
585
|
import { writeFileSync } from 'fs';
|
|
@@ -531,6 +606,7 @@ const dataUrl = await screenshot(browser, { format: 'jpeg', quality: 85 });
|
|
|
531
606
|
Find buttons, links, or any UI elements by their visible text without needing element IDs or CSS selectors. Returns exact pixel coordinates for each match.
|
|
532
607
|
|
|
533
608
|
**Example:**
|
|
609
|
+
|
|
534
610
|
```typescript
|
|
535
611
|
import { SentienceBrowser, findTextRect, clickRect } from 'sentienceapi';
|
|
536
612
|
|
|
@@ -538,8 +614,8 @@ const browser = await SentienceBrowser.create();
|
|
|
538
614
|
await browser.getPage().goto('https://example.com');
|
|
539
615
|
|
|
540
616
|
// Find "Sign In" button (simple string syntax)
|
|
541
|
-
const result = await findTextRect(browser.getPage(),
|
|
542
|
-
if (result.status ===
|
|
617
|
+
const result = await findTextRect(browser.getPage(), 'Sign In');
|
|
618
|
+
if (result.status === 'success' && result.results) {
|
|
543
619
|
const firstMatch = result.results[0];
|
|
544
620
|
console.log(`Found at: (${firstMatch.rect.x}, ${firstMatch.rect.y})`);
|
|
545
621
|
console.log(`In viewport: ${firstMatch.in_viewport}`);
|
|
@@ -550,30 +626,31 @@ if (result.status === "success" && result.results) {
|
|
|
550
626
|
x: firstMatch.rect.x,
|
|
551
627
|
y: firstMatch.rect.y,
|
|
552
628
|
w: firstMatch.rect.width,
|
|
553
|
-
h: firstMatch.rect.height
|
|
629
|
+
h: firstMatch.rect.height,
|
|
554
630
|
});
|
|
555
631
|
}
|
|
556
632
|
}
|
|
557
633
|
```
|
|
558
634
|
|
|
559
635
|
**Advanced Options:**
|
|
636
|
+
|
|
560
637
|
```typescript
|
|
561
638
|
// Case-sensitive search
|
|
562
639
|
const result = await findTextRect(browser.getPage(), {
|
|
563
|
-
text:
|
|
564
|
-
caseSensitive: true
|
|
640
|
+
text: 'LOGIN',
|
|
641
|
+
caseSensitive: true,
|
|
565
642
|
});
|
|
566
643
|
|
|
567
644
|
// Whole word only (won't match "login" as part of "loginButton")
|
|
568
645
|
const result = await findTextRect(browser.getPage(), {
|
|
569
|
-
text:
|
|
570
|
-
wholeWord: true
|
|
646
|
+
text: 'log',
|
|
647
|
+
wholeWord: true,
|
|
571
648
|
});
|
|
572
649
|
|
|
573
650
|
// Find multiple matches
|
|
574
651
|
const result = await findTextRect(browser.getPage(), {
|
|
575
|
-
text:
|
|
576
|
-
maxResults: 10
|
|
652
|
+
text: 'Buy',
|
|
653
|
+
maxResults: 10,
|
|
577
654
|
});
|
|
578
655
|
for (const match of result.results || []) {
|
|
579
656
|
if (match.in_viewport) {
|
|
@@ -584,6 +661,7 @@ for (const match of result.results || []) {
|
|
|
584
661
|
```
|
|
585
662
|
|
|
586
663
|
**Returns:** Promise<TextRectSearchResult> with:
|
|
664
|
+
|
|
587
665
|
- **`status`**: "success" or "error"
|
|
588
666
|
- **`results`**: Array of `TextMatch` objects with:
|
|
589
667
|
- `text` - The matched text
|
|
@@ -593,6 +671,7 @@ for (const match of result.results || []) {
|
|
|
593
671
|
- `in_viewport` - Whether visible in current viewport
|
|
594
672
|
|
|
595
673
|
**Use Cases:**
|
|
674
|
+
|
|
596
675
|
- Find buttons/links by visible text without CSS selectors
|
|
597
676
|
- Get exact pixel coordinates for click automation
|
|
598
677
|
- Verify text visibility and position on page
|
|
@@ -614,15 +693,15 @@ for (const match of result.results || []) {
|
|
|
614
693
|
Elements returned by `snapshot()` have the following properties:
|
|
615
694
|
|
|
616
695
|
```typescript
|
|
617
|
-
element.id
|
|
618
|
-
element.role
|
|
619
|
-
element.text
|
|
620
|
-
element.importance
|
|
621
|
-
element.bbox
|
|
622
|
-
element.visual_cues
|
|
623
|
-
element.in_viewport
|
|
624
|
-
element.is_occluded
|
|
625
|
-
element.z_index
|
|
696
|
+
element.id; // Unique identifier for interactions
|
|
697
|
+
element.role; // ARIA role (button, link, textbox, heading, etc.)
|
|
698
|
+
element.text; // Visible text content
|
|
699
|
+
element.importance; // AI importance score (0-1000)
|
|
700
|
+
element.bbox; // Bounding box (x, y, width, height)
|
|
701
|
+
element.visual_cues; // Visual analysis (is_primary, is_clickable, background_color)
|
|
702
|
+
element.in_viewport; // Is element visible in current viewport?
|
|
703
|
+
element.is_occluded; // Is element covered by other elements?
|
|
704
|
+
element.z_index; // CSS stacking order
|
|
626
705
|
```
|
|
627
706
|
|
|
628
707
|
</details>
|
|
@@ -632,15 +711,15 @@ element.z_index // CSS stacking order
|
|
|
632
711
|
|
|
633
712
|
### Basic Operators
|
|
634
713
|
|
|
635
|
-
| Operator
|
|
636
|
-
|
|
637
|
-
| `=`
|
|
638
|
-
| `!=`
|
|
639
|
-
| `~`
|
|
640
|
-
| `^=`
|
|
641
|
-
| `$=`
|
|
642
|
-
| `>`, `>=` | Greater than
|
|
643
|
-
| `<`, `<=` | Less than
|
|
714
|
+
| Operator | Description | Example |
|
|
715
|
+
| --------- | ---------------------------- | ---------------- |
|
|
716
|
+
| `=` | Exact match | `role=button` |
|
|
717
|
+
| `!=` | Exclusion | `role!=link` |
|
|
718
|
+
| `~` | Substring (case-insensitive) | `text~"sign in"` |
|
|
719
|
+
| `^=` | Prefix match | `text^="Add"` |
|
|
720
|
+
| `$=` | Suffix match | `text$="Cart"` |
|
|
721
|
+
| `>`, `>=` | Greater than | `importance>500` |
|
|
722
|
+
| `<`, `<=` | Less than | `bbox.y<600` |
|
|
644
723
|
|
|
645
724
|
### Supported Fields
|
|
646
725
|
|
|
@@ -685,7 +764,7 @@ const browser = new SentienceBrowser(undefined, undefined, false);
|
|
|
685
764
|
const browser = new SentienceBrowser(undefined, undefined, true);
|
|
686
765
|
|
|
687
766
|
// Auto-detect based on environment (default)
|
|
688
|
-
const browser = new SentienceBrowser();
|
|
767
|
+
const browser = new SentienceBrowser(); // headless=true if CI=true, else false
|
|
689
768
|
```
|
|
690
769
|
|
|
691
770
|
</details>
|
|
@@ -696,6 +775,7 @@ const browser = new SentienceBrowser(); // headless=true if CI=true, else false
|
|
|
696
775
|
For users running from datacenters (AWS, DigitalOcean, etc.), you can configure a residential proxy to prevent IP-based detection by Cloudflare, Akamai, and other anti-bot services.
|
|
697
776
|
|
|
698
777
|
**Supported Formats:**
|
|
778
|
+
|
|
699
779
|
- HTTP: `http://username:password@host:port`
|
|
700
780
|
- HTTPS: `https://username:password@host:port`
|
|
701
781
|
- SOCKS5: `socks5://username:password@host:port`
|
|
@@ -760,9 +840,9 @@ await saveStorageState(browser.getContext(), 'auth.json');
|
|
|
760
840
|
const browser2 = new SentienceBrowser(
|
|
761
841
|
undefined, // apiKey
|
|
762
842
|
undefined, // apiUrl
|
|
763
|
-
false,
|
|
764
|
-
undefined,
|
|
765
|
-
undefined,
|
|
843
|
+
false, // headless
|
|
844
|
+
undefined, // proxy
|
|
845
|
+
undefined, // userDataDir
|
|
766
846
|
'auth.json' // storageState - inject saved session
|
|
767
847
|
);
|
|
768
848
|
await browser2.start();
|
|
@@ -770,12 +850,12 @@ await browser2.start();
|
|
|
770
850
|
|
|
771
851
|
// Workflow 2: Persistent sessions (cookies persist across runs)
|
|
772
852
|
const browser3 = new SentienceBrowser(
|
|
773
|
-
undefined,
|
|
774
|
-
undefined,
|
|
775
|
-
false,
|
|
776
|
-
undefined,
|
|
853
|
+
undefined, // apiKey
|
|
854
|
+
undefined, // apiUrl
|
|
855
|
+
false, // headless
|
|
856
|
+
undefined, // proxy
|
|
777
857
|
'./chrome_profile', // userDataDir - persist cookies
|
|
778
|
-
undefined
|
|
858
|
+
undefined // storageState
|
|
779
859
|
);
|
|
780
860
|
await browser3.start();
|
|
781
861
|
// First run: Log in
|
|
@@ -783,6 +863,7 @@ await browser3.start();
|
|
|
783
863
|
```
|
|
784
864
|
|
|
785
865
|
**Benefits:**
|
|
866
|
+
|
|
786
867
|
- Bypass login screens and CAPTCHAs with valid sessions
|
|
787
868
|
- Save 5-10 agent steps and hundreds of tokens per run
|
|
788
869
|
- Maintain stateful sessions for accessing authenticated pages
|
|
@@ -800,13 +881,15 @@ See `examples/auth-injection-agent.ts` for complete examples.
|
|
|
800
881
|
<summary>Click to expand best practices</summary>
|
|
801
882
|
|
|
802
883
|
### 1. Wait for Dynamic Content
|
|
884
|
+
|
|
803
885
|
```typescript
|
|
804
886
|
await browser.goto('https://example.com');
|
|
805
887
|
await browser.getPage().waitForLoadState('networkidle');
|
|
806
|
-
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
888
|
+
await new Promise(resolve => setTimeout(resolve, 1000)); // Extra buffer
|
|
807
889
|
```
|
|
808
890
|
|
|
809
891
|
### 2. Use Multiple Strategies for Finding Elements
|
|
892
|
+
|
|
810
893
|
```typescript
|
|
811
894
|
// Try exact match first
|
|
812
895
|
let btn = find(snap, 'role=button text="Add to Cart"');
|
|
@@ -818,6 +901,7 @@ if (!btn) {
|
|
|
818
901
|
```
|
|
819
902
|
|
|
820
903
|
### 3. Check Element Visibility Before Clicking
|
|
904
|
+
|
|
821
905
|
```typescript
|
|
822
906
|
if (element.in_viewport && !element.is_occluded) {
|
|
823
907
|
await click(browser, element.id);
|
|
@@ -825,6 +909,7 @@ if (element.in_viewport && !element.is_occluded) {
|
|
|
825
909
|
```
|
|
826
910
|
|
|
827
911
|
### 4. Handle Navigation
|
|
912
|
+
|
|
828
913
|
```typescript
|
|
829
914
|
const result = await click(browser, linkId);
|
|
830
915
|
if (result.url_changed) {
|
|
@@ -833,6 +918,7 @@ if (result.url_changed) {
|
|
|
833
918
|
```
|
|
834
919
|
|
|
835
920
|
### 5. Use Screenshots Sparingly
|
|
921
|
+
|
|
836
922
|
```typescript
|
|
837
923
|
// Fast - no screenshot (only element data)
|
|
838
924
|
const snap = await snapshot(browser);
|
|
@@ -842,6 +928,7 @@ const snap = await snapshot(browser, { screenshot: true });
|
|
|
842
928
|
```
|
|
843
929
|
|
|
844
930
|
### 6. Always Close Browser
|
|
931
|
+
|
|
845
932
|
```typescript
|
|
846
933
|
const browser = new SentienceBrowser();
|
|
847
934
|
|
|
@@ -849,7 +936,7 @@ try {
|
|
|
849
936
|
await browser.start();
|
|
850
937
|
// ... your automation code
|
|
851
938
|
} finally {
|
|
852
|
-
await browser.close();
|
|
939
|
+
await browser.close(); // Always clean up
|
|
853
940
|
}
|
|
854
941
|
```
|
|
855
942
|
|
|
@@ -863,14 +950,18 @@ try {
|
|
|
863
950
|
<summary>Click to expand common issues and solutions</summary>
|
|
864
951
|
|
|
865
952
|
### "Extension failed to load"
|
|
953
|
+
|
|
866
954
|
**Solution:** Build the extension first:
|
|
955
|
+
|
|
867
956
|
```bash
|
|
868
957
|
cd sentience-chrome
|
|
869
958
|
./build.sh
|
|
870
959
|
```
|
|
871
960
|
|
|
872
961
|
### "Cannot use import statement outside a module"
|
|
962
|
+
|
|
873
963
|
**Solution:** Don't use `node` directly. Use `ts-node` or npm scripts:
|
|
964
|
+
|
|
874
965
|
```bash
|
|
875
966
|
npx ts-node examples/hello.ts
|
|
876
967
|
# or
|
|
@@ -878,13 +969,17 @@ npm run example:hello
|
|
|
878
969
|
```
|
|
879
970
|
|
|
880
971
|
### "Element not found"
|
|
972
|
+
|
|
881
973
|
**Solutions:**
|
|
974
|
+
|
|
882
975
|
- Ensure page is loaded: `await browser.getPage().waitForLoadState('networkidle')`
|
|
883
976
|
- Use `waitFor()`: `await waitFor(browser, 'role=button', 10000)`
|
|
884
977
|
- Debug elements: `console.log(snap.elements.map(el => el.text))`
|
|
885
978
|
|
|
886
979
|
### Button not clickable
|
|
980
|
+
|
|
887
981
|
**Solutions:**
|
|
982
|
+
|
|
888
983
|
- Check visibility: `element.in_viewport && !element.is_occluded`
|
|
889
984
|
- Scroll to element: ``await browser.getPage().evaluate(`window.sentience_registry[${element.id}].scrollIntoView()`)``
|
|
890
985
|
|
|
@@ -921,6 +1016,7 @@ npm run example:hello
|
|
|
921
1016
|
**⚠️ Important**: You cannot use `node` directly to run TypeScript files. Use one of these methods:
|
|
922
1017
|
|
|
923
1018
|
### Option 1: Using npm scripts (recommended)
|
|
1019
|
+
|
|
924
1020
|
```bash
|
|
925
1021
|
npm run example:hello
|
|
926
1022
|
npm run example:basic
|
|
@@ -929,6 +1025,7 @@ npm run example:wait
|
|
|
929
1025
|
```
|
|
930
1026
|
|
|
931
1027
|
### Option 2: Using ts-node directly
|
|
1028
|
+
|
|
932
1029
|
```bash
|
|
933
1030
|
npx ts-node examples/hello.ts
|
|
934
1031
|
# or if ts-node is installed globally:
|
|
@@ -936,6 +1033,7 @@ ts-node examples/hello.ts
|
|
|
936
1033
|
```
|
|
937
1034
|
|
|
938
1035
|
### Option 3: Compile then run
|
|
1036
|
+
|
|
939
1037
|
```bash
|
|
940
1038
|
npm run build
|
|
941
1039
|
# Then use compiled JavaScript from dist/
|
|
@@ -974,7 +1072,7 @@ npm test -- snapshot.test.ts
|
|
|
974
1072
|
|
|
975
1073
|
This project is licensed under either of:
|
|
976
1074
|
|
|
977
|
-
|
|
978
|
-
|
|
1075
|
+
- Apache License, Version 2.0, ([LICENSE-APACHE](./LICENSE-APACHE))
|
|
1076
|
+
- MIT license ([LICENSE-MIT](./LICENSE-MIT))
|
|
979
1077
|
|
|
980
1078
|
at your option.
|
package/dist/actions.d.ts
CHANGED
|
@@ -44,6 +44,7 @@ export declare function click(browser: IBrowser, elementId: number, useMouse?: b
|
|
|
44
44
|
* @param elementId - Element ID from snapshot (must be a text input element)
|
|
45
45
|
* @param text - Text to type
|
|
46
46
|
* @param takeSnapshot - Take snapshot after action (default: false)
|
|
47
|
+
* @param delayMs - Delay between keystrokes in milliseconds for human-like typing (default: 0)
|
|
47
48
|
* @returns ActionResult with success status, outcome, duration, and optional snapshot
|
|
48
49
|
*
|
|
49
50
|
* @example
|
|
@@ -51,11 +52,42 @@ export declare function click(browser: IBrowser, elementId: number, useMouse?: b
|
|
|
51
52
|
* const snap = await snapshot(browser);
|
|
52
53
|
* const searchBox = find(snap, 'role=searchbox');
|
|
53
54
|
* if (searchBox) {
|
|
55
|
+
* // Type instantly (default behavior)
|
|
54
56
|
* await typeText(browser, searchBox.id, 'Hello World');
|
|
57
|
+
*
|
|
58
|
+
* // Type with human-like delay (~10ms between keystrokes)
|
|
59
|
+
* await typeText(browser, searchBox.id, 'Hello World', false, 10);
|
|
60
|
+
* }
|
|
61
|
+
* ```
|
|
62
|
+
*/
|
|
63
|
+
export declare function typeText(browser: IBrowser, elementId: number, text: string, takeSnapshot?: boolean, delayMs?: number): Promise<ActionResult>;
|
|
64
|
+
/**
|
|
65
|
+
* Scroll an element into view
|
|
66
|
+
*
|
|
67
|
+
* Scrolls the page so that the specified element is visible in the viewport.
|
|
68
|
+
* Uses the element registry to find the element and scrollIntoView() to scroll it.
|
|
69
|
+
*
|
|
70
|
+
* @param browser - SentienceBrowser instance
|
|
71
|
+
* @param elementId - Element ID from snapshot to scroll into view
|
|
72
|
+
* @param behavior - Scroll behavior: 'smooth' for animated scroll, 'instant' for immediate (default: 'smooth')
|
|
73
|
+
* @param block - Vertical alignment: 'start', 'center', 'end', 'nearest' (default: 'center')
|
|
74
|
+
* @param takeSnapshot - Take snapshot after action (default: false)
|
|
75
|
+
* @returns ActionResult with success status, outcome, duration, and optional snapshot
|
|
76
|
+
*
|
|
77
|
+
* @example
|
|
78
|
+
* ```typescript
|
|
79
|
+
* const snap = await snapshot(browser);
|
|
80
|
+
* const button = find(snap, 'role=button[name="Submit"]');
|
|
81
|
+
* if (button) {
|
|
82
|
+
* // Scroll element into view with smooth animation
|
|
83
|
+
* await scrollTo(browser, button.id);
|
|
84
|
+
*
|
|
85
|
+
* // Scroll instantly to top of viewport
|
|
86
|
+
* await scrollTo(browser, button.id, 'instant', 'start');
|
|
55
87
|
* }
|
|
56
88
|
* ```
|
|
57
89
|
*/
|
|
58
|
-
export declare function
|
|
90
|
+
export declare function scrollTo(browser: IBrowser, elementId: number, behavior?: 'smooth' | 'instant' | 'auto', block?: 'start' | 'center' | 'end' | 'nearest', takeSnapshot?: boolean): Promise<ActionResult>;
|
|
59
91
|
/**
|
|
60
92
|
* Press a keyboard key
|
|
61
93
|
*
|
package/dist/actions.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAY,IAAI,EAAE,MAAM,SAAS,CAAC;AAIvD,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAoED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,KAAK,CACzB,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,QAAQ,GAAE,OAAc,EACxB,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAsGvB;AAED
|
|
1
|
+
{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAY,IAAI,EAAE,MAAM,SAAS,CAAC;AAIvD,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAoED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,KAAK,CACzB,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,QAAQ,GAAE,OAAc,EACxB,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAsGvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,YAAY,GAAE,OAAe,EAC7B,OAAO,GAAE,MAAU,GAClB,OAAO,CAAC,YAAY,CAAC,CAoDvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,QAAQ,EACjB,SAAS,EAAE,MAAM,EACjB,QAAQ,GAAE,QAAQ,GAAG,SAAS,GAAG,MAAiB,EAClD,KAAK,GAAE,OAAO,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAoB,EACxD,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAwDvB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,KAAK,CACzB,OAAO,EAAE,QAAQ,EACjB,GAAG,EAAE,MAAM,EACX,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAgCvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,QAAQ,EACjB,IAAI,EAAE,SAAS,GAAG,IAAI,EACtB,SAAS,GAAE,OAAc,EACzB,iBAAiB,GAAE,MAAY,EAC/B,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CA4FvB"}
|