stelo 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/LICENSE +184 -0
  2. package/README.md +853 -0
  3. package/dist/accessibility.d.ts +227 -0
  4. package/dist/accessibility.d.ts.map +1 -0
  5. package/dist/accessibility.js +602 -0
  6. package/dist/accessibility.js.map +1 -0
  7. package/dist/agent.d.ts +870 -0
  8. package/dist/agent.d.ts.map +1 -0
  9. package/dist/agent.js +1107 -0
  10. package/dist/agent.js.map +1 -0
  11. package/dist/audio-stream.d.ts +114 -0
  12. package/dist/audio-stream.d.ts.map +1 -0
  13. package/dist/audio-stream.js +167 -0
  14. package/dist/audio-stream.js.map +1 -0
  15. package/dist/clipboard.d.ts +99 -0
  16. package/dist/clipboard.d.ts.map +1 -0
  17. package/dist/clipboard.js +352 -0
  18. package/dist/clipboard.js.map +1 -0
  19. package/dist/config.d.ts +183 -0
  20. package/dist/config.d.ts.map +1 -0
  21. package/dist/config.js +477 -0
  22. package/dist/config.js.map +1 -0
  23. package/dist/context.d.ts +213 -0
  24. package/dist/context.d.ts.map +1 -0
  25. package/dist/context.js +387 -0
  26. package/dist/context.js.map +1 -0
  27. package/dist/cortex.d.ts +548 -0
  28. package/dist/cortex.d.ts.map +1 -0
  29. package/dist/cortex.js +1479 -0
  30. package/dist/cortex.js.map +1 -0
  31. package/dist/errors.d.ts +133 -0
  32. package/dist/errors.d.ts.map +1 -0
  33. package/dist/errors.js +278 -0
  34. package/dist/errors.js.map +1 -0
  35. package/dist/events.d.ts +227 -0
  36. package/dist/events.d.ts.map +1 -0
  37. package/dist/events.js +429 -0
  38. package/dist/events.js.map +1 -0
  39. package/dist/executor.d.ts +212 -0
  40. package/dist/executor.d.ts.map +1 -0
  41. package/dist/executor.js +545 -0
  42. package/dist/executor.js.map +1 -0
  43. package/dist/index.d.ts +69 -0
  44. package/dist/index.d.ts.map +1 -0
  45. package/dist/index.js +167 -0
  46. package/dist/index.js.map +1 -0
  47. package/dist/integration.d.ts +159 -0
  48. package/dist/integration.d.ts.map +1 -0
  49. package/dist/integration.js +533 -0
  50. package/dist/integration.js.map +1 -0
  51. package/dist/keyboard.d.ts +276 -0
  52. package/dist/keyboard.d.ts.map +1 -0
  53. package/dist/keyboard.js +404 -0
  54. package/dist/keyboard.js.map +1 -0
  55. package/dist/logger.d.ts +198 -0
  56. package/dist/logger.d.ts.map +1 -0
  57. package/dist/logger.js +516 -0
  58. package/dist/logger.js.map +1 -0
  59. package/dist/middleware.d.ts +183 -0
  60. package/dist/middleware.d.ts.map +1 -0
  61. package/dist/middleware.js +493 -0
  62. package/dist/middleware.js.map +1 -0
  63. package/dist/monitor.d.ts +136 -0
  64. package/dist/monitor.d.ts.map +1 -0
  65. package/dist/monitor.js +341 -0
  66. package/dist/monitor.js.map +1 -0
  67. package/dist/mouse.d.ts +290 -0
  68. package/dist/mouse.d.ts.map +1 -0
  69. package/dist/mouse.js +466 -0
  70. package/dist/mouse.js.map +1 -0
  71. package/dist/plugin.d.ts +157 -0
  72. package/dist/plugin.d.ts.map +1 -0
  73. package/dist/plugin.js +409 -0
  74. package/dist/plugin.js.map +1 -0
  75. package/dist/process.d.ts +106 -0
  76. package/dist/process.d.ts.map +1 -0
  77. package/dist/process.js +326 -0
  78. package/dist/process.js.map +1 -0
  79. package/dist/recorder.d.ts +100 -0
  80. package/dist/recorder.d.ts.map +1 -0
  81. package/dist/recorder.js +258 -0
  82. package/dist/recorder.js.map +1 -0
  83. package/dist/safety.d.ts +59 -0
  84. package/dist/safety.d.ts.map +1 -0
  85. package/dist/safety.js +98 -0
  86. package/dist/safety.js.map +1 -0
  87. package/dist/scheduler.d.ts +152 -0
  88. package/dist/scheduler.d.ts.map +1 -0
  89. package/dist/scheduler.js +615 -0
  90. package/dist/scheduler.js.map +1 -0
  91. package/dist/screen.d.ts +96 -0
  92. package/dist/screen.d.ts.map +1 -0
  93. package/dist/screen.js +154 -0
  94. package/dist/screen.js.map +1 -0
  95. package/dist/session.d.ts +209 -0
  96. package/dist/session.d.ts.map +1 -0
  97. package/dist/session.js +479 -0
  98. package/dist/session.js.map +1 -0
  99. package/dist/stream.d.ts +168 -0
  100. package/dist/stream.d.ts.map +1 -0
  101. package/dist/stream.js +298 -0
  102. package/dist/stream.js.map +1 -0
  103. package/dist/telemetry.d.ts +223 -0
  104. package/dist/telemetry.d.ts.map +1 -0
  105. package/dist/telemetry.js +433 -0
  106. package/dist/telemetry.js.map +1 -0
  107. package/dist/types.d.ts +165 -0
  108. package/dist/types.d.ts.map +1 -0
  109. package/dist/types.js +8 -0
  110. package/dist/types.js.map +1 -0
  111. package/dist/utils/bezier.d.ts +51 -0
  112. package/dist/utils/bezier.d.ts.map +1 -0
  113. package/dist/utils/bezier.js +117 -0
  114. package/dist/utils/bezier.js.map +1 -0
  115. package/dist/utils/helpers.d.ts +90 -0
  116. package/dist/utils/helpers.d.ts.map +1 -0
  117. package/dist/utils/helpers.js +143 -0
  118. package/dist/utils/helpers.js.map +1 -0
  119. package/dist/utils/index.d.ts +4 -0
  120. package/dist/utils/index.d.ts.map +1 -0
  121. package/dist/utils/index.js +18 -0
  122. package/dist/utils/index.js.map +1 -0
  123. package/dist/validation.d.ts +254 -0
  124. package/dist/validation.d.ts.map +1 -0
  125. package/dist/validation.js +478 -0
  126. package/dist/validation.js.map +1 -0
  127. package/dist/vision.d.ts +719 -0
  128. package/dist/vision.d.ts.map +1 -0
  129. package/dist/vision.js +1197 -0
  130. package/dist/vision.js.map +1 -0
  131. package/dist/window.d.ts +80 -0
  132. package/dist/window.d.ts.map +1 -0
  133. package/dist/window.js +170 -0
  134. package/dist/window.js.map +1 -0
  135. package/dist/workflow.d.ts +224 -0
  136. package/dist/workflow.d.ts.map +1 -0
  137. package/dist/workflow.js +578 -0
  138. package/dist/workflow.js.map +1 -0
  139. package/index.d.ts +840 -0
  140. package/index.js +495 -0
  141. package/package.json +91 -0
@@ -0,0 +1,870 @@
1
+ /**
2
+ * Real-time Agent Control Module
3
+ *
4
+ * Designed for low-latency control of mouse and keyboard with:
5
+ * - Action batching (minimizes round-trips)
6
+ * - Visual verification (confirms actions had effect)
7
+ * - Async non-blocking movement (cursor moves visually in real-time)
8
+ * - Parallel action execution (multiple things at once)
9
+ * - Streaming cursor control (continuous, cancellable)
10
+ * - Velocity-based streaming control (continuous, low-latency)
11
+ * - Gesture recording and playback
12
+ *
13
+ * @module agent
14
+ */
15
+ export interface Region {
16
+ x: number;
17
+ y: number;
18
+ width: number;
19
+ height: number;
20
+ }
21
+ /** Result of a single action in a batch */
22
+ export interface ActionResult {
23
+ /** Index of this action in the batch */
24
+ index: number;
25
+ /** Whether the action succeeded */
26
+ success: boolean;
27
+ /** Error message if failed */
28
+ error?: string;
29
+ /** Time spent on this action in milliseconds */
30
+ durationMs: number;
31
+ /** Mouse position after mouse actions */
32
+ mousePosition?: {
33
+ x: number;
34
+ y: number;
35
+ };
36
+ /** Whether visual change was detected (for wait actions) */
37
+ changeDetected?: boolean;
38
+ }
39
+ /** Result of executing an action batch */
40
+ export interface BatchResult {
41
+ /** Individual results for each action */
42
+ results: ActionResult[];
43
+ /** Total time spent in milliseconds */
44
+ totalDurationMs: number;
45
+ /** Number of successful actions */
46
+ successCount: number;
47
+ /** Number of failed actions */
48
+ failureCount: number;
49
+ /** Index of first failed action, or -1 if all succeeded */
50
+ firstFailureIndex: number;
51
+ /** Whether all actions succeeded */
52
+ allSucceeded: boolean;
53
+ }
54
+ /** Result of click/hotkey with verification */
55
+ export interface VerifyResult {
56
+ /** Whether visual change was detected */
57
+ verified: boolean;
58
+ /** Percentage of pixels changed */
59
+ changePercentage: number;
60
+ /** Time spent in milliseconds */
61
+ durationMs: number;
62
+ }
63
+ /** A point in a recorded mouse trail */
64
+ export interface TrailPoint {
65
+ /** Timestamp in milliseconds relative to start */
66
+ timestampMs: number;
67
+ /** X coordinate */
68
+ x: number;
69
+ /** Y coordinate */
70
+ y: number;
71
+ }
72
+ export type MouseButton = 'left' | 'right' | 'middle';
73
+ /** All possible actions that can be batched */
74
+ export type Action = {
75
+ type: 'mouseMove';
76
+ x: number;
77
+ y: number;
78
+ } | {
79
+ type: 'mouseMoveRel';
80
+ dx: number;
81
+ dy: number;
82
+ } | {
83
+ type: 'mouseMoveSmooth';
84
+ x: number;
85
+ y: number;
86
+ duration?: number;
87
+ } | {
88
+ type: 'mouseClick';
89
+ button?: MouseButton;
90
+ } | {
91
+ type: 'mouseClickAt';
92
+ x: number;
93
+ y: number;
94
+ button?: MouseButton;
95
+ } | {
96
+ type: 'mouseDoubleClick';
97
+ button?: MouseButton;
98
+ } | {
99
+ type: 'mouseDown';
100
+ button?: MouseButton;
101
+ } | {
102
+ type: 'mouseUp';
103
+ button?: MouseButton;
104
+ } | {
105
+ type: 'mouseScroll';
106
+ amount: number;
107
+ horizontal?: boolean;
108
+ } | {
109
+ type: 'mouseDrag';
110
+ toX: number;
111
+ toY: number;
112
+ button?: MouseButton;
113
+ } | {
114
+ type: 'type';
115
+ text: string;
116
+ } | {
117
+ type: 'typeHumanized';
118
+ text: string;
119
+ minDelay?: number;
120
+ maxDelay?: number;
121
+ } | {
122
+ type: 'keyPress';
123
+ key: string;
124
+ } | {
125
+ type: 'keyDown';
126
+ key: string;
127
+ } | {
128
+ type: 'keyUp';
129
+ key: string;
130
+ } | {
131
+ type: 'hotkey';
132
+ keys: string[];
133
+ } | {
134
+ type: 'delay';
135
+ ms: number;
136
+ } | {
137
+ type: 'waitForChange';
138
+ threshold?: number;
139
+ timeout?: number;
140
+ region?: Region;
141
+ } | {
142
+ type: 'waitForStable';
143
+ threshold?: number;
144
+ stableMs?: number;
145
+ timeout?: number;
146
+ region?: Region;
147
+ };
148
+ /**
149
+ * Execute multiple actions as an atomic batch.
150
+ *
151
+ * This is the primary way applications should interact with the system.
152
+ * All actions execute sequentially in a single native call,
153
+ * minimizing latency and round-trips.
154
+ *
155
+ * @example
156
+ * ```typescript
157
+ * // Click a button, wait for response, then type
158
+ * const result = await agent.executeBatch([
159
+ * { type: 'mouseClickAt', x: 500, y: 300 },
160
+ * { type: 'waitForStable', stableMs: 200 },
161
+ * { type: 'type', text: 'Hello World' },
162
+ * { type: 'keyPress', key: 'Enter' }
163
+ * ]);
164
+ *
165
+ * if (result.allSucceeded) {
166
+ * console.log('All actions completed in', result.totalDurationMs, 'ms');
167
+ * }
168
+ * ```
169
+ *
170
+ * @param actions Array of actions to execute
171
+ * @param options Execution options
172
+ * @returns Batch result with per-action details
173
+ */
174
+ export declare function executeBatch(actions: Action[], options?: {
175
+ stopOnError?: boolean;
176
+ }): BatchResult;
177
+ /**
178
+ * Click at a position and verify visual change occurred.
179
+ *
180
+ * Useful for confirming clicks had an effect.
181
+ * Captures screen before and after click, computes diff.
182
+ *
183
+ * @example
184
+ * ```typescript
185
+ * const result = await agent.clickAndVerify(500, 300);
186
+ * if (!result.verified) {
187
+ * console.log('Click had no visual effect - may need retry');
188
+ * }
189
+ * ```
190
+ *
191
+ * @param x Target X coordinate
192
+ * @param y Target Y coordinate
193
+ * @param options Click options
194
+ * @returns Verification result
195
+ */
196
+ export declare function clickAndVerify(x: number, y: number, options?: {
197
+ button?: MouseButton;
198
+ minChangePercent?: number;
199
+ timeoutMs?: number;
200
+ region?: Region;
201
+ }): VerifyResult;
202
+ /**
203
+ * Execute a hotkey combination and verify visual change.
204
+ *
205
+ * @example
206
+ * ```typescript
207
+ * // Open file dialog and verify it appeared
208
+ * const result = await agent.hotkeyAndVerify(['ctrl', 'o']);
209
+ * if (result.verified) {
210
+ * console.log('Dialog opened');
211
+ * }
212
+ * ```
213
+ *
214
+ * @param keys Array of keys to press together
215
+ * @param options Verification options
216
+ * @returns Verification result
217
+ */
218
+ export declare function hotkeyAndVerify(keys: string[], options?: {
219
+ minChangePercent?: number;
220
+ timeoutMs?: number;
221
+ region?: Region;
222
+ }): VerifyResult;
223
+ /**
224
+ * Move mouse relative to current position.
225
+ *
226
+ * @param dx Horizontal offset (positive = right)
227
+ * @param dy Vertical offset (positive = down)
228
+ * @returns New mouse position
229
+ */
230
+ export declare function moveRelative(dx: number, dy: number): {
231
+ x: number;
232
+ y: number;
233
+ };
234
+ /**
235
+ * Move mouse relative to current position with smooth animation.
236
+ *
237
+ * @param dx Horizontal offset
238
+ * @param dy Vertical offset
239
+ * @param durationMs Animation duration (default: 300ms)
240
+ * @returns New mouse position
241
+ */
242
+ export declare function moveRelativeSmooth(dx: number, dy: number, durationMs?: number): {
243
+ x: number;
244
+ y: number;
245
+ };
246
+ /**
247
+ * Type text and wait for screen to stabilize.
248
+ *
249
+ * Useful for form input where you want to wait for
250
+ * autocomplete or validation to finish.
251
+ *
252
+ * @param text Text to type
253
+ * @param options Wait options
254
+ * @returns True if screen stabilized within timeout
255
+ */
256
+ export declare function typeAndWaitStable(text: string, options?: {
257
+ stabilityThreshold?: number;
258
+ stableDurationMs?: number;
259
+ timeoutMs?: number;
260
+ }): boolean;
261
+ /**
262
+ * Record mouse movements for a duration.
263
+ *
264
+ * Captures mouse trail for gesture analysis or replay.
265
+ *
266
+ * @example
267
+ * ```typescript
268
+ * // Record 3 seconds of mouse movement
269
+ * const trail = agent.recordMouseTrail(3000);
270
+ * console.log(`Recorded ${trail.length} points`);
271
+ *
272
+ * // Replay at half speed
273
+ * agent.replayMouseTrail(trail, 0.5);
274
+ * ```
275
+ *
276
+ * @param durationMs How long to record
277
+ * @param sampleRateHz Sample rate (default: 60)
278
+ * @returns Array of trail points with timestamps
279
+ */
280
+ export declare function recordMouseTrail(durationMs: number, sampleRateHz?: number): TrailPoint[];
281
+ /**
282
+ * Replay a recorded mouse trail.
283
+ *
284
+ * @param trail Previously recorded trail
285
+ * @param speedMultiplier Playback speed (1.0 = normal, 0.5 = half speed)
286
+ */
287
+ export declare function replayMouseTrail(trail: TrailPoint[], speedMultiplier?: number): void;
288
+ /**
289
+ * Release all held modifier keys.
290
+ *
291
+ * Call this for error recovery when modifier keys might be stuck,
292
+ * or after complex keyboard sequences.
293
+ */
294
+ export declare function releaseAllModifiers(): void;
295
+ /**
296
+ * Fluent action builder for creating complex sequences.
297
+ *
298
+ * @example
299
+ * ```typescript
300
+ * const result = agent.sequence()
301
+ * .click(500, 300)
302
+ * .waitForStable()
303
+ * .type('Hello World')
304
+ * .press('Enter')
305
+ * .waitForChange()
306
+ * .execute();
307
+ * ```
308
+ */
309
+ export declare class ActionSequence {
310
+ private actions;
311
+ /** Move mouse to absolute position */
312
+ moveTo(x: number, y: number): this;
313
+ /** Move mouse smoothly to absolute position */
314
+ moveToSmooth(x: number, y: number, duration?: number): this;
315
+ /** Move mouse relative to current position */
316
+ moveBy(dx: number, dy: number): this;
317
+ /** Click at current position */
318
+ click(button?: MouseButton): this;
319
+ /** Click at specific coordinates */
320
+ clickAt(x: number, y: number, button?: MouseButton): this;
321
+ /** Double click */
322
+ doubleClick(button?: MouseButton): this;
323
+ /** Press mouse button down */
324
+ mouseDown(button?: MouseButton): this;
325
+ /** Release mouse button */
326
+ mouseUp(button?: MouseButton): this;
327
+ /** Scroll wheel */
328
+ scroll(amount: number, horizontal?: boolean): this;
329
+ /** Drag from current position to target */
330
+ dragTo(toX: number, toY: number, button?: MouseButton): this;
331
+ /** Type text instantly */
332
+ type(text: string): this;
333
+ /** Type text with human-like delays */
334
+ typeHuman(text: string, minDelay?: number, maxDelay?: number): this;
335
+ /** Press and release a key */
336
+ press(key: string): this;
337
+ /** Press key down (remember to release!) */
338
+ keyDown(key: string): this;
339
+ /** Release key */
340
+ keyUp(key: string): this;
341
+ /** Press a hotkey combination */
342
+ hotkey(...keys: string[]): this;
343
+ /** Wait for specified duration */
344
+ delay(ms: number): this;
345
+ /** Alias for delay */
346
+ wait(ms: number): this;
347
+ /** Wait for visual change */
348
+ waitForChange(options?: {
349
+ threshold?: number;
350
+ timeout?: number;
351
+ region?: Region;
352
+ }): this;
353
+ /** Wait for screen to stabilize */
354
+ waitForStable(options?: {
355
+ threshold?: number;
356
+ stableMs?: number;
357
+ timeout?: number;
358
+ region?: Region;
359
+ }): this;
360
+ /** Get the built actions array */
361
+ build(): Action[];
362
+ /** Execute all queued actions */
363
+ execute(options?: {
364
+ stopOnError?: boolean;
365
+ }): BatchResult;
366
+ /** Execute all queued actions — non-blocking async version */
367
+ executeAsync(options?: {
368
+ stopOnError?: boolean;
369
+ }): Promise<BatchResult>;
370
+ /** Clear all queued actions */
371
+ clear(): this;
372
+ }
373
+ /**
374
+ * Create a new action sequence builder.
375
+ *
376
+ * @example
377
+ * ```typescript
378
+ * const result = agent.sequence()
379
+ * .clickAt(500, 300)
380
+ * .waitForStable()
381
+ * .type('search query')
382
+ * .press('Enter')
383
+ * .execute();
384
+ * ```
385
+ */
386
+ export declare function sequence(): ActionSequence;
387
+ /**
388
+ * Simple click with optional wait for visual feedback.
389
+ * Combines click and waitForStable in one call.
390
+ *
391
+ * @param x Target X coordinate
392
+ * @param y Target Y coordinate
393
+ * @param options Click options
394
+ * @returns Batch result
395
+ */
396
+ export declare function clickAndWait(x: number, y: number, options?: {
397
+ button?: MouseButton;
398
+ stableMs?: number;
399
+ }): BatchResult;
400
+ /**
401
+ * Type text followed by Enter key.
402
+ *
403
+ * @param text Text to type before pressing Enter
404
+ * @param options Type options
405
+ * @returns Batch result
406
+ */
407
+ export declare function typeAndEnter(text: string, options?: {
408
+ humanized?: boolean;
409
+ }): BatchResult;
410
+ /** Result of a single parallel task */
411
+ export interface ParallelTaskResult {
412
+ /** Task name/label */
413
+ name: string;
414
+ /** Whether it succeeded */
415
+ success: boolean;
416
+ /** Error if failed */
417
+ error?: string;
418
+ /** Duration in ms */
419
+ durationMs: number;
420
+ /** Return value */
421
+ value?: unknown;
422
+ }
423
+ /** Result of parallel execution */
424
+ export interface ParallelResult {
425
+ /** Individual task results */
426
+ results: ParallelTaskResult[];
427
+ /** Total wall-clock time */
428
+ totalDurationMs: number;
429
+ /** Whether all tasks succeeded */
430
+ allSucceeded: boolean;
431
+ }
432
+ /**
433
+ * Execute multiple independent actions in parallel.
434
+ *
435
+ * This is the key primitive for AI agents that need to do multiple things
436
+ * simultaneously — e.g., move the mouse while monitoring the screen,
437
+ * or type while watching for visual changes.
438
+ *
439
+ * Each task is a named async function. All run concurrently via Promise.all.
440
+ *
441
+ * @example
442
+ * ```typescript
443
+ * // Move mouse and monitor screen simultaneously
444
+ * const result = await agent.parallel({
445
+ * move: async () => {
446
+ * // Non-blocking smooth movement
447
+ * const handle = mouse.moveSmoothAsync(500, 300, { duration: 600 });
448
+ * return handle.promise;
449
+ * },
450
+ * monitor: async () => {
451
+ * // Capture screen while mouse is moving
452
+ * await sleep(200);
453
+ * return screen.capture();
454
+ * },
455
+ * });
456
+ *
457
+ * // All done
458
+ * console.log(result.allSucceeded); // true
459
+ * ```
460
+ */
461
+ export declare function parallel(tasks: Record<string, () => unknown | Promise<unknown>>): Promise<ParallelResult>;
462
+ /** Handle for controlling a live cursor stream */
463
+ export interface CursorStreamHandle {
464
+ /** Send a new target position — cursor will move towards it smoothly */
465
+ moveTo(x: number, y: number): void;
466
+ /** Teleport immediately */
467
+ jumpTo(x: number, y: number): void;
468
+ /** Set velocity (pixels per second) */
469
+ setVelocity(vx: number, vy: number): void;
470
+ /** Stop all movement */
471
+ stop(): void;
472
+ /** Destroy the stream and release resources */
473
+ destroy(): void;
474
+ /** Whether the stream is still active */
475
+ readonly active: boolean;
476
+ /** Current cursor position */
477
+ readonly position: {
478
+ x: number;
479
+ y: number;
480
+ };
481
+ }
482
+ /**
483
+ * Create a real-time cursor control stream.
484
+ *
485
+ * Returns a handle that lets you continuously direct the cursor in real-time.
486
+ * The cursor moves smoothly towards targets with no blocking. Multiple
487
+ * `moveTo()` calls seamlessly redirect the cursor mid-flight.
488
+ *
489
+ * This is the preferred way for AI agents to control the cursor because:
490
+ * - Movement is always visible and smooth
491
+ * - Targets can be updated any time (no waiting for previous move to finish)
492
+ * - Velocity-based control for continuous steering
493
+ * - Zero blocking of the Node.js event loop
494
+ *
495
+ * @param updateRateHz How many times per second to update position (default: 120)
496
+ *
497
+ * @example
498
+ * ```typescript
499
+ * const cursor = agent.createCursorStream(120);
500
+ *
501
+ * // Smooth movement to target
502
+ * cursor.moveTo(500, 300);
503
+ *
504
+ * // Change target mid-flight — cursor redirects smoothly
505
+ * setTimeout(() => cursor.moveTo(800, 600), 200);
506
+ *
507
+ * // Velocity-based control (e.g., from joystick or AI output)
508
+ * cursor.setVelocity(200, -100); // 200px/s right, 100px/s up
509
+ *
510
+ * // When done
511
+ * cursor.destroy();
512
+ * ```
513
+ */
514
+ export declare function createCursorStream(updateRateHz?: number): CursorStreamHandle;
515
+ /**
516
+ * Move to a position smoothly, click, and verify — all non-blocking.
517
+ *
518
+ * The cursor visually moves to the target, clicks, then verifies
519
+ * that the screen changed. The entire operation is async.
520
+ *
521
+ * @example
522
+ * ```typescript
523
+ * const result = await agent.moveClickVerify(500, 300, {
524
+ * moveDuration: 400,
525
+ * button: 'left',
526
+ * });
527
+ * console.log(result.verified); // true if screen changed after click
528
+ * ```
529
+ */
530
+ export declare function moveClickVerify(x: number, y: number, options?: {
531
+ moveDuration?: number;
532
+ button?: MouseButton;
533
+ minChangePercent?: number;
534
+ verifyTimeoutMs?: number;
535
+ region?: Region;
536
+ }): Promise<VerifyResult & {
537
+ finalPosition: {
538
+ x: number;
539
+ y: number;
540
+ };
541
+ }>;
542
+ /**
543
+ * Execute a batch of actions with async wrapper.
544
+ *
545
+ * Same as `executeBatch()` but returns a Promise, so it can be used
546
+ * in `Promise.all()` with other async operations.
547
+ */
548
+ export declare function executeBatchAsync(actions: Action[], options?: {
549
+ stopOnError?: boolean;
550
+ }): Promise<BatchResult>;
551
+ /**
552
+ * Convenience: smoothly move to position and click — async.
553
+ * The cursor is visible moving on screen the whole time.
554
+ */
555
+ export declare function smoothClickAt(x: number, y: number, options?: {
556
+ duration?: number;
557
+ button?: MouseButton;
558
+ }): Promise<void>;
559
+ /**
560
+ * Convenience: smoothly move to a text field, click it, type text.
561
+ * Fully async and visually smooth.
562
+ */
563
+ export declare function smoothClickAndType(x: number, y: number, text: string, options?: {
564
+ duration?: number;
565
+ typeDelay?: number;
566
+ }): Promise<void>;
567
+ /** An action group to execute on its own OS thread */
568
+ export interface ActionGroup {
569
+ /** Unique group identifier */
570
+ groupId?: string;
571
+ /** Actions to execute sequentially within this group */
572
+ actions: Action[];
573
+ /** Stop this group on first error (default: true) */
574
+ stopOnError?: boolean;
575
+ /** Delay before starting this group (ms) — for interleaving */
576
+ startDelayMs?: number;
577
+ }
578
+ /** Result of a single parallel group */
579
+ export interface NativeGroupResultTS {
580
+ groupId: string;
581
+ results: ActionResult[];
582
+ durationMs: number;
583
+ successCount: number;
584
+ failureCount: number;
585
+ }
586
+ /** Result of native parallel execution */
587
+ export interface NativeParallelResultTS {
588
+ groups: NativeGroupResultTS[];
589
+ totalDurationMs: number;
590
+ totalActions: number;
591
+ totalSuccess: number;
592
+ totalFailures: number;
593
+ allSucceeded: boolean;
594
+ }
595
+ /**
596
+ * Execute multiple action groups in TRUE parallel on native OS threads.
597
+ *
598
+ * Each group runs on its own Rust thread. Within a group, actions execute
599
+ * sequentially. Groups start simultaneously (or with optional start delays).
600
+ *
601
+ * This is the core primitive that makes Stelo faster than any other SDK —
602
+ * true OS-level parallelism, not async pretend.
603
+ *
604
+ * @example
605
+ * ```typescript
606
+ * // Move mouse AND type AND capture screen — all at the same time
607
+ * const result = agent.nativeParallel([
608
+ * {
609
+ * groupId: 'mouse',
610
+ * actions: [
611
+ * { type: 'mouseMoveSmooth', x: 500, y: 300, duration: 500 },
612
+ * { type: 'mouseClick' },
613
+ * ],
614
+ * },
615
+ * {
616
+ * groupId: 'keyboard',
617
+ * actions: [
618
+ * { type: 'delay', ms: 200 }, // Start typing 200ms after mouse starts
619
+ * { type: 'type', text: 'hello world' },
620
+ * ],
621
+ * },
622
+ * ]);
623
+ * console.log(result.totalDurationMs); // ~500ms, not ~700ms sequential
624
+ * ```
625
+ */
626
+ export declare function nativeParallel(groups: ActionGroup[]): NativeParallelResultTS;
627
+ /** Timed action for interleaved execution */
628
+ export type TimedAction = Action & {
629
+ /** When to start this action (ms from beginning). Actions with same time run in parallel. */
630
+ startAtMs: number;
631
+ };
632
+ /**
633
+ * Execute actions with interleaved timing — human-like overlapping input.
634
+ *
635
+ * Each action has a startAtMs offset. Actions with the same start time
636
+ * run in parallel on separate OS threads. This enables realistic
637
+ * overlapping mouse + keyboard behavior.
638
+ *
639
+ * @example
640
+ * ```typescript
641
+ * // Start moving mouse immediately, start typing while mouse still moving
642
+ * agent.interleave([
643
+ * { startAtMs: 0, type: 'mouseMoveSmooth', x: 500, y: 300, duration: 500 },
644
+ * { startAtMs: 150, type: 'type', text: 'hello' }, // starts while mouse moving
645
+ * { startAtMs: 500, type: 'mouseClick' }, // after mouse arrives
646
+ * ]);
647
+ * ```
648
+ */
649
+ export declare function interleave(timedActions: TimedAction[]): NativeParallelResultTS;
650
+ /** A recognized word with its screen position */
651
+ export interface OcrWord {
652
+ text: string;
653
+ x: number;
654
+ y: number;
655
+ width: number;
656
+ height: number;
657
+ confidence: number;
658
+ /** Center point of this word's bounding box */
659
+ center: {
660
+ x: number;
661
+ y: number;
662
+ };
663
+ }
664
+ /** A recognized line of text */
665
+ export interface OcrLine {
666
+ text: string;
667
+ words: OcrWord[];
668
+ x: number;
669
+ y: number;
670
+ width: number;
671
+ height: number;
672
+ }
673
+ /** Full OCR result */
674
+ export interface OcrResult {
675
+ /** Complete recognized text */
676
+ text: string;
677
+ /** Lines with word-level detail */
678
+ lines: OcrLine[];
679
+ /** How long OCR took (ms) */
680
+ durationMs: number;
681
+ }
682
+ /**
683
+ * Perform OCR on a screen region. Returns recognized text with positions.
684
+ *
685
+ * @example
686
+ * ```typescript
687
+ * // OCR the entire screen
688
+ * const result = agent.ocr();
689
+ * console.log(result.text);
690
+ *
691
+ * // OCR a specific region
692
+ * const result = agent.ocr({ x: 100, y: 200, width: 400, height: 50 });
693
+ * for (const word of result.lines[0]?.words ?? []) {
694
+ * console.log(`"${word.text}" at (${word.center.x}, ${word.center.y})`);
695
+ * }
696
+ * ```
697
+ */
698
+ export declare function ocr(region?: Region): OcrResult;
699
+ /**
700
+ * Find text on screen via OCR and return its position.
701
+ *
702
+ * @example
703
+ * ```typescript
704
+ * const submit = agent.findText('Submit');
705
+ * if (submit) {
706
+ * mouse.clickAt(submit.center.x, submit.center.y);
707
+ * }
708
+ * ```
709
+ */
710
+ export declare function findText(needle: string, region?: Region): OcrWord | null;
711
+ /**
712
+ * Find text on screen and click its center. Game-changer for AI agents.
713
+ *
714
+ * @example
715
+ * ```typescript
716
+ * // Click the "Submit" button — no coordinates needed
717
+ * agent.clickText('Submit');
718
+ *
719
+ * // Click "Cancel" with right-click
720
+ * agent.clickText('Cancel', { button: 'right' });
721
+ * ```
722
+ *
723
+ * @throws Error if text not found on screen
724
+ */
725
+ export declare function clickText(needle: string, options?: {
726
+ button?: MouseButton;
727
+ region?: Region;
728
+ }): {
729
+ x: number;
730
+ y: number;
731
+ };
732
+ /**
733
+ * Wait until OCR detects the specified text on screen.
734
+ * Polls at a configurable interval. Returns the word position, or null on timeout.
735
+ *
736
+ * @example
737
+ * ```typescript
738
+ * // Wait up to 10s for "Loading complete" to appear
739
+ * const word = agent.waitForText('Loading complete', { timeout: 10000 });
740
+ * if (word) console.log('Found at', word.center);
741
+ * ```
742
+ */
743
+ export declare function waitForText(needle: string, options?: {
744
+ timeout?: number;
745
+ interval?: number;
746
+ region?: Region;
747
+ }): OcrWord | null;
748
+ /**
749
+ * Wait for text to appear on screen, then click it.
750
+ * Combines waitForText + click in a single native call for lower latency.
751
+ *
752
+ * @example
753
+ * ```typescript
754
+ * // Wait for "Submit" button to appear, then click it
755
+ * const pos = agent.waitForTextAndClick('Submit', { timeout: 5000 });
756
+ * if (pos) console.log('Clicked at', pos);
757
+ * ```
758
+ */
759
+ export declare function waitForTextAndClick(needle: string, options?: {
760
+ timeout?: number;
761
+ interval?: number;
762
+ button?: string;
763
+ region?: Region;
764
+ }): {
765
+ x: number;
766
+ y: number;
767
+ } | null;
768
+ /**
769
+ * Watch a screen region and wait for it to change.
770
+ * Returns the change percentage when threshold is exceeded, or null on timeout.
771
+ *
772
+ * @example
773
+ * ```typescript
774
+ * // Watch a button area for any visual change
775
+ * const change = agent.watchRegion(
776
+ * { x: 200, y: 300, width: 100, height: 40 },
777
+ * { threshold: 2.0, timeout: 5000 }
778
+ * );
779
+ * if (change) console.log(`Region changed by ${change.toFixed(1)}%`);
780
+ * ```
781
+ */
782
+ export declare function watchRegion(region: Region, options?: {
783
+ threshold?: number;
784
+ timeout?: number;
785
+ interval?: number;
786
+ }): number | null;
787
+ /**
788
+ * Continuously watch a screen region and call a handler when it changes.
789
+ * Returns a stop function.
790
+ *
791
+ * @example
792
+ * ```typescript
793
+ * const stop = agent.onRegionChange(
794
+ * { x: 200, y: 300, width: 100, height: 40 },
795
+ * (changePct) => console.log(`Changed: ${changePct}%`),
796
+ * { threshold: 1.0, interval: 300 }
797
+ * );
798
+ * // Later...
799
+ * stop();
800
+ * ```
801
+ */
802
+ export declare function onRegionChange(region: Region, handler: (changePercentage: number) => void, options?: {
803
+ threshold?: number;
804
+ interval?: number;
805
+ }): () => void;
806
+ /**
807
+ * Register a global hotkey handler. Starts listening for keyboard events
808
+ * and triggers the callback when the key combination is detected.
809
+ *
810
+ * @example
811
+ * ```typescript
812
+ * // Register Ctrl+Shift+A
813
+ * const unregister = agent.onHotkey('ctrl+shift+a', () => {
814
+ * console.log('Hotkey triggered!');
815
+ * });
816
+ *
817
+ * // Later: stop listening
818
+ * unregister();
819
+ * ```
820
+ */
821
+ export declare function onHotkey(combo: string, handler: () => void, options?: {
822
+ pollInterval?: number;
823
+ }): () => void;
824
+ /**
825
+ * Wait for a hotkey combo to be pressed. Returns a promise that resolves
826
+ * when the combo is detected, or rejects on timeout.
827
+ *
828
+ * @example
829
+ * ```typescript
830
+ * await agent.waitForHotkey('ctrl+shift+s', { timeout: 30000 });
831
+ * console.log('User pressed Ctrl+Shift+S!');
832
+ * ```
833
+ */
834
+ export declare function waitForHotkey(combo: string, options?: {
835
+ timeout?: number;
836
+ }): Promise<void>;
837
+ export declare const agent: {
838
+ executeBatch: typeof executeBatch;
839
+ executeBatchAsync: typeof executeBatchAsync;
840
+ clickAndVerify: typeof clickAndVerify;
841
+ hotkeyAndVerify: typeof hotkeyAndVerify;
842
+ parallel: typeof parallel;
843
+ moveClickVerify: typeof moveClickVerify;
844
+ smoothClickAt: typeof smoothClickAt;
845
+ smoothClickAndType: typeof smoothClickAndType;
846
+ createCursorStream: typeof createCursorStream;
847
+ moveRelative: typeof moveRelative;
848
+ moveRelativeSmooth: typeof moveRelativeSmooth;
849
+ typeAndWaitStable: typeof typeAndWaitStable;
850
+ recordMouseTrail: typeof recordMouseTrail;
851
+ replayMouseTrail: typeof replayMouseTrail;
852
+ releaseAllModifiers: typeof releaseAllModifiers;
853
+ sequence: typeof sequence;
854
+ ActionSequence: typeof ActionSequence;
855
+ clickAndWait: typeof clickAndWait;
856
+ typeAndEnter: typeof typeAndEnter;
857
+ nativeParallel: typeof nativeParallel;
858
+ interleave: typeof interleave;
859
+ ocr: typeof ocr;
860
+ findText: typeof findText;
861
+ clickText: typeof clickText;
862
+ waitForText: typeof waitForText;
863
+ waitForTextAndClick: typeof waitForTextAndClick;
864
+ watchRegion: typeof watchRegion;
865
+ onRegionChange: typeof onRegionChange;
866
+ onHotkey: typeof onHotkey;
867
+ waitForHotkey: typeof waitForHotkey;
868
+ };
869
+ export default agent;
870
+ //# sourceMappingURL=agent.d.ts.map