screencli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.d.ts +3 -0
- package/dist/bin/cli.d.ts.map +1 -0
- package/dist/bin/cli.js +14 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/src/agent/loop.d.ts +21 -0
- package/dist/src/agent/loop.d.ts.map +1 -0
- package/dist/src/agent/loop.js +119 -0
- package/dist/src/agent/loop.js.map +1 -0
- package/dist/src/agent/system-prompt.d.ts +2 -0
- package/dist/src/agent/system-prompt.d.ts.map +1 -0
- package/dist/src/agent/system-prompt.js +43 -0
- package/dist/src/agent/system-prompt.js.map +1 -0
- package/dist/src/agent/tool-handlers.d.ts +44 -0
- package/dist/src/agent/tool-handlers.d.ts.map +1 -0
- package/dist/src/agent/tool-handlers.js +242 -0
- package/dist/src/agent/tool-handlers.js.map +1 -0
- package/dist/src/agent/tools.d.ts +5 -0
- package/dist/src/agent/tools.d.ts.map +1 -0
- package/dist/src/agent/tools.js +251 -0
- package/dist/src/agent/tools.js.map +1 -0
- package/dist/src/browser/accessibility.d.ts +28 -0
- package/dist/src/browser/accessibility.d.ts.map +1 -0
- package/dist/src/browser/accessibility.js +47 -0
- package/dist/src/browser/accessibility.js.map +1 -0
- package/dist/src/browser/actions.d.ts +32 -0
- package/dist/src/browser/actions.d.ts.map +1 -0
- package/dist/src/browser/actions.js +122 -0
- package/dist/src/browser/actions.js.map +1 -0
- package/dist/src/browser/auth.d.ts +12 -0
- package/dist/src/browser/auth.d.ts.map +1 -0
- package/dist/src/browser/auth.js +53 -0
- package/dist/src/browser/auth.js.map +1 -0
- package/dist/src/browser/resolve-locator.d.ts +15 -0
- package/dist/src/browser/resolve-locator.d.ts.map +1 -0
- package/dist/src/browser/resolve-locator.js +27 -0
- package/dist/src/browser/resolve-locator.js.map +1 -0
- package/dist/src/browser/session.d.ts +16 -0
- package/dist/src/browser/session.d.ts.map +1 -0
- package/dist/src/browser/session.js +47 -0
- package/dist/src/browser/session.js.map +1 -0
- package/dist/src/cli/commands/export.d.ts +3 -0
- package/dist/src/cli/commands/export.d.ts.map +1 -0
- package/dist/src/cli/commands/export.js +76 -0
- package/dist/src/cli/commands/export.js.map +1 -0
- package/dist/src/cli/commands/init.d.ts +8 -0
- package/dist/src/cli/commands/init.d.ts.map +1 -0
- package/dist/src/cli/commands/init.js +64 -0
- package/dist/src/cli/commands/init.js.map +1 -0
- package/dist/src/cli/commands/record.d.ts +3 -0
- package/dist/src/cli/commands/record.d.ts.map +1 -0
- package/dist/src/cli/commands/record.js +215 -0
- package/dist/src/cli/commands/record.js.map +1 -0
- package/dist/src/cli/options.d.ts +24 -0
- package/dist/src/cli/options.d.ts.map +1 -0
- package/dist/src/cli/options.js +35 -0
- package/dist/src/cli/options.js.map +1 -0
- package/dist/src/cli/output.d.ts +10 -0
- package/dist/src/cli/output.d.ts.map +1 -0
- package/dist/src/cli/output.js +29 -0
- package/dist/src/cli/output.js.map +1 -0
- package/dist/src/export/exporter.d.ts +16 -0
- package/dist/src/export/exporter.d.ts.map +1 -0
- package/dist/src/export/exporter.js +66 -0
- package/dist/src/export/exporter.js.map +1 -0
- package/dist/src/export/gif.d.ts +3 -0
- package/dist/src/export/gif.d.ts.map +1 -0
- package/dist/src/export/gif.js +36 -0
- package/dist/src/export/gif.js.map +1 -0
- package/dist/src/export/presets.d.ts +4 -0
- package/dist/src/export/presets.d.ts.map +1 -0
- package/dist/src/export/presets.js +68 -0
- package/dist/src/export/presets.js.map +1 -0
- package/dist/src/export/smart-crop.d.ts +10 -0
- package/dist/src/export/smart-crop.d.ts.map +1 -0
- package/dist/src/export/smart-crop.js +50 -0
- package/dist/src/export/smart-crop.js.map +1 -0
- package/dist/src/index.d.ts +10 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +9 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/recording/chapters.d.ts +3 -0
- package/dist/src/recording/chapters.d.ts.map +1 -0
- package/dist/src/recording/chapters.js +37 -0
- package/dist/src/recording/chapters.js.map +1 -0
- package/dist/src/recording/event-log.d.ts +20 -0
- package/dist/src/recording/event-log.d.ts.map +1 -0
- package/dist/src/recording/event-log.js +37 -0
- package/dist/src/recording/event-log.js.map +1 -0
- package/dist/src/recording/metadata.d.ts +4 -0
- package/dist/src/recording/metadata.d.ts.map +1 -0
- package/dist/src/recording/metadata.js +8 -0
- package/dist/src/recording/metadata.js.map +1 -0
- package/dist/src/recording/types.d.ts +81 -0
- package/dist/src/recording/types.d.ts.map +1 -0
- package/dist/src/recording/types.js +2 -0
- package/dist/src/recording/types.js.map +1 -0
- package/dist/src/utils/config.d.ts +13 -0
- package/dist/src/utils/config.d.ts.map +1 -0
- package/dist/src/utils/config.js +37 -0
- package/dist/src/utils/config.js.map +1 -0
- package/dist/src/utils/errors.d.ts +20 -0
- package/dist/src/utils/errors.d.ts.map +1 -0
- package/dist/src/utils/errors.js +39 -0
- package/dist/src/utils/errors.js.map +1 -0
- package/dist/src/utils/logger.d.ts +9 -0
- package/dist/src/utils/logger.d.ts.map +1 -0
- package/dist/src/utils/logger.js +40 -0
- package/dist/src/utils/logger.js.map +1 -0
- package/dist/src/utils/paths.d.ts +9 -0
- package/dist/src/utils/paths.d.ts.map +1 -0
- package/dist/src/utils/paths.js +28 -0
- package/dist/src/utils/paths.js.map +1 -0
- package/dist/src/video/background.d.ts +39 -0
- package/dist/src/video/background.d.ts.map +1 -0
- package/dist/src/video/background.js +141 -0
- package/dist/src/video/background.js.map +1 -0
- package/dist/src/video/compose.d.ts +14 -0
- package/dist/src/video/compose.d.ts.map +1 -0
- package/dist/src/video/compose.js +149 -0
- package/dist/src/video/compose.js.map +1 -0
- package/dist/src/video/cursor.d.ts +10 -0
- package/dist/src/video/cursor.d.ts.map +1 -0
- package/dist/src/video/cursor.js +31 -0
- package/dist/src/video/cursor.js.map +1 -0
- package/dist/src/video/ffmpeg.d.ts +10 -0
- package/dist/src/video/ffmpeg.d.ts.map +1 -0
- package/dist/src/video/ffmpeg.js +71 -0
- package/dist/src/video/ffmpeg.js.map +1 -0
- package/dist/src/video/highlight.d.ts +3 -0
- package/dist/src/video/highlight.d.ts.map +1 -0
- package/dist/src/video/highlight.js +21 -0
- package/dist/src/video/highlight.js.map +1 -0
- package/dist/src/video/trim.d.ts +19 -0
- package/dist/src/video/trim.d.ts.map +1 -0
- package/dist/src/video/trim.js +47 -0
- package/dist/src/video/trim.js.map +1 -0
- package/dist/src/video/zoom.d.ts +11 -0
- package/dist/src/video/zoom.d.ts.map +1 -0
- package/dist/src/video/zoom.js +88 -0
- package/dist/src/video/zoom.js.map +1 -0
- package/package.json +63 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../bin/cli.ts"],"names":[],"mappings":""}
|
package/dist/bin/cli.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from 'commander';
|
|
3
|
+
import { recordCommand } from '../src/cli/commands/record.js';
|
|
4
|
+
import { exportCommand } from '../src/cli/commands/export.js';
|
|
5
|
+
import { initCommand } from '../src/cli/commands/init.js';
|
|
6
|
+
const program = new Command()
|
|
7
|
+
.name('screencli')
|
|
8
|
+
.description('AI-powered screen recording CLI tool')
|
|
9
|
+
.version('0.1.0');
|
|
10
|
+
program.addCommand(initCommand);
|
|
11
|
+
program.addCommand(recordCommand);
|
|
12
|
+
program.addCommand(exportCommand);
|
|
13
|
+
program.parse();
|
|
14
|
+
//# sourceMappingURL=cli.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../bin/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE1D,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE;KAC1B,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,sCAAsC,CAAC;KACnD,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;AAChC,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;AAClC,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;AAElC,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
import { EventLog } from '../recording/event-log.js';
|
|
3
|
+
import type { AgentStats } from '../recording/types.js';
|
|
4
|
+
export interface AgentLoopOptions {
|
|
5
|
+
apiKey: string;
|
|
6
|
+
model: string;
|
|
7
|
+
url: string;
|
|
8
|
+
prompt: string;
|
|
9
|
+
page: Page;
|
|
10
|
+
eventLog: EventLog;
|
|
11
|
+
recordingDir: string;
|
|
12
|
+
actionDelayMs: number;
|
|
13
|
+
maxSteps: number;
|
|
14
|
+
onAction?: (step: number, toolName: string, description: string) => void;
|
|
15
|
+
}
|
|
16
|
+
export interface AgentLoopResult {
|
|
17
|
+
summary: string;
|
|
18
|
+
stats: AgentStats;
|
|
19
|
+
}
|
|
20
|
+
export declare function runAgentLoop(options: AgentLoopOptions): Promise<AgentLoopResult>;
|
|
21
|
+
//# sourceMappingURL=loop.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loop.d.ts","sourceRoot":"","sources":["../../../src/agent/loop.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAIvC,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAGrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAExD,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,UAAU,CAAC;CACnB;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAsItF"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
import { tools } from './tools.js';
|
|
3
|
+
import { buildSystemPrompt } from './system-prompt.js';
|
|
4
|
+
import { ToolHandlers } from './tool-handlers.js';
|
|
5
|
+
import { logger } from '../utils/logger.js';
|
|
6
|
+
import { AgentError } from '../utils/errors.js';
|
|
7
|
+
export async function runAgentLoop(options) {
|
|
8
|
+
const client = new Anthropic({ apiKey: options.apiKey });
|
|
9
|
+
const handlers = new ToolHandlers(options.page, options.eventLog, options.recordingDir, options.actionDelayMs);
|
|
10
|
+
const systemPrompt = buildSystemPrompt(options.url, options.prompt);
|
|
11
|
+
const messages = [];
|
|
12
|
+
let totalActions = 0;
|
|
13
|
+
let inputTokens = 0;
|
|
14
|
+
let outputTokens = 0;
|
|
15
|
+
// Initial observation: navigate to URL
|
|
16
|
+
logger.info(`Navigating to ${options.url}`);
|
|
17
|
+
await options.page.goto(options.url, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
|
18
|
+
await options.page.waitForTimeout(1000);
|
|
19
|
+
options.eventLog.append({
|
|
20
|
+
type: 'navigate',
|
|
21
|
+
description: `Navigate to ${options.url}`,
|
|
22
|
+
viewport: options.page.viewportSize() ?? { width: 1920, height: 1080 },
|
|
23
|
+
value: options.url,
|
|
24
|
+
url: options.url,
|
|
25
|
+
});
|
|
26
|
+
// Take initial screenshot
|
|
27
|
+
const initialScreenshot = await options.page.screenshot({ type: 'png' });
|
|
28
|
+
messages.push({
|
|
29
|
+
role: 'user',
|
|
30
|
+
content: [
|
|
31
|
+
{
|
|
32
|
+
type: 'text',
|
|
33
|
+
text: 'I have navigated to the starting URL. Here is the current page. Begin the task.',
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
type: 'image',
|
|
37
|
+
source: {
|
|
38
|
+
type: 'base64',
|
|
39
|
+
media_type: 'image/png',
|
|
40
|
+
data: initialScreenshot.toString('base64'),
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
],
|
|
44
|
+
});
|
|
45
|
+
for (let step = 0; step < options.maxSteps; step++) {
|
|
46
|
+
logger.debug(`Agent step ${step + 1}/${options.maxSteps}`);
|
|
47
|
+
let response;
|
|
48
|
+
try {
|
|
49
|
+
response = await client.messages.create({
|
|
50
|
+
model: options.model,
|
|
51
|
+
max_tokens: 4096,
|
|
52
|
+
system: systemPrompt,
|
|
53
|
+
tools,
|
|
54
|
+
messages,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
throw new AgentError(`Claude API error: ${err}`);
|
|
59
|
+
}
|
|
60
|
+
inputTokens += response.usage.input_tokens;
|
|
61
|
+
outputTokens += response.usage.output_tokens;
|
|
62
|
+
// Add assistant response to conversation
|
|
63
|
+
messages.push({ role: 'assistant', content: response.content });
|
|
64
|
+
// Check if there are tool calls
|
|
65
|
+
const toolUses = response.content.filter((block) => block.type === 'tool_use');
|
|
66
|
+
if (toolUses.length === 0) {
|
|
67
|
+
// No tool calls — model is just talking. Check if stop reason indicates end.
|
|
68
|
+
if (response.stop_reason === 'end_turn') {
|
|
69
|
+
logger.info('Agent ended without calling done. Finishing.');
|
|
70
|
+
return {
|
|
71
|
+
summary: 'Agent completed without explicit done signal.',
|
|
72
|
+
stats: { total_actions: totalActions, input_tokens: inputTokens, output_tokens: outputTokens },
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
// Process each tool call
|
|
78
|
+
const toolResults = [];
|
|
79
|
+
for (const toolUse of toolUses) {
|
|
80
|
+
totalActions++;
|
|
81
|
+
const description = toolUse.input.description ??
|
|
82
|
+
toolUse.input.summary ??
|
|
83
|
+
toolUse.input.text ??
|
|
84
|
+
toolUse.name;
|
|
85
|
+
options.onAction?.(step + 1, toolUse.name, String(description));
|
|
86
|
+
logger.info(`[${step + 1}] ${toolUse.name}: ${description}`);
|
|
87
|
+
try {
|
|
88
|
+
const result = await handlers.handle(toolUse.name, toolUse.input);
|
|
89
|
+
toolResults.push({
|
|
90
|
+
type: 'tool_result',
|
|
91
|
+
tool_use_id: toolUse.id,
|
|
92
|
+
content: result.content,
|
|
93
|
+
});
|
|
94
|
+
if (result.isDone) {
|
|
95
|
+
return {
|
|
96
|
+
summary: result.summary ?? 'Task completed.',
|
|
97
|
+
stats: { total_actions: totalActions, input_tokens: inputTokens, output_tokens: outputTokens },
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
catch (err) {
|
|
102
|
+
logger.warn(`Tool ${toolUse.name} failed: ${err}`);
|
|
103
|
+
toolResults.push({
|
|
104
|
+
type: 'tool_result',
|
|
105
|
+
tool_use_id: toolUse.id,
|
|
106
|
+
content: [{ type: 'text', text: `Error: ${err instanceof Error ? err.message : String(err)}` }],
|
|
107
|
+
is_error: true,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
messages.push({ role: 'user', content: toolResults });
|
|
112
|
+
}
|
|
113
|
+
logger.warn('Agent reached max steps limit.');
|
|
114
|
+
return {
|
|
115
|
+
summary: 'Agent reached maximum steps without completing.',
|
|
116
|
+
stats: { total_actions: totalActions, input_tokens: inputTokens, output_tokens: outputTokens },
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=loop.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loop.js","sourceRoot":"","sources":["../../../src/agent/loop.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAE1C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAqBhD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAyB;IAC1D,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,IAAI,YAAY,CAC/B,OAAO,CAAC,IAAI,EACZ,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,YAAY,EACpB,OAAO,CAAC,aAAa,CACtB,CAAC;IAEF,MAAM,YAAY,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACpE,MAAM,QAAQ,GAA6B,EAAE,CAAC;IAC9C,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,uCAAuC;IACvC,MAAM,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAExC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QACtB,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,eAAe,OAAO,CAAC,GAAG,EAAE;QACzC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;QACtE,KAAK,EAAE,OAAO,CAAC,GAAG;QAClB,GAAG,EAAE,OAAO,CAAC,GAAG;KACjB,CAAC,CAAC;IAEH,0BAA0B;IAC1B,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzE,QAAQ,CAAC,IAAI,CAAC;QACZ,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,iFAAiF;aACxF;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE,WAAW;oBACvB,IAAI,EAAE,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC;iBAC3C;aACF;SACF;KACF,CAAC,CAAC;IAEH,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;QACnD,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE3D,IAAI,QAA2B,CAAC;QAChC,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACtC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,IAAI;gBAChB,MAAM,EAAE,YAAY;gBACpB,KAAK;gBACL,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,WAAW,IAAI,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC;QAC3C,YAAY,IAAI,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;QAE7C,yCAAyC;QACzC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QAEhE,gCAAgC;QAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CACtC,CAAC,KAAK,EAAqG,EAAE,CAC3G,KAAK,CAAC,IAAI,KAAK,UAAU,CAC5B,CAAC;QAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,6EAA6E;YAC7E,IAAI,QAAQ,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;gBAC5D,OAAO;oBACL,OAAO,EAAE,+CAA+C;oBACxD,KAAK,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE;iBAC/F,CAAC;YACJ,CAAC;YACD,SAAS;QACX,CAAC;QAED,yBAAyB;QACzB,MAAM,WAAW,GAAqC,EAAE,CAAC;QAEzD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,YAAY,EAAE,CAAC;YACf,MAAM,WAAW,GAAI,OAAO,CAAC,KAA6B,CAAC,WAAW;gBACnE,OAAO,CAAC,KAA6B,CAAC,OAAO;gBAC7C,OAAO,CAAC,KAA6B,CAAC,IAAI;gBAC3C,OAAO,CAAC,IAAI,CAAC;YACf,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,GAAG,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YAChE,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC,CAAC;YAE7D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,KAA4B,CAAC,CAAC;gBAEzF,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,aAAa;oBACnB,WAAW,EAAE,OAAO,CAAC,EAAE;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAc;iBAC/B,CAAC,CAAC;gBAEH,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;oBAClB,OAAO;wBACL,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,iBAAiB;wBAC5C,KAAK,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE;qBAC/F,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,QAAQ,OAAO,CAAC,IAAI,YAAY,GAAG,EAAE,CAAC,CAAC;gBACnD,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,aAAa;oBACnB,WAAW,EAAE,OAAO,CAAC,EAAE;oBACvB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAQ;oBACtG,QAAQ,EAAE,IAAI;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IAC9C,OAAO;QACL,OAAO,EAAE,iDAAiD;QAC1D,KAAK,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE;KAC/F,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"system-prompt.d.ts","sourceRoot":"","sources":["../../../src/agent/system-prompt.ts"],"names":[],"mappings":"AAAA,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAyCrE"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export function buildSystemPrompt(url, prompt) {
|
|
2
|
+
return `You are a browser automation agent creating a screen recording demo. You control a browser via tool calls while the screen is being recorded.
|
|
3
|
+
|
|
4
|
+
## Your Task
|
|
5
|
+
Starting URL: ${url}
|
|
6
|
+
Instructions: ${prompt}
|
|
7
|
+
|
|
8
|
+
## Guidelines
|
|
9
|
+
|
|
10
|
+
### Observation
|
|
11
|
+
- ALWAYS start by taking a screenshot and getting the accessibility tree to understand the page.
|
|
12
|
+
- After every action, a screenshot is automatically returned — no need to call screenshot again.
|
|
13
|
+
|
|
14
|
+
### Targeting Elements
|
|
15
|
+
- PREFER role + name targeting (from the accessibility tree) — it is the most reliable.
|
|
16
|
+
- Use text matching as a fallback.
|
|
17
|
+
- Use CSS selectors only as a last resort.
|
|
18
|
+
|
|
19
|
+
### Actions
|
|
20
|
+
- You MUST provide a "description" for every action — it is used for narration and the event log.
|
|
21
|
+
- Work at a deliberate pace. Each action should be visible in the recording.
|
|
22
|
+
- After visual changes (page load, modal open, etc.), let the page settle so viewers can see the result.
|
|
23
|
+
- Use the "wait" tool if you need to wait for content to load.
|
|
24
|
+
|
|
25
|
+
### Narration
|
|
26
|
+
- Use "narrate" before major sections to add viewer-facing captions.
|
|
27
|
+
- Example: narrate("Step 1: Navigate to the settings page")
|
|
28
|
+
- Keep narrations concise and descriptive.
|
|
29
|
+
|
|
30
|
+
### Error Handling
|
|
31
|
+
- If an action fails, try an alternative approach (different targeting, scroll to element, wait for it).
|
|
32
|
+
- Max 3 retries on any single action before moving on or reporting failure.
|
|
33
|
+
|
|
34
|
+
### Completion
|
|
35
|
+
- When you have completed the task, call "done" with a summary of what was accomplished.
|
|
36
|
+
- If you cannot complete the task, call "done" explaining what went wrong.
|
|
37
|
+
|
|
38
|
+
## Important
|
|
39
|
+
- Do NOT call screenshot after action tools — they already return one.
|
|
40
|
+
- Aim for smooth, demonstrable interactions that will look good in a recording.
|
|
41
|
+
- Prefer clicking visible buttons/links over keyboard shortcuts when possible.`;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=system-prompt.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"system-prompt.js","sourceRoot":"","sources":["../../../src/agent/system-prompt.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,MAAc;IAC3D,OAAO;;;gBAGO,GAAG;gBACH,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+EAmCyD,CAAC;AAChF,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
import { EventLog } from '../recording/event-log.js';
|
|
3
|
+
export interface ToolResult {
|
|
4
|
+
content: Array<{
|
|
5
|
+
type: 'text';
|
|
6
|
+
text: string;
|
|
7
|
+
} | {
|
|
8
|
+
type: 'image';
|
|
9
|
+
source: {
|
|
10
|
+
type: 'base64';
|
|
11
|
+
media_type: 'image/png';
|
|
12
|
+
data: string;
|
|
13
|
+
};
|
|
14
|
+
}>;
|
|
15
|
+
isDone?: boolean;
|
|
16
|
+
summary?: string;
|
|
17
|
+
}
|
|
18
|
+
export declare class ToolHandlers {
|
|
19
|
+
private page;
|
|
20
|
+
private eventLog;
|
|
21
|
+
private recordingDir;
|
|
22
|
+
private actionDelayMs;
|
|
23
|
+
private actionCount;
|
|
24
|
+
constructor(page: Page, eventLog: EventLog, recordingDir: string, actionDelayMs: number);
|
|
25
|
+
handle(toolName: string, input: Record<string, any>): Promise<ToolResult>;
|
|
26
|
+
private viewport;
|
|
27
|
+
private saveScreenshot;
|
|
28
|
+
private screenshotContent;
|
|
29
|
+
private extractTarget;
|
|
30
|
+
private handleScreenshot;
|
|
31
|
+
private handleAccessibilityTree;
|
|
32
|
+
private handlePageInfo;
|
|
33
|
+
private handleClick;
|
|
34
|
+
private handleType;
|
|
35
|
+
private handlePressKey;
|
|
36
|
+
private handleScroll;
|
|
37
|
+
private handleHover;
|
|
38
|
+
private handleNavigate;
|
|
39
|
+
private handleWait;
|
|
40
|
+
private handleSelectOption;
|
|
41
|
+
private handleDone;
|
|
42
|
+
private handleNarrate;
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=tool-handlers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-handlers.d.ts","sourceRoot":"","sources":["../../../src/agent/tool-handlers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAOrD,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE;YAAE,IAAI,EAAE,QAAQ,CAAC;YAAC,UAAU,EAAE,WAAW,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;IACtI,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,YAAY;IAIrB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,aAAa;IANvB,OAAO,CAAC,WAAW,CAAK;gBAGd,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,QAAQ,EAClB,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM;IAGzB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAiC/E,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,aAAa;YASP,gBAAgB;YAKhB,uBAAuB;YAOvB,cAAc;YAOd,WAAW;YAaX,UAAU;YAoCV,cAAc;YAYd,YAAY;YAoBZ,WAAW;YAaX,cAAc;YAYd,UAAU;YAgBV,kBAAkB;YAmBlB,UAAU;YAeV,aAAa;CAY5B"}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import * as actions from '../browser/actions.js';
|
|
2
|
+
import { getAccessibilityTree, getPageInfo } from '../browser/accessibility.js';
|
|
3
|
+
import { screenshotsDir } from '../utils/paths.js';
|
|
4
|
+
import { writeFileSync } from 'node:fs';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
export class ToolHandlers {
|
|
7
|
+
page;
|
|
8
|
+
eventLog;
|
|
9
|
+
recordingDir;
|
|
10
|
+
actionDelayMs;
|
|
11
|
+
actionCount = 0;
|
|
12
|
+
constructor(page, eventLog, recordingDir, actionDelayMs) {
|
|
13
|
+
this.page = page;
|
|
14
|
+
this.eventLog = eventLog;
|
|
15
|
+
this.recordingDir = recordingDir;
|
|
16
|
+
this.actionDelayMs = actionDelayMs;
|
|
17
|
+
}
|
|
18
|
+
async handle(toolName, input) {
|
|
19
|
+
switch (toolName) {
|
|
20
|
+
case 'screenshot':
|
|
21
|
+
return this.handleScreenshot();
|
|
22
|
+
case 'get_accessibility_tree':
|
|
23
|
+
return this.handleAccessibilityTree();
|
|
24
|
+
case 'get_page_info':
|
|
25
|
+
return this.handlePageInfo();
|
|
26
|
+
case 'click':
|
|
27
|
+
return this.handleClick(input);
|
|
28
|
+
case 'type':
|
|
29
|
+
return this.handleType(input);
|
|
30
|
+
case 'press_key':
|
|
31
|
+
return this.handlePressKey(input);
|
|
32
|
+
case 'scroll':
|
|
33
|
+
return this.handleScroll(input);
|
|
34
|
+
case 'hover':
|
|
35
|
+
return this.handleHover(input);
|
|
36
|
+
case 'navigate':
|
|
37
|
+
return this.handleNavigate(input);
|
|
38
|
+
case 'wait':
|
|
39
|
+
return this.handleWait(input);
|
|
40
|
+
case 'select_option':
|
|
41
|
+
return this.handleSelectOption(input);
|
|
42
|
+
case 'done':
|
|
43
|
+
return this.handleDone(input);
|
|
44
|
+
case 'narrate':
|
|
45
|
+
return this.handleNarrate(input);
|
|
46
|
+
default:
|
|
47
|
+
return { content: [{ type: 'text', text: `Unknown tool: ${toolName}` }] };
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
viewport() {
|
|
51
|
+
return this.page.viewportSize() ?? { width: 1920, height: 1080 };
|
|
52
|
+
}
|
|
53
|
+
saveScreenshot(buffer) {
|
|
54
|
+
this.actionCount++;
|
|
55
|
+
const filename = `step-${String(this.actionCount).padStart(3, '0')}.png`;
|
|
56
|
+
const dir = screenshotsDir(this.recordingDir);
|
|
57
|
+
const path = join(dir, filename);
|
|
58
|
+
writeFileSync(path, buffer);
|
|
59
|
+
return path;
|
|
60
|
+
}
|
|
61
|
+
screenshotContent(buffer) {
|
|
62
|
+
this.saveScreenshot(buffer);
|
|
63
|
+
return [
|
|
64
|
+
{
|
|
65
|
+
type: 'image',
|
|
66
|
+
source: {
|
|
67
|
+
type: 'base64',
|
|
68
|
+
media_type: 'image/png',
|
|
69
|
+
data: buffer.toString('base64'),
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
];
|
|
73
|
+
}
|
|
74
|
+
extractTarget(input) {
|
|
75
|
+
return {
|
|
76
|
+
role: input.role,
|
|
77
|
+
name: input.name,
|
|
78
|
+
text: input.target_text ?? (input.role || input.selector ? input.text : undefined),
|
|
79
|
+
selector: input.selector,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
async handleScreenshot() {
|
|
83
|
+
const screenshot = await this.page.screenshot({ type: 'png' });
|
|
84
|
+
return { content: this.screenshotContent(screenshot) };
|
|
85
|
+
}
|
|
86
|
+
async handleAccessibilityTree() {
|
|
87
|
+
const { tree, elementCount } = await getAccessibilityTree(this.page);
|
|
88
|
+
return {
|
|
89
|
+
content: [{ type: 'text', text: `Accessibility tree (${elementCount} elements):\n${tree}` }],
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
async handlePageInfo() {
|
|
93
|
+
const info = await getPageInfo(this.page);
|
|
94
|
+
return {
|
|
95
|
+
content: [{ type: 'text', text: JSON.stringify(info, null, 2) }],
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
async handleClick(input) {
|
|
99
|
+
const target = this.extractTarget(input);
|
|
100
|
+
const result = await actions.click(this.page, target, input.click_type ?? 'left', this.actionDelayMs);
|
|
101
|
+
this.eventLog.append({
|
|
102
|
+
type: 'click',
|
|
103
|
+
description: input.description,
|
|
104
|
+
bounding_box: result.bounding_box,
|
|
105
|
+
viewport: this.viewport(),
|
|
106
|
+
url: result.url,
|
|
107
|
+
});
|
|
108
|
+
return { content: this.screenshotContent(result.screenshot) };
|
|
109
|
+
}
|
|
110
|
+
async handleType(input) {
|
|
111
|
+
const target = this.extractTarget(input);
|
|
112
|
+
// For type tool, if text is the value to type, use target_text for element matching
|
|
113
|
+
if (!target.role && !target.selector && !target.text) {
|
|
114
|
+
// No explicit target — focus is likely already on the element
|
|
115
|
+
// Type directly into focused element
|
|
116
|
+
await this.page.keyboard.type(input.text, { delay: input.character_by_character ? 50 : undefined });
|
|
117
|
+
await this.page.waitForTimeout(this.actionDelayMs);
|
|
118
|
+
const screenshot = await this.page.screenshot({ type: 'png' });
|
|
119
|
+
this.eventLog.append({
|
|
120
|
+
type: 'type',
|
|
121
|
+
description: input.description,
|
|
122
|
+
viewport: this.viewport(),
|
|
123
|
+
value: input.text,
|
|
124
|
+
url: this.page.url(),
|
|
125
|
+
});
|
|
126
|
+
return { content: this.screenshotContent(screenshot) };
|
|
127
|
+
}
|
|
128
|
+
const result = await actions.type(this.page, target, input.text, { clearFirst: input.clear_first, characterByCharacter: input.character_by_character }, this.actionDelayMs);
|
|
129
|
+
this.eventLog.append({
|
|
130
|
+
type: 'type',
|
|
131
|
+
description: input.description,
|
|
132
|
+
bounding_box: result.bounding_box,
|
|
133
|
+
viewport: this.viewport(),
|
|
134
|
+
value: input.text,
|
|
135
|
+
url: result.url,
|
|
136
|
+
});
|
|
137
|
+
return { content: this.screenshotContent(result.screenshot) };
|
|
138
|
+
}
|
|
139
|
+
async handlePressKey(input) {
|
|
140
|
+
const result = await actions.pressKey(this.page, input.key, this.actionDelayMs);
|
|
141
|
+
this.eventLog.append({
|
|
142
|
+
type: 'press_key',
|
|
143
|
+
description: input.description,
|
|
144
|
+
viewport: this.viewport(),
|
|
145
|
+
value: input.key,
|
|
146
|
+
url: result.url,
|
|
147
|
+
});
|
|
148
|
+
return { content: this.screenshotContent(result.screenshot) };
|
|
149
|
+
}
|
|
150
|
+
async handleScroll(input) {
|
|
151
|
+
const toElement = input.to_role || input.to_name || input.to_text || input.to_selector
|
|
152
|
+
? { role: input.to_role, name: input.to_name, text: input.to_text, selector: input.to_selector }
|
|
153
|
+
: undefined;
|
|
154
|
+
const result = await actions.scroll(this.page, { direction: input.direction, amount: input.amount, toElement }, this.actionDelayMs);
|
|
155
|
+
this.eventLog.append({
|
|
156
|
+
type: 'scroll',
|
|
157
|
+
description: input.description,
|
|
158
|
+
bounding_box: result.bounding_box,
|
|
159
|
+
viewport: this.viewport(),
|
|
160
|
+
url: result.url,
|
|
161
|
+
});
|
|
162
|
+
return { content: this.screenshotContent(result.screenshot) };
|
|
163
|
+
}
|
|
164
|
+
async handleHover(input) {
|
|
165
|
+
const target = this.extractTarget(input);
|
|
166
|
+
const result = await actions.hover(this.page, target, this.actionDelayMs);
|
|
167
|
+
this.eventLog.append({
|
|
168
|
+
type: 'hover',
|
|
169
|
+
description: input.description,
|
|
170
|
+
bounding_box: result.bounding_box,
|
|
171
|
+
viewport: this.viewport(),
|
|
172
|
+
url: result.url,
|
|
173
|
+
});
|
|
174
|
+
return { content: this.screenshotContent(result.screenshot) };
|
|
175
|
+
}
|
|
176
|
+
async handleNavigate(input) {
|
|
177
|
+
const result = await actions.navigate(this.page, input.url, this.actionDelayMs);
|
|
178
|
+
this.eventLog.append({
|
|
179
|
+
type: 'navigate',
|
|
180
|
+
description: input.description,
|
|
181
|
+
viewport: this.viewport(),
|
|
182
|
+
value: input.url,
|
|
183
|
+
url: result.url,
|
|
184
|
+
});
|
|
185
|
+
return { content: this.screenshotContent(result.screenshot) };
|
|
186
|
+
}
|
|
187
|
+
async handleWait(input) {
|
|
188
|
+
const result = await actions.waitFor(this.page, {
|
|
189
|
+
time: input.time,
|
|
190
|
+
elementVisible: input.element_visible,
|
|
191
|
+
elementHidden: input.element_hidden,
|
|
192
|
+
networkIdle: input.network_idle,
|
|
193
|
+
});
|
|
194
|
+
this.eventLog.append({
|
|
195
|
+
type: 'wait',
|
|
196
|
+
description: input.description,
|
|
197
|
+
viewport: this.viewport(),
|
|
198
|
+
url: result.url,
|
|
199
|
+
});
|
|
200
|
+
return { content: this.screenshotContent(result.screenshot) };
|
|
201
|
+
}
|
|
202
|
+
async handleSelectOption(input) {
|
|
203
|
+
const target = this.extractTarget(input);
|
|
204
|
+
const result = await actions.selectOption(this.page, target, { label: input.option_label, value: input.option_value }, this.actionDelayMs);
|
|
205
|
+
this.eventLog.append({
|
|
206
|
+
type: 'select_option',
|
|
207
|
+
description: input.description,
|
|
208
|
+
bounding_box: result.bounding_box,
|
|
209
|
+
viewport: this.viewport(),
|
|
210
|
+
value: input.option_label ?? input.option_value,
|
|
211
|
+
url: result.url,
|
|
212
|
+
});
|
|
213
|
+
return { content: this.screenshotContent(result.screenshot) };
|
|
214
|
+
}
|
|
215
|
+
async handleDone(input) {
|
|
216
|
+
this.eventLog.append({
|
|
217
|
+
type: 'done',
|
|
218
|
+
description: input.summary,
|
|
219
|
+
viewport: this.viewport(),
|
|
220
|
+
url: this.page.url(),
|
|
221
|
+
});
|
|
222
|
+
this.eventLog.flush();
|
|
223
|
+
return {
|
|
224
|
+
content: [{ type: 'text', text: `Recording complete: ${input.summary}` }],
|
|
225
|
+
isDone: true,
|
|
226
|
+
summary: input.summary,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
async handleNarrate(input) {
|
|
230
|
+
this.eventLog.append({
|
|
231
|
+
type: 'narrate',
|
|
232
|
+
description: input.text,
|
|
233
|
+
viewport: this.viewport(),
|
|
234
|
+
value: input.text,
|
|
235
|
+
url: this.page.url(),
|
|
236
|
+
});
|
|
237
|
+
return {
|
|
238
|
+
content: [{ type: 'text', text: `Narration added: "${input.text}"` }],
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
//# sourceMappingURL=tool-handlers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-handlers.js","sourceRoot":"","sources":["../../../src/agent/tool-handlers.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAEhF,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAUjC,MAAM,OAAO,YAAY;IAIb;IACA;IACA;IACA;IANF,WAAW,GAAG,CAAC,CAAC;IAExB,YACU,IAAU,EACV,QAAkB,EAClB,YAAoB,EACpB,aAAqB;QAHrB,SAAI,GAAJ,IAAI,CAAM;QACV,aAAQ,GAAR,QAAQ,CAAU;QAClB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,kBAAa,GAAb,aAAa,CAAQ;IAC5B,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,KAA0B;QACvD,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,YAAY;gBACf,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,wBAAwB;gBAC3B,OAAO,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACxC,KAAK,eAAe;gBAClB,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/B,KAAK,OAAO;gBACV,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACjC,KAAK,MAAM;gBACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAChC,KAAK,WAAW;gBACd,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YACpC,KAAK,QAAQ;gBACX,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YAClC,KAAK,OAAO;gBACV,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACjC,KAAK,UAAU;gBACb,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YACpC,KAAK,MAAM;gBACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAChC,KAAK,eAAe;gBAClB,OAAO,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YACxC,KAAK,MAAM;gBACT,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAChC,KAAK,SAAS;gBACZ,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACnC;gBACE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;QAC9E,CAAC;IACH,CAAC;IAEO,QAAQ;QACd,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACnE,CAAC;IAEO,cAAc,CAAC,MAAc;QACnC,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,MAAM,QAAQ,GAAG,QAAQ,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC;QACzE,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACjC,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iBAAiB,CAAC,MAAc;QACtC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC5B,OAAO;YACL;gBACE,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE,WAAW;oBACvB,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;iBAChC;aACF;SACF,CAAC;IACJ,CAAC;IAEO,aAAa,CAAC,KAA0B;QAC9C,OAAO;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YAClF,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;IACzD,CAAC;IAEO,KAAK,CAAC,uBAAuB;QACnC,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrE,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uBAAuB,YAAY,gBAAgB,IAAI,EAAE,EAAE,CAAC;SAC7F,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;SACjE,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAA0B;QAClD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,UAAU,IAAI,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QACtG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAA0B;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACzC,oFAAoF;QACpF,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACrD,8DAA8D;YAC9D,qCAAqC;YACrC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;YACpG,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACnD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACnB,IAAI,EAAE,MAAM;gBACZ,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;gBACzB,KAAK,EAAE,KAAK,CAAC,IAAI;gBACjB,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;aACrB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;QACzD,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAC/B,IAAI,CAAC,IAAI,EACT,MAAM,EACN,KAAK,CAAC,IAAI,EACV,EAAE,UAAU,EAAE,KAAK,CAAC,WAAW,EAAE,oBAAoB,EAAE,KAAK,CAAC,sBAAsB,EAAE,EACrF,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,MAAM;YACZ,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAA0B;QACrD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAChF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,KAAK,EAAE,KAAK,CAAC,GAAG;YAChB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,KAA0B;QACnD,MAAM,SAAS,GACb,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,WAAW;YAClE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,WAAW,EAAE;YAChG,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CACjC,IAAI,CAAC,IAAI,EACT,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAC/D,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAA0B;QAClD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAA0B;QACrD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAChF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,UAAU;YAChB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,KAAK,EAAE,KAAK,CAAC,GAAG;YAChB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAA0B;QACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE;YAC9C,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,cAAc,EAAE,KAAK,CAAC,eAAe;YACrC,aAAa,EAAE,KAAK,CAAC,cAAc;YACnC,WAAW,EAAE,KAAK,CAAC,YAAY;SAChC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,MAAM;YACZ,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,KAA0B;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,YAAY,CACvC,IAAI,CAAC,IAAI,EACT,MAAM,EACN,EAAE,KAAK,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,YAAY,EAAE,EACxD,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,eAAe;YACrB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,KAAK,EAAE,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY;YAC/C,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAA0B;QACjD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,MAAM;YACZ,WAAW,EAAE,KAAK,CAAC,OAAO;YAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;SACrB,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACzE,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,KAA0B;QACpD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnB,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,KAAK,CAAC,IAAI;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE;YACzB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;SACrB,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,KAAK,CAAC,IAAI,GAAG,EAAE,CAAC;SACtE,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../../src/agent/tools.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,mBAAmB,CAAC;AAE/C,KAAK,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC;AAS9B,eAAO,MAAM,KAAK,EAAE,OAAO,EAqP1B,CAAC"}
|