testronaut 2.1.3 → 2.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/agent.js +7 -0
- package/core/turnLoop.js +21 -1
- package/package.json +1 -1
- package/tests/coreTests/turnLoop.test.js +50 -0
- package/tests/toolsTests/chromeBrowserDialogs.test.js +102 -0
- package/tests/toolsTests/generateHtmlReport.test.js +30 -0
- package/tests/toolsTests/toolSchema.test.js +8 -0
- package/tools/chromeBrowser.js +120 -0
- package/tools/generateHtmlReport.js +53 -0
- package/tools/toolSchema.js +20 -0
package/core/agent.js
CHANGED
|
@@ -221,6 +221,13 @@ Email invitation and magic links:
|
|
|
221
221
|
- open_email_link enforces the operator-configured emailLinks.allowedHosts policy. If the host is not configured or navigation is blocked, report a graceful FAILURE; do not use the ordinary navigate tool as a workaround.
|
|
222
222
|
`.trim();
|
|
223
223
|
|
|
224
|
+
systemContent += '\n\n' + `
|
|
225
|
+
JavaScript dialogs:
|
|
226
|
+
- Browser-native alert, confirm, prompt, and beforeunload dialogs block their triggering action.
|
|
227
|
+
- When a mission requires a particular response, call prepare_dialog_response immediately before the click, reload, or navigation that opens it. Specify the expected type, accept/dismiss decision, and an exact or partial message when known; provide input only when accepting a prompt.
|
|
228
|
+
- Prepared responses are one-time and expire. Unexpected dialogs are dismissed fail-closed and reported to you and the mission report. Never assume an unexpected dialog was accepted.
|
|
229
|
+
`.trim();
|
|
230
|
+
|
|
224
231
|
const permittedRecoveries = Array.isArray(opts.recovery?.permitted)
|
|
225
232
|
? opts.recovery.permitted.map(value => String(value).trim().toLowerCase()).filter(Boolean)
|
|
226
233
|
: null;
|
package/core/turnLoop.js
CHANGED
|
@@ -653,6 +653,24 @@ export const turnLoop = async (
|
|
|
653
653
|
hadToolIssues = true;
|
|
654
654
|
}
|
|
655
655
|
|
|
656
|
+
const dialogEvents = browser?.drainDialogEvents?.() || [];
|
|
657
|
+
if (dialogEvents.length) {
|
|
658
|
+
step.dialogs = step.dialogs || [];
|
|
659
|
+
step.dialogs.push(...dialogEvents);
|
|
660
|
+
for (const dialogEvent of dialogEvents) {
|
|
661
|
+
if (dialogEvent.event === 'dialog_rule_expired') {
|
|
662
|
+
step.events.push(`⏱️ Dialog response ${dialogEvent.ruleId} expired before a matching ${dialogEvent.expectedType} appeared.`);
|
|
663
|
+
} else {
|
|
664
|
+
const expectation = dialogEvent.matched ? 'expected' : 'unexpected';
|
|
665
|
+
step.events.push(`💬 ${expectation} ${dialogEvent.type} dialog: ${JSON.stringify(dialogEvent.message)} → ${dialogEvent.outcome}`);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
result = JSON.stringify({
|
|
669
|
+
result: (() => { try { return JSON.parse(result); } catch { return result; } })(),
|
|
670
|
+
dialogs: dialogEvents,
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
|
|
656
674
|
if (fnName === 'request_human_input') {
|
|
657
675
|
step.humanInput = step.humanInput || {};
|
|
658
676
|
step.humanInput.requested = true;
|
|
@@ -936,7 +954,9 @@ export const turnLoop = async (
|
|
|
936
954
|
// bloating context with large payloads (file JSON, etc.).
|
|
937
955
|
// - For semantic tools (get_dom, check_text, etc.), send full result.
|
|
938
956
|
let contentForModel;
|
|
939
|
-
if (
|
|
957
|
+
if (dialogEvents.length) {
|
|
958
|
+
contentForModel = result;
|
|
959
|
+
} else if (FIRE_AND_FORGET_TOOLS.has(fnName)) {
|
|
940
960
|
contentForModel = errorMessage || 'OK';
|
|
941
961
|
} else {
|
|
942
962
|
contentForModel = result;
|
package/package.json
CHANGED
|
@@ -182,6 +182,56 @@ describe('turnLoop', () => {
|
|
|
182
182
|
expect(shared.chromeToolSpies.get_dom).not.toHaveBeenCalled();
|
|
183
183
|
});
|
|
184
184
|
|
|
185
|
+
it('reports browser-native dialog appearance and response to the model and mission step', async () => {
|
|
186
|
+
const dialog = {
|
|
187
|
+
event: 'javascript_dialog',
|
|
188
|
+
type: 'confirm',
|
|
189
|
+
message: 'Emergency meeting! Attend?',
|
|
190
|
+
defaultValue: '',
|
|
191
|
+
pageUrl: 'https://example.test/dashboard',
|
|
192
|
+
matched: true,
|
|
193
|
+
expected: true,
|
|
194
|
+
action: 'accept',
|
|
195
|
+
outcome: 'accept',
|
|
196
|
+
};
|
|
197
|
+
browser.drainDialogEvents = vi.fn()
|
|
198
|
+
.mockReturnValueOnce([dialog])
|
|
199
|
+
.mockReturnValue([]);
|
|
200
|
+
shared.chatMock
|
|
201
|
+
.mockResolvedValueOnce({
|
|
202
|
+
message: {
|
|
203
|
+
role: 'assistant',
|
|
204
|
+
content: '',
|
|
205
|
+
tool_calls: [{
|
|
206
|
+
id: 'dialog_click',
|
|
207
|
+
type: 'function',
|
|
208
|
+
function: { name: 'click_text', arguments: '{"text":"Request attendance"}' },
|
|
209
|
+
}],
|
|
210
|
+
},
|
|
211
|
+
usage: { total_tokens: 10 },
|
|
212
|
+
headers: {},
|
|
213
|
+
})
|
|
214
|
+
.mockResolvedValueOnce({
|
|
215
|
+
message: { role: 'assistant', content: 'FINAL: success' },
|
|
216
|
+
usage: { total_tokens: 5 },
|
|
217
|
+
headers: {},
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
const result = await turnLoop(browser, baseMessages(), 2, 0, 0, {}, {
|
|
221
|
+
steps: [],
|
|
222
|
+
missionName: 'dialogs',
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
expect(result.success).toBe(true);
|
|
226
|
+
expect(result.steps[0].dialogs).toEqual([dialog]);
|
|
227
|
+
expect(result.steps[0].events).toContain(
|
|
228
|
+
'💬 expected confirm dialog: "Emergency meeting! Attend?" → accept',
|
|
229
|
+
);
|
|
230
|
+
const secondRequest = shared.chatMock.mock.calls[1][0].messages;
|
|
231
|
+
const response = secondRequest.find(message => message.tool_call_id === 'dialog_click');
|
|
232
|
+
expect(JSON.parse(response.content).dialogs).toEqual([dialog]);
|
|
233
|
+
});
|
|
234
|
+
|
|
185
235
|
it('uses one tool-free closure turn after the working-turn budget is exhausted', async () => {
|
|
186
236
|
shared.chatMock
|
|
187
237
|
.mockResolvedValueOnce({
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
3
|
+
import { CHROME_TOOL_MAP, ChromeBrowser } from '../../tools/chromeBrowser.js';
|
|
4
|
+
|
|
5
|
+
class FakePage extends EventEmitter {
|
|
6
|
+
url() { return 'https://example.test/dialogs'; }
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function fakeDialog(type, message, defaultValue = '') {
|
|
10
|
+
return {
|
|
11
|
+
type: () => type,
|
|
12
|
+
message: () => message,
|
|
13
|
+
defaultValue: () => defaultValue,
|
|
14
|
+
accept: vi.fn().mockResolvedValue(undefined),
|
|
15
|
+
dismiss: vi.fn().mockResolvedValue(undefined),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const flush = () => new Promise(resolve => setTimeout(resolve, 0));
|
|
20
|
+
|
|
21
|
+
describe('ChromeBrowser JavaScript dialogs', () => {
|
|
22
|
+
it.each([
|
|
23
|
+
['alert', 'accept', undefined],
|
|
24
|
+
['confirm', 'dismiss', undefined],
|
|
25
|
+
['prompt', 'accept', 'red crewmate'],
|
|
26
|
+
['beforeunload', 'accept', undefined],
|
|
27
|
+
])('handles an expected %s without blocking', async (type, action, input) => {
|
|
28
|
+
const browser = new ChromeBrowser();
|
|
29
|
+
const page = new FakePage();
|
|
30
|
+
browser._addAndFocus(page);
|
|
31
|
+
browser.prepare_dialog_response({
|
|
32
|
+
type,
|
|
33
|
+
action,
|
|
34
|
+
input,
|
|
35
|
+
messageIncludes: 'Emergency',
|
|
36
|
+
});
|
|
37
|
+
const dialog = fakeDialog(type, 'Emergency meeting!', input ? 'anonymous' : '');
|
|
38
|
+
|
|
39
|
+
page.emit('dialog', dialog);
|
|
40
|
+
await flush();
|
|
41
|
+
|
|
42
|
+
expect(dialog[action]).toHaveBeenCalledOnce();
|
|
43
|
+
if (input !== undefined) expect(dialog.accept).toHaveBeenCalledWith(input);
|
|
44
|
+
const [event] = browser.drainDialogEvents();
|
|
45
|
+
expect(event).toMatchObject({
|
|
46
|
+
event: 'javascript_dialog',
|
|
47
|
+
type,
|
|
48
|
+
message: 'Emergency meeting!',
|
|
49
|
+
action,
|
|
50
|
+
matched: true,
|
|
51
|
+
outcome: action,
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('dismisses and reports an unexpected dialog while retaining a nonmatching rule', async () => {
|
|
56
|
+
const browser = new ChromeBrowser();
|
|
57
|
+
const page = new FakePage();
|
|
58
|
+
browser._addAndFocus(page);
|
|
59
|
+
browser.prepare_dialog_response({ type: 'confirm', action: 'accept', message: 'Expected' });
|
|
60
|
+
const dialog = fakeDialog('confirm', 'Suspicious impostor dialog');
|
|
61
|
+
|
|
62
|
+
page.emit('dialog', dialog);
|
|
63
|
+
await flush();
|
|
64
|
+
|
|
65
|
+
expect(dialog.dismiss).toHaveBeenCalledOnce();
|
|
66
|
+
expect(dialog.accept).not.toHaveBeenCalled();
|
|
67
|
+
expect(browser.dialogRules).toHaveLength(1);
|
|
68
|
+
expect(browser.drainDialogEvents()[0]).toMatchObject({
|
|
69
|
+
matched: false,
|
|
70
|
+
action: 'dismiss',
|
|
71
|
+
outcome: 'unexpected_dialog_dismissed',
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it('reports an armed response that times out', () => {
|
|
76
|
+
vi.useFakeTimers();
|
|
77
|
+
try {
|
|
78
|
+
const browser = new ChromeBrowser();
|
|
79
|
+
browser.prepare_dialog_response({ type: 'alert', action: 'accept', timeoutMs: 250 });
|
|
80
|
+
vi.advanceTimersByTime(251);
|
|
81
|
+
expect(browser.drainDialogEvents()[0]).toMatchObject({
|
|
82
|
+
event: 'dialog_rule_expired',
|
|
83
|
+
expectedType: 'alert',
|
|
84
|
+
outcome: 'timeout',
|
|
85
|
+
});
|
|
86
|
+
} finally {
|
|
87
|
+
vi.useRealTimers();
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('rejects unscoped prompt input and exposes the dispatcher', async () => {
|
|
92
|
+
const browser = new ChromeBrowser();
|
|
93
|
+
expect(() => browser.prepare_dialog_response({ type: 'confirm', action: 'accept', input: 'secret' }))
|
|
94
|
+
.toThrow(/only supported.*prompt/i);
|
|
95
|
+
|
|
96
|
+
const prepare_dialog_response = vi.fn().mockReturnValue({ armed: true });
|
|
97
|
+
expect(CHROME_TOOL_MAP.prepare_dialog_response(
|
|
98
|
+
{ prepare_dialog_response },
|
|
99
|
+
{ type: 'alert', action: 'accept' },
|
|
100
|
+
)).toEqual({ armed: true });
|
|
101
|
+
});
|
|
102
|
+
});
|
|
@@ -165,6 +165,36 @@ describe('generateHtmlReport', () => {
|
|
|
165
165
|
expect(html).not.toContain('<strong>LLM:</strong>');
|
|
166
166
|
});
|
|
167
167
|
|
|
168
|
+
it('renders native dialog evidence as a clearly labeled report visualization', () => {
|
|
169
|
+
const outPath = path.join(tmpDir, 'dialogs.html');
|
|
170
|
+
generateHtmlReport({ missions: [{
|
|
171
|
+
missionName: 'Dialogs', status: 'passed', steps: [{
|
|
172
|
+
turn: 0,
|
|
173
|
+
result: '✅ Passed',
|
|
174
|
+
events: ['dialog handled'],
|
|
175
|
+
dialogs: [{
|
|
176
|
+
event: 'javascript_dialog',
|
|
177
|
+
type: 'prompt',
|
|
178
|
+
message: 'Name the suspicious crewmate:',
|
|
179
|
+
defaultValue: 'Red',
|
|
180
|
+
inputProvided: true,
|
|
181
|
+
action: 'accept',
|
|
182
|
+
matched: true,
|
|
183
|
+
outcome: 'accept',
|
|
184
|
+
pageUrl: 'https://example.test/dashboard',
|
|
185
|
+
}],
|
|
186
|
+
}],
|
|
187
|
+
}] }, outPath);
|
|
188
|
+
|
|
189
|
+
const html = fs.readFileSync(outPath, 'utf8');
|
|
190
|
+
expect(html).toContain('Report visualization · not a literal browser screenshot');
|
|
191
|
+
expect(html).toContain('Browser-native prompt');
|
|
192
|
+
expect(html).toContain('Name the suspicious crewmate:');
|
|
193
|
+
expect(html).toContain('•••••••• · input provided');
|
|
194
|
+
expect(html).toContain('dialog-button chosen">Accept');
|
|
195
|
+
expect(html).not.toContain('>Red</div>');
|
|
196
|
+
});
|
|
197
|
+
|
|
168
198
|
it('renders run, mission, and step Jev token spend', () => {
|
|
169
199
|
const steps = [{
|
|
170
200
|
turn: 0,
|
|
@@ -27,6 +27,14 @@ describe('tools/toolSchema', () => {
|
|
|
27
27
|
expect(entry.function.parameters.properties.key.type).toBe('string');
|
|
28
28
|
expect(entry.function.parameters.properties.selector.type).toBe('string');
|
|
29
29
|
});
|
|
30
|
+
it('includes a scoped one-shot JavaScript dialog response tool', () => {
|
|
31
|
+
const entry = toolsSchema.find(tool => tool.function.name === 'prepare_dialog_response');
|
|
32
|
+
expect(entry?.function.parameters.required).toEqual(['type', 'action']);
|
|
33
|
+
expect(entry.function.parameters.properties.type.enum)
|
|
34
|
+
.toEqual(['alert', 'confirm', 'prompt', 'beforeunload']);
|
|
35
|
+
expect(entry.function.parameters.properties.action.enum).toEqual(['accept', 'dismiss']);
|
|
36
|
+
expect(entry.function.parameters.properties.input.type).toBe('string');
|
|
37
|
+
});
|
|
30
38
|
it('exposes iframe targeting and iframe text search', () => {
|
|
31
39
|
const fill = toolsSchema.find(tool => tool.function.name === 'fill');
|
|
32
40
|
const click = toolsSchema.find(tool => tool.function.name === 'click');
|
package/tools/chromeBrowser.js
CHANGED
|
@@ -59,6 +59,9 @@ export class ChromeBrowser {
|
|
|
59
59
|
this.debugEnabled = !!options.debug;
|
|
60
60
|
this._domLimitLogged = false;
|
|
61
61
|
this.storageStatePath = options.storageStatePath;
|
|
62
|
+
this.dialogRules = [];
|
|
63
|
+
this.dialogEvents = [];
|
|
64
|
+
this._dialogRuleSequence = 0;
|
|
62
65
|
|
|
63
66
|
const defaultResourceGuard = {
|
|
64
67
|
enabled: true,
|
|
@@ -98,6 +101,13 @@ export class ChromeBrowser {
|
|
|
98
101
|
this.currentId = id;
|
|
99
102
|
this.page = page;
|
|
100
103
|
|
|
104
|
+
// Playwright blocks the action that opened a JavaScript dialog until the
|
|
105
|
+
// dialog is handled. Keep this listener synchronous and always resolve the
|
|
106
|
+
// dialog so an unexpected alert can never hang a mission.
|
|
107
|
+
page.on('dialog', dialog => {
|
|
108
|
+
void this._handleDialog(dialog, page);
|
|
109
|
+
});
|
|
110
|
+
|
|
101
111
|
page.on('close', () => {
|
|
102
112
|
// remove from registry
|
|
103
113
|
this.pages = this.pages.filter(p => p.id !== id);
|
|
@@ -115,6 +125,115 @@ export class ChromeBrowser {
|
|
|
115
125
|
return id;
|
|
116
126
|
}
|
|
117
127
|
|
|
128
|
+
_expireDialogRules(now = Date.now()) {
|
|
129
|
+
const active = [];
|
|
130
|
+
for (const rule of this.dialogRules) {
|
|
131
|
+
if (rule.expiresAt > now) {
|
|
132
|
+
active.push(rule);
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
this.dialogEvents.push({
|
|
136
|
+
event: 'dialog_rule_expired',
|
|
137
|
+
ruleId: rule.id,
|
|
138
|
+
expectedType: rule.type,
|
|
139
|
+
expectedMessage: rule.message,
|
|
140
|
+
messageIncludes: rule.messageIncludes,
|
|
141
|
+
action: rule.action,
|
|
142
|
+
armedAt: new Date(rule.armedAt).toISOString(),
|
|
143
|
+
expiredAt: new Date(rule.expiresAt).toISOString(),
|
|
144
|
+
outcome: 'timeout',
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
this.dialogRules = active;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
async _handleDialog(dialog, page) {
|
|
151
|
+
const now = Date.now();
|
|
152
|
+
this._expireDialogRules(now);
|
|
153
|
+
const type = dialog.type();
|
|
154
|
+
const message = dialog.message();
|
|
155
|
+
const matchingIndex = this.dialogRules.findIndex(rule =>
|
|
156
|
+
rule.type === type
|
|
157
|
+
&& (rule.message === undefined || rule.message === message)
|
|
158
|
+
&& (rule.messageIncludes === undefined || message.includes(rule.messageIncludes))
|
|
159
|
+
);
|
|
160
|
+
const rule = matchingIndex >= 0 ? this.dialogRules.splice(matchingIndex, 1)[0] : null;
|
|
161
|
+
const action = rule?.action || 'dismiss';
|
|
162
|
+
const record = {
|
|
163
|
+
event: 'javascript_dialog',
|
|
164
|
+
type,
|
|
165
|
+
message,
|
|
166
|
+
defaultValue: dialog.defaultValue(),
|
|
167
|
+
pageUrl: page?.url?.() || null,
|
|
168
|
+
appearedAt: new Date(now).toISOString(),
|
|
169
|
+
ruleId: rule?.id || null,
|
|
170
|
+
matched: Boolean(rule),
|
|
171
|
+
expected: Boolean(rule),
|
|
172
|
+
action,
|
|
173
|
+
inputProvided: rule?.input !== undefined,
|
|
174
|
+
outcome: 'pending',
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
try {
|
|
178
|
+
if (action === 'accept') await dialog.accept(rule?.input);
|
|
179
|
+
else await dialog.dismiss();
|
|
180
|
+
record.outcome = rule ? action : 'unexpected_dialog_dismissed';
|
|
181
|
+
} catch (error) {
|
|
182
|
+
record.outcome = 'response_failed';
|
|
183
|
+
record.error = error?.message || String(error);
|
|
184
|
+
// A second response can fail if the page closed. Never mask that fact.
|
|
185
|
+
}
|
|
186
|
+
record.respondedAt = new Date().toISOString();
|
|
187
|
+
this.dialogEvents.push(record);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
prepare_dialog_response({
|
|
191
|
+
action,
|
|
192
|
+
type,
|
|
193
|
+
message,
|
|
194
|
+
messageIncludes,
|
|
195
|
+
input,
|
|
196
|
+
timeoutMs = 10_000,
|
|
197
|
+
} = {}) {
|
|
198
|
+
const supportedTypes = new Set(['alert', 'confirm', 'prompt', 'beforeunload']);
|
|
199
|
+
if (!supportedTypes.has(type)) throw new Error(`Unsupported dialog type: ${type || '(missing)'}`);
|
|
200
|
+
if (!['accept', 'dismiss'].includes(action)) throw new Error('Dialog action must be "accept" or "dismiss".');
|
|
201
|
+
if (message !== undefined && messageIncludes !== undefined) {
|
|
202
|
+
throw new Error('Use either message or messageIncludes, not both.');
|
|
203
|
+
}
|
|
204
|
+
if (input !== undefined && (type !== 'prompt' || action !== 'accept')) {
|
|
205
|
+
throw new Error('Dialog input is only supported when accepting a prompt.');
|
|
206
|
+
}
|
|
207
|
+
const boundedTimeout = Math.min(30_000, Math.max(250, Number(timeoutMs) || 10_000));
|
|
208
|
+
const armedAt = Date.now();
|
|
209
|
+
const rule = {
|
|
210
|
+
id: `dialog_${++this._dialogRuleSequence}`,
|
|
211
|
+
action,
|
|
212
|
+
type,
|
|
213
|
+
message,
|
|
214
|
+
messageIncludes,
|
|
215
|
+
input,
|
|
216
|
+
armedAt,
|
|
217
|
+
expiresAt: armedAt + boundedTimeout,
|
|
218
|
+
};
|
|
219
|
+
this.dialogRules.push(rule);
|
|
220
|
+
return {
|
|
221
|
+
armed: true,
|
|
222
|
+
ruleId: rule.id,
|
|
223
|
+
action,
|
|
224
|
+
type,
|
|
225
|
+
message,
|
|
226
|
+
messageIncludes,
|
|
227
|
+
inputProvided: input !== undefined,
|
|
228
|
+
timeoutMs: boundedTimeout,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
drainDialogEvents() {
|
|
233
|
+
this._expireDialogRules();
|
|
234
|
+
return this.dialogEvents.splice(0);
|
|
235
|
+
}
|
|
236
|
+
|
|
118
237
|
_switchTo(target) {
|
|
119
238
|
if (target === 'main') {
|
|
120
239
|
this.currentId = this.pages[0]?.id ?? '';
|
|
@@ -1538,6 +1657,7 @@ export const CHROME_TOOL_MAP = {
|
|
|
1538
1657
|
fill: (b, args) => b.fill(args),
|
|
1539
1658
|
click: (b, args) => b.click(args),
|
|
1540
1659
|
press_key: (b, args) => b.press_key(args),
|
|
1660
|
+
prepare_dialog_response: (b, args) => b.prepare_dialog_response(args),
|
|
1541
1661
|
get_dom: (b, args, agentMemory) => b.get_dom(args, agentMemory),
|
|
1542
1662
|
check_text: (b, args) => b.check_text(args),
|
|
1543
1663
|
click_text: async (browser, args, agentMemory) => {
|
|
@@ -104,6 +104,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
104
104
|
const missionModelUsage = summarizeModelTokens(steps);
|
|
105
105
|
const stepItems = steps.map((step, idx) => {
|
|
106
106
|
const events = Array.isArray(step.events) ? step.events : [];
|
|
107
|
+
const dialogs = Array.isArray(step.dialogs) ? step.dialogs : [];
|
|
107
108
|
const ok = /✅|Passed|Mission Success/i.test(step.result || '');
|
|
108
109
|
const resultRaw = step.result || '—';
|
|
109
110
|
const resultTooltip = resultRaw.includes('⚠️ Turn Issues')
|
|
@@ -122,6 +123,36 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
122
123
|
const imgTag = step.screenshotPath
|
|
123
124
|
? `<img src="${esc(step.screenshotPath)}" alt="screenshot turn ${esc(step.turn ?? idx)}">`
|
|
124
125
|
: '';
|
|
126
|
+
const dialogCards = dialogs.map(dialog => {
|
|
127
|
+
if (dialog.event === 'dialog_rule_expired') {
|
|
128
|
+
return `<article class="dialog-card dialog-timeout">
|
|
129
|
+
<div class="dialog-card-label">Report visualization · response timeout</div>
|
|
130
|
+
<strong>${esc(dialog.expectedType || 'dialog')} rule expired</strong>
|
|
131
|
+
<span>No matching dialog appeared before ${esc(dialog.ruleId || 'the prepared response')} expired.</span>
|
|
132
|
+
</article>`;
|
|
133
|
+
}
|
|
134
|
+
const type = String(dialog.type || 'dialog');
|
|
135
|
+
const accepted = dialog.action === 'accept';
|
|
136
|
+
const acceptLabel = type === 'beforeunload' ? 'Leave' : 'Accept';
|
|
137
|
+
const dismissLabel = type === 'beforeunload' ? 'Stay' : 'Dismiss';
|
|
138
|
+
return `<article class="dialog-card ${dialog.matched ? 'dialog-expected' : 'dialog-unexpected'}">
|
|
139
|
+
<div class="dialog-card-label">Report visualization · not a literal browser screenshot</div>
|
|
140
|
+
<div class="dialog-window">
|
|
141
|
+
<div class="dialog-title"><span class="dialog-icon">!</span><strong>Browser-native ${esc(type)}</strong></div>
|
|
142
|
+
<div class="dialog-message">${esc(dialog.message || '(browser supplied no message)')}</div>
|
|
143
|
+
${type === 'prompt' ? `<div class="dialog-input">${dialog.inputProvided ? '•••••••• · input provided' : esc(dialog.defaultValue || 'No input provided')}</div>` : ''}
|
|
144
|
+
<div class="dialog-actions">
|
|
145
|
+
<span class="dialog-button ${accepted ? 'chosen' : ''}">${esc(acceptLabel)}</span>
|
|
146
|
+
<span class="dialog-button ${!accepted ? 'chosen' : ''}">${esc(dismissLabel)}</span>
|
|
147
|
+
</div>
|
|
148
|
+
</div>
|
|
149
|
+
<div class="dialog-meta">
|
|
150
|
+
<span>${dialog.matched ? '✓ Expected response matched' : '⚠ Unexpected dialog; fail-closed response'}</span>
|
|
151
|
+
<span>Outcome: ${esc(dialog.outcome || 'unknown')}</span>
|
|
152
|
+
${dialog.pageUrl ? `<span>Page: ${esc(dialog.pageUrl)}</span>` : ''}
|
|
153
|
+
</div>
|
|
154
|
+
</article>`;
|
|
155
|
+
}).join('');
|
|
125
156
|
const stepJev = step.jevShadow?.status === 'ok'
|
|
126
157
|
? ` • Jev: ${formatTokens(step.jevShadow.usage?.input_tokens)} in / ${formatTokens(step.jevShadow.usage?.output_tokens)} out`
|
|
127
158
|
: step.jevShadow
|
|
@@ -152,6 +183,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
152
183
|
<span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
|
|
153
184
|
<span class="tokens">${esc(step.model || llm.model || 'Model')} tokens: ${esc(step.tokensUsed ?? '—')} (${esc(step.inputTokens ?? '—')} in / ${esc(step.outputTokens ?? '—')} out)${routeLabel}${stepJev}</span>
|
|
154
185
|
</summary>
|
|
186
|
+
${dialogCards ? `<div class="dialog-evidence">${dialogCards}</div>` : ''}
|
|
155
187
|
<pre class="events">${esc(events.join('\n')) || '(no events)'}</pre>
|
|
156
188
|
${imgTag}
|
|
157
189
|
</details>
|
|
@@ -428,6 +460,27 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
428
460
|
border: 1px solid var(--hairline-strong);
|
|
429
461
|
}
|
|
430
462
|
|
|
463
|
+
.dialog-evidence{display:grid;gap:12px;margin:12px 0 0;}
|
|
464
|
+
.dialog-card{
|
|
465
|
+
display:grid;gap:10px;padding:12px;border:1px solid rgba(96,165,250,.34);
|
|
466
|
+
border-radius:12px;background:linear-gradient(145deg,rgba(15,23,42,.96),rgba(30,41,59,.92));
|
|
467
|
+
}
|
|
468
|
+
.dialog-card.dialog-unexpected,.dialog-card.dialog-timeout{border-color:rgba(251,191,36,.48);}
|
|
469
|
+
.dialog-card-label{font-size:10px;font-weight:800;letter-spacing:.12em;text-transform:uppercase;color:#93c5fd;}
|
|
470
|
+
.dialog-window{
|
|
471
|
+
width:min(100%,520px);margin:auto;padding:18px;border-radius:13px;color:#e5e7eb;
|
|
472
|
+
background:#343540;border:1px solid rgba(255,255,255,.14);box-shadow:0 18px 40px rgba(0,0,0,.35);
|
|
473
|
+
}
|
|
474
|
+
.dialog-title{display:flex;align-items:center;gap:9px;font-size:14px;}
|
|
475
|
+
.dialog-icon{display:grid;place-items:center;width:22px;height:22px;border-radius:50%;background:#f59e0b;color:#111827;font-weight:900;}
|
|
476
|
+
.dialog-message{padding:18px 0;font-size:15px;line-height:1.45;white-space:pre-wrap;}
|
|
477
|
+
.dialog-input{padding:9px 11px;border:2px solid #22d3ee;border-radius:8px;background:#171821;color:#cbd5e1;font:12px ui-monospace,monospace;}
|
|
478
|
+
.dialog-actions{display:flex;justify-content:flex-end;gap:9px;margin-top:16px;}
|
|
479
|
+
.dialog-button{padding:7px 15px;border-radius:8px;background:#454753;border:1px solid rgba(255,255,255,.13);font-size:12px;font-weight:750;}
|
|
480
|
+
.dialog-button.chosen{color:#07111e;background:#22d3ee;border-color:#67e8f9;box-shadow:0 0 0 2px rgba(34,211,238,.18);}
|
|
481
|
+
.dialog-meta{display:flex;gap:8px;flex-wrap:wrap;color:var(--text-muted);font-size:11px;}
|
|
482
|
+
.dialog-meta span{padding:4px 7px;border:1px solid var(--hairline);border-radius:999px;background:rgba(255,255,255,.035);}
|
|
483
|
+
|
|
431
484
|
img{
|
|
432
485
|
display:block; max-width:100%;
|
|
433
486
|
border:1px solid var(--hairline); border-radius:12px; margin-top:10px;
|
package/tools/toolSchema.js
CHANGED
|
@@ -96,6 +96,26 @@ const toolsSchema = [
|
|
|
96
96
|
},
|
|
97
97
|
},
|
|
98
98
|
},
|
|
99
|
+
{
|
|
100
|
+
type: 'function',
|
|
101
|
+
function: {
|
|
102
|
+
name: 'prepare_dialog_response',
|
|
103
|
+
description: 'Arm a one-time response for the next matching JavaScript alert, confirm, prompt, or beforeunload dialog. Call immediately before the action that opens it. Unexpected dialogs are dismissed and reported.',
|
|
104
|
+
parameters: {
|
|
105
|
+
type: 'object',
|
|
106
|
+
properties: {
|
|
107
|
+
type: { type: 'string', enum: ['alert', 'confirm', 'prompt', 'beforeunload'] },
|
|
108
|
+
action: { type: 'string', enum: ['accept', 'dismiss'] },
|
|
109
|
+
message: { type: 'string', description: 'Optional exact message match.' },
|
|
110
|
+
messageIncludes: { type: 'string', description: 'Optional substring message match. Cannot be combined with message.' },
|
|
111
|
+
input: { type: 'string', description: 'Prompt text; valid only when accepting a prompt.' },
|
|
112
|
+
timeoutMs: { type: 'number', default: 10000, minimum: 250, maximum: 30000 },
|
|
113
|
+
},
|
|
114
|
+
required: ['type', 'action'],
|
|
115
|
+
additionalProperties: false,
|
|
116
|
+
},
|
|
117
|
+
},
|
|
118
|
+
},
|
|
99
119
|
{
|
|
100
120
|
type: 'function',
|
|
101
121
|
function: {
|