testronaut 1.3.4 ā 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devcontainer/devcontainer.json +21 -3
- package/.devcontainer/setup-github-auth.sh +36 -0
- package/.github/workflows/prepare-release.yml +168 -0
- package/.github/workflows/publish.yml +133 -0
- package/bin/cli.js +67 -0
- package/bin/init.js +1 -0
- package/bin/initHelpers.js +4 -0
- package/core/agent.js +13 -1
- package/core/config.js +65 -0
- package/core/redaction.js +30 -7
- package/core/turnLoop.js +35 -4
- package/package.json +5 -1
- package/runner/testronaut.js +15 -2
- package/tests/binTests/init.test.js +6 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +2 -0
- package/tests/coreTests/config.test.js +40 -0
- package/tests/coreTests/redation.test.js +30 -0
- package/tests/coreTests/turnLoop.test.js +1 -0
- package/tests/runnerTests/testronaut.test.js +7 -5
- package/tests/toolsTests/humanInput.test.js +26 -0
- package/tests/toolsTests/toolSchema.test.js +8 -0
- package/tests/toolsTests/turnLoopUtils.test.js +24 -0
- package/tools/chromeBrowser.js +2 -0
- package/tools/generateHtmlReport.js +13 -4
- package/tools/humanInput.js +165 -0
- package/tools/toolSchema.js +28 -0
- package/tools/turnLoopUtils.js +10 -6
package/core/turnLoop.js
CHANGED
|
@@ -46,7 +46,7 @@ import {
|
|
|
46
46
|
import { resolveProviderModel } from '../llm/modelResolver.js';
|
|
47
47
|
import { getLLM } from '../llm/llmFactory.js';
|
|
48
48
|
import { summarizeTurnIntentFromMessage } from './turnIntent.js';
|
|
49
|
-
import { redactArgs } from './redaction.js';
|
|
49
|
+
import { maskPreview, redactArgs } from './redaction.js';
|
|
50
50
|
import {
|
|
51
51
|
sanitizeHeavyToolHistory,
|
|
52
52
|
pruneConversationContext,
|
|
@@ -132,6 +132,7 @@ const FIRE_AND_FORGET_TOOLS = new Set([
|
|
|
132
132
|
'click_and_follow_popup',
|
|
133
133
|
'switch_to_page',
|
|
134
134
|
'close_current_page',
|
|
135
|
+
'request_human_input',
|
|
135
136
|
]);
|
|
136
137
|
|
|
137
138
|
/**
|
|
@@ -227,7 +228,11 @@ export const turnLoop = async (
|
|
|
227
228
|
const retryLimitClamped = Math.min(10, Math.max(1, Number.isFinite(retryLimitRaw) ? retryLimitRaw : DEFAULT_TURN_RETRY_LIMIT)); // retries allowed (excludes initial)
|
|
228
229
|
const maxAttempts = retryLimitClamped + 1; // includes initial attempt
|
|
229
230
|
ctx.groundControl = groundControl;
|
|
230
|
-
|
|
231
|
+
const humanInput = ctx.humanInput || { enabled: true, timeoutSeconds: 60 };
|
|
232
|
+
const activeToolsSchema = humanInput.enabled === false
|
|
233
|
+
? toolsSchema.filter(t => t?.function?.name !== 'request_human_input')
|
|
234
|
+
: toolsSchema;
|
|
235
|
+
let agentMemory = { lastMenuExpanded: false, humanInput };
|
|
231
236
|
ensureDocProgress(agentMemory, resourceGuardCfg);
|
|
232
237
|
let turnRetries = 0;
|
|
233
238
|
let stepSeq = ctx._stepSeq || 0;
|
|
@@ -325,7 +330,7 @@ export const turnLoop = async (
|
|
|
325
330
|
const { message, usage, headers } = await llm.chat({
|
|
326
331
|
model: MODEL_ID,
|
|
327
332
|
messages,
|
|
328
|
-
tools:
|
|
333
|
+
tools: activeToolsSchema,
|
|
329
334
|
});
|
|
330
335
|
response = { message, usage, headers };
|
|
331
336
|
|
|
@@ -412,6 +417,19 @@ export const turnLoop = async (
|
|
|
412
417
|
hadToolIssues = true;
|
|
413
418
|
}
|
|
414
419
|
|
|
420
|
+
if (fnName === 'request_human_input') {
|
|
421
|
+
step.humanInput = step.humanInput || {};
|
|
422
|
+
step.humanInput.requested = true;
|
|
423
|
+
step.humanInput.codeType = args.codeType || 'verification_code';
|
|
424
|
+
step.humanInput.timeoutSeconds = humanInput.timeoutSeconds;
|
|
425
|
+
step.humanInput.status = errorMessage
|
|
426
|
+
? (String(errorMessage).toLowerCase().includes('timed out') ? 'timeout' : 'invalid')
|
|
427
|
+
: 'provided';
|
|
428
|
+
step.events.push(errorMessage
|
|
429
|
+
? `š¤ Human-in-the-loop input ${step.humanInput.status}: ${errorMessage.replace(/^ERROR:\s*/, '')}`
|
|
430
|
+
: 'š¤ Human-in-the-loop input provided.');
|
|
431
|
+
}
|
|
432
|
+
|
|
415
433
|
// Capture and log any file upload/download events (for reports)
|
|
416
434
|
try {
|
|
417
435
|
const maybeJson = JSON.parse(result);
|
|
@@ -443,7 +461,20 @@ export const turnLoop = async (
|
|
|
443
461
|
}
|
|
444
462
|
|
|
445
463
|
console.log(`[tool ] ā ${fnName} result:`, errorMessage ? 'ā Failed' : 'ā
Success');
|
|
446
|
-
|
|
464
|
+
let resultForLog = result;
|
|
465
|
+
if (fnName === 'request_human_input') {
|
|
466
|
+
try {
|
|
467
|
+
const parsed = JSON.parse(result);
|
|
468
|
+
resultForLog = JSON.stringify({
|
|
469
|
+
...parsed,
|
|
470
|
+
value: maskPreview(parsed.value),
|
|
471
|
+
redactedValue: maskPreview(parsed.value),
|
|
472
|
+
});
|
|
473
|
+
} catch {
|
|
474
|
+
resultForLog = errorMessage || 'Human input received.';
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
const truncated = resultForLog.length > 1000 ? resultForLog.slice(0, 1000) + 'ā¦' : resultForLog;
|
|
447
478
|
step.events.push(`[tool ] ā ${fnName} result: ${errorMessage ? 'ā Failed' : 'ā
Success'}`);
|
|
448
479
|
step.events.push(`[tool ] ā ${truncated}`);
|
|
449
480
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testronaut",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.6",
|
|
4
4
|
"main": "./runner/testronaut.js",
|
|
5
5
|
"bin": {
|
|
6
6
|
"testronaut": "bin/cli.js"
|
|
@@ -19,6 +19,10 @@
|
|
|
19
19
|
"author": "",
|
|
20
20
|
"license": "MIT",
|
|
21
21
|
"description": "",
|
|
22
|
+
"repository": {
|
|
23
|
+
"type": "git",
|
|
24
|
+
"url": "https://github.com/mission-testronaut/testronaut-cli"
|
|
25
|
+
},
|
|
22
26
|
"dependencies": {
|
|
23
27
|
"@dqbd/tiktoken": "^1.0.21",
|
|
24
28
|
"@google/generative-ai": "^0.24.1",
|
package/runner/testronaut.js
CHANGED
|
@@ -27,7 +27,7 @@ import fs from 'fs';
|
|
|
27
27
|
import path from 'path';
|
|
28
28
|
import { runAgent } from '../core/agent.js';
|
|
29
29
|
import { redactPasswordInText } from '../core/redaction.js';
|
|
30
|
-
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig } from '../core/config.js';
|
|
30
|
+
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../core/config.js';
|
|
31
31
|
|
|
32
32
|
// Check process env for debug toggles (shared helper for tests and CLI).
|
|
33
33
|
const isDebugEnabled = () => {
|
|
@@ -53,6 +53,7 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
53
53
|
const retryLimit = retryInfo.value;
|
|
54
54
|
const domListLimitInfo = getDomListLimit(cfg);
|
|
55
55
|
const resourceGuard = getResourceGuardConfig(cfg);
|
|
56
|
+
const humanInput = getHumanInputConfig(cfg);
|
|
56
57
|
const debugEnabled = isDebugEnabled();
|
|
57
58
|
if (notes.length) {
|
|
58
59
|
console.warn(notes.join('\n'));
|
|
@@ -66,6 +67,12 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
66
67
|
if (domListLimitInfo?.mode === 'all') {
|
|
67
68
|
console.warn('ā ļø DOM list limit is set to "all". This can dramatically increase token usage and may break flows on heavy pages.');
|
|
68
69
|
}
|
|
70
|
+
if (humanInput.clamped) {
|
|
71
|
+
console.warn(`ā ļø Human input timeout clamped to ${humanInput.timeoutSeconds}s (allowed 5-300).`);
|
|
72
|
+
}
|
|
73
|
+
if (!humanInput.enabled) {
|
|
74
|
+
console.log('š¤ Human-in-the-loop input tool disabled for this run.');
|
|
75
|
+
}
|
|
69
76
|
if (debugEnabled) {
|
|
70
77
|
const cfgDomRaw = cfg?.dom?.listItemLimit ?? cfg?.dom?.listLimit ?? cfg?.domListLimit;
|
|
71
78
|
console.log('[debug] Debug mode enabled');
|
|
@@ -89,6 +96,12 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
89
96
|
hrefIncludes: resourceGuard.hrefIncludes,
|
|
90
97
|
dataTypes: resourceGuard.dataTypes,
|
|
91
98
|
});
|
|
99
|
+
console.log('[debug] Human input', {
|
|
100
|
+
enabled: humanInput.enabled,
|
|
101
|
+
timeoutSeconds: humanInput.timeoutSeconds,
|
|
102
|
+
source: humanInput.source,
|
|
103
|
+
clamped: humanInput.clamped,
|
|
104
|
+
});
|
|
92
105
|
// Helpful for upload missions: show missions/files contents
|
|
93
106
|
const filesDir = path.resolve(process.cwd(), 'missions/files');
|
|
94
107
|
try {
|
|
@@ -188,7 +201,7 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
188
201
|
missionName,
|
|
189
202
|
maxTurns,
|
|
190
203
|
retryLimit,
|
|
191
|
-
{ domListLimit: domListLimitInfo?.value, debug: debugEnabled, resourceGuard }
|
|
204
|
+
{ domListLimit: domListLimitInfo?.value, debug: debugEnabled, resourceGuard, humanInput }
|
|
192
205
|
);
|
|
193
206
|
if (!success) {
|
|
194
207
|
console.log(`ā Aborting after failed goal.`);
|
|
@@ -79,6 +79,8 @@ describe('initializeTestronautProject', () => {
|
|
|
79
79
|
expect(typeof cfg.maxTurns).toBe('number');
|
|
80
80
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
81
81
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
82
|
+
expect(cfg.humanInput?.enabled).toBe(true);
|
|
83
|
+
expect(cfg.humanInput?.timeoutSeconds).toBe(60);
|
|
82
84
|
|
|
83
85
|
// .env
|
|
84
86
|
const envPath = path.join(temp, '.env');
|
|
@@ -112,6 +114,8 @@ describe('initializeTestronautProject', () => {
|
|
|
112
114
|
expect(cfg.model).toBe('gemini-2.5-flash');
|
|
113
115
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
114
116
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
117
|
+
expect(cfg.humanInput?.enabled).toBe(true);
|
|
118
|
+
expect(cfg.humanInput?.timeoutSeconds).toBe(60);
|
|
115
119
|
|
|
116
120
|
const envTxt = fs.readFileSync(path.join(temp, '.env'), 'utf8');
|
|
117
121
|
expect(envTxt).toMatch(/GEMINI_API_KEY=AIza/);
|
|
@@ -191,5 +195,7 @@ describe('initializeTestronautProject', () => {
|
|
|
191
195
|
expect(cfg.maxTurns).toBe(20); // default filled in
|
|
192
196
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
193
197
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
198
|
+
expect(cfg.humanInput?.enabled).toBe(true);
|
|
199
|
+
expect(cfg.humanInput?.timeoutSeconds).toBe(60);
|
|
194
200
|
});
|
|
195
201
|
});
|
|
@@ -11,5 +11,7 @@ describe('defaultConfig', () => {
|
|
|
11
11
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
12
12
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
13
13
|
expect(cfg.resourceGuard?.hrefIncludes).toContain('/document/');
|
|
14
|
+
expect(cfg.humanInput?.enabled).toBe(true);
|
|
15
|
+
expect(cfg.humanInput?.timeoutSeconds).toBe(60);
|
|
14
16
|
});
|
|
15
17
|
});
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
enforceTurnBudget,
|
|
14
14
|
getDomListLimit,
|
|
15
15
|
getResourceGuardConfig,
|
|
16
|
+
getHumanInputConfig,
|
|
16
17
|
} from '../../core/config.js';
|
|
17
18
|
|
|
18
19
|
describe('core/config', () => {
|
|
@@ -201,6 +202,45 @@ describe('core/config', () => {
|
|
|
201
202
|
});
|
|
202
203
|
});
|
|
203
204
|
|
|
205
|
+
describe('getHumanInputConfig', () => {
|
|
206
|
+
const OLD_ENV = { ...process.env };
|
|
207
|
+
beforeEach(() => {
|
|
208
|
+
process.env = { ...OLD_ENV };
|
|
209
|
+
delete process.env.TESTRONAUT_HUMAN_INPUT;
|
|
210
|
+
delete process.env.TESTRONAUT_HUMAN_INPUT_TIMEOUT_SECONDS;
|
|
211
|
+
});
|
|
212
|
+
afterEach(() => {
|
|
213
|
+
process.env = { ...OLD_ENV };
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
it('defaults to enabled with a 60 second timeout', () => {
|
|
217
|
+
const res = getHumanInputConfig({});
|
|
218
|
+
expect(res.enabled).toBe(true);
|
|
219
|
+
expect(res.timeoutSeconds).toBe(60);
|
|
220
|
+
expect(res.source.enabled).toBe('default');
|
|
221
|
+
expect(res.source.timeout).toBe('default');
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
it('uses config values and clamps timeout', () => {
|
|
225
|
+
const res = getHumanInputConfig({ humanInput: { enabled: false, timeoutSeconds: 999 } });
|
|
226
|
+
expect(res.enabled).toBe(false);
|
|
227
|
+
expect(res.timeoutSeconds).toBe(300);
|
|
228
|
+
expect(res.source.enabled).toBe('config');
|
|
229
|
+
expect(res.source.timeout).toBe('config');
|
|
230
|
+
expect(res.clamped).toBe(true);
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it('prefers env overrides over config', () => {
|
|
234
|
+
process.env.TESTRONAUT_HUMAN_INPUT = 'true';
|
|
235
|
+
process.env.TESTRONAUT_HUMAN_INPUT_TIMEOUT_SECONDS = '7';
|
|
236
|
+
const res = getHumanInputConfig({ humanInput: { enabled: false, timeoutSeconds: 120 } });
|
|
237
|
+
expect(res.enabled).toBe(true);
|
|
238
|
+
expect(res.timeoutSeconds).toBe(7);
|
|
239
|
+
expect(res.source.enabled).toBe('env');
|
|
240
|
+
expect(res.source.timeout).toBe('env');
|
|
241
|
+
});
|
|
242
|
+
});
|
|
243
|
+
|
|
204
244
|
describe('enforceTurnBudget (lenient)', () => {
|
|
205
245
|
it('clamps > hardMaxTurns', () => {
|
|
206
246
|
const cfg = { maxTurns: 1000 };
|
|
@@ -75,6 +75,14 @@ describe('redaction utilities', () => {
|
|
|
75
75
|
const redacted = redactArgs('fill', args);
|
|
76
76
|
expect(redacted.text).toBe('hello world');
|
|
77
77
|
});
|
|
78
|
+
|
|
79
|
+
it('masks human input tool values', () => {
|
|
80
|
+
const args = { prompt: 'Enter code', value: '123456', code: '654321' };
|
|
81
|
+
const redacted = redactArgs('request_human_input', args);
|
|
82
|
+
expect(redacted.prompt).toBe('Enter code');
|
|
83
|
+
expect(redacted.value).toBe('ā¢ā¢ā¢ā¢ā¢ā¢ (6)');
|
|
84
|
+
expect(redacted.code).toBe('ā¢ā¢ā¢ā¢ā¢ā¢ (6)');
|
|
85
|
+
});
|
|
78
86
|
});
|
|
79
87
|
|
|
80
88
|
describe('redactPasswordInText', () => {
|
|
@@ -91,6 +99,13 @@ describe('redaction utilities', () => {
|
|
|
91
99
|
expect(out).toContain(`auth with ā¢ā¢ā¢ā¢ā¢ā¢ (3)`);
|
|
92
100
|
});
|
|
93
101
|
|
|
102
|
+
it('masks MFA codes mentioned as entered in final text', () => {
|
|
103
|
+
const input = `SUCCESS: Logged in as ISAAC, completed MFA (entered 123456), and reached the dashboard.`;
|
|
104
|
+
const out = redactPasswordInText(input);
|
|
105
|
+
expect(out).toContain(`MFA (entered ā¢ā¢ā¢ā¢ā¢ā¢ (6))`);
|
|
106
|
+
expect(out).not.toContain('123456');
|
|
107
|
+
});
|
|
108
|
+
|
|
94
109
|
it('Case 3: "VALUE" into sensitive target (#access)', () => {
|
|
95
110
|
const input = `Type "Imp0st3r123!" into #access`;
|
|
96
111
|
const out = redactPasswordInText(input);
|
|
@@ -103,6 +118,21 @@ describe('redaction utilities', () => {
|
|
|
103
118
|
expect(out).toBe(input);
|
|
104
119
|
});
|
|
105
120
|
|
|
121
|
+
it('does not mangle MFA mission instructions while masking the access value', () => {
|
|
122
|
+
const input = `Fill the access code field (#access) with sword.
|
|
123
|
+
Check the "Challenge with MFA code" checkbox.
|
|
124
|
+
Set MFA code digits to 8.
|
|
125
|
+
There should be an MFA challenge. Prompt the user for an MFA code, then fill the MFA Code field (#mfa-code) with that code.
|
|
126
|
+
Click the "Verify Code" button (#mfa-submit).`;
|
|
127
|
+
const out = redactPasswordInText(input);
|
|
128
|
+
expect(out).toContain('Fill the access code field (#access) with ā¢ā¢ā¢ā¢ā¢ā¢ (5)');
|
|
129
|
+
expect(out).toContain('Check the "Challenge with MFA code" checkbox.');
|
|
130
|
+
expect(out).toContain('Set MFA code digits to 8.');
|
|
131
|
+
expect(out).toContain('with that code.');
|
|
132
|
+
expect(out).toContain('Click the "Verify Code" button (#mfa-submit).');
|
|
133
|
+
expect(out).not.toContain('sword');
|
|
134
|
+
});
|
|
135
|
+
|
|
106
136
|
it('handles api key forms with spaces/underscores/hyphens', () => {
|
|
107
137
|
const cases = [
|
|
108
138
|
`api-key "abcDEF123"`,
|
|
@@ -12,10 +12,11 @@ vi.mock('../../core/config.js', () => ({
|
|
|
12
12
|
getRetryLimit: vi.fn(),
|
|
13
13
|
getDomListLimit: vi.fn(),
|
|
14
14
|
getResourceGuardConfig: vi.fn(),
|
|
15
|
+
getHumanInputConfig: vi.fn(),
|
|
15
16
|
}));
|
|
16
17
|
|
|
17
18
|
import { runAgent } from '../../core/agent.js';
|
|
18
|
-
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig } from '../../core/config.js';
|
|
19
|
+
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../../core/config.js';
|
|
19
20
|
|
|
20
21
|
// Adjust the import path if your file lives elsewhere
|
|
21
22
|
import { runMissions, __test__ as testronautInternals } from '../../runner/testronaut.js';
|
|
@@ -29,6 +30,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
29
30
|
getRetryLimit.mockReturnValue({ value: 2, source: 'default', clamped: false });
|
|
30
31
|
getDomListLimit.mockReturnValue({ value: 3, mode: 'number', source: 'default', clamped: false });
|
|
31
32
|
getResourceGuardConfig.mockReturnValue({ enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] });
|
|
33
|
+
getHumanInputConfig.mockReturnValue({ enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false });
|
|
32
34
|
});
|
|
33
35
|
|
|
34
36
|
it('passes effectiveMax to runAgent and logs any notes', async () => {
|
|
@@ -56,7 +58,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
56
58
|
expect.stringContaining('Clamping to 200')
|
|
57
59
|
);
|
|
58
60
|
expect(runAgent).toHaveBeenCalledWith(
|
|
59
|
-
expect.any(Array), 'Budgeted Run', 200, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] } }
|
|
61
|
+
expect.any(Array), 'Budgeted Run', 200, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } }
|
|
60
62
|
);
|
|
61
63
|
|
|
62
64
|
warn.mockRestore();
|
|
@@ -83,7 +85,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
83
85
|
await runMissions({ mission: 'No warnings' }, 'Clean');
|
|
84
86
|
|
|
85
87
|
expect(warn).not.toHaveBeenCalled();
|
|
86
|
-
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20, 3, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] } });
|
|
88
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20, 3, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
87
89
|
|
|
88
90
|
warn.mockRestore();
|
|
89
91
|
log.mockRestore();
|
|
@@ -176,7 +178,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
176
178
|
expect(goals[1].submissionName).toMatch(/^My Mission/);
|
|
177
179
|
|
|
178
180
|
// Effective max turns and retry limit passed through
|
|
179
|
-
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] } });
|
|
181
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
180
182
|
|
|
181
183
|
log.mockRestore();
|
|
182
184
|
});
|
|
@@ -215,7 +217,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
215
217
|
|
|
216
218
|
await runMissions({ mission: 'Debug' }, 'Debug Mission');
|
|
217
219
|
|
|
218
|
-
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Debug Mission', 20, 2, { domListLimit: 3, debug: true, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] } });
|
|
220
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Debug Mission', 20, 2, { domListLimit: 3, debug: true, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
219
221
|
|
|
220
222
|
log.mockRestore();
|
|
221
223
|
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import {
|
|
3
|
+
sanitizeHumanCodeInput,
|
|
4
|
+
normalizeHumanInputOptions,
|
|
5
|
+
} from '../../tools/humanInput.js';
|
|
6
|
+
|
|
7
|
+
describe('tools/humanInput', () => {
|
|
8
|
+
it('accepts short verification codes and removes whitespace', () => {
|
|
9
|
+
expect(sanitizeHumanCodeInput(' 123 456 ').value).toBe('123456');
|
|
10
|
+
expect(sanitizeHumanCodeInput('AB-12_cd').ok).toBe(true);
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it('rejects empty, long, and unsafe input', () => {
|
|
14
|
+
expect(sanitizeHumanCodeInput('')).toMatchObject({ ok: false, reason: 'empty' });
|
|
15
|
+
expect(sanitizeHumanCodeInput('a'.repeat(65))).toMatchObject({ ok: false, reason: 'too_long' });
|
|
16
|
+
expect(sanitizeHumanCodeInput('abc@example.com')).toMatchObject({ ok: false, reason: 'invalid_characters' });
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it('normalizes timeout and length limits', () => {
|
|
20
|
+
const low = normalizeHumanInputOptions({ timeoutSeconds: 1, maxLength: 1000 });
|
|
21
|
+
expect(low.timeoutSeconds).toBe(5);
|
|
22
|
+
expect(low.maxLength).toBe(64);
|
|
23
|
+
expect(low.timeoutClamped).toBe(true);
|
|
24
|
+
expect(low.maxLengthClamped).toBe(true);
|
|
25
|
+
});
|
|
26
|
+
});
|
|
@@ -15,6 +15,14 @@ describe('tools/toolSchema', () => {
|
|
|
15
15
|
expect(entry.function.parameters?.properties).toHaveProperty('dir');
|
|
16
16
|
});
|
|
17
17
|
|
|
18
|
+
it('includes request_human_input for verification codes', () => {
|
|
19
|
+
const entry = toolsSchema.find(t => t.function?.name === 'request_human_input');
|
|
20
|
+
expect(entry?.type).toBe('function');
|
|
21
|
+
expect(entry.function.description).toMatch(/verification code/i);
|
|
22
|
+
expect(entry.function.parameters?.properties?.codeType?.enum).toContain('totp');
|
|
23
|
+
expect(entry.function.parameters?.properties?.maxLength?.default).toBe(64);
|
|
24
|
+
});
|
|
25
|
+
|
|
18
26
|
it('defines navigate and download_file with required fields', () => {
|
|
19
27
|
const nav = toolsSchema.find(t => t.function?.name === 'navigate');
|
|
20
28
|
const download = toolsSchema.find(t => t.function?.name === 'download_file');
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { finalResponseHandler } from '../../tools/turnLoopUtils.js';
|
|
3
|
+
|
|
4
|
+
describe('tools/turnLoopUtils', () => {
|
|
5
|
+
it('redacts verification codes before printing and returning final responses', () => {
|
|
6
|
+
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
7
|
+
|
|
8
|
+
try {
|
|
9
|
+
const result = finalResponseHandler({
|
|
10
|
+
content: 'SUCCESS: Logged in, completed MFA (entered 123456), and reached the dashboard.',
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
expect(result).toMatchObject({ success: true });
|
|
14
|
+
expect(result.finalMessage).toContain('MFA (entered ā¢ā¢ā¢ā¢ā¢ā¢ (6))');
|
|
15
|
+
expect(result.finalMessage).not.toContain('123456');
|
|
16
|
+
|
|
17
|
+
const printed = logSpy.mock.calls.flat().join('\n');
|
|
18
|
+
expect(printed).toContain('MFA (entered ā¢ā¢ā¢ā¢ā¢ā¢ (6))');
|
|
19
|
+
expect(printed).not.toContain('123456');
|
|
20
|
+
} finally {
|
|
21
|
+
logSpy.mockRestore();
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
});
|
package/tools/chromeBrowser.js
CHANGED
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
import fs from 'fs';
|
|
23
23
|
import path from 'path';
|
|
24
24
|
import { ensureBrowsers } from '../tools/playwrightSetup.js';
|
|
25
|
+
import { requestHumanInput } from './humanInput.js';
|
|
25
26
|
|
|
26
27
|
const FILES_DIR = path.join('missions', 'files');
|
|
27
28
|
const REPORTS_DIR = path.join('missions', 'mission_reports');
|
|
@@ -1361,6 +1362,7 @@ export const CHROME_TOOL_MAP = {
|
|
|
1361
1362
|
upload_file: (b, args) => b.upload_file(args),
|
|
1362
1363
|
download_file: (b, args) => b.download_file(args),
|
|
1363
1364
|
list_local_files: (b, args) => b.list_local_files(args),
|
|
1365
|
+
request_human_input: (b, args, agentMemory) => requestHumanInput(args, agentMemory?.humanInput),
|
|
1364
1366
|
resource_progress: (b, args, agentMemory) => {
|
|
1365
1367
|
const prog = agentMemory?.docProgress;
|
|
1366
1368
|
if (!prog) return JSON.stringify({ enabled: false, total: 0, downloaded: 0, remaining: [] });
|
|
@@ -35,8 +35,8 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
35
35
|
// Escaping helpers for text and HTML attributes
|
|
36
36
|
const esc = (s) => String(s ?? '')
|
|
37
37
|
.replace(/&/g, '&').replace(/</g, '<')
|
|
38
|
-
.replace(/>/g, '>')
|
|
39
|
-
|
|
38
|
+
.replace(/>/g, '>')
|
|
39
|
+
.replace(/"/g, '"');
|
|
40
40
|
const badge = (status) =>
|
|
41
41
|
status === 'passed' ? 'ā
Passed' :
|
|
42
42
|
status === 'failed' ? 'ā Failed' : (status || 'ā');
|
|
@@ -62,6 +62,9 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
62
62
|
const attemptLabel = retryNumber > 0
|
|
63
63
|
? ` (re-attempt ${retryNumber}/${retryTotal || retryNumber})`
|
|
64
64
|
: '';
|
|
65
|
+
const humanInput = step.humanInput?.requested
|
|
66
|
+
? `<span class="hitl" title="Human-in-the-loop input: ${esc(step.humanInput.status || 'requested')}">š¤ Human in-the-loop</span>`
|
|
67
|
+
: '';
|
|
65
68
|
const imgTag = step.screenshotPath
|
|
66
69
|
? `<img src="${esc(step.screenshotPath)}" alt="screenshot turn ${esc(step.turn ?? idx)}">`
|
|
67
70
|
: '';
|
|
@@ -70,15 +73,16 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
70
73
|
? step.summary.trim()
|
|
71
74
|
: '';
|
|
72
75
|
const planSpan = plan
|
|
73
|
-
? `<span class="plan" title="${
|
|
76
|
+
? `<span class="plan" title="${esc(plan)}">${esc(plan)}</span>`
|
|
74
77
|
: '';
|
|
75
78
|
|
|
76
79
|
return `
|
|
77
80
|
<details class="step" ${ok ? '' : 'open'}>
|
|
78
81
|
<summary>
|
|
79
82
|
<span class="turn">Turn ${esc((step.turn ?? idx) + 1)}${esc(attemptLabel)}</span>
|
|
83
|
+
${humanInput}
|
|
80
84
|
${planSpan}
|
|
81
|
-
<span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${
|
|
85
|
+
<span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
|
|
82
86
|
<span class="tokens">tokens: ${esc(step.tokensUsed ?? 'ā')} / total: ${esc(step.totalTokensUsed ?? 'ā')}</span>
|
|
83
87
|
</summary>
|
|
84
88
|
<pre class="events">${esc(events.join('\n')) || '(no events)'}</pre>
|
|
@@ -276,6 +280,11 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
276
280
|
.step summary:hover{ background: rgba(255,255,255,.08); }
|
|
277
281
|
|
|
278
282
|
.turn{ font-weight:700; }
|
|
283
|
+
.hitl{
|
|
284
|
+
color:#fbbf24; border:1px solid rgba(251,191,36,.45);
|
|
285
|
+
background:rgba(251,191,36,.12); border-radius:999px;
|
|
286
|
+
padding:2px 8px; font-size:12px; font-weight:700;
|
|
287
|
+
}
|
|
279
288
|
.plan{
|
|
280
289
|
flex:1; color: var(--text-muted); font-size:12px;
|
|
281
290
|
white-space:nowrap; overflow:hidden; text-overflow:ellipsis;
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { createInterface } from 'node:readline';
|
|
2
|
+
import { Writable } from 'node:stream';
|
|
3
|
+
import { stdin as input, stdout as output } from 'node:process';
|
|
4
|
+
import { maskPreview } from '../core/redaction.js';
|
|
5
|
+
|
|
6
|
+
const DEFAULT_TIMEOUT_SECONDS = 60;
|
|
7
|
+
const MIN_TIMEOUT_SECONDS = 5;
|
|
8
|
+
const MAX_TIMEOUT_SECONDS = 300;
|
|
9
|
+
const DEFAULT_MAX_CODE_LENGTH = 64;
|
|
10
|
+
const MIN_CODE_LENGTH = 1;
|
|
11
|
+
const MAX_CODE_LENGTH = 64;
|
|
12
|
+
const CODE_RE = /^[A-Za-z0-9_-]+$/;
|
|
13
|
+
|
|
14
|
+
class HiddenInputOutput extends Writable {
|
|
15
|
+
constructor(realOutput) {
|
|
16
|
+
super();
|
|
17
|
+
this.realOutput = realOutput;
|
|
18
|
+
this.hidden = false;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
_write(chunk, encoding, callback) {
|
|
22
|
+
if (!this.hidden) {
|
|
23
|
+
this.realOutput.write(chunk, encoding);
|
|
24
|
+
}
|
|
25
|
+
callback();
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function clampNumber(raw, fallback, min, max) {
|
|
30
|
+
const n = Number(raw);
|
|
31
|
+
if (!Number.isFinite(n)) return { value: fallback, clamped: false };
|
|
32
|
+
const value = Math.min(max, Math.max(min, Math.trunc(n)));
|
|
33
|
+
return { value, clamped: value !== n };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function normalizeHumanInputOptions(opts = {}) {
|
|
37
|
+
const timeout = clampNumber(
|
|
38
|
+
opts.timeoutSeconds,
|
|
39
|
+
DEFAULT_TIMEOUT_SECONDS,
|
|
40
|
+
MIN_TIMEOUT_SECONDS,
|
|
41
|
+
MAX_TIMEOUT_SECONDS
|
|
42
|
+
);
|
|
43
|
+
const maxLength = clampNumber(
|
|
44
|
+
opts.maxLength,
|
|
45
|
+
DEFAULT_MAX_CODE_LENGTH,
|
|
46
|
+
MIN_CODE_LENGTH,
|
|
47
|
+
MAX_CODE_LENGTH
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
return {
|
|
51
|
+
enabled: opts.enabled !== false,
|
|
52
|
+
timeoutSeconds: timeout.value,
|
|
53
|
+
timeoutClamped: timeout.clamped,
|
|
54
|
+
maxLength: maxLength.value,
|
|
55
|
+
maxLengthClamped: maxLength.clamped,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function sanitizeHumanCodeInput(raw, { maxLength = DEFAULT_MAX_CODE_LENGTH } = {}) {
|
|
60
|
+
const normalizedMax = clampNumber(maxLength, DEFAULT_MAX_CODE_LENGTH, MIN_CODE_LENGTH, MAX_CODE_LENGTH).value;
|
|
61
|
+
const value = String(raw ?? '').trim().replace(/\s+/g, '');
|
|
62
|
+
|
|
63
|
+
if (!value) {
|
|
64
|
+
return { ok: false, reason: 'empty', value: '' };
|
|
65
|
+
}
|
|
66
|
+
if (value.length > normalizedMax) {
|
|
67
|
+
return { ok: false, reason: 'too_long', value: '' };
|
|
68
|
+
}
|
|
69
|
+
if (!CODE_RE.test(value)) {
|
|
70
|
+
return { ok: false, reason: 'invalid_characters', value: '' };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return { ok: true, value };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function questionWithHiddenInput(prompt, { timeoutMs } = {}) {
|
|
77
|
+
return new Promise((resolve, reject) => {
|
|
78
|
+
const maskedOutput = new HiddenInputOutput(output);
|
|
79
|
+
const rl = createInterface({
|
|
80
|
+
input,
|
|
81
|
+
output: maskedOutput,
|
|
82
|
+
terminal: Boolean(output.isTTY),
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
let settled = false;
|
|
86
|
+
let timeout;
|
|
87
|
+
|
|
88
|
+
function finish(err, answer) {
|
|
89
|
+
if (settled) return;
|
|
90
|
+
settled = true;
|
|
91
|
+
if (timeout) clearTimeout(timeout);
|
|
92
|
+
maskedOutput.hidden = false;
|
|
93
|
+
output.write('\n');
|
|
94
|
+
rl.close();
|
|
95
|
+
if (err) reject(err);
|
|
96
|
+
else resolve(answer);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
if (timeoutMs) {
|
|
100
|
+
timeout = setTimeout(() => {
|
|
101
|
+
const err = new Error('Human input timed out.');
|
|
102
|
+
err.name = 'AbortError';
|
|
103
|
+
finish(err);
|
|
104
|
+
}, timeoutMs);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
output.write(prompt);
|
|
108
|
+
maskedOutput.hidden = true;
|
|
109
|
+
rl.question('', (answer) => finish(null, answer));
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export async function requestHumanInput(args = {}, options = {}) {
|
|
114
|
+
const opts = normalizeHumanInputOptions({
|
|
115
|
+
...options,
|
|
116
|
+
maxLength: args.maxLength ?? options.maxLength,
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
if (!opts.enabled) {
|
|
120
|
+
throw new Error('Human input tool is disabled for this run.');
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const codeType = String(args.codeType || 'verification_code')
|
|
124
|
+
.trim()
|
|
125
|
+
.toLowerCase()
|
|
126
|
+
.replace(/[^a-z0-9_-]/g, '')
|
|
127
|
+
.slice(0, 40) || 'verification_code';
|
|
128
|
+
const prompt = String(args.prompt || `Enter ${codeType.replace(/_/g, ' ')}:`).trim();
|
|
129
|
+
const label = prompt.endsWith(':') ? prompt : `${prompt}:`;
|
|
130
|
+
const timeoutMs = opts.timeoutSeconds * 1000;
|
|
131
|
+
|
|
132
|
+
let answer;
|
|
133
|
+
try {
|
|
134
|
+
answer = await questionWithHiddenInput(`\nš¤ ${label} `, { timeoutMs });
|
|
135
|
+
} catch (err) {
|
|
136
|
+
if (err?.name === 'AbortError') {
|
|
137
|
+
throw new Error(`Human input timed out after ${opts.timeoutSeconds}s.`);
|
|
138
|
+
}
|
|
139
|
+
throw err;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const sanitized = sanitizeHumanCodeInput(answer, { maxLength: opts.maxLength });
|
|
143
|
+
if (!sanitized.ok) {
|
|
144
|
+
throw new Error(`Human input rejected: ${sanitized.reason}.`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return {
|
|
148
|
+
ok: true,
|
|
149
|
+
humanInputProvided: true,
|
|
150
|
+
codeType,
|
|
151
|
+
value: sanitized.value,
|
|
152
|
+
redactedValue: maskPreview(sanitized.value),
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export const __test__ = {
|
|
157
|
+
DEFAULT_TIMEOUT_SECONDS,
|
|
158
|
+
MIN_TIMEOUT_SECONDS,
|
|
159
|
+
MAX_TIMEOUT_SECONDS,
|
|
160
|
+
DEFAULT_MAX_CODE_LENGTH,
|
|
161
|
+
MAX_CODE_LENGTH,
|
|
162
|
+
sanitizeHumanCodeInput,
|
|
163
|
+
normalizeHumanInputOptions,
|
|
164
|
+
questionWithHiddenInput,
|
|
165
|
+
};
|