testronaut 1.3.5 ā 1.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devcontainer/devcontainer.json +19 -2
- package/.devcontainer/setup-github-auth.sh +36 -0
- package/bin/cli.js +67 -0
- package/bin/init.js +1 -0
- package/bin/initHelpers.js +4 -0
- package/core/agent.js +13 -1
- package/core/config.js +65 -0
- package/core/redaction.js +30 -7
- package/core/turnLoop.js +35 -4
- package/package.json +1 -1
- package/runner/testronaut.js +15 -2
- package/tests/binTests/init.test.js +6 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +2 -0
- package/tests/coreTests/config.test.js +43 -2
- package/tests/coreTests/redation.test.js +30 -0
- package/tests/coreTests/turnLoop.test.js +1 -0
- package/tests/runnerTests/testronaut.test.js +7 -5
- package/tests/toolsTests/humanInput.test.js +26 -0
- package/tests/toolsTests/toolSchema.test.js +8 -0
- package/tests/toolsTests/turnLoopUtils.test.js +24 -0
- package/tools/chromeBrowser.js +2 -0
- package/tools/generateHtmlReport.js +9 -0
- package/tools/humanInput.js +165 -0
- package/tools/toolSchema.js +28 -0
- package/tools/turnLoopUtils.js +10 -6
|
@@ -3,6 +3,20 @@
|
|
|
3
3
|
|
|
4
4
|
"image": "mcr.microsoft.com/devcontainers/javascript-node:22",
|
|
5
5
|
|
|
6
|
+
"features": {
|
|
7
|
+
"ghcr.io/devcontainers/features/github-cli:1": {}
|
|
8
|
+
},
|
|
9
|
+
|
|
10
|
+
"secrets": {
|
|
11
|
+
"GH_TOKEN": {
|
|
12
|
+
"description": "A GitHub PAT with read access to mission-testronaut/testronaut-demo."
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
|
|
16
|
+
"containerEnv": {
|
|
17
|
+
"GH_TOKEN": "${localEnv:GH_TOKEN}"
|
|
18
|
+
},
|
|
19
|
+
|
|
6
20
|
"customizations": {
|
|
7
21
|
"codespaces": {
|
|
8
22
|
"repositories": {
|
|
@@ -11,6 +25,9 @@
|
|
|
11
25
|
},
|
|
12
26
|
"mission-testronaut/testronaut-examples": {
|
|
13
27
|
"permissions": "write-all"
|
|
28
|
+
},
|
|
29
|
+
"mission-testronaut/testronaut-demo": {
|
|
30
|
+
"permissions": "write-all"
|
|
14
31
|
}
|
|
15
32
|
}
|
|
16
33
|
},
|
|
@@ -37,9 +54,9 @@
|
|
|
37
54
|
|
|
38
55
|
"workspaceFolder": "/workspaces",
|
|
39
56
|
|
|
40
|
-
"postCreateCommand": "bash
|
|
57
|
+
"postCreateCommand": "bash /workspaces/testronaut-cli/.devcontainer/setup-github-auth.sh",
|
|
41
58
|
|
|
42
|
-
"postStartCommand": "bash -
|
|
59
|
+
"postStartCommand": "bash /workspaces/testronaut-cli/.devcontainer/setup-github-auth.sh",
|
|
43
60
|
|
|
44
61
|
"hostRequirements": {
|
|
45
62
|
"cpus": 4,
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
cd /workspaces
|
|
5
|
+
|
|
6
|
+
git config --global pull.rebase false
|
|
7
|
+
git config --global init.defaultBranch main
|
|
8
|
+
|
|
9
|
+
if ! command -v gh >/dev/null 2>&1; then
|
|
10
|
+
echo "GitHub CLI is not installed. Rebuild the devcontainer so the github-cli feature is installed."
|
|
11
|
+
exit 1
|
|
12
|
+
fi
|
|
13
|
+
|
|
14
|
+
if [ -n "${GH_TOKEN:-}" ] && ! gh auth status --hostname github.com >/dev/null 2>&1; then
|
|
15
|
+
printf '%s\n' "$GH_TOKEN" | gh auth login --hostname github.com --with-token >/dev/null
|
|
16
|
+
fi
|
|
17
|
+
|
|
18
|
+
if gh auth status --hostname github.com >/dev/null 2>&1; then
|
|
19
|
+
gh auth setup-git --hostname github.com
|
|
20
|
+
else
|
|
21
|
+
echo "GitHub CLI is not authenticated."
|
|
22
|
+
echo "Add GH_TOKEN as a Codespaces secret with access to mission-testronaut/testronaut-demo."
|
|
23
|
+
exit 1
|
|
24
|
+
fi
|
|
25
|
+
|
|
26
|
+
clone_if_missing() {
|
|
27
|
+
local repo="$1"
|
|
28
|
+
local directory="$2"
|
|
29
|
+
|
|
30
|
+
if [ ! -d "$directory/.git" ]; then
|
|
31
|
+
gh repo clone "$repo" "$directory"
|
|
32
|
+
fi
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
clone_if_missing mission-testronaut/testronaut-examples testronaut-examples
|
|
36
|
+
clone_if_missing mission-testronaut/testronaut-demo testronaut-demo
|
package/bin/cli.js
CHANGED
|
@@ -364,6 +364,70 @@ if (retryFlagIndex >= 0) {
|
|
|
364
364
|
args.splice(retryFlagIndex, retryOverride ? 2 : 1);
|
|
365
365
|
}
|
|
366
366
|
|
|
367
|
+
// Look for --human-input / --no-human-input / --disable-human-input
|
|
368
|
+
let humanInputOverride;
|
|
369
|
+
let humanInputConsumesNext = false;
|
|
370
|
+
const humanInputFlagIndex = args.findIndex(a =>
|
|
371
|
+
a === '--human-input' ||
|
|
372
|
+
a.startsWith('--human-input=') ||
|
|
373
|
+
a === '--human_input' ||
|
|
374
|
+
a.startsWith('--human_input=') ||
|
|
375
|
+
a === '--no-human-input' ||
|
|
376
|
+
a === '--disable-human-input'
|
|
377
|
+
);
|
|
378
|
+
if (humanInputFlagIndex >= 0) {
|
|
379
|
+
const rawArg = args[humanInputFlagIndex];
|
|
380
|
+
if (rawArg.includes('=')) {
|
|
381
|
+
humanInputOverride = parseBool(rawArg.split('=')[1]);
|
|
382
|
+
} else if (rawArg === '--no-human-input' || rawArg === '--disable-human-input') {
|
|
383
|
+
humanInputOverride = false;
|
|
384
|
+
} else if (args[humanInputFlagIndex + 1] && !args[humanInputFlagIndex + 1].startsWith('-')) {
|
|
385
|
+
humanInputOverride = parseBool(args[humanInputFlagIndex + 1]);
|
|
386
|
+
humanInputConsumesNext = true;
|
|
387
|
+
} else {
|
|
388
|
+
humanInputOverride = true;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
if (humanInputOverride !== null) {
|
|
392
|
+
process.env.TESTRONAUT_HUMAN_INPUT = humanInputOverride ? '1' : '0';
|
|
393
|
+
console.log(`š¤ Human input tool ${humanInputOverride ? 'enabled' : 'disabled'} (TESTRONAUT_HUMAN_INPUT=${process.env.TESTRONAUT_HUMAN_INPUT})`);
|
|
394
|
+
} else {
|
|
395
|
+
console.warn('ā ļø Invalid --human-input value. Use true/false, 1/0, yes/no.');
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
const consume = 1 + (humanInputConsumesNext && humanInputOverride !== null ? 1 : 0);
|
|
399
|
+
args.splice(humanInputFlagIndex, consume);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
// Look for --human-input-timeout / --human_input_timeout in seconds
|
|
403
|
+
let humanInputTimeoutOverride;
|
|
404
|
+
const humanInputTimeoutFlagIndex = args.findIndex(a =>
|
|
405
|
+
a === '--human-input-timeout' ||
|
|
406
|
+
a.startsWith('--human-input-timeout=') ||
|
|
407
|
+
a === '--human_input_timeout' ||
|
|
408
|
+
a.startsWith('--human_input_timeout=')
|
|
409
|
+
);
|
|
410
|
+
if (humanInputTimeoutFlagIndex >= 0) {
|
|
411
|
+
const rawArg = args[humanInputTimeoutFlagIndex];
|
|
412
|
+
if (rawArg.includes('=')) {
|
|
413
|
+
humanInputTimeoutOverride = rawArg.split('=')[1];
|
|
414
|
+
} else if (args[humanInputTimeoutFlagIndex + 1]) {
|
|
415
|
+
humanInputTimeoutOverride = args[humanInputTimeoutFlagIndex + 1];
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
if (humanInputTimeoutOverride) {
|
|
419
|
+
const n = Number(humanInputTimeoutOverride.trim());
|
|
420
|
+
if (Number.isFinite(n) && n > 0) {
|
|
421
|
+
process.env.TESTRONAUT_HUMAN_INPUT_TIMEOUT_SECONDS = String(n);
|
|
422
|
+
console.log(`ā±ļø Human input timeout override: ${process.env.TESTRONAUT_HUMAN_INPUT_TIMEOUT_SECONDS}s`);
|
|
423
|
+
} else {
|
|
424
|
+
console.warn(`ā ļø Invalid --human-input-timeout value "${humanInputTimeoutOverride}". Ignoring.`);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
args.splice(humanInputTimeoutFlagIndex, humanInputTimeoutOverride ? 2 : 1);
|
|
429
|
+
}
|
|
430
|
+
|
|
367
431
|
const allResults = [];
|
|
368
432
|
const runId = `run_${Date.now()}`;
|
|
369
433
|
const startTime = new Date();
|
|
@@ -386,6 +450,9 @@ Options:
|
|
|
386
450
|
--provider=<id> Override LLM provider for this run (e.g., --provider=openai)
|
|
387
451
|
--dev Use the staging API base URL
|
|
388
452
|
--vercel-bypass=<secret> Send Vercel protection bypass header for protected deployments
|
|
453
|
+
--human-input[=<bool>] Allow the agent to pause for short verification codes (default: true)
|
|
454
|
+
--no-human-input Disable human-in-the-loop verification prompts for automated runs
|
|
455
|
+
--human-input-timeout=<s> Override human input wait timeout in seconds (default: 60)
|
|
389
456
|
--help Show this help message
|
|
390
457
|
--retry_limit=<n> Override agent turn retry limits (minimum 1, maximum 10)
|
|
391
458
|
|
package/bin/init.js
CHANGED
|
@@ -141,6 +141,7 @@ export async function initializeTestronautProject() {
|
|
|
141
141
|
config.maxTurns = config.maxTurns ?? defaults.maxTurns;
|
|
142
142
|
config.dom = { ...defaults.dom, ...(config.dom || {}) };
|
|
143
143
|
config.resourceGuard = { ...defaults.resourceGuard, ...(config.resourceGuard || {}) };
|
|
144
|
+
config.humanInput = { ...defaults.humanInput, ...(config.humanInput || {}) };
|
|
144
145
|
|
|
145
146
|
// āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā
|
|
146
147
|
// STEP 4: Persist config and scaffold .env (first run only)
|
package/bin/initHelpers.js
CHANGED
package/core/agent.js
CHANGED
|
@@ -36,7 +36,7 @@ import path from 'path';
|
|
|
36
36
|
* @param {string} missionName
|
|
37
37
|
* @param {number} [maxTurns=20] - upper bound for turnLoop per goal
|
|
38
38
|
* @param {number} [retryLimit]
|
|
39
|
-
* @param {{ domListLimit?: number|typeof Infinity, debug?: boolean, resourceGuard?: { enabled:boolean, hrefIncludes:string[], dataTypes:string[] } }} [opts]
|
|
39
|
+
* @param {{ domListLimit?: number|typeof Infinity, debug?: boolean, resourceGuard?: { enabled:boolean, hrefIncludes:string[], dataTypes:string[] }, humanInput?: { enabled:boolean, timeoutSeconds:number } }} [opts]
|
|
40
40
|
* @returns {Promise<Array<{missionName:string, submissionType:string, submissionName:string|null, status:'passed'|'failed', steps:any[], startTime:number, endTime:number}>>}
|
|
41
41
|
*/
|
|
42
42
|
export async function runAgent(goals, missionName, maxTurns = 20, retryLimit, opts = {}) {
|
|
@@ -135,6 +135,17 @@ Use Ground Control as your persistent mission memory about:
|
|
|
135
135
|
- Any constraints about staying on the correct site.
|
|
136
136
|
`.trim();
|
|
137
137
|
|
|
138
|
+
if (opts.humanInput?.enabled !== false) {
|
|
139
|
+
systemContent +=
|
|
140
|
+
'\n\n' +
|
|
141
|
+
`
|
|
142
|
+
Human-in-the-loop verification:
|
|
143
|
+
- If the app requires a TOTP, SMS, email, or similar short verification code that you cannot retrieve yourself, call request_human_input.
|
|
144
|
+
- Do not ask for passwords, API keys, or long free-form text with this tool.
|
|
145
|
+
- After receiving the code from the tool result, enter it into the appropriate field and continue the mission.
|
|
146
|
+
`.trim();
|
|
147
|
+
}
|
|
148
|
+
|
|
138
149
|
if (groundSummary) {
|
|
139
150
|
systemContent +=
|
|
140
151
|
'\n\n' +
|
|
@@ -164,6 +175,7 @@ Use Ground Control as your persistent mission memory about:
|
|
|
164
175
|
retryLimit,
|
|
165
176
|
groundControl,
|
|
166
177
|
resourceGuard: opts.resourceGuard,
|
|
178
|
+
humanInput: opts.humanInput,
|
|
167
179
|
onStep: (s) => {
|
|
168
180
|
// Append each step as a JSON line
|
|
169
181
|
try {
|
package/core/config.js
CHANGED
|
@@ -173,6 +173,71 @@ export function getResourceGuardConfig(cfg) {
|
|
|
173
173
|
return { enabled, hrefIncludes: finalHref, dataTypes: finalTypes };
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
+
function parseBool(raw) {
|
|
177
|
+
if (raw === undefined || raw === null) return null;
|
|
178
|
+
const lower = String(raw).trim().toLowerCase();
|
|
179
|
+
if (['1', 'true', 'yes', 'on', 'allow', 'enabled'].includes(lower)) return true;
|
|
180
|
+
if (['0', 'false', 'no', 'off', 'deny', 'disabled'].includes(lower)) return false;
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Resolve human-in-the-loop verification input settings.
|
|
186
|
+
* Priority: CLI/env overrides ā testronaut-config.json ā built-in defaults.
|
|
187
|
+
*
|
|
188
|
+
* Supported config:
|
|
189
|
+
* humanInput: { enabled: true, timeoutSeconds: 60 }
|
|
190
|
+
* humanInputEnabled / allowHumanInput
|
|
191
|
+
* humanInputTimeoutSeconds / humanInputTimeout
|
|
192
|
+
*
|
|
193
|
+
* Supported env (set by CLI flags too):
|
|
194
|
+
* TESTRONAUT_HUMAN_INPUT=true|false
|
|
195
|
+
* TESTRONAUT_HUMAN_INPUT_TIMEOUT_SECONDS=60
|
|
196
|
+
*
|
|
197
|
+
* @param {object} cfg
|
|
198
|
+
* @param {{ enabled?: boolean, timeoutSeconds?: number }} [fallback]
|
|
199
|
+
* @returns {{ enabled:boolean, timeoutSeconds:number, source:{ enabled:'env'|'config'|'default', timeout:'env'|'config'|'default' }, clamped:boolean }}
|
|
200
|
+
*/
|
|
201
|
+
export function getHumanInputConfig(cfg, fallback = { enabled: true, timeoutSeconds: 60 }) {
|
|
202
|
+
const clampTimeout = (raw, fb) => {
|
|
203
|
+
const n = Number(raw);
|
|
204
|
+
if (!Number.isFinite(n)) return { value: fb, clamped: false };
|
|
205
|
+
const clamped = Math.min(300, Math.max(5, Math.trunc(n)));
|
|
206
|
+
return { value: clamped, clamped: clamped !== n };
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
const envEnabled = parseBool(process.env.TESTRONAUT_HUMAN_INPUT);
|
|
210
|
+
const cfgEnabled = parseBool(cfg?.humanInput?.enabled ?? cfg?.humanInputEnabled ?? cfg?.allowHumanInput);
|
|
211
|
+
const enabled =
|
|
212
|
+
envEnabled !== null ? envEnabled :
|
|
213
|
+
cfgEnabled !== null ? cfgEnabled :
|
|
214
|
+
fallback.enabled !== false;
|
|
215
|
+
const enabledSource =
|
|
216
|
+
envEnabled !== null ? 'env' :
|
|
217
|
+
cfgEnabled !== null ? 'config' :
|
|
218
|
+
'default';
|
|
219
|
+
|
|
220
|
+
const cfgTimeoutRaw = cfg?.humanInput?.timeoutSeconds ?? cfg?.humanInput?.timeout ?? cfg?.humanInputTimeoutSeconds ?? cfg?.humanInputTimeout;
|
|
221
|
+
const envTimeoutRaw = process.env.TESTRONAUT_HUMAN_INPUT_TIMEOUT_SECONDS;
|
|
222
|
+
let timeoutSource = 'default';
|
|
223
|
+
let timeoutRaw = fallback.timeoutSeconds ?? 60;
|
|
224
|
+
if (envTimeoutRaw !== undefined) {
|
|
225
|
+
timeoutRaw = envTimeoutRaw;
|
|
226
|
+
timeoutSource = 'env';
|
|
227
|
+
} else if (cfgTimeoutRaw !== undefined) {
|
|
228
|
+
timeoutRaw = cfgTimeoutRaw;
|
|
229
|
+
timeoutSource = 'config';
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const timeout = clampTimeout(timeoutRaw, 60);
|
|
233
|
+
return {
|
|
234
|
+
enabled,
|
|
235
|
+
timeoutSeconds: timeout.value,
|
|
236
|
+
source: { enabled: enabledSource, timeout: timeoutSource },
|
|
237
|
+
clamped: timeout.clamped,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
|
|
176
241
|
/**
|
|
177
242
|
* Baseline limits for mission runs. Adjust here for global defaults.
|
|
178
243
|
*
|
package/core/redaction.js
CHANGED
|
@@ -44,8 +44,13 @@ export const SENSITIVE_KEYWORDS = [
|
|
|
44
44
|
'auth',
|
|
45
45
|
'pin',
|
|
46
46
|
'otp',
|
|
47
|
+
'code',
|
|
48
|
+
'totp',
|
|
49
|
+
'mfa',
|
|
47
50
|
];
|
|
48
51
|
|
|
52
|
+
const VERIFICATION_CODE_KEYWORDS = new Set(['otp', 'code', 'totp', 'mfa']);
|
|
53
|
+
|
|
49
54
|
/**
|
|
50
55
|
* Builds a single RegExp that treats hyphens as `[-_\s]?` to catch variants.
|
|
51
56
|
* @returns {RegExp}
|
|
@@ -141,6 +146,13 @@ function isSensitiveCall(fnName, args = {}) {
|
|
|
141
146
|
export function redactArgs(fnName, args = {}, { showLength = true } = {}) {
|
|
142
147
|
const out = clone(args);
|
|
143
148
|
|
|
149
|
+
if (fnName === 'request_human_input') {
|
|
150
|
+
for (const key of ['value', 'code', 'input', 'answer', 'redactedValue']) {
|
|
151
|
+
if (key in out) out[key] = maskPreview(out[key], showLength);
|
|
152
|
+
}
|
|
153
|
+
return out;
|
|
154
|
+
}
|
|
155
|
+
|
|
144
156
|
// Mask direct props with sensitive-looking keys
|
|
145
157
|
for (const k of Object.keys(out || {})) {
|
|
146
158
|
if (hasSensitiveHint(k)) {
|
|
@@ -165,8 +177,8 @@ export function redactArgs(fnName, args = {}, { showLength = true } = {}) {
|
|
|
165
177
|
* 1) Sensitive-word ... "VALUE"
|
|
166
178
|
* e.g., `password is "hunter2"`
|
|
167
179
|
*
|
|
168
|
-
* 2) Sensitive-word ... (with|as|to
|
|
169
|
-
* e.g., `token=abcd1234`, `auth with xyz`
|
|
180
|
+
* 2) Sensitive-word ... (with|as|to|=|entered|provided|used|submitted|filled) VALUE
|
|
181
|
+
* e.g., `token=abcd1234`, `auth with xyz`, `MFA (entered 123456)`
|
|
170
182
|
*
|
|
171
183
|
* 3) "VALUE" ... into/in/on/to TARGET (TARGET contains a sensitive hint)
|
|
172
184
|
* e.g., `Type "Imp0st3r123!" into #access`
|
|
@@ -179,18 +191,29 @@ export function redactPasswordInText(text = '', { showLength = true } = {}) {
|
|
|
179
191
|
let s = String(text ?? '');
|
|
180
192
|
|
|
181
193
|
// Build a reusable source pattern for SENSITIVE_KEYWORDS with hyphen expansion
|
|
182
|
-
const
|
|
183
|
-
|
|
194
|
+
const strongKwParts = SENSITIVE_KEYWORDS
|
|
195
|
+
.filter(k => !VERIFICATION_CODE_KEYWORDS.has(k))
|
|
196
|
+
.map(k => k.replace(/-/g, '[-_\\s]?'));
|
|
197
|
+
const verificationKwParts = [...VERIFICATION_CODE_KEYWORDS].map(k => k.replace(/-/g, '[-_\\s]?'));
|
|
198
|
+
const strongKwGroup = `(${strongKwParts.join('|')})`;
|
|
199
|
+
const verificationKwGroup = `(${verificationKwParts.join('|')})`;
|
|
184
200
|
|
|
185
201
|
// Case 1: keyword ... "VALUE"
|
|
186
202
|
s = s.replace(
|
|
187
|
-
new RegExp(`(\\b${
|
|
203
|
+
new RegExp(`(\\b${strongKwGroup}\\b[^"'\\\\\\n]{0,80}(?:[:=]|\\s)["'])([^"'\\n]+)(["'])`, 'gi'),
|
|
188
204
|
(_m, pre, _kw, secret, post) => pre + maskPreview(secret, showLength) + post
|
|
189
205
|
);
|
|
190
206
|
|
|
191
|
-
// Case 2: keyword ... (with|as|to
|
|
207
|
+
// Case 2: keyword ... (with|as|to|=|entered|provided|used|submitted|filled) VALUE
|
|
208
|
+
s = s.replace(
|
|
209
|
+
new RegExp(`(\\b${strongKwGroup}\\b[^.\\n]{0,80}?(?:\\b(with|as|to|entered|provided|used|submitted|filled)\\b|=)\\s*\\(?)([^\\s"'\\\`),.;:]+)`, 'gi'),
|
|
210
|
+
(_m, pre, _kw1, _kw2, secret) => pre + maskPreview(secret, showLength)
|
|
211
|
+
);
|
|
212
|
+
|
|
213
|
+
// Verification-code words are common UI text. Only redact them when the
|
|
214
|
+
// phrasing says a concrete value was entered/provided/etc.
|
|
192
215
|
s = s.replace(
|
|
193
|
-
new RegExp(`(\\b${
|
|
216
|
+
new RegExp(`(\\b${verificationKwGroup}\\b[^.\\n]{0,80}?\\b(entered|provided|used|submitted|filled)\\b\\s*\\(?)([A-Za-z0-9_-]{4,64})`, 'gi'),
|
|
194
217
|
(_m, pre, _kw1, _kw2, secret) => pre + maskPreview(secret, showLength)
|
|
195
218
|
);
|
|
196
219
|
|
package/core/turnLoop.js
CHANGED
|
@@ -46,7 +46,7 @@ import {
|
|
|
46
46
|
import { resolveProviderModel } from '../llm/modelResolver.js';
|
|
47
47
|
import { getLLM } from '../llm/llmFactory.js';
|
|
48
48
|
import { summarizeTurnIntentFromMessage } from './turnIntent.js';
|
|
49
|
-
import { redactArgs } from './redaction.js';
|
|
49
|
+
import { maskPreview, redactArgs } from './redaction.js';
|
|
50
50
|
import {
|
|
51
51
|
sanitizeHeavyToolHistory,
|
|
52
52
|
pruneConversationContext,
|
|
@@ -132,6 +132,7 @@ const FIRE_AND_FORGET_TOOLS = new Set([
|
|
|
132
132
|
'click_and_follow_popup',
|
|
133
133
|
'switch_to_page',
|
|
134
134
|
'close_current_page',
|
|
135
|
+
'request_human_input',
|
|
135
136
|
]);
|
|
136
137
|
|
|
137
138
|
/**
|
|
@@ -227,7 +228,11 @@ export const turnLoop = async (
|
|
|
227
228
|
const retryLimitClamped = Math.min(10, Math.max(1, Number.isFinite(retryLimitRaw) ? retryLimitRaw : DEFAULT_TURN_RETRY_LIMIT)); // retries allowed (excludes initial)
|
|
228
229
|
const maxAttempts = retryLimitClamped + 1; // includes initial attempt
|
|
229
230
|
ctx.groundControl = groundControl;
|
|
230
|
-
|
|
231
|
+
const humanInput = ctx.humanInput || { enabled: true, timeoutSeconds: 60 };
|
|
232
|
+
const activeToolsSchema = humanInput.enabled === false
|
|
233
|
+
? toolsSchema.filter(t => t?.function?.name !== 'request_human_input')
|
|
234
|
+
: toolsSchema;
|
|
235
|
+
let agentMemory = { lastMenuExpanded: false, humanInput };
|
|
231
236
|
ensureDocProgress(agentMemory, resourceGuardCfg);
|
|
232
237
|
let turnRetries = 0;
|
|
233
238
|
let stepSeq = ctx._stepSeq || 0;
|
|
@@ -325,7 +330,7 @@ export const turnLoop = async (
|
|
|
325
330
|
const { message, usage, headers } = await llm.chat({
|
|
326
331
|
model: MODEL_ID,
|
|
327
332
|
messages,
|
|
328
|
-
tools:
|
|
333
|
+
tools: activeToolsSchema,
|
|
329
334
|
});
|
|
330
335
|
response = { message, usage, headers };
|
|
331
336
|
|
|
@@ -412,6 +417,19 @@ export const turnLoop = async (
|
|
|
412
417
|
hadToolIssues = true;
|
|
413
418
|
}
|
|
414
419
|
|
|
420
|
+
if (fnName === 'request_human_input') {
|
|
421
|
+
step.humanInput = step.humanInput || {};
|
|
422
|
+
step.humanInput.requested = true;
|
|
423
|
+
step.humanInput.codeType = args.codeType || 'verification_code';
|
|
424
|
+
step.humanInput.timeoutSeconds = humanInput.timeoutSeconds;
|
|
425
|
+
step.humanInput.status = errorMessage
|
|
426
|
+
? (String(errorMessage).toLowerCase().includes('timed out') ? 'timeout' : 'invalid')
|
|
427
|
+
: 'provided';
|
|
428
|
+
step.events.push(errorMessage
|
|
429
|
+
? `š¤ Human-in-the-loop input ${step.humanInput.status}: ${errorMessage.replace(/^ERROR:\s*/, '')}`
|
|
430
|
+
: 'š¤ Human-in-the-loop input provided.');
|
|
431
|
+
}
|
|
432
|
+
|
|
415
433
|
// Capture and log any file upload/download events (for reports)
|
|
416
434
|
try {
|
|
417
435
|
const maybeJson = JSON.parse(result);
|
|
@@ -443,7 +461,20 @@ export const turnLoop = async (
|
|
|
443
461
|
}
|
|
444
462
|
|
|
445
463
|
console.log(`[tool ] ā ${fnName} result:`, errorMessage ? 'ā Failed' : 'ā
Success');
|
|
446
|
-
|
|
464
|
+
let resultForLog = result;
|
|
465
|
+
if (fnName === 'request_human_input') {
|
|
466
|
+
try {
|
|
467
|
+
const parsed = JSON.parse(result);
|
|
468
|
+
resultForLog = JSON.stringify({
|
|
469
|
+
...parsed,
|
|
470
|
+
value: maskPreview(parsed.value),
|
|
471
|
+
redactedValue: maskPreview(parsed.value),
|
|
472
|
+
});
|
|
473
|
+
} catch {
|
|
474
|
+
resultForLog = errorMessage || 'Human input received.';
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
const truncated = resultForLog.length > 1000 ? resultForLog.slice(0, 1000) + 'ā¦' : resultForLog;
|
|
447
478
|
step.events.push(`[tool ] ā ${fnName} result: ${errorMessage ? 'ā Failed' : 'ā
Success'}`);
|
|
448
479
|
step.events.push(`[tool ] ā ${truncated}`);
|
|
449
480
|
|
package/package.json
CHANGED
package/runner/testronaut.js
CHANGED
|
@@ -27,7 +27,7 @@ import fs from 'fs';
|
|
|
27
27
|
import path from 'path';
|
|
28
28
|
import { runAgent } from '../core/agent.js';
|
|
29
29
|
import { redactPasswordInText } from '../core/redaction.js';
|
|
30
|
-
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig } from '../core/config.js';
|
|
30
|
+
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../core/config.js';
|
|
31
31
|
|
|
32
32
|
// Check process env for debug toggles (shared helper for tests and CLI).
|
|
33
33
|
const isDebugEnabled = () => {
|
|
@@ -53,6 +53,7 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
53
53
|
const retryLimit = retryInfo.value;
|
|
54
54
|
const domListLimitInfo = getDomListLimit(cfg);
|
|
55
55
|
const resourceGuard = getResourceGuardConfig(cfg);
|
|
56
|
+
const humanInput = getHumanInputConfig(cfg);
|
|
56
57
|
const debugEnabled = isDebugEnabled();
|
|
57
58
|
if (notes.length) {
|
|
58
59
|
console.warn(notes.join('\n'));
|
|
@@ -66,6 +67,12 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
66
67
|
if (domListLimitInfo?.mode === 'all') {
|
|
67
68
|
console.warn('ā ļø DOM list limit is set to "all". This can dramatically increase token usage and may break flows on heavy pages.');
|
|
68
69
|
}
|
|
70
|
+
if (humanInput.clamped) {
|
|
71
|
+
console.warn(`ā ļø Human input timeout clamped to ${humanInput.timeoutSeconds}s (allowed 5-300).`);
|
|
72
|
+
}
|
|
73
|
+
if (!humanInput.enabled) {
|
|
74
|
+
console.log('š¤ Human-in-the-loop input tool disabled for this run.');
|
|
75
|
+
}
|
|
69
76
|
if (debugEnabled) {
|
|
70
77
|
const cfgDomRaw = cfg?.dom?.listItemLimit ?? cfg?.dom?.listLimit ?? cfg?.domListLimit;
|
|
71
78
|
console.log('[debug] Debug mode enabled');
|
|
@@ -89,6 +96,12 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
89
96
|
hrefIncludes: resourceGuard.hrefIncludes,
|
|
90
97
|
dataTypes: resourceGuard.dataTypes,
|
|
91
98
|
});
|
|
99
|
+
console.log('[debug] Human input', {
|
|
100
|
+
enabled: humanInput.enabled,
|
|
101
|
+
timeoutSeconds: humanInput.timeoutSeconds,
|
|
102
|
+
source: humanInput.source,
|
|
103
|
+
clamped: humanInput.clamped,
|
|
104
|
+
});
|
|
92
105
|
// Helpful for upload missions: show missions/files contents
|
|
93
106
|
const filesDir = path.resolve(process.cwd(), 'missions/files');
|
|
94
107
|
try {
|
|
@@ -188,7 +201,7 @@ export async function runMissions({ preMission, mission, postMission }, missionN
|
|
|
188
201
|
missionName,
|
|
189
202
|
maxTurns,
|
|
190
203
|
retryLimit,
|
|
191
|
-
{ domListLimit: domListLimitInfo?.value, debug: debugEnabled, resourceGuard }
|
|
204
|
+
{ domListLimit: domListLimitInfo?.value, debug: debugEnabled, resourceGuard, humanInput }
|
|
192
205
|
);
|
|
193
206
|
if (!success) {
|
|
194
207
|
console.log(`ā Aborting after failed goal.`);
|
|
@@ -79,6 +79,8 @@ describe('initializeTestronautProject', () => {
|
|
|
79
79
|
expect(typeof cfg.maxTurns).toBe('number');
|
|
80
80
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
81
81
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
82
|
+
expect(cfg.humanInput?.enabled).toBe(true);
|
|
83
|
+
expect(cfg.humanInput?.timeoutSeconds).toBe(60);
|
|
82
84
|
|
|
83
85
|
// .env
|
|
84
86
|
const envPath = path.join(temp, '.env');
|
|
@@ -112,6 +114,8 @@ describe('initializeTestronautProject', () => {
|
|
|
112
114
|
expect(cfg.model).toBe('gemini-2.5-flash');
|
|
113
115
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
114
116
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
117
|
+
expect(cfg.humanInput?.enabled).toBe(true);
|
|
118
|
+
expect(cfg.humanInput?.timeoutSeconds).toBe(60);
|
|
115
119
|
|
|
116
120
|
const envTxt = fs.readFileSync(path.join(temp, '.env'), 'utf8');
|
|
117
121
|
expect(envTxt).toMatch(/GEMINI_API_KEY=AIza/);
|
|
@@ -191,5 +195,7 @@ describe('initializeTestronautProject', () => {
|
|
|
191
195
|
expect(cfg.maxTurns).toBe(20); // default filled in
|
|
192
196
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
193
197
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
198
|
+
expect(cfg.humanInput?.enabled).toBe(true);
|
|
199
|
+
expect(cfg.humanInput?.timeoutSeconds).toBe(60);
|
|
194
200
|
});
|
|
195
201
|
});
|
|
@@ -11,5 +11,7 @@ describe('defaultConfig', () => {
|
|
|
11
11
|
expect(cfg.dom?.listItemLimit).toBe(3);
|
|
12
12
|
expect(cfg.resourceGuard?.enabled).toBe(true);
|
|
13
13
|
expect(cfg.resourceGuard?.hrefIncludes).toContain('/document/');
|
|
14
|
+
expect(cfg.humanInput?.enabled).toBe(true);
|
|
15
|
+
expect(cfg.humanInput?.timeoutSeconds).toBe(60);
|
|
14
16
|
});
|
|
15
17
|
});
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
2
|
|
|
3
|
+
const { readFile } = vi.hoisted(() => ({ readFile: vi.fn() }));
|
|
4
|
+
|
|
3
5
|
vi.mock('node:fs/promises', () => ({
|
|
4
|
-
readFile
|
|
6
|
+
readFile,
|
|
5
7
|
}));
|
|
6
8
|
|
|
7
|
-
import { readFile } from 'node:fs/promises';
|
|
8
9
|
import {
|
|
9
10
|
loadConfig,
|
|
10
11
|
getMaxTurns,
|
|
@@ -13,6 +14,7 @@ import {
|
|
|
13
14
|
enforceTurnBudget,
|
|
14
15
|
getDomListLimit,
|
|
15
16
|
getResourceGuardConfig,
|
|
17
|
+
getHumanInputConfig,
|
|
16
18
|
} from '../../core/config.js';
|
|
17
19
|
|
|
18
20
|
describe('core/config', () => {
|
|
@@ -201,6 +203,45 @@ describe('core/config', () => {
|
|
|
201
203
|
});
|
|
202
204
|
});
|
|
203
205
|
|
|
206
|
+
describe('getHumanInputConfig', () => {
|
|
207
|
+
const OLD_ENV = { ...process.env };
|
|
208
|
+
beforeEach(() => {
|
|
209
|
+
process.env = { ...OLD_ENV };
|
|
210
|
+
delete process.env.TESTRONAUT_HUMAN_INPUT;
|
|
211
|
+
delete process.env.TESTRONAUT_HUMAN_INPUT_TIMEOUT_SECONDS;
|
|
212
|
+
});
|
|
213
|
+
afterEach(() => {
|
|
214
|
+
process.env = { ...OLD_ENV };
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it('defaults to enabled with a 60 second timeout', () => {
|
|
218
|
+
const res = getHumanInputConfig({});
|
|
219
|
+
expect(res.enabled).toBe(true);
|
|
220
|
+
expect(res.timeoutSeconds).toBe(60);
|
|
221
|
+
expect(res.source.enabled).toBe('default');
|
|
222
|
+
expect(res.source.timeout).toBe('default');
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it('uses config values and clamps timeout', () => {
|
|
226
|
+
const res = getHumanInputConfig({ humanInput: { enabled: false, timeoutSeconds: 999 } });
|
|
227
|
+
expect(res.enabled).toBe(false);
|
|
228
|
+
expect(res.timeoutSeconds).toBe(300);
|
|
229
|
+
expect(res.source.enabled).toBe('config');
|
|
230
|
+
expect(res.source.timeout).toBe('config');
|
|
231
|
+
expect(res.clamped).toBe(true);
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
it('prefers env overrides over config', () => {
|
|
235
|
+
process.env.TESTRONAUT_HUMAN_INPUT = 'true';
|
|
236
|
+
process.env.TESTRONAUT_HUMAN_INPUT_TIMEOUT_SECONDS = '7';
|
|
237
|
+
const res = getHumanInputConfig({ humanInput: { enabled: false, timeoutSeconds: 120 } });
|
|
238
|
+
expect(res.enabled).toBe(true);
|
|
239
|
+
expect(res.timeoutSeconds).toBe(7);
|
|
240
|
+
expect(res.source.enabled).toBe('env');
|
|
241
|
+
expect(res.source.timeout).toBe('env');
|
|
242
|
+
});
|
|
243
|
+
});
|
|
244
|
+
|
|
204
245
|
describe('enforceTurnBudget (lenient)', () => {
|
|
205
246
|
it('clamps > hardMaxTurns', () => {
|
|
206
247
|
const cfg = { maxTurns: 1000 };
|
|
@@ -75,6 +75,14 @@ describe('redaction utilities', () => {
|
|
|
75
75
|
const redacted = redactArgs('fill', args);
|
|
76
76
|
expect(redacted.text).toBe('hello world');
|
|
77
77
|
});
|
|
78
|
+
|
|
79
|
+
it('masks human input tool values', () => {
|
|
80
|
+
const args = { prompt: 'Enter code', value: '123456', code: '654321' };
|
|
81
|
+
const redacted = redactArgs('request_human_input', args);
|
|
82
|
+
expect(redacted.prompt).toBe('Enter code');
|
|
83
|
+
expect(redacted.value).toBe('ā¢ā¢ā¢ā¢ā¢ā¢ (6)');
|
|
84
|
+
expect(redacted.code).toBe('ā¢ā¢ā¢ā¢ā¢ā¢ (6)');
|
|
85
|
+
});
|
|
78
86
|
});
|
|
79
87
|
|
|
80
88
|
describe('redactPasswordInText', () => {
|
|
@@ -91,6 +99,13 @@ describe('redaction utilities', () => {
|
|
|
91
99
|
expect(out).toContain(`auth with ā¢ā¢ā¢ā¢ā¢ā¢ (3)`);
|
|
92
100
|
});
|
|
93
101
|
|
|
102
|
+
it('masks MFA codes mentioned as entered in final text', () => {
|
|
103
|
+
const input = `SUCCESS: Logged in as ISAAC, completed MFA (entered 123456), and reached the dashboard.`;
|
|
104
|
+
const out = redactPasswordInText(input);
|
|
105
|
+
expect(out).toContain(`MFA (entered ā¢ā¢ā¢ā¢ā¢ā¢ (6))`);
|
|
106
|
+
expect(out).not.toContain('123456');
|
|
107
|
+
});
|
|
108
|
+
|
|
94
109
|
it('Case 3: "VALUE" into sensitive target (#access)', () => {
|
|
95
110
|
const input = `Type "Imp0st3r123!" into #access`;
|
|
96
111
|
const out = redactPasswordInText(input);
|
|
@@ -103,6 +118,21 @@ describe('redaction utilities', () => {
|
|
|
103
118
|
expect(out).toBe(input);
|
|
104
119
|
});
|
|
105
120
|
|
|
121
|
+
it('does not mangle MFA mission instructions while masking the access value', () => {
|
|
122
|
+
const input = `Fill the access code field (#access) with sword.
|
|
123
|
+
Check the "Challenge with MFA code" checkbox.
|
|
124
|
+
Set MFA code digits to 8.
|
|
125
|
+
There should be an MFA challenge. Prompt the user for an MFA code, then fill the MFA Code field (#mfa-code) with that code.
|
|
126
|
+
Click the "Verify Code" button (#mfa-submit).`;
|
|
127
|
+
const out = redactPasswordInText(input);
|
|
128
|
+
expect(out).toContain('Fill the access code field (#access) with ā¢ā¢ā¢ā¢ā¢ā¢ (5)');
|
|
129
|
+
expect(out).toContain('Check the "Challenge with MFA code" checkbox.');
|
|
130
|
+
expect(out).toContain('Set MFA code digits to 8.');
|
|
131
|
+
expect(out).toContain('with that code.');
|
|
132
|
+
expect(out).toContain('Click the "Verify Code" button (#mfa-submit).');
|
|
133
|
+
expect(out).not.toContain('sword');
|
|
134
|
+
});
|
|
135
|
+
|
|
106
136
|
it('handles api key forms with spaces/underscores/hyphens', () => {
|
|
107
137
|
const cases = [
|
|
108
138
|
`api-key "abcDEF123"`,
|
|
@@ -12,10 +12,11 @@ vi.mock('../../core/config.js', () => ({
|
|
|
12
12
|
getRetryLimit: vi.fn(),
|
|
13
13
|
getDomListLimit: vi.fn(),
|
|
14
14
|
getResourceGuardConfig: vi.fn(),
|
|
15
|
+
getHumanInputConfig: vi.fn(),
|
|
15
16
|
}));
|
|
16
17
|
|
|
17
18
|
import { runAgent } from '../../core/agent.js';
|
|
18
|
-
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig } from '../../core/config.js';
|
|
19
|
+
import { loadConfig, enforceTurnBudget, getRetryLimit, getDomListLimit, getResourceGuardConfig, getHumanInputConfig } from '../../core/config.js';
|
|
19
20
|
|
|
20
21
|
// Adjust the import path if your file lives elsewhere
|
|
21
22
|
import { runMissions, __test__ as testronautInternals } from '../../runner/testronaut.js';
|
|
@@ -29,6 +30,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
29
30
|
getRetryLimit.mockReturnValue({ value: 2, source: 'default', clamped: false });
|
|
30
31
|
getDomListLimit.mockReturnValue({ value: 3, mode: 'number', source: 'default', clamped: false });
|
|
31
32
|
getResourceGuardConfig.mockReturnValue({ enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] });
|
|
33
|
+
getHumanInputConfig.mockReturnValue({ enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false });
|
|
32
34
|
});
|
|
33
35
|
|
|
34
36
|
it('passes effectiveMax to runAgent and logs any notes', async () => {
|
|
@@ -56,7 +58,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
56
58
|
expect.stringContaining('Clamping to 200')
|
|
57
59
|
);
|
|
58
60
|
expect(runAgent).toHaveBeenCalledWith(
|
|
59
|
-
expect.any(Array), 'Budgeted Run', 200, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] } }
|
|
61
|
+
expect.any(Array), 'Budgeted Run', 200, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } }
|
|
60
62
|
);
|
|
61
63
|
|
|
62
64
|
warn.mockRestore();
|
|
@@ -83,7 +85,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
83
85
|
await runMissions({ mission: 'No warnings' }, 'Clean');
|
|
84
86
|
|
|
85
87
|
expect(warn).not.toHaveBeenCalled();
|
|
86
|
-
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20, 3, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] } });
|
|
88
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Clean', 20, 3, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
87
89
|
|
|
88
90
|
warn.mockRestore();
|
|
89
91
|
log.mockRestore();
|
|
@@ -176,7 +178,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
176
178
|
expect(goals[1].submissionName).toMatch(/^My Mission/);
|
|
177
179
|
|
|
178
180
|
// Effective max turns and retry limit passed through
|
|
179
|
-
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] } });
|
|
181
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'My Mission', 15, 2, { domListLimit: 3, debug: false, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
180
182
|
|
|
181
183
|
log.mockRestore();
|
|
182
184
|
});
|
|
@@ -215,7 +217,7 @@ describe('cli/testronaut.runMissions (with enforceTurnBudget)', () => {
|
|
|
215
217
|
|
|
216
218
|
await runMissions({ mission: 'Debug' }, 'Debug Mission');
|
|
217
219
|
|
|
218
|
-
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Debug Mission', 20, 2, { domListLimit: 3, debug: true, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] } });
|
|
220
|
+
expect(runAgent).toHaveBeenCalledWith(expect.any(Array), 'Debug Mission', 20, 2, { domListLimit: 3, debug: true, resourceGuard: { enabled: true, hrefIncludes: ['/document/'], dataTypes: ['document'] }, humanInput: { enabled: true, timeoutSeconds: 60, source: { enabled: 'default', timeout: 'default' }, clamped: false } });
|
|
219
221
|
|
|
220
222
|
log.mockRestore();
|
|
221
223
|
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import {
|
|
3
|
+
sanitizeHumanCodeInput,
|
|
4
|
+
normalizeHumanInputOptions,
|
|
5
|
+
} from '../../tools/humanInput.js';
|
|
6
|
+
|
|
7
|
+
describe('tools/humanInput', () => {
|
|
8
|
+
it('accepts short verification codes and removes whitespace', () => {
|
|
9
|
+
expect(sanitizeHumanCodeInput(' 123 456 ').value).toBe('123456');
|
|
10
|
+
expect(sanitizeHumanCodeInput('AB-12_cd').ok).toBe(true);
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it('rejects empty, long, and unsafe input', () => {
|
|
14
|
+
expect(sanitizeHumanCodeInput('')).toMatchObject({ ok: false, reason: 'empty' });
|
|
15
|
+
expect(sanitizeHumanCodeInput('a'.repeat(65))).toMatchObject({ ok: false, reason: 'too_long' });
|
|
16
|
+
expect(sanitizeHumanCodeInput('abc@example.com')).toMatchObject({ ok: false, reason: 'invalid_characters' });
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it('normalizes timeout and length limits', () => {
|
|
20
|
+
const low = normalizeHumanInputOptions({ timeoutSeconds: 1, maxLength: 1000 });
|
|
21
|
+
expect(low.timeoutSeconds).toBe(5);
|
|
22
|
+
expect(low.maxLength).toBe(64);
|
|
23
|
+
expect(low.timeoutClamped).toBe(true);
|
|
24
|
+
expect(low.maxLengthClamped).toBe(true);
|
|
25
|
+
});
|
|
26
|
+
});
|
|
@@ -15,6 +15,14 @@ describe('tools/toolSchema', () => {
|
|
|
15
15
|
expect(entry.function.parameters?.properties).toHaveProperty('dir');
|
|
16
16
|
});
|
|
17
17
|
|
|
18
|
+
it('includes request_human_input for verification codes', () => {
|
|
19
|
+
const entry = toolsSchema.find(t => t.function?.name === 'request_human_input');
|
|
20
|
+
expect(entry?.type).toBe('function');
|
|
21
|
+
expect(entry.function.description).toMatch(/verification code/i);
|
|
22
|
+
expect(entry.function.parameters?.properties?.codeType?.enum).toContain('totp');
|
|
23
|
+
expect(entry.function.parameters?.properties?.maxLength?.default).toBe(64);
|
|
24
|
+
});
|
|
25
|
+
|
|
18
26
|
it('defines navigate and download_file with required fields', () => {
|
|
19
27
|
const nav = toolsSchema.find(t => t.function?.name === 'navigate');
|
|
20
28
|
const download = toolsSchema.find(t => t.function?.name === 'download_file');
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { finalResponseHandler } from '../../tools/turnLoopUtils.js';
|
|
3
|
+
|
|
4
|
+
describe('tools/turnLoopUtils', () => {
|
|
5
|
+
it('redacts verification codes before printing and returning final responses', () => {
|
|
6
|
+
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
7
|
+
|
|
8
|
+
try {
|
|
9
|
+
const result = finalResponseHandler({
|
|
10
|
+
content: 'SUCCESS: Logged in, completed MFA (entered 123456), and reached the dashboard.',
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
expect(result).toMatchObject({ success: true });
|
|
14
|
+
expect(result.finalMessage).toContain('MFA (entered ā¢ā¢ā¢ā¢ā¢ā¢ (6))');
|
|
15
|
+
expect(result.finalMessage).not.toContain('123456');
|
|
16
|
+
|
|
17
|
+
const printed = logSpy.mock.calls.flat().join('\n');
|
|
18
|
+
expect(printed).toContain('MFA (entered ā¢ā¢ā¢ā¢ā¢ā¢ (6))');
|
|
19
|
+
expect(printed).not.toContain('123456');
|
|
20
|
+
} finally {
|
|
21
|
+
logSpy.mockRestore();
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
});
|
package/tools/chromeBrowser.js
CHANGED
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
import fs from 'fs';
|
|
23
23
|
import path from 'path';
|
|
24
24
|
import { ensureBrowsers } from '../tools/playwrightSetup.js';
|
|
25
|
+
import { requestHumanInput } from './humanInput.js';
|
|
25
26
|
|
|
26
27
|
const FILES_DIR = path.join('missions', 'files');
|
|
27
28
|
const REPORTS_DIR = path.join('missions', 'mission_reports');
|
|
@@ -1361,6 +1362,7 @@ export const CHROME_TOOL_MAP = {
|
|
|
1361
1362
|
upload_file: (b, args) => b.upload_file(args),
|
|
1362
1363
|
download_file: (b, args) => b.download_file(args),
|
|
1363
1364
|
list_local_files: (b, args) => b.list_local_files(args),
|
|
1365
|
+
request_human_input: (b, args, agentMemory) => requestHumanInput(args, agentMemory?.humanInput),
|
|
1364
1366
|
resource_progress: (b, args, agentMemory) => {
|
|
1365
1367
|
const prog = agentMemory?.docProgress;
|
|
1366
1368
|
if (!prog) return JSON.stringify({ enabled: false, total: 0, downloaded: 0, remaining: [] });
|
|
@@ -62,6 +62,9 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
62
62
|
const attemptLabel = retryNumber > 0
|
|
63
63
|
? ` (re-attempt ${retryNumber}/${retryTotal || retryNumber})`
|
|
64
64
|
: '';
|
|
65
|
+
const humanInput = step.humanInput?.requested
|
|
66
|
+
? `<span class="hitl" title="Human-in-the-loop input: ${esc(step.humanInput.status || 'requested')}">š¤ Human in-the-loop</span>`
|
|
67
|
+
: '';
|
|
65
68
|
const imgTag = step.screenshotPath
|
|
66
69
|
? `<img src="${esc(step.screenshotPath)}" alt="screenshot turn ${esc(step.turn ?? idx)}">`
|
|
67
70
|
: '';
|
|
@@ -77,6 +80,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
77
80
|
<details class="step" ${ok ? '' : 'open'}>
|
|
78
81
|
<summary>
|
|
79
82
|
<span class="turn">Turn ${esc((step.turn ?? idx) + 1)}${esc(attemptLabel)}</span>
|
|
83
|
+
${humanInput}
|
|
80
84
|
${planSpan}
|
|
81
85
|
<span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
|
|
82
86
|
<span class="tokens">tokens: ${esc(step.tokensUsed ?? 'ā')} / total: ${esc(step.totalTokensUsed ?? 'ā')}</span>
|
|
@@ -276,6 +280,11 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
276
280
|
.step summary:hover{ background: rgba(255,255,255,.08); }
|
|
277
281
|
|
|
278
282
|
.turn{ font-weight:700; }
|
|
283
|
+
.hitl{
|
|
284
|
+
color:#fbbf24; border:1px solid rgba(251,191,36,.45);
|
|
285
|
+
background:rgba(251,191,36,.12); border-radius:999px;
|
|
286
|
+
padding:2px 8px; font-size:12px; font-weight:700;
|
|
287
|
+
}
|
|
279
288
|
.plan{
|
|
280
289
|
flex:1; color: var(--text-muted); font-size:12px;
|
|
281
290
|
white-space:nowrap; overflow:hidden; text-overflow:ellipsis;
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { createInterface } from 'node:readline';
|
|
2
|
+
import { Writable } from 'node:stream';
|
|
3
|
+
import { stdin as input, stdout as output } from 'node:process';
|
|
4
|
+
import { maskPreview } from '../core/redaction.js';
|
|
5
|
+
|
|
6
|
+
const DEFAULT_TIMEOUT_SECONDS = 60;
|
|
7
|
+
const MIN_TIMEOUT_SECONDS = 5;
|
|
8
|
+
const MAX_TIMEOUT_SECONDS = 300;
|
|
9
|
+
const DEFAULT_MAX_CODE_LENGTH = 64;
|
|
10
|
+
const MIN_CODE_LENGTH = 1;
|
|
11
|
+
const MAX_CODE_LENGTH = 64;
|
|
12
|
+
const CODE_RE = /^[A-Za-z0-9_-]+$/;
|
|
13
|
+
|
|
14
|
+
class HiddenInputOutput extends Writable {
|
|
15
|
+
constructor(realOutput) {
|
|
16
|
+
super();
|
|
17
|
+
this.realOutput = realOutput;
|
|
18
|
+
this.hidden = false;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
_write(chunk, encoding, callback) {
|
|
22
|
+
if (!this.hidden) {
|
|
23
|
+
this.realOutput.write(chunk, encoding);
|
|
24
|
+
}
|
|
25
|
+
callback();
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function clampNumber(raw, fallback, min, max) {
|
|
30
|
+
const n = Number(raw);
|
|
31
|
+
if (!Number.isFinite(n)) return { value: fallback, clamped: false };
|
|
32
|
+
const value = Math.min(max, Math.max(min, Math.trunc(n)));
|
|
33
|
+
return { value, clamped: value !== n };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function normalizeHumanInputOptions(opts = {}) {
|
|
37
|
+
const timeout = clampNumber(
|
|
38
|
+
opts.timeoutSeconds,
|
|
39
|
+
DEFAULT_TIMEOUT_SECONDS,
|
|
40
|
+
MIN_TIMEOUT_SECONDS,
|
|
41
|
+
MAX_TIMEOUT_SECONDS
|
|
42
|
+
);
|
|
43
|
+
const maxLength = clampNumber(
|
|
44
|
+
opts.maxLength,
|
|
45
|
+
DEFAULT_MAX_CODE_LENGTH,
|
|
46
|
+
MIN_CODE_LENGTH,
|
|
47
|
+
MAX_CODE_LENGTH
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
return {
|
|
51
|
+
enabled: opts.enabled !== false,
|
|
52
|
+
timeoutSeconds: timeout.value,
|
|
53
|
+
timeoutClamped: timeout.clamped,
|
|
54
|
+
maxLength: maxLength.value,
|
|
55
|
+
maxLengthClamped: maxLength.clamped,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function sanitizeHumanCodeInput(raw, { maxLength = DEFAULT_MAX_CODE_LENGTH } = {}) {
|
|
60
|
+
const normalizedMax = clampNumber(maxLength, DEFAULT_MAX_CODE_LENGTH, MIN_CODE_LENGTH, MAX_CODE_LENGTH).value;
|
|
61
|
+
const value = String(raw ?? '').trim().replace(/\s+/g, '');
|
|
62
|
+
|
|
63
|
+
if (!value) {
|
|
64
|
+
return { ok: false, reason: 'empty', value: '' };
|
|
65
|
+
}
|
|
66
|
+
if (value.length > normalizedMax) {
|
|
67
|
+
return { ok: false, reason: 'too_long', value: '' };
|
|
68
|
+
}
|
|
69
|
+
if (!CODE_RE.test(value)) {
|
|
70
|
+
return { ok: false, reason: 'invalid_characters', value: '' };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return { ok: true, value };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function questionWithHiddenInput(prompt, { timeoutMs } = {}) {
|
|
77
|
+
return new Promise((resolve, reject) => {
|
|
78
|
+
const maskedOutput = new HiddenInputOutput(output);
|
|
79
|
+
const rl = createInterface({
|
|
80
|
+
input,
|
|
81
|
+
output: maskedOutput,
|
|
82
|
+
terminal: Boolean(output.isTTY),
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
let settled = false;
|
|
86
|
+
let timeout;
|
|
87
|
+
|
|
88
|
+
function finish(err, answer) {
|
|
89
|
+
if (settled) return;
|
|
90
|
+
settled = true;
|
|
91
|
+
if (timeout) clearTimeout(timeout);
|
|
92
|
+
maskedOutput.hidden = false;
|
|
93
|
+
output.write('\n');
|
|
94
|
+
rl.close();
|
|
95
|
+
if (err) reject(err);
|
|
96
|
+
else resolve(answer);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
if (timeoutMs) {
|
|
100
|
+
timeout = setTimeout(() => {
|
|
101
|
+
const err = new Error('Human input timed out.');
|
|
102
|
+
err.name = 'AbortError';
|
|
103
|
+
finish(err);
|
|
104
|
+
}, timeoutMs);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
output.write(prompt);
|
|
108
|
+
maskedOutput.hidden = true;
|
|
109
|
+
rl.question('', (answer) => finish(null, answer));
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export async function requestHumanInput(args = {}, options = {}) {
|
|
114
|
+
const opts = normalizeHumanInputOptions({
|
|
115
|
+
...options,
|
|
116
|
+
maxLength: args.maxLength ?? options.maxLength,
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
if (!opts.enabled) {
|
|
120
|
+
throw new Error('Human input tool is disabled for this run.');
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const codeType = String(args.codeType || 'verification_code')
|
|
124
|
+
.trim()
|
|
125
|
+
.toLowerCase()
|
|
126
|
+
.replace(/[^a-z0-9_-]/g, '')
|
|
127
|
+
.slice(0, 40) || 'verification_code';
|
|
128
|
+
const prompt = String(args.prompt || `Enter ${codeType.replace(/_/g, ' ')}:`).trim();
|
|
129
|
+
const label = prompt.endsWith(':') ? prompt : `${prompt}:`;
|
|
130
|
+
const timeoutMs = opts.timeoutSeconds * 1000;
|
|
131
|
+
|
|
132
|
+
let answer;
|
|
133
|
+
try {
|
|
134
|
+
answer = await questionWithHiddenInput(`\nš¤ ${label} `, { timeoutMs });
|
|
135
|
+
} catch (err) {
|
|
136
|
+
if (err?.name === 'AbortError') {
|
|
137
|
+
throw new Error(`Human input timed out after ${opts.timeoutSeconds}s.`);
|
|
138
|
+
}
|
|
139
|
+
throw err;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const sanitized = sanitizeHumanCodeInput(answer, { maxLength: opts.maxLength });
|
|
143
|
+
if (!sanitized.ok) {
|
|
144
|
+
throw new Error(`Human input rejected: ${sanitized.reason}.`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return {
|
|
148
|
+
ok: true,
|
|
149
|
+
humanInputProvided: true,
|
|
150
|
+
codeType,
|
|
151
|
+
value: sanitized.value,
|
|
152
|
+
redactedValue: maskPreview(sanitized.value),
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export const __test__ = {
|
|
157
|
+
DEFAULT_TIMEOUT_SECONDS,
|
|
158
|
+
MIN_TIMEOUT_SECONDS,
|
|
159
|
+
MAX_TIMEOUT_SECONDS,
|
|
160
|
+
DEFAULT_MAX_CODE_LENGTH,
|
|
161
|
+
MAX_CODE_LENGTH,
|
|
162
|
+
sanitizeHumanCodeInput,
|
|
163
|
+
normalizeHumanInputOptions,
|
|
164
|
+
questionWithHiddenInput,
|
|
165
|
+
};
|
package/tools/toolSchema.js
CHANGED
|
@@ -328,6 +328,34 @@ const toolsSchema = [
|
|
|
328
328
|
},
|
|
329
329
|
},
|
|
330
330
|
},
|
|
331
|
+
{
|
|
332
|
+
type: 'function',
|
|
333
|
+
function: {
|
|
334
|
+
name: 'request_human_input',
|
|
335
|
+
description:
|
|
336
|
+
'Pause for a human to enter a short verification code such as TOTP, SMS, or email confirmation. Use only when the page requires a code the agent cannot obtain itself.',
|
|
337
|
+
parameters: {
|
|
338
|
+
type: 'object',
|
|
339
|
+
properties: {
|
|
340
|
+
prompt: {
|
|
341
|
+
type: 'string',
|
|
342
|
+
description: 'Short prompt shown to the human. Do not ask for passwords or long free-form text.'
|
|
343
|
+
},
|
|
344
|
+
codeType: {
|
|
345
|
+
type: 'string',
|
|
346
|
+
enum: ['totp', 'sms', 'email', 'verification_code', 'other_code'],
|
|
347
|
+
default: 'verification_code'
|
|
348
|
+
},
|
|
349
|
+
maxLength: {
|
|
350
|
+
type: 'number',
|
|
351
|
+
default: 64,
|
|
352
|
+
description: 'Maximum accepted code length. Values are clamped to 1-64.'
|
|
353
|
+
}
|
|
354
|
+
},
|
|
355
|
+
required: ['prompt']
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
},
|
|
331
359
|
{
|
|
332
360
|
type: 'function',
|
|
333
361
|
function: {
|
package/tools/turnLoopUtils.js
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
|
+
import { redactPasswordInText } from '../core/redaction.js';
|
|
2
|
+
|
|
1
3
|
export const finalResponseHandler = (msg) => {
|
|
2
|
-
const
|
|
4
|
+
const content = String(msg.content ?? '');
|
|
5
|
+
const safeContent = redactPasswordInText(content);
|
|
6
|
+
const final = content.trim().toLowerCase();
|
|
3
7
|
if (final?.startsWith('success')) {
|
|
4
8
|
console.log('\nāā FINAL AGENT RESPONSE āāāāāāāāāāāāāāāāāāā');
|
|
5
|
-
console.log(
|
|
9
|
+
console.log(safeContent);
|
|
6
10
|
console.log('āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā');
|
|
7
|
-
return { finalMessage:
|
|
11
|
+
return { finalMessage: safeContent, success: true};
|
|
8
12
|
}
|
|
9
13
|
if (final?.startsWith('failure')) {
|
|
10
14
|
console.log('\nāā FINAL AGENT RESPONSE āāāāāāāāāāāāāāāāāāā');
|
|
11
|
-
console.log(
|
|
15
|
+
console.log(safeContent);
|
|
12
16
|
console.log('āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā');
|
|
13
|
-
return { finalMessage:
|
|
17
|
+
return { finalMessage: safeContent, success: false};
|
|
14
18
|
}
|
|
15
19
|
return null;
|
|
16
20
|
}
|
|
@@ -62,4 +66,4 @@ export function validateAndInsertMissingToolResponses(messages, options = {}) {
|
|
|
62
66
|
}
|
|
63
67
|
|
|
64
68
|
return true;
|
|
65
|
-
}
|
|
69
|
+
}
|