testronaut 2.1.7 โ 2.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/core/agent.js +10 -2
- package/core/turnLoop.js +56 -9
- package/package.json +1 -1
- package/runner/testronaut.js +18 -5
- package/tests/coreTests/turnLoop.test.js +46 -11
- package/tests/toolsTests/generateHtmlReport.test.js +30 -8
- package/tools/generateHtmlReport.js +25 -8
package/README.md
CHANGED
|
@@ -622,6 +622,20 @@ Each includes:
|
|
|
622
622
|
- Screenshots
|
|
623
623
|
- Pass/Fail summaries
|
|
624
624
|
|
|
625
|
+
### Status language
|
|
626
|
+
|
|
627
|
+
Turn and tool statuses describe different layers of a run:
|
|
628
|
+
|
|
629
|
+
- `๐ฐ๏ธ Engaged` means a tool invocation ran without a structured operation verdict. It does not mean the goal was achieved.
|
|
630
|
+
- `โ
Action confirmed` and `โ ๏ธ Action unconfirmed` reflect an explicit structured tool outcome such as `ok` or `passed`.
|
|
631
|
+
- `๐ Progressing` marks an intermediate turn. Intermediate turns are not pass/fail verdicts.
|
|
632
|
+
- `๐ Replotting course` and `โ ๏ธ Navigation issue` identify retries and tool/action problems.
|
|
633
|
+
- `โ
Mission accomplished ยท evidence recorded` means the successful verdict is supported by recorded browser observations, screenshots, matched dialogs, or assertion results.
|
|
634
|
+
- `โ
Mission accomplished ยท independently verified` means a separate verifier such as the Jev completion gate confirmed the outcome.
|
|
635
|
+
- `โ ๏ธ Completion reported ยท limited evidence` means the model declared success without enough supporting evidence in the trace. `โ Mission incomplete` is the final failure outcome. Reports retain the machine-readable `passed`, `recovered`, and `failed` mission statuses for compatibility.
|
|
636
|
+
|
|
637
|
+
For example, clicking **Create Work Package** can be an engaged or confirmed action while the mission remains **Progressing**. Creation should only be described as confirmed after evidence shows that the work package exists.
|
|
638
|
+
|
|
625
639
|
---
|
|
626
640
|
|
|
627
641
|
## ๐งช Under the Hood
|
package/core/agent.js
CHANGED
|
@@ -99,7 +99,7 @@ export async function runAgent(goals, missionName, maxTurns = 20, retryLimit, op
|
|
|
99
99
|
const step = {
|
|
100
100
|
turn: 0,
|
|
101
101
|
events: [`๐ก Launch protocol probe passed: ${protocol.id}`],
|
|
102
|
-
result: 'โ
Mission
|
|
102
|
+
result: 'โ
Mission accomplished',
|
|
103
103
|
summary: `Reused launch protocol ${protocol.id}`,
|
|
104
104
|
missionName,
|
|
105
105
|
tokensUsed: 0,
|
|
@@ -117,6 +117,7 @@ export async function runAgent(goals, missionName, maxTurns = 20, retryLimit, op
|
|
|
117
117
|
startTime: now,
|
|
118
118
|
endTime: now,
|
|
119
119
|
durationMs: 0,
|
|
120
|
+
verification: 'verified',
|
|
120
121
|
telemetry: [],
|
|
121
122
|
launchProtocol: { id: protocol.id, status: 'reused', probe: 'passed' },
|
|
122
123
|
});
|
|
@@ -321,6 +322,10 @@ JavaScript dialogs:
|
|
|
321
322
|
submissionType: goal.submissionType || goal.label,
|
|
322
323
|
submissionName: goal.submissionName || null,
|
|
323
324
|
status: succeeded ? (recoveries.length ? 'recovered' : 'passed') : 'failed',
|
|
325
|
+
verification: succeeded
|
|
326
|
+
? (result?.completedBy === 'jev_gate' ? 'verified' : result?.evidenceRecorded ? 'evidence_recorded' : 'limited_evidence')
|
|
327
|
+
: 'failed',
|
|
328
|
+
...(result?.evidenceSources?.length ? { evidenceSources: result.evidenceSources } : {}),
|
|
324
329
|
steps: JSON.parse(JSON.stringify(compact)), // last 20 (by memory design)
|
|
325
330
|
stepFile, // full history is in this JSONL (one step per line)
|
|
326
331
|
startTime: goalStartTime,
|
|
@@ -342,7 +347,10 @@ JavaScript dialogs:
|
|
|
342
347
|
}
|
|
343
348
|
}
|
|
344
349
|
|
|
345
|
-
|
|
350
|
+
const limitedEvidence = missionResults.some(mission => mission.verification === 'limited_evidence');
|
|
351
|
+
console.log(limitedEvidence
|
|
352
|
+
? 'โ ๏ธ Completion reported ยท limited evidence.\n'
|
|
353
|
+
: 'โ
Mission accomplished.\n');
|
|
346
354
|
return missionResults;
|
|
347
355
|
} finally {
|
|
348
356
|
await browser.close();
|
package/core/turnLoop.js
CHANGED
|
@@ -189,6 +189,27 @@ async function captureBrowserEvidence(browser, html, previous) {
|
|
|
189
189
|
return mergeBrowserControlState(evidence, states);
|
|
190
190
|
}
|
|
191
191
|
|
|
192
|
+
function collectMissionEvidenceSources(recordedSteps, agentMemory) {
|
|
193
|
+
const sources = new Set();
|
|
194
|
+
if (agentMemory?.browserEvidence) sources.add('browser_state');
|
|
195
|
+
|
|
196
|
+
for (const recordedStep of recordedSteps) {
|
|
197
|
+
if (recordedStep?.screenshotPath) sources.add('screenshot');
|
|
198
|
+
if (recordedStep?.dialogs?.some(dialog => dialog?.matched && dialog?.expected)) {
|
|
199
|
+
sources.add('matched_dialog');
|
|
200
|
+
}
|
|
201
|
+
const events = Array.isArray(recordedStep?.events) ? recordedStep.events : [];
|
|
202
|
+
if (events.some(event => /\[model\] โ (?:get_dom|check_text|probe)\b/.test(event))) {
|
|
203
|
+
sources.add('browser_observation');
|
|
204
|
+
}
|
|
205
|
+
if (events.some(event => /\[tool \] โ FOUND:/.test(event))) {
|
|
206
|
+
sources.add('assertion_result');
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return [...sources];
|
|
211
|
+
}
|
|
212
|
+
|
|
192
213
|
/**
|
|
193
214
|
* Utility: format byte counts into human-readable strings.
|
|
194
215
|
* Used when reporting upload/download events in mission logs.
|
|
@@ -402,7 +423,7 @@ export const turnLoop = async (
|
|
|
402
423
|
|
|
403
424
|
step.jevGate = { triggered: true, ...gate };
|
|
404
425
|
step.events.push(`๐ฌ Jev completion gate triggered at ${Math.round(gate.threshold * 100)}% threshold`);
|
|
405
|
-
step.result = 'โ
Mission
|
|
426
|
+
step.result = 'โ
Mission accomplished (Jev verified)';
|
|
406
427
|
recordStep(step);
|
|
407
428
|
const finalMessage = 'SUCCESS: Jev completion gate found sufficient browser evidence.';
|
|
408
429
|
return { success: true, finalMessage, steps: stepsArchive, completedBy: 'jev_gate' };
|
|
@@ -837,7 +858,21 @@ export const turnLoop = async (
|
|
|
837
858
|
// non-JSON results ignored
|
|
838
859
|
}
|
|
839
860
|
|
|
840
|
-
|
|
861
|
+
// A tool returning without throwing only proves that its invocation ran.
|
|
862
|
+
// It does not prove that the requested operation or mission goal succeeded.
|
|
863
|
+
let toolStatusLabel = errorMessage ? 'โ System fault' : '๐ฐ๏ธ Engaged';
|
|
864
|
+
if (!errorMessage) {
|
|
865
|
+
try {
|
|
866
|
+
const parsed = JSON.parse(result);
|
|
867
|
+
const operationOutcome = typeof parsed?.ok === 'boolean'
|
|
868
|
+
? parsed.ok
|
|
869
|
+
: (typeof parsed?.passed === 'boolean' ? parsed.passed : null);
|
|
870
|
+
if (operationOutcome === true) toolStatusLabel = 'โ
Action confirmed';
|
|
871
|
+
if (operationOutcome === false) toolStatusLabel = 'โ ๏ธ Action unconfirmed';
|
|
872
|
+
} catch {
|
|
873
|
+
// Strings and other unstructured results remain transport-level "Engaged".
|
|
874
|
+
}
|
|
875
|
+
}
|
|
841
876
|
if (fnName === 'get_mfa_code' && !errorMessage) {
|
|
842
877
|
try {
|
|
843
878
|
const parsed = JSON.parse(result);
|
|
@@ -1029,7 +1064,7 @@ export const turnLoop = async (
|
|
|
1029
1064
|
if (gatedResult) return gatedResult;
|
|
1030
1065
|
}
|
|
1031
1066
|
if (hadToolIssues && turnRetries < retryLimitClamped) {
|
|
1032
|
-
step.result = '
|
|
1067
|
+
step.result = '๐ Replotting course';
|
|
1033
1068
|
const retryNumber = attempt - 1;
|
|
1034
1069
|
step.events.push(`๐ Re-attempt ${retryNumber}/${retryLimitClamped} after tool issues`);
|
|
1035
1070
|
recordStep(step);
|
|
@@ -1043,7 +1078,7 @@ export const turnLoop = async (
|
|
|
1043
1078
|
|
|
1044
1079
|
// reset retries after a clean turn or after exhausting retries
|
|
1045
1080
|
turnRetries = 0;
|
|
1046
|
-
step.result = hadToolIssues ? 'โ ๏ธ
|
|
1081
|
+
step.result = hadToolIssues ? 'โ ๏ธ Navigation issue' : '๐ Progressing';
|
|
1047
1082
|
recordStep(step);
|
|
1048
1083
|
continue;
|
|
1049
1084
|
}
|
|
@@ -1073,10 +1108,15 @@ export const turnLoop = async (
|
|
|
1073
1108
|
}
|
|
1074
1109
|
await recordJevShadow(step, msg, 'final_response');
|
|
1075
1110
|
step.events.push(finalResponse.finalMessage);
|
|
1076
|
-
|
|
1111
|
+
const evidenceSources = collectMissionEvidenceSources([...stepsArchive, step], agentMemory);
|
|
1112
|
+
const evidenceRecorded = finalResponse.success && evidenceSources.length > 0;
|
|
1113
|
+
step.result = finalResponse.success
|
|
1114
|
+
? evidenceRecorded ? 'โ
Mission accomplished ยท evidence recorded' : 'โ ๏ธ Completion reported ยท limited evidence'
|
|
1115
|
+
: 'โ Mission incomplete';
|
|
1116
|
+
if (finalResponse.success) step.verification = { level: evidenceRecorded ? 'evidence_recorded' : 'limited_evidence', sources: evidenceSources };
|
|
1077
1117
|
recordStep(step);
|
|
1078
1118
|
turnRetries = 0;
|
|
1079
|
-
return { success: finalResponse.success, finalMessage: finalResponse.finalMessage, steps: stepsArchive };
|
|
1119
|
+
return { success: finalResponse.success, finalMessage: finalResponse.finalMessage, steps: stepsArchive, evidenceRecorded, evidenceSources };
|
|
1080
1120
|
}
|
|
1081
1121
|
|
|
1082
1122
|
// โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
@@ -1104,7 +1144,7 @@ export const turnLoop = async (
|
|
|
1104
1144
|
retryAttempt: 1,
|
|
1105
1145
|
retryLimit: 0,
|
|
1106
1146
|
events: ['โฑ๏ธ Working-turn budget exhausted; requesting final mission verdict.'],
|
|
1107
|
-
result: '
|
|
1147
|
+
result: '๐ก Awaiting mission verdict',
|
|
1108
1148
|
missionName,
|
|
1109
1149
|
provider: PROVIDER_ID,
|
|
1110
1150
|
model: MODEL_ID,
|
|
@@ -1128,13 +1168,20 @@ export const turnLoop = async (
|
|
|
1128
1168
|
const finalResponse = finalResponseHandler(msg);
|
|
1129
1169
|
if (finalResponse) {
|
|
1130
1170
|
closureStep.events.push(finalResponse.finalMessage);
|
|
1131
|
-
|
|
1171
|
+
const evidenceSources = collectMissionEvidenceSources([...stepsArchive, closureStep], agentMemory);
|
|
1172
|
+
const evidenceRecorded = finalResponse.success && evidenceSources.length > 0;
|
|
1173
|
+
closureStep.result = finalResponse.success
|
|
1174
|
+
? evidenceRecorded ? 'โ
Mission accomplished ยท evidence recorded' : 'โ ๏ธ Completion reported ยท limited evidence'
|
|
1175
|
+
: 'โ Mission incomplete';
|
|
1176
|
+
if (finalResponse.success) closureStep.verification = { level: evidenceRecorded ? 'evidence_recorded' : 'limited_evidence', sources: evidenceSources };
|
|
1132
1177
|
recordStep(closureStep);
|
|
1133
1178
|
return {
|
|
1134
1179
|
success: finalResponse.success,
|
|
1135
1180
|
finalMessage: finalResponse.finalMessage,
|
|
1136
1181
|
steps: stepsArchive,
|
|
1137
1182
|
completedBy: 'budget_closure',
|
|
1183
|
+
evidenceRecorded,
|
|
1184
|
+
evidenceSources,
|
|
1138
1185
|
};
|
|
1139
1186
|
}
|
|
1140
1187
|
closureStep.events.push('FAILURE: The closure response did not provide an explicit SUCCESS or FAILURE verdict.');
|
|
@@ -1142,7 +1189,7 @@ export const turnLoop = async (
|
|
|
1142
1189
|
closureStep.events.push(`FAILURE: Unable to obtain the closure verdict: ${error?.message || error}`);
|
|
1143
1190
|
}
|
|
1144
1191
|
|
|
1145
|
-
closureStep.result = 'โ Mission
|
|
1192
|
+
closureStep.result = 'โ Mission incomplete';
|
|
1146
1193
|
recordStep(closureStep);
|
|
1147
1194
|
return {
|
|
1148
1195
|
success: false,
|
package/package.json
CHANGED
package/runner/testronaut.js
CHANGED
|
@@ -237,13 +237,26 @@ export async function runMissions({ preMission, mission, postMission, tags = [],
|
|
|
237
237
|
|
|
238
238
|
for (const result of success) {
|
|
239
239
|
result.tags = result.submissionType === 'mission' ? missionTags : [];
|
|
240
|
+
const terminalResult = result.steps?.at(-1)?.result || '';
|
|
241
|
+
// Preserve compatibility with older/custom agents that communicate the
|
|
242
|
+
// terminal verdict through step text but return an inconsistent status.
|
|
243
|
+
if (/\bfailure\b|mission incomplete/i.test(terminalResult)) {
|
|
244
|
+
result.status = 'failed';
|
|
245
|
+
}
|
|
240
246
|
}
|
|
241
247
|
|
|
242
|
-
const
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
248
|
+
const failed = success.some(result => result.status === 'failed');
|
|
249
|
+
const recovered = success.some(result => result.status === 'recovered');
|
|
250
|
+
const limitedEvidence = success.some(result => result.verification === 'limited_evidence');
|
|
251
|
+
console.log(failed
|
|
252
|
+
? '\nโ Mission incomplete.'
|
|
253
|
+
: limitedEvidence
|
|
254
|
+
? recovered
|
|
255
|
+
? '\nโ ๏ธ Completion reported after recovery ยท limited evidence.'
|
|
256
|
+
: '\nโ ๏ธ Completion reported ยท limited evidence.'
|
|
257
|
+
: recovered
|
|
258
|
+
? '\nโ ๏ธ Mission accomplished after recovery.'
|
|
259
|
+
: '\nโ
Mission accomplished.');
|
|
247
260
|
return success;
|
|
248
261
|
}
|
|
249
262
|
|
|
@@ -176,7 +176,8 @@ describe('turnLoop', () => {
|
|
|
176
176
|
const step = result.steps[0];
|
|
177
177
|
expect(step.tokensUsed).toBe(42);
|
|
178
178
|
expect(step.summary).toContain('plan:FINAL: success');
|
|
179
|
-
expect(step.result).
|
|
179
|
+
expect(step.result).toBe('โ ๏ธ Completion reported ยท limited evidence');
|
|
180
|
+
expect(result.evidenceRecorded).toBe(false);
|
|
180
181
|
|
|
181
182
|
expect(shared.chromeToolSpies.click_text).not.toHaveBeenCalled();
|
|
182
183
|
expect(shared.chromeToolSpies.get_dom).not.toHaveBeenCalled();
|
|
@@ -255,11 +256,43 @@ describe('turnLoop', () => {
|
|
|
255
256
|
expect(shared.chatMock.mock.calls[1][0].messages.at(-1).content).toContain('working-turn budget is exhausted');
|
|
256
257
|
expect(result.steps.at(-1)).toMatchObject({
|
|
257
258
|
isClosureTurn: true,
|
|
258
|
-
result: 'โ Mission
|
|
259
|
+
result: 'โ Mission incomplete',
|
|
259
260
|
tokensUsed: 4,
|
|
260
261
|
});
|
|
261
262
|
});
|
|
262
263
|
|
|
264
|
+
it('does not report mission success when a tool succeeds but the goal remains unfulfilled', async () => {
|
|
265
|
+
shared.chatMock
|
|
266
|
+
.mockResolvedValueOnce({
|
|
267
|
+
message: {
|
|
268
|
+
role: 'assistant',
|
|
269
|
+
content: '',
|
|
270
|
+
tool_calls: [{
|
|
271
|
+
id: 'create_work_package',
|
|
272
|
+
type: 'function',
|
|
273
|
+
function: { name: 'click_text', arguments: JSON.stringify({ text: 'Create Work Package' }) },
|
|
274
|
+
}],
|
|
275
|
+
},
|
|
276
|
+
usage: { total_tokens: 10 },
|
|
277
|
+
})
|
|
278
|
+
.mockResolvedValueOnce({
|
|
279
|
+
message: { role: 'assistant', content: 'FINAL: FAILURE: work package creation was not verified' },
|
|
280
|
+
usage: { total_tokens: 4 },
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
const result = await turnLoop(browser, baseMessages(), 1, 0, 0, {}, {
|
|
284
|
+
steps: [], missionName: 'work-package',
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
expect(result.success).toBe(false);
|
|
288
|
+
expect(result.steps[0].result).toBe('๐ Progressing');
|
|
289
|
+
expect(result.steps[0].events).toContain(
|
|
290
|
+
'[tool ] โ click_text result: โ
Action confirmed',
|
|
291
|
+
);
|
|
292
|
+
expect(result.steps.at(-1).result).toBe('โ Mission incomplete');
|
|
293
|
+
expect(result.steps.map(step => step.result)).not.toContain('โ
Mission accomplished');
|
|
294
|
+
});
|
|
295
|
+
|
|
263
296
|
it('records elapsed milliseconds for normal and closure turns', async () => {
|
|
264
297
|
shared.chatMock
|
|
265
298
|
.mockResolvedValueOnce({ message: { role: 'assistant', content: 'Continue' }, usage: {} })
|
|
@@ -312,7 +345,8 @@ describe('turnLoop', () => {
|
|
|
312
345
|
});
|
|
313
346
|
|
|
314
347
|
expect(result.success).toBe(true);
|
|
315
|
-
expect(result.steps[0].result).
|
|
348
|
+
expect(result.steps[0].result).toBe('โ
Mission accomplished ยท evidence recorded');
|
|
349
|
+
expect(result.evidenceRecorded).toBe(true);
|
|
316
350
|
expect(result.steps[0].jevShadow).toMatchObject({
|
|
317
351
|
status: 'ok',
|
|
318
352
|
model: 'jev-test',
|
|
@@ -504,7 +538,7 @@ describe('turnLoop', () => {
|
|
|
504
538
|
expect(shared.chatMock).toHaveBeenCalledTimes(1);
|
|
505
539
|
expect(shared.chromeToolSpies.screenshot).toHaveBeenCalledTimes(1);
|
|
506
540
|
expect(result.steps[0]).toMatchObject({
|
|
507
|
-
result: 'โ
Mission
|
|
541
|
+
result: 'โ
Mission accomplished (Jev verified)',
|
|
508
542
|
screenshotPath: './screenshots/jev-gate.png',
|
|
509
543
|
jevGate: { triggered: true, candidate: true, threshold: 0.8 },
|
|
510
544
|
});
|
|
@@ -602,7 +636,7 @@ describe('turnLoop', () => {
|
|
|
602
636
|
expect(res.steps.length).toBe(2);
|
|
603
637
|
|
|
604
638
|
const step1 = res.steps[0];
|
|
605
|
-
expect(step1.result).toBe('
|
|
639
|
+
expect(step1.result).toBe('๐ Progressing');
|
|
606
640
|
expect(shared.chromeToolSpies.click_text).toHaveBeenCalledTimes(1);
|
|
607
641
|
expect(shared.chromeToolSpies.get_dom).toHaveBeenCalled();
|
|
608
642
|
expect(shared.chromeToolSpies.get_dom).toHaveBeenCalledWith(
|
|
@@ -628,7 +662,8 @@ describe('turnLoop', () => {
|
|
|
628
662
|
.toEqual({ limit: 15000, exclude: true });
|
|
629
663
|
|
|
630
664
|
const step2 = res.steps[1];
|
|
631
|
-
expect(step2.result).
|
|
665
|
+
expect(step2.result).toBe('โ
Mission accomplished ยท evidence recorded');
|
|
666
|
+
expect(res.evidenceSources).toContain('browser_state');
|
|
632
667
|
});
|
|
633
668
|
|
|
634
669
|
it('handles Ground Control state locally without dispatching it as a browser tool', async () => {
|
|
@@ -653,7 +688,7 @@ describe('turnLoop', () => {
|
|
|
653
688
|
expect(res.success).toBe(true);
|
|
654
689
|
expect(ctx.groundControl.app.baseUrl).toBe('https://example.com');
|
|
655
690
|
expect(ctx.groundControl.session.loggedIn).toBe(false);
|
|
656
|
-
expect(res.steps[0].events).toContain('[tool ] โ set_ground_control_state result: โ
|
|
691
|
+
expect(res.steps[0].events).toContain('[tool ] โ set_ground_control_state result: โ
Action confirmed');
|
|
657
692
|
});
|
|
658
693
|
|
|
659
694
|
it('retries a turn on tool error before marking issues', async () => {
|
|
@@ -703,9 +738,9 @@ describe('turnLoop', () => {
|
|
|
703
738
|
|
|
704
739
|
expect(res.success).toBe(true);
|
|
705
740
|
expect(res.steps.length).toBe(3);
|
|
706
|
-
expect(res.steps[0].result).toBe('
|
|
707
|
-
expect(res.steps[1].result).toBe('
|
|
708
|
-
expect(res.steps[2].result).
|
|
741
|
+
expect(res.steps[0].result).toBe('๐ Replotting course');
|
|
742
|
+
expect(res.steps[1].result).toBe('๐ Progressing');
|
|
743
|
+
expect(res.steps[2].result).toBe('โ
Mission accomplished ยท evidence recorded');
|
|
709
744
|
});
|
|
710
745
|
|
|
711
746
|
it('logs unavailable MFA lookups without marking the tool as a success', async () => {
|
|
@@ -754,7 +789,7 @@ describe('turnLoop', () => {
|
|
|
754
789
|
expect(res.steps[0].events).toContain('๐ MFA list endpoint status: available');
|
|
755
790
|
expect(res.steps[0].events).toContain('๐ MFA API response keys: nickname, message');
|
|
756
791
|
expect(res.steps[0].events.join('\n')).not.toContain(
|
|
757
|
-
'[tool ] โ get_mfa_code result:
|
|
792
|
+
'[tool ] โ get_mfa_code result: ๐ฐ๏ธ Engaged'
|
|
758
793
|
);
|
|
759
794
|
});
|
|
760
795
|
|
|
@@ -18,6 +18,28 @@ afterEach(() => {
|
|
|
18
18
|
});
|
|
19
19
|
|
|
20
20
|
describe('generateHtmlReport', () => {
|
|
21
|
+
it('distinguishes evidence-backed completion from limited-evidence reports', () => {
|
|
22
|
+
const outPath = path.join(tmpDir, 'verification-levels.html');
|
|
23
|
+
generateHtmlReport({ missions: [{
|
|
24
|
+
missionName: 'Work package',
|
|
25
|
+
status: 'passed',
|
|
26
|
+
verification: 'limited_evidence',
|
|
27
|
+
steps: [{ turn: 0, result: 'โ ๏ธ Completion reported ยท limited evidence' }],
|
|
28
|
+
}, {
|
|
29
|
+
missionName: 'Dialogs',
|
|
30
|
+
status: 'passed',
|
|
31
|
+
verification: 'evidence_recorded',
|
|
32
|
+
steps: [{ turn: 0, result: 'โ
Mission accomplished ยท evidence recorded' }],
|
|
33
|
+
}] }, outPath);
|
|
34
|
+
|
|
35
|
+
const html = fs.readFileSync(outPath, 'utf8');
|
|
36
|
+
expect(html).toContain('โ ๏ธ Passed ยท limited evidence');
|
|
37
|
+
expect(html).toContain('โ ๏ธ Completion reported ยท limited evidence');
|
|
38
|
+
expect(html).toContain('โ
Passed ยท evidence recorded');
|
|
39
|
+
expect(html).toContain('โ
Mission accomplished ยท evidence recorded');
|
|
40
|
+
expect(html).toContain('step-result warn');
|
|
41
|
+
});
|
|
42
|
+
|
|
21
43
|
it('renders recovered status and recovery evidence', () => {
|
|
22
44
|
const outPath = path.join(tmpDir, 'recovered.html');
|
|
23
45
|
generateHtmlReport({ summary: { passed: 0, recovered: 1, failed: 0 }, missions: [{
|
|
@@ -55,7 +77,7 @@ describe('generateHtmlReport', () => {
|
|
|
55
77
|
submissionType: 'mission',
|
|
56
78
|
status: 'passed',
|
|
57
79
|
durationMs: 2500,
|
|
58
|
-
steps: [{ turn: 0, result: 'โ
Mission
|
|
80
|
+
steps: [{ turn: 0, result: 'โ
Mission accomplished', durationMs: 1250 }],
|
|
59
81
|
}],
|
|
60
82
|
};
|
|
61
83
|
const outPath = path.join(tmpDir, 'timed.html');
|
|
@@ -85,7 +107,7 @@ describe('generateHtmlReport', () => {
|
|
|
85
107
|
retryAttempt: 2,
|
|
86
108
|
retryLimit: 5,
|
|
87
109
|
events: ['did something'],
|
|
88
|
-
result: 'โ ๏ธ
|
|
110
|
+
result: 'โ ๏ธ Navigation issue',
|
|
89
111
|
summary: 'Do thing',
|
|
90
112
|
tokensUsed: 10,
|
|
91
113
|
totalTokensUsed: 20,
|
|
@@ -108,7 +130,7 @@ describe('generateHtmlReport', () => {
|
|
|
108
130
|
expect(html).toContain('--cyan:#79d7df');
|
|
109
131
|
expect(html).toContain('Mission A');
|
|
110
132
|
expect(html).toContain('Turn 1 (re-attempt 1/5)');
|
|
111
|
-
expect(html).toContain('โ ๏ธ
|
|
133
|
+
expect(html).toContain('โ ๏ธ Navigation issue');
|
|
112
134
|
expect(html).toContain('data-tag="smoke"');
|
|
113
135
|
expect(html).toContain('data-tags="smoke"');
|
|
114
136
|
expect(html).toContain('CLI version:</strong> 1.7.0');
|
|
@@ -137,7 +159,7 @@ describe('generateHtmlReport', () => {
|
|
|
137
159
|
turn: 0,
|
|
138
160
|
retryAttempt: 1,
|
|
139
161
|
events: ['<script>alert(1)</script>'],
|
|
140
|
-
result: '
|
|
162
|
+
result: '๐ Progressing',
|
|
141
163
|
summary: '<b>bold</b>',
|
|
142
164
|
},
|
|
143
165
|
],
|
|
@@ -170,7 +192,7 @@ describe('generateHtmlReport', () => {
|
|
|
170
192
|
generateHtmlReport({ missions: [{
|
|
171
193
|
missionName: 'Dialogs', status: 'passed', steps: [{
|
|
172
194
|
turn: 0,
|
|
173
|
-
result: '
|
|
195
|
+
result: '๐ Progressing',
|
|
174
196
|
events: ['dialog handled'],
|
|
175
197
|
dialogs: [{
|
|
176
198
|
event: 'javascript_dialog',
|
|
@@ -198,7 +220,7 @@ describe('generateHtmlReport', () => {
|
|
|
198
220
|
it('renders run, mission, and step Jev token spend', () => {
|
|
199
221
|
const steps = [{
|
|
200
222
|
turn: 0,
|
|
201
|
-
result: 'โ
Mission
|
|
223
|
+
result: 'โ
Mission accomplished (Jev verified)',
|
|
202
224
|
tokensUsed: 500,
|
|
203
225
|
inputTokens: 450,
|
|
204
226
|
outputTokens: 50,
|
|
@@ -248,8 +270,8 @@ describe('generateHtmlReport', () => {
|
|
|
248
270
|
optimization: { configured: true, objective: 'cost', mode: 'live' },
|
|
249
271
|
missions: [{
|
|
250
272
|
missionName: 'Routed Mission', status: 'passed', steps: [
|
|
251
|
-
{ turn: 0, model: 'gpt-5.6-terra', tokensUsed: 100, inputTokens: 90, outputTokens: 10, result: '
|
|
252
|
-
{ turn: 1, model: 'gpt-5.6-luna', tokensUsed: 40, inputTokens: 35, outputTokens: 5, result: '
|
|
273
|
+
{ turn: 0, model: 'gpt-5.6-terra', tokensUsed: 100, inputTokens: 90, outputTokens: 10, result: '๐ Progressing' },
|
|
274
|
+
{ turn: 1, model: 'gpt-5.6-luna', tokensUsed: 40, inputTokens: 35, outputTokens: 5, result: '๐ Progressing', modelRouting: { applied: true } },
|
|
253
275
|
],
|
|
254
276
|
}],
|
|
255
277
|
}, outPath);
|
|
@@ -41,7 +41,10 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
41
41
|
.replace(/&/g, '&').replace(/</g, '<')
|
|
42
42
|
.replace(/>/g, '>')
|
|
43
43
|
.replace(/"/g, '"');
|
|
44
|
-
const badge = (status) =>
|
|
44
|
+
const badge = (status, verification) =>
|
|
45
|
+
status === 'passed' && verification === 'verified' ? 'โ
Passed ยท independently verified' :
|
|
46
|
+
status === 'passed' && verification === 'evidence_recorded' ? 'โ
Passed ยท evidence recorded' :
|
|
47
|
+
status === 'passed' && verification === 'limited_evidence' ? 'โ ๏ธ Passed ยท limited evidence' :
|
|
45
48
|
status === 'passed' ? 'โ
Passed' :
|
|
46
49
|
status === 'recovered' ? 'โ ๏ธ Passed after recovery' :
|
|
47
50
|
status === 'failed' ? 'โ Failed' : (status || 'โ');
|
|
@@ -105,10 +108,14 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
105
108
|
const stepItems = steps.map((step, idx) => {
|
|
106
109
|
const events = Array.isArray(step.events) ? step.events : [];
|
|
107
110
|
const dialogs = Array.isArray(step.dialogs) ? step.dialogs : [];
|
|
108
|
-
const ok =
|
|
111
|
+
const ok = /Mission accomplished|Mission Success|Action confirmed/i.test(step.result || '');
|
|
112
|
+
const progressing = /Progressing|Replotting course|Awaiting mission verdict|^โ
Passed$/i.test(step.result || '');
|
|
113
|
+
const reported = /Completion reported|limited evidence/i.test(step.result || '');
|
|
109
114
|
const resultRaw = step.result || 'โ';
|
|
110
|
-
const resultTooltip =
|
|
115
|
+
const resultTooltip = /Navigation issue|Turn Issues/.test(resultRaw)
|
|
111
116
|
? 'Turn had tool/action issues (e.g., selector/timeouts). It may not indicate a product bug.'
|
|
117
|
+
: progressing
|
|
118
|
+
? 'Intermediate activity only; this does not indicate mission completion.'
|
|
112
119
|
: '';
|
|
113
120
|
const retryAttempt = step.retryAttempt || 1; // includes initial
|
|
114
121
|
const retryLimit = step.retryLimit; // number of retries allowed (excludes initial)
|
|
@@ -175,12 +182,12 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
175
182
|
: '';
|
|
176
183
|
|
|
177
184
|
return `
|
|
178
|
-
<details class="step" ${ok ? '' : 'open'}>
|
|
185
|
+
<details class="step" ${ok || progressing || reported ? '' : 'open'}>
|
|
179
186
|
<summary>
|
|
180
187
|
<span class="turn">${step.isClosureTurn ? 'Closure' : `Turn ${esc((step.turn ?? idx) + 1)}`}${esc(attemptLabel)}${stepDuration}</span>
|
|
181
188
|
${humanInput}
|
|
182
189
|
${planSpan}
|
|
183
|
-
<span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
|
|
190
|
+
<span class="step-result ${ok ? 'ok' : reported ? 'warn' : progressing ? '' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
|
|
184
191
|
<span class="tokens">${esc(step.model || llm.model || 'Model')} tokens: ${esc(step.tokensUsed ?? 'โ')} (${esc(step.inputTokens ?? 'โ')} in / ${esc(step.outputTokens ?? 'โ')} out)${routeLabel}${stepJev}</span>
|
|
185
192
|
</summary>
|
|
186
193
|
${dialogCards ? `<div class="dialog-evidence">${dialogCards}</div>` : ''}
|
|
@@ -193,7 +200,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
193
200
|
const type = String(m.submissionType || 'mission').toLowerCase();
|
|
194
201
|
const typeLabel = (type === 'premission' || type === 'postmission') ? type : 'mission';
|
|
195
202
|
const prettyTitle = `${esc(m.missionName || 'Mission')} โ ${typeLabel}${m.submissionName ? `: ${esc(m.submissionName)}` : ''}`;
|
|
196
|
-
const statusClass = m.status === 'failed' ? 'bad' : (m.status === 'recovered' ? 'warn' : (m.status === 'passed' ? 'ok' : ''));
|
|
203
|
+
const statusClass = m.status === 'failed' ? 'bad' : (m.status === 'recovered' || m.verification === 'limited_evidence' ? 'warn' : (m.status === 'passed' ? 'ok' : ''));
|
|
197
204
|
const recoveries = Array.isArray(m.recoveries) ? m.recoveries : [];
|
|
198
205
|
const recoveryItems = recoveries.length ? `
|
|
199
206
|
<div class="recoveries">
|
|
@@ -205,7 +212,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
205
212
|
<details class="mission-submission">
|
|
206
213
|
<summary>
|
|
207
214
|
<span class="name">${prettyTitle}</span>
|
|
208
|
-
<span class="status ${statusClass}">${badge(m.status)}</span>
|
|
215
|
+
<span class="status ${statusClass}">${badge(m.status, m.verification)}</span>
|
|
209
216
|
<span class="meta">steps: ${steps.length} โข duration: ${mDurationSec}s โข ${esc(tokenMeta(missionModelUsage, missionJevUsage))}</span>
|
|
210
217
|
<span class="toolbar">
|
|
211
218
|
<button class="btn-mini toggle" data-scope="submission" aria-label="Expand">โผ</button>
|
|
@@ -232,6 +239,15 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
232
239
|
|
|
233
240
|
const missionGroupBlock = (missionName, subs) => {
|
|
234
241
|
const status = groupStatus(subs);
|
|
242
|
+
const verification = status === 'passed'
|
|
243
|
+
? subs.some(s => s.verification === 'limited_evidence')
|
|
244
|
+
? 'limited_evidence'
|
|
245
|
+
: subs.every(s => s.verification === 'verified')
|
|
246
|
+
? 'verified'
|
|
247
|
+
: subs.some(s => s.verification === 'evidence_recorded')
|
|
248
|
+
? 'evidence_recorded'
|
|
249
|
+
: undefined
|
|
250
|
+
: undefined;
|
|
235
251
|
const totalSteps = subs.reduce((n, s) => n + (Array.isArray(s.steps) ? s.steps.length : 0), 0);
|
|
236
252
|
const startTimes = subs.map(s => s.startTime || 0).filter(Boolean);
|
|
237
253
|
const endTimes = subs.map(s => s.endTime || 0).filter(Boolean);
|
|
@@ -255,7 +271,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
255
271
|
<summary>
|
|
256
272
|
<span class="name">${esc(missionName)}</span>
|
|
257
273
|
<span class="mission-tags">${tags.map(tag => `<span class="tag-small" style="${tagStyle(tag)}">${esc(tag)}</span>`).join('')}</span>
|
|
258
|
-
<span class="status ${status === 'failed' ? 'bad' : status === 'recovered' ? 'warn' : 'ok'}">${badge(status)}</span>
|
|
274
|
+
<span class="status ${status === 'failed' ? 'bad' : status === 'recovered' || verification === 'limited_evidence' ? 'warn' : 'ok'}">${badge(status, verification)}</span>
|
|
259
275
|
<span class="meta">${sourceFiles.length ? `${sourceFiles.map(esc).join(', ')} โข ` : ''}submissions: ${subs.length} โข steps: ${totalSteps} โข duration: ${groupDur}s โข ${esc(tokenMeta(groupModelUsage, groupJevUsage))}</span>
|
|
260
276
|
<span class="toolbar">
|
|
261
277
|
<button class="btn-mini toggle" data-scope="mission" aria-label="Expand">โผ</button>
|
|
@@ -448,6 +464,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
448
464
|
white-space:nowrap; overflow:hidden; text-overflow:ellipsis;
|
|
449
465
|
}
|
|
450
466
|
.step-result.ok{ color: var(--ok); font-weight:700; }
|
|
467
|
+
.step-result.warn{ color: var(--amber); font-weight:700; }
|
|
451
468
|
.step-result.bad{ color: var(--bad); font-weight:700; }
|
|
452
469
|
.tokens{ color: var(--text-muted); font-size:12px; margin-left:auto; font-variant-numeric: tabular-nums; }
|
|
453
470
|
|