testronaut 2.1.7 โ†’ 2.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -622,6 +622,20 @@ Each includes:
622
622
  - Screenshots
623
623
  - Pass/Fail summaries
624
624
 
625
+ ### Status language
626
+
627
+ Turn and tool statuses describe different layers of a run:
628
+
629
+ - `๐Ÿ›ฐ๏ธ Engaged` means a tool invocation ran without a structured operation verdict. It does not mean the goal was achieved.
630
+ - `โœ… Action confirmed` and `โš ๏ธ Action unconfirmed` reflect an explicit structured tool outcome such as `ok` or `passed`.
631
+ - `๐Ÿš€ Progressing` marks an intermediate turn. Intermediate turns are not pass/fail verdicts.
632
+ - `๐Ÿ”„ Replotting course` and `โš ๏ธ Navigation issue` identify retries and tool/action problems.
633
+ - `โœ… Mission accomplished ยท evidence recorded` means the successful verdict is supported by recorded browser observations, screenshots, matched dialogs, or assertion results.
634
+ - `โœ… Mission accomplished ยท independently verified` means a separate verifier such as the Jev completion gate confirmed the outcome.
635
+ - `โš ๏ธ Completion reported ยท limited evidence` means the model declared success without enough supporting evidence in the trace. `โŒ Mission incomplete` is the final failure outcome. Reports retain the machine-readable `passed`, `recovered`, and `failed` mission statuses for compatibility.
636
+
637
+ For example, clicking **Create Work Package** can be an engaged or confirmed action while the mission remains **Progressing**. Creation should only be described as confirmed after evidence shows that the work package exists.
638
+
625
639
  ---
626
640
 
627
641
  ## ๐Ÿงช Under the Hood
package/core/agent.js CHANGED
@@ -99,7 +99,7 @@ export async function runAgent(goals, missionName, maxTurns = 20, retryLimit, op
99
99
  const step = {
100
100
  turn: 0,
101
101
  events: [`๐Ÿ“ก Launch protocol probe passed: ${protocol.id}`],
102
- result: 'โœ… Mission Success',
102
+ result: 'โœ… Mission accomplished',
103
103
  summary: `Reused launch protocol ${protocol.id}`,
104
104
  missionName,
105
105
  tokensUsed: 0,
@@ -117,6 +117,7 @@ export async function runAgent(goals, missionName, maxTurns = 20, retryLimit, op
117
117
  startTime: now,
118
118
  endTime: now,
119
119
  durationMs: 0,
120
+ verification: 'verified',
120
121
  telemetry: [],
121
122
  launchProtocol: { id: protocol.id, status: 'reused', probe: 'passed' },
122
123
  });
@@ -321,6 +322,10 @@ JavaScript dialogs:
321
322
  submissionType: goal.submissionType || goal.label,
322
323
  submissionName: goal.submissionName || null,
323
324
  status: succeeded ? (recoveries.length ? 'recovered' : 'passed') : 'failed',
325
+ verification: succeeded
326
+ ? (result?.completedBy === 'jev_gate' ? 'verified' : result?.evidenceRecorded ? 'evidence_recorded' : 'limited_evidence')
327
+ : 'failed',
328
+ ...(result?.evidenceSources?.length ? { evidenceSources: result.evidenceSources } : {}),
324
329
  steps: JSON.parse(JSON.stringify(compact)), // last 20 (by memory design)
325
330
  stepFile, // full history is in this JSONL (one step per line)
326
331
  startTime: goalStartTime,
@@ -342,7 +347,10 @@ JavaScript dialogs:
342
347
  }
343
348
  }
344
349
 
345
- console.log('โœ… All goals completed successfully.\n');
350
+ const limitedEvidence = missionResults.some(mission => mission.verification === 'limited_evidence');
351
+ console.log(limitedEvidence
352
+ ? 'โš ๏ธ Completion reported ยท limited evidence.\n'
353
+ : 'โœ… Mission accomplished.\n');
346
354
  return missionResults;
347
355
  } finally {
348
356
  await browser.close();
package/core/turnLoop.js CHANGED
@@ -189,6 +189,27 @@ async function captureBrowserEvidence(browser, html, previous) {
189
189
  return mergeBrowserControlState(evidence, states);
190
190
  }
191
191
 
192
+ function collectMissionEvidenceSources(recordedSteps, agentMemory) {
193
+ const sources = new Set();
194
+ if (agentMemory?.browserEvidence) sources.add('browser_state');
195
+
196
+ for (const recordedStep of recordedSteps) {
197
+ if (recordedStep?.screenshotPath) sources.add('screenshot');
198
+ if (recordedStep?.dialogs?.some(dialog => dialog?.matched && dialog?.expected)) {
199
+ sources.add('matched_dialog');
200
+ }
201
+ const events = Array.isArray(recordedStep?.events) ? recordedStep.events : [];
202
+ if (events.some(event => /\[model\] โ†’ (?:get_dom|check_text|probe)\b/.test(event))) {
203
+ sources.add('browser_observation');
204
+ }
205
+ if (events.some(event => /\[tool \] โ† FOUND:/.test(event))) {
206
+ sources.add('assertion_result');
207
+ }
208
+ }
209
+
210
+ return [...sources];
211
+ }
212
+
192
213
  /**
193
214
  * Utility: format byte counts into human-readable strings.
194
215
  * Used when reporting upload/download events in mission logs.
@@ -402,7 +423,7 @@ export const turnLoop = async (
402
423
 
403
424
  step.jevGate = { triggered: true, ...gate };
404
425
  step.events.push(`๐Ÿ”ฌ Jev completion gate triggered at ${Math.round(gate.threshold * 100)}% threshold`);
405
- step.result = 'โœ… Mission Success (Jev gate)';
426
+ step.result = 'โœ… Mission accomplished (Jev verified)';
406
427
  recordStep(step);
407
428
  const finalMessage = 'SUCCESS: Jev completion gate found sufficient browser evidence.';
408
429
  return { success: true, finalMessage, steps: stepsArchive, completedBy: 'jev_gate' };
@@ -837,7 +858,21 @@ export const turnLoop = async (
837
858
  // non-JSON results ignored
838
859
  }
839
860
 
840
- let toolStatusLabel = errorMessage ? 'โŒ Failed' : 'โœ… Success';
861
+ // A tool returning without throwing only proves that its invocation ran.
862
+ // It does not prove that the requested operation or mission goal succeeded.
863
+ let toolStatusLabel = errorMessage ? 'โŒ System fault' : '๐Ÿ›ฐ๏ธ Engaged';
864
+ if (!errorMessage) {
865
+ try {
866
+ const parsed = JSON.parse(result);
867
+ const operationOutcome = typeof parsed?.ok === 'boolean'
868
+ ? parsed.ok
869
+ : (typeof parsed?.passed === 'boolean' ? parsed.passed : null);
870
+ if (operationOutcome === true) toolStatusLabel = 'โœ… Action confirmed';
871
+ if (operationOutcome === false) toolStatusLabel = 'โš ๏ธ Action unconfirmed';
872
+ } catch {
873
+ // Strings and other unstructured results remain transport-level "Engaged".
874
+ }
875
+ }
841
876
  if (fnName === 'get_mfa_code' && !errorMessage) {
842
877
  try {
843
878
  const parsed = JSON.parse(result);
@@ -1029,7 +1064,7 @@ export const turnLoop = async (
1029
1064
  if (gatedResult) return gatedResult;
1030
1065
  }
1031
1066
  if (hadToolIssues && turnRetries < retryLimitClamped) {
1032
- step.result = 'โณ Retrying turn';
1067
+ step.result = '๐Ÿ”„ Replotting course';
1033
1068
  const retryNumber = attempt - 1;
1034
1069
  step.events.push(`๐Ÿ” Re-attempt ${retryNumber}/${retryLimitClamped} after tool issues`);
1035
1070
  recordStep(step);
@@ -1043,7 +1078,7 @@ export const turnLoop = async (
1043
1078
 
1044
1079
  // reset retries after a clean turn or after exhausting retries
1045
1080
  turnRetries = 0;
1046
- step.result = hadToolIssues ? 'โš ๏ธ Turn Issues' : 'โœ… Passed';
1081
+ step.result = hadToolIssues ? 'โš ๏ธ Navigation issue' : '๐Ÿš€ Progressing';
1047
1082
  recordStep(step);
1048
1083
  continue;
1049
1084
  }
@@ -1073,10 +1108,15 @@ export const turnLoop = async (
1073
1108
  }
1074
1109
  await recordJevShadow(step, msg, 'final_response');
1075
1110
  step.events.push(finalResponse.finalMessage);
1076
- step.result = finalResponse.success ? 'โœ… Mission Success' : 'โŒ Mission Failure';
1111
+ const evidenceSources = collectMissionEvidenceSources([...stepsArchive, step], agentMemory);
1112
+ const evidenceRecorded = finalResponse.success && evidenceSources.length > 0;
1113
+ step.result = finalResponse.success
1114
+ ? evidenceRecorded ? 'โœ… Mission accomplished ยท evidence recorded' : 'โš ๏ธ Completion reported ยท limited evidence'
1115
+ : 'โŒ Mission incomplete';
1116
+ if (finalResponse.success) step.verification = { level: evidenceRecorded ? 'evidence_recorded' : 'limited_evidence', sources: evidenceSources };
1077
1117
  recordStep(step);
1078
1118
  turnRetries = 0;
1079
- return { success: finalResponse.success, finalMessage: finalResponse.finalMessage, steps: stepsArchive };
1119
+ return { success: finalResponse.success, finalMessage: finalResponse.finalMessage, steps: stepsArchive, evidenceRecorded, evidenceSources };
1080
1120
  }
1081
1121
 
1082
1122
  // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
@@ -1104,7 +1144,7 @@ export const turnLoop = async (
1104
1144
  retryAttempt: 1,
1105
1145
  retryLimit: 0,
1106
1146
  events: ['โฑ๏ธ Working-turn budget exhausted; requesting final mission verdict.'],
1107
- result: '๐ŸŸก Awaiting Closure',
1147
+ result: '๐Ÿ“ก Awaiting mission verdict',
1108
1148
  missionName,
1109
1149
  provider: PROVIDER_ID,
1110
1150
  model: MODEL_ID,
@@ -1128,13 +1168,20 @@ export const turnLoop = async (
1128
1168
  const finalResponse = finalResponseHandler(msg);
1129
1169
  if (finalResponse) {
1130
1170
  closureStep.events.push(finalResponse.finalMessage);
1131
- closureStep.result = finalResponse.success ? 'โœ… Mission Success' : 'โŒ Mission Failure';
1171
+ const evidenceSources = collectMissionEvidenceSources([...stepsArchive, closureStep], agentMemory);
1172
+ const evidenceRecorded = finalResponse.success && evidenceSources.length > 0;
1173
+ closureStep.result = finalResponse.success
1174
+ ? evidenceRecorded ? 'โœ… Mission accomplished ยท evidence recorded' : 'โš ๏ธ Completion reported ยท limited evidence'
1175
+ : 'โŒ Mission incomplete';
1176
+ if (finalResponse.success) closureStep.verification = { level: evidenceRecorded ? 'evidence_recorded' : 'limited_evidence', sources: evidenceSources };
1132
1177
  recordStep(closureStep);
1133
1178
  return {
1134
1179
  success: finalResponse.success,
1135
1180
  finalMessage: finalResponse.finalMessage,
1136
1181
  steps: stepsArchive,
1137
1182
  completedBy: 'budget_closure',
1183
+ evidenceRecorded,
1184
+ evidenceSources,
1138
1185
  };
1139
1186
  }
1140
1187
  closureStep.events.push('FAILURE: The closure response did not provide an explicit SUCCESS or FAILURE verdict.');
@@ -1142,7 +1189,7 @@ export const turnLoop = async (
1142
1189
  closureStep.events.push(`FAILURE: Unable to obtain the closure verdict: ${error?.message || error}`);
1143
1190
  }
1144
1191
 
1145
- closureStep.result = 'โŒ Mission Failure';
1192
+ closureStep.result = 'โŒ Mission incomplete';
1146
1193
  recordStep(closureStep);
1147
1194
  return {
1148
1195
  success: false,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "testronaut",
3
- "version": "2.1.7",
3
+ "version": "2.1.9",
4
4
  "main": "./runner/testronaut.js",
5
5
  "bin": {
6
6
  "testronaut": "bin/cli.js"
@@ -237,13 +237,26 @@ export async function runMissions({ preMission, mission, postMission, tags = [],
237
237
 
238
238
  for (const result of success) {
239
239
  result.tags = result.submissionType === 'mission' ? missionTags : [];
240
+ const terminalResult = result.steps?.at(-1)?.result || '';
241
+ // Preserve compatibility with older/custom agents that communicate the
242
+ // terminal verdict through step text but return an inconsistent status.
243
+ if (/\bfailure\b|mission incomplete/i.test(terminalResult)) {
244
+ result.status = 'failed';
245
+ }
240
246
  }
241
247
 
242
- const missionStatus = success[0].steps[success[0].steps.length - 1].result;
243
- if (missionStatus.toLowerCase().includes('failure')) {
244
- success[0].status = 'failed';
245
- }
246
- console.log('\nโœ… Mission flow complete.');
248
+ const failed = success.some(result => result.status === 'failed');
249
+ const recovered = success.some(result => result.status === 'recovered');
250
+ const limitedEvidence = success.some(result => result.verification === 'limited_evidence');
251
+ console.log(failed
252
+ ? '\nโŒ Mission incomplete.'
253
+ : limitedEvidence
254
+ ? recovered
255
+ ? '\nโš ๏ธ Completion reported after recovery ยท limited evidence.'
256
+ : '\nโš ๏ธ Completion reported ยท limited evidence.'
257
+ : recovered
258
+ ? '\nโš ๏ธ Mission accomplished after recovery.'
259
+ : '\nโœ… Mission accomplished.');
247
260
  return success;
248
261
  }
249
262
 
@@ -176,7 +176,8 @@ describe('turnLoop', () => {
176
176
  const step = result.steps[0];
177
177
  expect(step.tokensUsed).toBe(42);
178
178
  expect(step.summary).toContain('plan:FINAL: success');
179
- expect(step.result).toMatch(/โœ…|Success/);
179
+ expect(step.result).toBe('โš ๏ธ Completion reported ยท limited evidence');
180
+ expect(result.evidenceRecorded).toBe(false);
180
181
 
181
182
  expect(shared.chromeToolSpies.click_text).not.toHaveBeenCalled();
182
183
  expect(shared.chromeToolSpies.get_dom).not.toHaveBeenCalled();
@@ -255,11 +256,43 @@ describe('turnLoop', () => {
255
256
  expect(shared.chatMock.mock.calls[1][0].messages.at(-1).content).toContain('working-turn budget is exhausted');
256
257
  expect(result.steps.at(-1)).toMatchObject({
257
258
  isClosureTurn: true,
258
- result: 'โŒ Mission Failure',
259
+ result: 'โŒ Mission incomplete',
259
260
  tokensUsed: 4,
260
261
  });
261
262
  });
262
263
 
264
+ it('does not report mission success when a tool succeeds but the goal remains unfulfilled', async () => {
265
+ shared.chatMock
266
+ .mockResolvedValueOnce({
267
+ message: {
268
+ role: 'assistant',
269
+ content: '',
270
+ tool_calls: [{
271
+ id: 'create_work_package',
272
+ type: 'function',
273
+ function: { name: 'click_text', arguments: JSON.stringify({ text: 'Create Work Package' }) },
274
+ }],
275
+ },
276
+ usage: { total_tokens: 10 },
277
+ })
278
+ .mockResolvedValueOnce({
279
+ message: { role: 'assistant', content: 'FINAL: FAILURE: work package creation was not verified' },
280
+ usage: { total_tokens: 4 },
281
+ });
282
+
283
+ const result = await turnLoop(browser, baseMessages(), 1, 0, 0, {}, {
284
+ steps: [], missionName: 'work-package',
285
+ });
286
+
287
+ expect(result.success).toBe(false);
288
+ expect(result.steps[0].result).toBe('๐Ÿš€ Progressing');
289
+ expect(result.steps[0].events).toContain(
290
+ '[tool ] โ† click_text result: โœ… Action confirmed',
291
+ );
292
+ expect(result.steps.at(-1).result).toBe('โŒ Mission incomplete');
293
+ expect(result.steps.map(step => step.result)).not.toContain('โœ… Mission accomplished');
294
+ });
295
+
263
296
  it('records elapsed milliseconds for normal and closure turns', async () => {
264
297
  shared.chatMock
265
298
  .mockResolvedValueOnce({ message: { role: 'assistant', content: 'Continue' }, usage: {} })
@@ -312,7 +345,8 @@ describe('turnLoop', () => {
312
345
  });
313
346
 
314
347
  expect(result.success).toBe(true);
315
- expect(result.steps[0].result).toMatch(/Success/);
348
+ expect(result.steps[0].result).toBe('โœ… Mission accomplished ยท evidence recorded');
349
+ expect(result.evidenceRecorded).toBe(true);
316
350
  expect(result.steps[0].jevShadow).toMatchObject({
317
351
  status: 'ok',
318
352
  model: 'jev-test',
@@ -504,7 +538,7 @@ describe('turnLoop', () => {
504
538
  expect(shared.chatMock).toHaveBeenCalledTimes(1);
505
539
  expect(shared.chromeToolSpies.screenshot).toHaveBeenCalledTimes(1);
506
540
  expect(result.steps[0]).toMatchObject({
507
- result: 'โœ… Mission Success (Jev gate)',
541
+ result: 'โœ… Mission accomplished (Jev verified)',
508
542
  screenshotPath: './screenshots/jev-gate.png',
509
543
  jevGate: { triggered: true, candidate: true, threshold: 0.8 },
510
544
  });
@@ -602,7 +636,7 @@ describe('turnLoop', () => {
602
636
  expect(res.steps.length).toBe(2);
603
637
 
604
638
  const step1 = res.steps[0];
605
- expect(step1.result).toBe('โœ… Passed');
639
+ expect(step1.result).toBe('๐Ÿš€ Progressing');
606
640
  expect(shared.chromeToolSpies.click_text).toHaveBeenCalledTimes(1);
607
641
  expect(shared.chromeToolSpies.get_dom).toHaveBeenCalled();
608
642
  expect(shared.chromeToolSpies.get_dom).toHaveBeenCalledWith(
@@ -628,7 +662,8 @@ describe('turnLoop', () => {
628
662
  .toEqual({ limit: 15000, exclude: true });
629
663
 
630
664
  const step2 = res.steps[1];
631
- expect(step2.result).toMatch(/Success/);
665
+ expect(step2.result).toBe('โœ… Mission accomplished ยท evidence recorded');
666
+ expect(res.evidenceSources).toContain('browser_state');
632
667
  });
633
668
 
634
669
  it('handles Ground Control state locally without dispatching it as a browser tool', async () => {
@@ -653,7 +688,7 @@ describe('turnLoop', () => {
653
688
  expect(res.success).toBe(true);
654
689
  expect(ctx.groundControl.app.baseUrl).toBe('https://example.com');
655
690
  expect(ctx.groundControl.session.loggedIn).toBe(false);
656
- expect(res.steps[0].events).toContain('[tool ] โ† set_ground_control_state result: โœ… Success');
691
+ expect(res.steps[0].events).toContain('[tool ] โ† set_ground_control_state result: โœ… Action confirmed');
657
692
  });
658
693
 
659
694
  it('retries a turn on tool error before marking issues', async () => {
@@ -703,9 +738,9 @@ describe('turnLoop', () => {
703
738
 
704
739
  expect(res.success).toBe(true);
705
740
  expect(res.steps.length).toBe(3);
706
- expect(res.steps[0].result).toBe('โณ Retrying turn');
707
- expect(res.steps[1].result).toBe('โœ… Passed');
708
- expect(res.steps[2].result).toMatch(/Success/);
741
+ expect(res.steps[0].result).toBe('๐Ÿ”„ Replotting course');
742
+ expect(res.steps[1].result).toBe('๐Ÿš€ Progressing');
743
+ expect(res.steps[2].result).toBe('โœ… Mission accomplished ยท evidence recorded');
709
744
  });
710
745
 
711
746
  it('logs unavailable MFA lookups without marking the tool as a success', async () => {
@@ -754,7 +789,7 @@ describe('turnLoop', () => {
754
789
  expect(res.steps[0].events).toContain('๐Ÿ” MFA list endpoint status: available');
755
790
  expect(res.steps[0].events).toContain('๐Ÿ” MFA API response keys: nickname, message');
756
791
  expect(res.steps[0].events.join('\n')).not.toContain(
757
- '[tool ] โ† get_mfa_code result: โœ… Success'
792
+ '[tool ] โ† get_mfa_code result: ๐Ÿ›ฐ๏ธ Engaged'
758
793
  );
759
794
  });
760
795
 
@@ -51,3 +51,17 @@ describe('runtime dependency compatibility', () => {
51
51
  expect(rolldown.dev).toBe(true);
52
52
  });
53
53
  });
54
+
55
+ describe('development dependency compatibility', () => {
56
+ it('uses the reviewed fast-uri release through the Periapsis AJV toolchain', () => {
57
+ const periapsis = lock.packages['node_modules/periapsis'];
58
+ const ajv = lock.packages['node_modules/ajv'];
59
+ const fastUri = lock.packages['node_modules/fast-uri'];
60
+
61
+ expect(pkg.devDependencies.periapsis).toBe('^2.1.2');
62
+ expect(periapsis.dependencies.ajv).toBe('^8.17.1');
63
+ expect(ajv.dependencies['fast-uri']).toBe('^3.0.1');
64
+ expect(fastUri.version).toBe('3.1.9');
65
+ expect(fastUri.dev).toBe(true);
66
+ });
67
+ });
@@ -18,6 +18,28 @@ afterEach(() => {
18
18
  });
19
19
 
20
20
  describe('generateHtmlReport', () => {
21
+ it('distinguishes evidence-backed completion from limited-evidence reports', () => {
22
+ const outPath = path.join(tmpDir, 'verification-levels.html');
23
+ generateHtmlReport({ missions: [{
24
+ missionName: 'Work package',
25
+ status: 'passed',
26
+ verification: 'limited_evidence',
27
+ steps: [{ turn: 0, result: 'โš ๏ธ Completion reported ยท limited evidence' }],
28
+ }, {
29
+ missionName: 'Dialogs',
30
+ status: 'passed',
31
+ verification: 'evidence_recorded',
32
+ steps: [{ turn: 0, result: 'โœ… Mission accomplished ยท evidence recorded' }],
33
+ }] }, outPath);
34
+
35
+ const html = fs.readFileSync(outPath, 'utf8');
36
+ expect(html).toContain('โš ๏ธ Passed ยท limited evidence');
37
+ expect(html).toContain('โš ๏ธ Completion reported ยท limited evidence');
38
+ expect(html).toContain('โœ… Passed ยท evidence recorded');
39
+ expect(html).toContain('โœ… Mission accomplished ยท evidence recorded');
40
+ expect(html).toContain('step-result warn');
41
+ });
42
+
21
43
  it('renders recovered status and recovery evidence', () => {
22
44
  const outPath = path.join(tmpDir, 'recovered.html');
23
45
  generateHtmlReport({ summary: { passed: 0, recovered: 1, failed: 0 }, missions: [{
@@ -55,7 +77,7 @@ describe('generateHtmlReport', () => {
55
77
  submissionType: 'mission',
56
78
  status: 'passed',
57
79
  durationMs: 2500,
58
- steps: [{ turn: 0, result: 'โœ… Mission Success', durationMs: 1250 }],
80
+ steps: [{ turn: 0, result: 'โœ… Mission accomplished', durationMs: 1250 }],
59
81
  }],
60
82
  };
61
83
  const outPath = path.join(tmpDir, 'timed.html');
@@ -85,7 +107,7 @@ describe('generateHtmlReport', () => {
85
107
  retryAttempt: 2,
86
108
  retryLimit: 5,
87
109
  events: ['did something'],
88
- result: 'โš ๏ธ Turn Issues',
110
+ result: 'โš ๏ธ Navigation issue',
89
111
  summary: 'Do thing',
90
112
  tokensUsed: 10,
91
113
  totalTokensUsed: 20,
@@ -108,7 +130,7 @@ describe('generateHtmlReport', () => {
108
130
  expect(html).toContain('--cyan:#79d7df');
109
131
  expect(html).toContain('Mission A');
110
132
  expect(html).toContain('Turn 1 (re-attempt 1/5)');
111
- expect(html).toContain('โš ๏ธ Turn Issues');
133
+ expect(html).toContain('โš ๏ธ Navigation issue');
112
134
  expect(html).toContain('data-tag="smoke"');
113
135
  expect(html).toContain('data-tags="smoke"');
114
136
  expect(html).toContain('CLI version:</strong> 1.7.0');
@@ -137,7 +159,7 @@ describe('generateHtmlReport', () => {
137
159
  turn: 0,
138
160
  retryAttempt: 1,
139
161
  events: ['<script>alert(1)</script>'],
140
- result: 'โœ… Passed',
162
+ result: '๐Ÿš€ Progressing',
141
163
  summary: '<b>bold</b>',
142
164
  },
143
165
  ],
@@ -170,7 +192,7 @@ describe('generateHtmlReport', () => {
170
192
  generateHtmlReport({ missions: [{
171
193
  missionName: 'Dialogs', status: 'passed', steps: [{
172
194
  turn: 0,
173
- result: 'โœ… Passed',
195
+ result: '๐Ÿš€ Progressing',
174
196
  events: ['dialog handled'],
175
197
  dialogs: [{
176
198
  event: 'javascript_dialog',
@@ -198,7 +220,7 @@ describe('generateHtmlReport', () => {
198
220
  it('renders run, mission, and step Jev token spend', () => {
199
221
  const steps = [{
200
222
  turn: 0,
201
- result: 'โœ… Mission Success (Jev gate)',
223
+ result: 'โœ… Mission accomplished (Jev verified)',
202
224
  tokensUsed: 500,
203
225
  inputTokens: 450,
204
226
  outputTokens: 50,
@@ -248,8 +270,8 @@ describe('generateHtmlReport', () => {
248
270
  optimization: { configured: true, objective: 'cost', mode: 'live' },
249
271
  missions: [{
250
272
  missionName: 'Routed Mission', status: 'passed', steps: [
251
- { turn: 0, model: 'gpt-5.6-terra', tokensUsed: 100, inputTokens: 90, outputTokens: 10, result: 'โœ… Passed' },
252
- { turn: 1, model: 'gpt-5.6-luna', tokensUsed: 40, inputTokens: 35, outputTokens: 5, result: 'โœ… Passed', modelRouting: { applied: true } },
273
+ { turn: 0, model: 'gpt-5.6-terra', tokensUsed: 100, inputTokens: 90, outputTokens: 10, result: '๐Ÿš€ Progressing' },
274
+ { turn: 1, model: 'gpt-5.6-luna', tokensUsed: 40, inputTokens: 35, outputTokens: 5, result: '๐Ÿš€ Progressing', modelRouting: { applied: true } },
253
275
  ],
254
276
  }],
255
277
  }, outPath);
@@ -41,7 +41,10 @@ export function generateHtmlReport(report, outputPath) {
41
41
  .replace(/&/g, '&amp;').replace(/</g, '&lt;')
42
42
  .replace(/>/g, '&gt;')
43
43
  .replace(/"/g, '&quot;');
44
- const badge = (status) =>
44
+ const badge = (status, verification) =>
45
+ status === 'passed' && verification === 'verified' ? 'โœ… Passed ยท independently verified' :
46
+ status === 'passed' && verification === 'evidence_recorded' ? 'โœ… Passed ยท evidence recorded' :
47
+ status === 'passed' && verification === 'limited_evidence' ? 'โš ๏ธ Passed ยท limited evidence' :
45
48
  status === 'passed' ? 'โœ… Passed' :
46
49
  status === 'recovered' ? 'โš ๏ธ Passed after recovery' :
47
50
  status === 'failed' ? 'โŒ Failed' : (status || 'โ€”');
@@ -105,10 +108,14 @@ export function generateHtmlReport(report, outputPath) {
105
108
  const stepItems = steps.map((step, idx) => {
106
109
  const events = Array.isArray(step.events) ? step.events : [];
107
110
  const dialogs = Array.isArray(step.dialogs) ? step.dialogs : [];
108
- const ok = /โœ…|Passed|Mission Success/i.test(step.result || '');
111
+ const ok = /Mission accomplished|Mission Success|Action confirmed/i.test(step.result || '');
112
+ const progressing = /Progressing|Replotting course|Awaiting mission verdict|^โœ… Passed$/i.test(step.result || '');
113
+ const reported = /Completion reported|limited evidence/i.test(step.result || '');
109
114
  const resultRaw = step.result || 'โ€”';
110
- const resultTooltip = resultRaw.includes('โš ๏ธ Turn Issues')
115
+ const resultTooltip = /Navigation issue|Turn Issues/.test(resultRaw)
111
116
  ? 'Turn had tool/action issues (e.g., selector/timeouts). It may not indicate a product bug.'
117
+ : progressing
118
+ ? 'Intermediate activity only; this does not indicate mission completion.'
112
119
  : '';
113
120
  const retryAttempt = step.retryAttempt || 1; // includes initial
114
121
  const retryLimit = step.retryLimit; // number of retries allowed (excludes initial)
@@ -175,12 +182,12 @@ export function generateHtmlReport(report, outputPath) {
175
182
  : '';
176
183
 
177
184
  return `
178
- <details class="step" ${ok ? '' : 'open'}>
185
+ <details class="step" ${ok || progressing || reported ? '' : 'open'}>
179
186
  <summary>
180
187
  <span class="turn">${step.isClosureTurn ? 'Closure' : `Turn ${esc((step.turn ?? idx) + 1)}`}${esc(attemptLabel)}${stepDuration}</span>
181
188
  ${humanInput}
182
189
  ${planSpan}
183
- <span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
190
+ <span class="step-result ${ok ? 'ok' : reported ? 'warn' : progressing ? '' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
184
191
  <span class="tokens">${esc(step.model || llm.model || 'Model')} tokens: ${esc(step.tokensUsed ?? 'โ€”')} (${esc(step.inputTokens ?? 'โ€”')} in / ${esc(step.outputTokens ?? 'โ€”')} out)${routeLabel}${stepJev}</span>
185
192
  </summary>
186
193
  ${dialogCards ? `<div class="dialog-evidence">${dialogCards}</div>` : ''}
@@ -193,7 +200,7 @@ export function generateHtmlReport(report, outputPath) {
193
200
  const type = String(m.submissionType || 'mission').toLowerCase();
194
201
  const typeLabel = (type === 'premission' || type === 'postmission') ? type : 'mission';
195
202
  const prettyTitle = `${esc(m.missionName || 'Mission')} โ€” ${typeLabel}${m.submissionName ? `: ${esc(m.submissionName)}` : ''}`;
196
- const statusClass = m.status === 'failed' ? 'bad' : (m.status === 'recovered' ? 'warn' : (m.status === 'passed' ? 'ok' : ''));
203
+ const statusClass = m.status === 'failed' ? 'bad' : (m.status === 'recovered' || m.verification === 'limited_evidence' ? 'warn' : (m.status === 'passed' ? 'ok' : ''));
197
204
  const recoveries = Array.isArray(m.recoveries) ? m.recoveries : [];
198
205
  const recoveryItems = recoveries.length ? `
199
206
  <div class="recoveries">
@@ -205,7 +212,7 @@ export function generateHtmlReport(report, outputPath) {
205
212
  <details class="mission-submission">
206
213
  <summary>
207
214
  <span class="name">${prettyTitle}</span>
208
- <span class="status ${statusClass}">${badge(m.status)}</span>
215
+ <span class="status ${statusClass}">${badge(m.status, m.verification)}</span>
209
216
  <span class="meta">steps: ${steps.length} โ€ข duration: ${mDurationSec}s โ€ข ${esc(tokenMeta(missionModelUsage, missionJevUsage))}</span>
210
217
  <span class="toolbar">
211
218
  <button class="btn-mini toggle" data-scope="submission" aria-label="Expand">โ–ผ</button>
@@ -232,6 +239,15 @@ export function generateHtmlReport(report, outputPath) {
232
239
 
233
240
  const missionGroupBlock = (missionName, subs) => {
234
241
  const status = groupStatus(subs);
242
+ const verification = status === 'passed'
243
+ ? subs.some(s => s.verification === 'limited_evidence')
244
+ ? 'limited_evidence'
245
+ : subs.every(s => s.verification === 'verified')
246
+ ? 'verified'
247
+ : subs.some(s => s.verification === 'evidence_recorded')
248
+ ? 'evidence_recorded'
249
+ : undefined
250
+ : undefined;
235
251
  const totalSteps = subs.reduce((n, s) => n + (Array.isArray(s.steps) ? s.steps.length : 0), 0);
236
252
  const startTimes = subs.map(s => s.startTime || 0).filter(Boolean);
237
253
  const endTimes = subs.map(s => s.endTime || 0).filter(Boolean);
@@ -255,7 +271,7 @@ export function generateHtmlReport(report, outputPath) {
255
271
  <summary>
256
272
  <span class="name">${esc(missionName)}</span>
257
273
  <span class="mission-tags">${tags.map(tag => `<span class="tag-small" style="${tagStyle(tag)}">${esc(tag)}</span>`).join('')}</span>
258
- <span class="status ${status === 'failed' ? 'bad' : status === 'recovered' ? 'warn' : 'ok'}">${badge(status)}</span>
274
+ <span class="status ${status === 'failed' ? 'bad' : status === 'recovered' || verification === 'limited_evidence' ? 'warn' : 'ok'}">${badge(status, verification)}</span>
259
275
  <span class="meta">${sourceFiles.length ? `${sourceFiles.map(esc).join(', ')} โ€ข ` : ''}submissions: ${subs.length} โ€ข steps: ${totalSteps} โ€ข duration: ${groupDur}s โ€ข ${esc(tokenMeta(groupModelUsage, groupJevUsage))}</span>
260
276
  <span class="toolbar">
261
277
  <button class="btn-mini toggle" data-scope="mission" aria-label="Expand">โ–ผ</button>
@@ -448,6 +464,7 @@ export function generateHtmlReport(report, outputPath) {
448
464
  white-space:nowrap; overflow:hidden; text-overflow:ellipsis;
449
465
  }
450
466
  .step-result.ok{ color: var(--ok); font-weight:700; }
467
+ .step-result.warn{ color: var(--amber); font-weight:700; }
451
468
  .step-result.bad{ color: var(--bad); font-weight:700; }
452
469
  .tokens{ color: var(--text-muted); font-size:12px; margin-left:auto; font-variant-numeric: tabular-nums; }
453
470