testronaut 1.10.2 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +198 -0
- package/TODO.md +5 -0
- package/bin/cli.js +234 -5
- package/bin/terminalUi.js +51 -0
- package/core/agent.js +1 -0
- package/core/browserEvidence.js +184 -0
- package/core/jevRouting.js +77 -0
- package/core/jevShadow.js +325 -0
- package/core/jevSuiteSelection.js +39 -0
- package/core/jevUsage.js +62 -0
- package/core/optimization.js +98 -0
- package/core/turnLoop.js +174 -10
- package/experiments/CROSS_PROVIDER.md +19 -0
- package/experiments/README.md +62 -0
- package/experiments/buildExperimentPublicationPackage.js +219 -0
- package/experiments/buildOptimizationDataPackage.js +390 -0
- package/experiments/crossProviderBenchmark.js +227 -0
- package/experiments/optimizationBenchmark.js +339 -0
- package/package.json +5 -1
- package/tests/binTests/cli.helpers.test.js +16 -0
- package/tests/binTests/cli.integration.test.js +14 -0
- package/tests/binTests/terminalUi.test.js +34 -0
- package/tests/coreTests/browserEvidence.test.js +70 -0
- package/tests/coreTests/jevRouting.test.js +57 -0
- package/tests/coreTests/jevShadow.test.js +158 -0
- package/tests/coreTests/jevSuiteSelection.test.js +24 -0
- package/tests/coreTests/jevUsage.test.js +58 -0
- package/tests/coreTests/optimization.test.js +80 -0
- package/tests/coreTests/turnLoop.test.js +304 -0
- package/tests/experimentTests/buildOptimizationDataPackage.test.js +25 -0
- package/tests/experimentTests/crossProviderBenchmark.test.js +37 -0
- package/tests/experimentTests/optimizationBenchmark.test.js +49 -0
- package/tests/toolsTests/generateHtmlReport.test.js +83 -0
- package/tests/toolsTests/turnLoopUtils.test.js +17 -0
- package/tools/generateHtmlReport.js +96 -26
- package/tools/turnLoopUtils.js +7 -11
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import fs from 'fs';
|
|
19
19
|
import path from 'path';
|
|
20
20
|
import { normalizeTags } from '../core/tags.js';
|
|
21
|
+
import { summarizeJevUsage } from '../core/jevUsage.js';
|
|
21
22
|
|
|
22
23
|
/**
|
|
23
24
|
* Render and write a Testronaut run report to disk.
|
|
@@ -27,7 +28,7 @@ import { normalizeTags } from '../core/tags.js';
|
|
|
27
28
|
* @returns {string} absolute path to the written HTML file
|
|
28
29
|
*/
|
|
29
30
|
export function generateHtmlReport(report, outputPath) {
|
|
30
|
-
const { runId, startTime, endTime, missions = [], summary = {}, llm = {}, cli = {} } = report;
|
|
31
|
+
const { runId, startTime, endTime, missions = [], summary = {}, llm = {}, cli = {}, optimization } = report;
|
|
31
32
|
const reportTags = normalizeTags(report.tags ?? missions.flatMap(m => m.submissionType === 'mission' ? (m.tags ?? []) : []));
|
|
32
33
|
const durationSec =
|
|
33
34
|
(startTime && endTime)
|
|
@@ -47,6 +48,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
47
48
|
: String(llm.provider ?? '').trim().toLowerCase();
|
|
48
49
|
const provider = {
|
|
49
50
|
openai: { name: 'OpenAI', logo: '<svg viewBox="0 0 24 24" aria-hidden="true"><path fill="currentColor" d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"/></svg>' },
|
|
51
|
+
gemini: { name: 'Google Gemini', logo: '<svg viewBox="0 0 24 24" aria-hidden="true"><defs><linearGradient id="gemini-gradient" x1="4" y1="20" x2="20" y2="4"><stop stop-color="#3186ff"/><stop offset=".5" stop-color="#8e68ff"/><stop offset="1" stop-color="#ef78b5"/></linearGradient></defs><path fill="url(#gemini-gradient)" d="M12 2c.7 5.8 4.2 9.3 10 10-5.8.7-9.3 4.2-10 10-.7-5.8-4.2-9.3-10-10 5.8-.7 9.3-4.2 10-10z"/></svg>' },
|
|
50
52
|
anthropic: { name: 'Anthropic', logo: '<img src="data:image/svg+xml;base64,PHN2ZyByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICA8dGl0bGU+Q2xhdWRlPC90aXRsZT4KICA8cmVjdCB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHJ4PSI1IiBmaWxsPSIjRDk3NzU3Ii8+CiAgPHBhdGggZmlsbD0iI2ZmZiIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoMyAzKSBzY2FsZSguNzUpIiBkPSJNNC43MDkgMTUuOTU1bDQuNzItMi42NDcuMDgtLjIzLS4wOC0uMTI4SDkuMmwtLjc5LS4wNDgtMi42OTgtLjA3My0yLjMzOS0uMDk3LTIuMjY2LS4xMjItLjU3MS0uMTIxTDAgMTEuNzg0bC4wNTUtLjM1Mi40OC0uMzIxLjY4Ni4wNiAxLjUyLjEwMyAyLjI3OC4xNTggMS42NTIuMDk3IDIuNDQ5LjI1NWguMzg5bC4wNTUtLjE1Ny0uMTM0LS4wOTgtLjEwMy0uMDk3LTIuMzU4LTEuNTk2LTIuNTUyLTEuNjg4LTEuMzM2LS45NzItLjcyNC0uNDkxLS4zNjQtLjQ2Mi0uMTU4LTEuMDA4LjY1Ni0uNzIyLjg4MS4wNi4yMjUuMDYxLjg5My42ODYgMS45MDggMS40NzYgMi40OTEgMS44MzMuMzY1LjMwNC4xNDUtLjEwMy4wMTktLjA3My0uMTY0LS4yNzQtMS4zNTUtMi40NDYtMS40NDYtMi40OS0uNjQ0LTEuMDMyLS4xNy0uNjE5YTIuOTcgMi45NyAwIDAxLS4xMDQtLjcyOUw2LjI4My4xMzQgNi42OTYgMGwuOTk2LjEzNC40Mi4zNjQuNjIgMS40MTQgMS4wMDIgMi4yMjkgMS41NTUgMy4wMy40NTYuODk4LjI0My44MzIuMDkxLjI1NWguMTU4VjkuMDFsLjEyOC0xLjcwNi4yMzctMi4wOTUuMjMtMi42OTUuMDgtLjc2LjM3Ni0uOTEuNzQ3LS40OTIuNTg0LjI4LjQ4LjY4NS0uMDY3LjQ0NC0uMjg2IDEuODUxLS41NTkgMi45MDMtLjM2NCAxLjk0MmguMjEybC4yNDMtLjI0Mi45ODUtMS4zMDYgMS42NTItMi4wNjQuNzMtLjgyLjg1LS45MDQuNTQ3LS40MzFoMS4wMzNsLjc2IDEuMTI5LS4zNCAxLjE2Ni0xLjA2NCAxLjM0Ny0uODgxIDEuMTQyLTEuMjY0IDEuNy0uNzkgMS4zNi4wNzMuMTEuMTg4LS4wMiAyLjg1Ni0uNjA2IDEuNTQzLS4yOCAxLjg0MS0uMzE1LjgzMy4zODguMDkxLjM5NS0uMzI4LjgwNy0xLjk2OS40ODYtMi4zMDkuNDYyLTMuNDM5LjgxMy0uMDQyLjAzLjA0OS4wNjEgMS41NDkuMTQ2LjY2Mi4wMzZoMS42MjJsMy4wMi4yMjUuNzkuNTIyLjQ3NC42MzgtLjA3OS40ODUtMS4yMTUuNjItMS42NC0uMzg5LTMuODI5LS45MS0xLjMxMi0uMzI5aC0uMTgydi4xMWwxLjA5MyAxLjA2OCAyLjAwNiAxLjgxIDIuNTA5IDIuMzMuMTI3LjU3OC0uMzIyLjQ1NS0uMzQtLjA0OS0yLjIwNS0xLjY1Ny0uODUxLS43NDctMS45MjYtMS42MmgtLjEyOHYuMTdsLjQ0NC42NDkgMi4zNDUgMy41MjEuMTIyIDEuMDgtLjE3LjM1My0uNjA4LjIxMy0uNjY4LS4xMjItMS4zNzQtMS45MjUtMS40MTUtMi4xNjctMS4xNDMtMS45NDMtLjE0LjA4LS42NzQgNy4yNTQtLjMxNi4zNy0uNzI5LjI4LS42MDctLjQ2MS0uMzIyLS43NDcuMzIyLTEuNDc2LjM4OS0xLjkyNC4zMTUtMS41My4yODYtMS45LjE3LS42MzItLjAxMi0uMDQyLS4xNC4wMTgtMS40MzQgMS45NjctMi4xOCAyLjk0NS0xLjcyNiAxLjg0NS0uNDE0LjE2NC0uNzE3LS4zNy4wNjctLjY2Mi40MDEtLjU4OSAyLjM4OC0zLjAzNiAxLjQ0LTEuODgyLjkzLTEuMDg2LS4wMDYtLjE1OGgtLjA1NUw0LjEzMiAxOC41NmwtMS4xMy4xNDYtLjQ4Ny0uNDU2LjA2MS0uNzQ2LjIzMS0uMjQzIDEuOTA4LTEuMzEyLS4wMDYuMDA2eiIvPgo8L3N2Zz4K" alt="" />' },
|
|
51
53
|
}[providerKey] ?? { name: llm.provider || 'Unknown provider', logo: '' };
|
|
52
54
|
const tagPalette = [
|
|
@@ -59,6 +61,34 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
59
61
|
const [color, background] = tagPalette[index];
|
|
60
62
|
return `color:${color};background:${background};border-color:${color}66`;
|
|
61
63
|
};
|
|
64
|
+
const formatTokens = value => Number(value || 0).toLocaleString('en-US');
|
|
65
|
+
const summarizeModelTokens = (steps) => steps.reduce((usage, step) => {
|
|
66
|
+
const input = Number(step.inputTokens || 0);
|
|
67
|
+
const output = Number(step.outputTokens || 0);
|
|
68
|
+
const model = String(step.model || llm.model || 'Model');
|
|
69
|
+
usage.input += input;
|
|
70
|
+
usage.output += output;
|
|
71
|
+
usage.total += Number(step.tokensUsed ?? (input + output));
|
|
72
|
+
usage.byModel[model] ||= { input: 0, output: 0, total: 0, turns: 0 };
|
|
73
|
+
usage.byModel[model].input += input;
|
|
74
|
+
usage.byModel[model].output += output;
|
|
75
|
+
usage.byModel[model].total += Number(step.tokensUsed ?? (input + output));
|
|
76
|
+
usage.byModel[model].turns += 1;
|
|
77
|
+
return usage;
|
|
78
|
+
}, { input: 0, output: 0, total: 0, byModel: {} });
|
|
79
|
+
const tokenMeta = (modelUsage, jevUsage) => [
|
|
80
|
+
`Model tokens: ${formatTokens(modelUsage.total)} (${formatTokens(modelUsage.input)} in / ${formatTokens(modelUsage.output)} out)`,
|
|
81
|
+
Object.keys(modelUsage.byModel).length > 1
|
|
82
|
+
? `By model: ${Object.entries(modelUsage.byModel).map(([model, usage]) => `${model} ${formatTokens(usage.total)}`).join(' / ')}`
|
|
83
|
+
: '',
|
|
84
|
+
jevUsage?.completed ? `Jev tokens: ${formatTokens(jevUsage.inputTokens)} in / ${formatTokens(jevUsage.outputTokens)} out` : '',
|
|
85
|
+
].filter(Boolean).join(' • ');
|
|
86
|
+
const reportJevUsage = report.jevShadow?.evaluations
|
|
87
|
+
? report.jevShadow
|
|
88
|
+
: summarizeJevUsage(missions.flatMap(m => Array.isArray(m.steps) ? m.steps : []));
|
|
89
|
+
const reportModelUsage = summarizeModelTokens(missions.flatMap(m => Array.isArray(m.steps) ? m.steps : []));
|
|
90
|
+
const reportModels = Object.keys(reportModelUsage.byModel);
|
|
91
|
+
const jevLogo = '<svg viewBox="0 0 24 24" role="img" aria-hidden="true"><rect width="24" height="24" rx="4" fill="#e547b5"/><g fill="none" stroke="#181a1b" stroke-width="2.25" stroke-linecap="square" stroke-linejoin="round"><path d="M12 2.5 6.1 6v10.1l3.2 1.9 2.7-1.6V9.6l3-1.8V4.3z"/><path d="m9.3 9.6 2.7 1.6 3-1.8 3 1.8v7L12 21.5l-3-1.8V13z"/><path d="m12 11.2 3 1.8 3-1.8M15 13v5.2l-6 3.3M9.3 9.6 12 8l3 1.8"/></g></svg>';
|
|
62
92
|
|
|
63
93
|
const submissionBlock = (m) => {
|
|
64
94
|
const mDurationSec =
|
|
@@ -67,6 +97,8 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
67
97
|
: (typeof m.durationSeconds === 'number' ? m.durationSeconds.toFixed(2) : '—');
|
|
68
98
|
|
|
69
99
|
const steps = Array.isArray(m.steps) ? m.steps : [];
|
|
100
|
+
const missionJevUsage = m.jevShadow?.evaluations ? m.jevShadow : summarizeJevUsage(steps);
|
|
101
|
+
const missionModelUsage = summarizeModelTokens(steps);
|
|
70
102
|
const stepItems = steps.map((step, idx) => {
|
|
71
103
|
const events = Array.isArray(step.events) ? step.events : [];
|
|
72
104
|
const ok = /✅|Passed|Mission Success/i.test(step.result || '');
|
|
@@ -87,6 +119,16 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
87
119
|
const imgTag = step.screenshotPath
|
|
88
120
|
? `<img src="${esc(step.screenshotPath)}" alt="screenshot turn ${esc(step.turn ?? idx)}">`
|
|
89
121
|
: '';
|
|
122
|
+
const stepJev = step.jevShadow?.status === 'ok'
|
|
123
|
+
? ` • Jev: ${formatTokens(step.jevShadow.usage?.input_tokens)} in / ${formatTokens(step.jevShadow.usage?.output_tokens)} out`
|
|
124
|
+
: step.jevShadow
|
|
125
|
+
? ` • Jev: ${esc(step.jevShadow.status)}${step.jevShadow.reason ? ` (${esc(step.jevShadow.reason)})` : ''}`
|
|
126
|
+
: '';
|
|
127
|
+
const routeLabel = step.modelRouting?.fallback
|
|
128
|
+
? ' • Jev route fallback'
|
|
129
|
+
: step.modelRouting?.applied
|
|
130
|
+
? ' • routed by Jev'
|
|
131
|
+
: '';
|
|
90
132
|
|
|
91
133
|
const plan = (typeof step.summary === 'string' && step.summary.trim())
|
|
92
134
|
? step.summary.trim()
|
|
@@ -102,7 +144,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
102
144
|
${humanInput}
|
|
103
145
|
${planSpan}
|
|
104
146
|
<span class="step-result ${ok ? 'ok' : 'bad'}" ${resultTooltip ? `title="${esc(resultTooltip)}"` : ''}>${esc(resultRaw)}</span>
|
|
105
|
-
<span class="tokens"
|
|
147
|
+
<span class="tokens">${esc(step.model || llm.model || 'Model')} tokens: ${esc(step.tokensUsed ?? '—')} (${esc(step.inputTokens ?? '—')} in / ${esc(step.outputTokens ?? '—')} out)${routeLabel}${stepJev}</span>
|
|
106
148
|
</summary>
|
|
107
149
|
<pre class="events">${esc(events.join('\n')) || '(no events)'}</pre>
|
|
108
150
|
${imgTag}
|
|
@@ -120,7 +162,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
120
162
|
<summary>
|
|
121
163
|
<span class="name">${prettyTitle}</span>
|
|
122
164
|
<span class="status ${statusClass}">${badge(m.status)}</span>
|
|
123
|
-
<span class="meta">steps: ${steps.length} • duration: ${mDurationSec}s</span>
|
|
165
|
+
<span class="meta">steps: ${steps.length} • duration: ${mDurationSec}s • ${esc(tokenMeta(missionModelUsage, missionJevUsage))}</span>
|
|
124
166
|
<span class="toolbar">
|
|
125
167
|
<button class="btn-mini toggle" data-scope="submission" aria-label="Expand">▼</button>
|
|
126
168
|
</span>
|
|
@@ -151,6 +193,8 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
151
193
|
const groupDur = (firstStart != null && lastEnd != null) ? ((lastEnd - firstStart) / 1000).toFixed(2) : '—';
|
|
152
194
|
const tags = normalizeTags(subs.flatMap(s => s.submissionType === 'mission' ? (s.tags ?? []) : []));
|
|
153
195
|
const sourceFiles = [...new Set(subs.map(s => s.file).filter(Boolean))];
|
|
196
|
+
const groupJevUsage = summarizeJevUsage(subs.flatMap(s => Array.isArray(s.steps) ? s.steps : []));
|
|
197
|
+
const groupModelUsage = summarizeModelTokens(subs.flatMap(s => Array.isArray(s.steps) ? s.steps : []));
|
|
154
198
|
|
|
155
199
|
// pre → mission → post
|
|
156
200
|
const order = { premission: 0, mission: 1, postmission: 2 };
|
|
@@ -165,7 +209,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
165
209
|
<span class="name">${esc(missionName)}</span>
|
|
166
210
|
<span class="mission-tags">${tags.map(tag => `<span class="tag-small" style="${tagStyle(tag)}">${esc(tag)}</span>`).join('')}</span>
|
|
167
211
|
<span class="status ${status === 'failed' ? 'bad' : 'ok'}">${badge(status)}</span>
|
|
168
|
-
<span class="meta">${sourceFiles.length ? `${sourceFiles.map(esc).join(', ')} • ` : ''}submissions: ${subs.length} • steps: ${totalSteps} • duration: ${groupDur}s</span>
|
|
212
|
+
<span class="meta">${sourceFiles.length ? `${sourceFiles.map(esc).join(', ')} • ` : ''}submissions: ${subs.length} • steps: ${totalSteps} • duration: ${groupDur}s • ${esc(tokenMeta(groupModelUsage, groupJevUsage))}</span>
|
|
169
213
|
<span class="toolbar">
|
|
170
214
|
<button class="btn-mini toggle" data-scope="mission" aria-label="Expand">▼</button>
|
|
171
215
|
</span>
|
|
@@ -196,55 +240,71 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
196
240
|
<title>Testronaut Report – ${esc(runId)}</title>
|
|
197
241
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
198
242
|
<style>
|
|
199
|
-
/* =====
|
|
243
|
+
/* ===== Testronaut launch-control tokens ===== */
|
|
200
244
|
:root{
|
|
201
|
-
--bg-top:#
|
|
202
|
-
--bg-mid:#
|
|
203
|
-
--bg-btm:#
|
|
204
|
-
|
|
205
|
-
--
|
|
206
|
-
--
|
|
245
|
+
--bg-top:#050b18;
|
|
246
|
+
--bg-mid:#081426;
|
|
247
|
+
--bg-btm:#030712;
|
|
248
|
+
--panel:rgba(8,24,45,.78);
|
|
249
|
+
--panel-strong:rgba(7,19,36,.94);
|
|
250
|
+
--orange:#ff8a43;
|
|
251
|
+
--amber:#ffc15b;
|
|
252
|
+
--cyan:#79d7df;
|
|
253
|
+
|
|
254
|
+
--hairline:rgba(121,215,223,.20);
|
|
255
|
+
--hairline-strong:rgba(121,215,223,.34);
|
|
207
256
|
|
|
208
257
|
--text:rgba(255,255,255,.92);
|
|
209
258
|
--text-muted:rgba(255,255,255,.65);
|
|
210
259
|
|
|
211
|
-
--ok:#
|
|
212
|
-
--bad:#
|
|
260
|
+
--ok:#39d9aa;
|
|
261
|
+
--bad:#ff6678;
|
|
213
262
|
|
|
214
263
|
--chip-ok-bg:rgba(34,197,94,.14);
|
|
215
264
|
--chip-bad-bg:rgba(239,68,68,.16);
|
|
216
|
-
--chip-border:rgba(
|
|
265
|
+
--chip-border:rgba(121,215,223,.30);
|
|
217
266
|
--chip-ok-border:rgba(34,197,94,.40);
|
|
218
267
|
--chip-bad-border:rgba(239,68,68,.45);
|
|
219
268
|
}
|
|
220
269
|
|
|
221
270
|
/* ===== Base ===== */
|
|
222
|
-
html,body{height:100%;}
|
|
271
|
+
html,body{min-height:100%;}
|
|
223
272
|
body{
|
|
224
273
|
margin:0; padding:24px;
|
|
225
274
|
font-family: ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,Ubuntu,Helvetica,Arial;
|
|
226
|
-
background:
|
|
275
|
+
background:
|
|
276
|
+
radial-gradient(circle at 84% 8%,rgba(255,138,67,.16),transparent 25rem),
|
|
277
|
+
radial-gradient(circle at 10% 32%,rgba(121,215,223,.10),transparent 28rem),
|
|
278
|
+
linear-gradient(180deg,var(--bg-top) 0%,var(--bg-mid) 60%,var(--bg-btm) 100%);
|
|
279
|
+
background-attachment:fixed;
|
|
227
280
|
color: var(--text);
|
|
228
281
|
-webkit-font-smoothing: antialiased;
|
|
229
282
|
-moz-osx-font-smoothing: grayscale;
|
|
230
283
|
}
|
|
284
|
+
body::before{
|
|
285
|
+
content:"";position:fixed;inset:0;pointer-events:none;opacity:.22;
|
|
286
|
+
background-image:linear-gradient(rgba(121,215,223,.08) 1px,transparent 1px),linear-gradient(90deg,rgba(121,215,223,.08) 1px,transparent 1px);
|
|
287
|
+
background-size:42px 42px;mask-image:linear-gradient(to bottom,black,transparent 82%);
|
|
288
|
+
}
|
|
231
289
|
|
|
232
290
|
/* ===== Utilities ===== */
|
|
233
291
|
.gradient-text{
|
|
234
|
-
background: linear-gradient(90deg,#
|
|
292
|
+
background: linear-gradient(90deg,#fff7e7,var(--amber),var(--orange));
|
|
235
293
|
-webkit-background-clip:text; background-clip:text; color:transparent;
|
|
236
294
|
}
|
|
237
295
|
.glass{
|
|
238
|
-
background:
|
|
296
|
+
background: var(--panel);
|
|
239
297
|
border: 1px solid var(--hairline);
|
|
240
298
|
border-radius:16px;
|
|
241
299
|
backdrop-filter: blur(8px); -webkit-backdrop-filter: blur(8px);
|
|
242
|
-
box-shadow: 0
|
|
300
|
+
box-shadow: 0 18px 45px rgba(0,0,0,.28), inset 0 1px rgba(255,255,255,.035);
|
|
243
301
|
}
|
|
244
302
|
|
|
245
303
|
/* ===== Header ===== */
|
|
246
|
-
.header{ max-width:960px; margin:0 auto 16px; padding:
|
|
247
|
-
|
|
304
|
+
.header{ max-width:960px; margin:0 auto 16px; padding:22px 24px; position:relative; overflow:hidden; }
|
|
305
|
+
.header::after{content:"";position:absolute;right:0;top:0;width:92px;height:4px;background:linear-gradient(90deg,var(--orange),var(--amber));}
|
|
306
|
+
.eyebrow{margin-bottom:7px;color:var(--cyan);font-size:11px;font-weight:800;letter-spacing:.18em;text-transform:uppercase;}
|
|
307
|
+
h1{ margin:0 0 12px; font-size:27px; font-weight:850; color:#fff; letter-spacing:-.02em; }
|
|
248
308
|
.run-meta{ color:var(--text-muted); display:grid; gap:4px; }
|
|
249
309
|
.provider-meta{ display:inline-flex; align-items:center; gap:7px; }
|
|
250
310
|
.provider-logo{ width:20px; height:20px; color:var(--text); flex:none; }
|
|
@@ -258,8 +318,10 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
258
318
|
padding:6px 12px; border-radius:999px; font-size:12px; color:var(--text);
|
|
259
319
|
letter-spacing:.2px;
|
|
260
320
|
}
|
|
321
|
+
.pill:first-child{border-color:rgba(255,138,67,.5);box-shadow:inset 3px 0 var(--orange);}
|
|
261
322
|
.pill.ok{ background: var(--chip-ok-bg); color: var(--ok); border-color: var(--chip-ok-border); font-weight:700; }
|
|
262
323
|
.pill.bad{ background: var(--chip-bad-bg); color: var(--bad); border-color: var(--chip-bad-border); font-weight:700; }
|
|
324
|
+
.pill.warn{ background:rgba(251,191,36,.14);color:#fbbf24;border-color:rgba(251,191,36,.4);font-weight:700; }
|
|
263
325
|
.tag-filter{max-width:928px;margin:0 auto 20px;padding:18px 16px;background:rgba(2,6,23,.30);}
|
|
264
326
|
.tag-filter-heading{display:flex;align-items:center;gap:7px;margin-bottom:12px;flex-wrap:wrap;}
|
|
265
327
|
.tag-filter-title{font-size:14px;color:var(--text);font-weight:800;}
|
|
@@ -267,7 +329,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
267
329
|
.match-count{margin-left:auto;color:var(--text-muted);font-size:12px;}
|
|
268
330
|
.tag-buttons,.mission-tags{display:flex;gap:6px;flex-wrap:wrap;align-items:center;}
|
|
269
331
|
.tag-button,.tag-small{border:1px solid var(--chip-border);background:rgba(255,255,255,.06);color:var(--text);border-radius:999px;padding:5px 10px;font-size:11px;font-weight:700;}
|
|
270
|
-
.tag-button{cursor:pointer;transition:transform .15s ease,background .15s ease}.tag-button:hover{transform:translateY(-1px);background:rgba(
|
|
332
|
+
.tag-button{cursor:pointer;transition:transform .15s ease,background .15s ease}.tag-button:hover{transform:translateY(-1px);background:rgba(121,215,223,.12)}.tag-button.active{box-shadow:0 0 0 2px rgba(255,138,67,.48);}
|
|
271
333
|
.filter-controls{display:flex;gap:16px;align-items:center;margin-top:14px;font-size:12px;color:var(--text-muted);flex-wrap:wrap;}
|
|
272
334
|
.filter-controls select{color:var(--text);background:#111a31;border:1px solid var(--chip-border);border-radius:8px;padding:6px 24px 6px 8px;}
|
|
273
335
|
.show-control{cursor:help;display:flex;align-items:center;gap:5px;}
|
|
@@ -277,12 +339,12 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
277
339
|
@media(max-width:640px){.match-count{width:100%;margin-left:0}.tag-filter{padding:14px 12px}.meta{display:none}}
|
|
278
340
|
|
|
279
341
|
/* ===== Disclosure blocks ===== */
|
|
280
|
-
details{ background:
|
|
342
|
+
details{ background: var(--panel); border:1px solid var(--hairline); border-radius:16px; margin:10px 0; overflow:hidden; }
|
|
281
343
|
summary{
|
|
282
344
|
cursor:pointer; padding:14px 16px; display:flex; align-items:center; gap:12px;
|
|
283
345
|
color:#fff; background: rgba(255,255,255,.04);
|
|
284
346
|
}
|
|
285
|
-
.mission-group > summary{ font-weight:800; font-size:15px; letter-spacing:.2px; }
|
|
347
|
+
.mission-group > summary{ font-weight:800; font-size:15px; letter-spacing:.2px; border-left:3px solid var(--orange); }
|
|
286
348
|
.mission-group > summary:hover{ background: rgba(255,255,255,.08); }
|
|
287
349
|
|
|
288
350
|
.group-body{ padding:12px; display:grid; gap:10px; }
|
|
@@ -336,7 +398,7 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
336
398
|
.tokens{ color: var(--text-muted); font-size:12px; margin-left:auto; font-variant-numeric: tabular-nums; }
|
|
337
399
|
|
|
338
400
|
.events{
|
|
339
|
-
background:
|
|
401
|
+
background: var(--panel-strong);
|
|
340
402
|
color: rgba(255,255,255,.92);
|
|
341
403
|
padding:12px; border-radius:12px; margin:12px 0 0;
|
|
342
404
|
font:12px/1.45 ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New";
|
|
@@ -357,9 +419,10 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
357
419
|
</head>
|
|
358
420
|
<body>
|
|
359
421
|
<div class="header glass">
|
|
422
|
+
<div class="eyebrow">Mission Control · Flight Report</div>
|
|
360
423
|
<h1>🧑🚀 <span class="gradient-text">Testronaut Report</span></h1>
|
|
361
424
|
<div class="run-meta">
|
|
362
|
-
<div class="provider-meta"><span class="provider-logo" title="${esc(provider.name)}" aria-label="${esc(provider.name)}">${provider.logo}</span><strong>${esc(llm.model ?? '—')}</strong
|
|
425
|
+
<div class="provider-meta"><span class="provider-logo" title="${esc(provider.name)}" aria-label="${esc(provider.name)}">${provider.logo}</span><strong>${esc((reportModels.length ? reportModels : [llm.model ?? '—']).join(' + '))}</strong>${reportJevUsage.evaluations ? ` <span aria-hidden="true">+</span> <span class="provider-logo" title="Jev" aria-label="Jev">${jevLogo}</span><strong>Jev</strong>` : ''}</div>
|
|
363
426
|
<div><strong>Run ID:</strong> ${esc(runId ?? '—')}</div>
|
|
364
427
|
<div><strong>Start:</strong> ${esc(startTime ?? '—')}</div>
|
|
365
428
|
<div><strong>End:</strong> ${esc(endTime ?? '—')}</div>
|
|
@@ -372,6 +435,13 @@ export function generateHtmlReport(report, outputPath) {
|
|
|
372
435
|
<div class="pill">Missions: ${esc(summary.totalMissions ?? totals.total)}</div>
|
|
373
436
|
<div class="pill ok">Passed: ${esc(summary.passed ?? totals.passed)}</div>
|
|
374
437
|
<div class="pill bad">Failed: ${esc(summary.failed ?? totals.failed)}</div>
|
|
438
|
+
${optimization?.configured ? `<div class="pill">Optimization: ${esc(optimization.objective)} (${esc(optimization.mode)})</div>` : ''}
|
|
439
|
+
${Object.entries(reportModelUsage.byModel).length > 1
|
|
440
|
+
? Object.entries(reportModelUsage.byModel).map(([model, usage]) => `<div class="pill">${esc(model)} tokens: ${esc(formatTokens(usage.total))} (${esc(formatTokens(usage.input))} in / ${esc(formatTokens(usage.output))} out)</div>`).join('')
|
|
441
|
+
: `<div class="pill">Model tokens: ${esc(formatTokens(reportModelUsage.total))} (${esc(formatTokens(reportModelUsage.input))} in / ${esc(formatTokens(reportModelUsage.output))} out)</div>`}
|
|
442
|
+
${reportJevUsage.completed ? `<div class="pill">Jev tokens: ${esc(formatTokens(reportJevUsage.inputTokens))} in / ${esc(formatTokens(reportJevUsage.outputTokens))} out</div>` : ''}
|
|
443
|
+
${reportJevUsage.skipped ? `<div class="pill warn">Jev skipped: ${esc(formatTokens(reportJevUsage.skipped))}</div>` : ''}
|
|
444
|
+
${reportJevUsage.errors ? `<div class="pill warn">Jev errors: ${esc(formatTokens(reportJevUsage.errors))}</div>` : ''}
|
|
375
445
|
</div>
|
|
376
446
|
|
|
377
447
|
<div class="tag-filter glass">
|
package/tools/turnLoopUtils.js
CHANGED
|
@@ -5,17 +5,13 @@ export const finalResponseHandler = (msg) => {
|
|
|
5
5
|
const safeContent = redactPasswordInText(content);
|
|
6
6
|
const final = content.trim().replace(/^[#*_`\s]+/, '');
|
|
7
7
|
const verdict = final.match(/^(?:(?:result|final)\s*:\s*)?(success|failure)\b/i)?.[1]?.toLowerCase();
|
|
8
|
-
if (verdict === 'success') {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
console.log(
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
console.log('\n┏━ FINAL AGENT RESPONSE ━━━━━━━━━━━━━━━━━━━');
|
|
16
|
-
console.log(safeContent);
|
|
17
|
-
console.log('┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
|
|
18
|
-
return { finalMessage: safeContent, success: false};
|
|
8
|
+
if (verdict === 'success' || verdict === 'failure') {
|
|
9
|
+
const success = verdict === 'success';
|
|
10
|
+
const signal = success ? 'SUCCESS' : 'FAILURE';
|
|
11
|
+
console.log(`\n╭─ 🧑🚀 MISSION TRANSMISSION / ${signal} ─────────────────`);
|
|
12
|
+
safeContent.split('\n').forEach(line => console.log(`│ ${line}`));
|
|
13
|
+
console.log('╰─ END TRANSMISSION ─────────────────────────────');
|
|
14
|
+
return { finalMessage: safeContent, success };
|
|
19
15
|
}
|
|
20
16
|
return null;
|
|
21
17
|
}
|