runledger 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runledger/__init__.py +3 -0
- runledger/__main__.py +11 -0
- runledger/artifacts/__init__.py +13 -0
- runledger/artifacts/junit.py +51 -0
- runledger/artifacts/report.py +424 -0
- runledger/artifacts/run_log.py +24 -0
- runledger/artifacts/summary.py +205 -0
- runledger/assertions/__init__.py +14 -0
- runledger/assertions/base.py +11 -0
- runledger/assertions/engine.py +116 -0
- runledger/assertions/json_schema.py +51 -0
- runledger/assertions/required_fields.py +19 -0
- runledger/assertions/tool_contract.py +69 -0
- runledger/baseline/__init__.py +4 -0
- runledger/baseline/io.py +26 -0
- runledger/baseline/models.py +77 -0
- runledger/cassette/__init__.py +6 -0
- runledger/cassette/loader.py +53 -0
- runledger/cassette/match.py +61 -0
- runledger/cassette/models.py +13 -0
- runledger/cassette/writer.py +22 -0
- runledger/cli.py +487 -0
- runledger/config/__init__.py +3 -0
- runledger/config/loader.py +85 -0
- runledger/config/models.py +65 -0
- runledger/protocol/__init__.py +25 -0
- runledger/protocol/jsonl.py +50 -0
- runledger/protocol/messages.py +87 -0
- runledger/regression/__init__.py +3 -0
- runledger/regression/engine.py +197 -0
- runledger/runner/__init__.py +16 -0
- runledger/runner/budgets.py +35 -0
- runledger/runner/engine.py +340 -0
- runledger/runner/models.py +47 -0
- runledger/runner/subprocess.py +161 -0
- runledger/tools/__init__.py +3 -0
- runledger/tools/builtin.py +21 -0
- runledger/tools/registry.py +58 -0
- runledger/util/__init__.py +4 -0
- runledger/util/canonical_json.py +21 -0
- runledger/util/redaction.py +63 -0
- runledger-0.1.0.dist-info/METADATA +406 -0
- runledger-0.1.0.dist-info/RECORD +47 -0
- runledger-0.1.0.dist-info/WHEEL +5 -0
- runledger-0.1.0.dist-info/entry_points.txt +2 -0
- runledger-0.1.0.dist-info/licenses/LICENSE +21 -0
- runledger-0.1.0.dist-info/top_level.txt +1 -0
runledger/__init__.py
ADDED
runledger/__main__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from .junit import write_junit
|
|
2
|
+
from .report import write_report
|
|
3
|
+
from .run_log import write_run_log
|
|
4
|
+
from .summary import build_summary, create_run_dir, write_summary
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"build_summary",
|
|
8
|
+
"create_run_dir",
|
|
9
|
+
"write_junit",
|
|
10
|
+
"write_report",
|
|
11
|
+
"write_run_log",
|
|
12
|
+
"write_summary",
|
|
13
|
+
]
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Iterable
|
|
5
|
+
import xml.etree.ElementTree as ET
|
|
6
|
+
|
|
7
|
+
from runledger.runner.models import CaseResult
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def write_junit(run_dir: Path, suite_name: str, cases: Iterable[CaseResult]) -> Path:
|
|
11
|
+
junit_path = run_dir / "junit.xml"
|
|
12
|
+
run_dir.mkdir(parents=True, exist_ok=True)
|
|
13
|
+
|
|
14
|
+
cases_list = list(cases)
|
|
15
|
+
tests = len(cases_list)
|
|
16
|
+
failures = sum(1 for case in cases_list if not case.passed)
|
|
17
|
+
time_seconds = sum(case.wall_ms for case in cases_list) / 1000.0
|
|
18
|
+
|
|
19
|
+
suite = ET.Element(
|
|
20
|
+
"testsuite",
|
|
21
|
+
attrib={
|
|
22
|
+
"name": suite_name,
|
|
23
|
+
"tests": str(tests),
|
|
24
|
+
"failures": str(failures),
|
|
25
|
+
"time": f"{time_seconds:.3f}",
|
|
26
|
+
},
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
for case in cases_list:
|
|
30
|
+
testcase = ET.SubElement(
|
|
31
|
+
suite,
|
|
32
|
+
"testcase",
|
|
33
|
+
attrib={
|
|
34
|
+
"name": case.case_id,
|
|
35
|
+
"time": f"{case.wall_ms / 1000.0:.3f}",
|
|
36
|
+
},
|
|
37
|
+
)
|
|
38
|
+
if case.failure is not None:
|
|
39
|
+
failure = ET.SubElement(
|
|
40
|
+
testcase,
|
|
41
|
+
"failure",
|
|
42
|
+
attrib={"message": case.failure.message},
|
|
43
|
+
)
|
|
44
|
+
failure.text = case.failure.type
|
|
45
|
+
|
|
46
|
+
tree = ET.ElementTree(suite)
|
|
47
|
+
junit_path.write_text(
|
|
48
|
+
ET.tostring(tree.getroot(), encoding="unicode"),
|
|
49
|
+
encoding="utf-8",
|
|
50
|
+
)
|
|
51
|
+
return junit_path
|
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from jinja2 import Template
|
|
8
|
+
from markupsafe import Markup
|
|
9
|
+
|
|
10
|
+
from runledger.util.redaction import redact
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
_REPORT_TEMPLATE = """<!doctype html>
|
|
14
|
+
<html lang="en">
|
|
15
|
+
<head>
|
|
16
|
+
<meta charset="utf-8" />
|
|
17
|
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
18
|
+
<title>RunLedger Report</title>
|
|
19
|
+
<style>
|
|
20
|
+
:root {
|
|
21
|
+
--bg: #f5efe6;
|
|
22
|
+
--bg2: #e6f0f5;
|
|
23
|
+
--ink: #1f2a30;
|
|
24
|
+
--muted: #5b6b75;
|
|
25
|
+
--accent: #1f7a8c;
|
|
26
|
+
--accent-2: #e07a5f;
|
|
27
|
+
--card: #fffaf3;
|
|
28
|
+
--border: rgba(31, 42, 48, 0.12);
|
|
29
|
+
--shadow: 0 12px 30px rgba(31, 42, 48, 0.12);
|
|
30
|
+
}
|
|
31
|
+
* { box-sizing: border-box; }
|
|
32
|
+
body {
|
|
33
|
+
margin: 0;
|
|
34
|
+
font-family: "Space Grotesk", "Avenir Next", "Trebuchet MS", sans-serif;
|
|
35
|
+
color: var(--ink);
|
|
36
|
+
background:
|
|
37
|
+
radial-gradient(1200px 500px at 10% -10%, #f8d9c0, transparent 60%),
|
|
38
|
+
radial-gradient(900px 400px at 100% 0%, #cfe6f3, transparent 55%),
|
|
39
|
+
linear-gradient(120deg, var(--bg), var(--bg2));
|
|
40
|
+
min-height: 100vh;
|
|
41
|
+
}
|
|
42
|
+
header {
|
|
43
|
+
padding: 32px 24px 12px;
|
|
44
|
+
max-width: 1200px;
|
|
45
|
+
margin: 0 auto;
|
|
46
|
+
animation: rise 0.7s ease forwards;
|
|
47
|
+
}
|
|
48
|
+
h1 {
|
|
49
|
+
margin: 0;
|
|
50
|
+
font-size: clamp(28px, 4vw, 42px);
|
|
51
|
+
letter-spacing: -0.02em;
|
|
52
|
+
}
|
|
53
|
+
.subtitle {
|
|
54
|
+
color: var(--muted);
|
|
55
|
+
margin-top: 6px;
|
|
56
|
+
}
|
|
57
|
+
.pill {
|
|
58
|
+
display: inline-flex;
|
|
59
|
+
align-items: center;
|
|
60
|
+
gap: 6px;
|
|
61
|
+
border-radius: 999px;
|
|
62
|
+
padding: 6px 12px;
|
|
63
|
+
font-size: 12px;
|
|
64
|
+
background: rgba(31, 42, 48, 0.08);
|
|
65
|
+
color: var(--ink);
|
|
66
|
+
margin-right: 8px;
|
|
67
|
+
}
|
|
68
|
+
main {
|
|
69
|
+
max-width: 1200px;
|
|
70
|
+
margin: 0 auto;
|
|
71
|
+
padding: 0 24px 40px;
|
|
72
|
+
display: grid;
|
|
73
|
+
gap: 24px;
|
|
74
|
+
}
|
|
75
|
+
.grid {
|
|
76
|
+
display: grid;
|
|
77
|
+
gap: 16px;
|
|
78
|
+
grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
|
|
79
|
+
}
|
|
80
|
+
.card {
|
|
81
|
+
background: var(--card);
|
|
82
|
+
border: 1px solid var(--border);
|
|
83
|
+
border-radius: 18px;
|
|
84
|
+
padding: 16px;
|
|
85
|
+
box-shadow: var(--shadow);
|
|
86
|
+
animation: rise 0.7s ease forwards;
|
|
87
|
+
}
|
|
88
|
+
.card h3 {
|
|
89
|
+
margin: 0 0 6px;
|
|
90
|
+
font-size: 16px;
|
|
91
|
+
color: var(--muted);
|
|
92
|
+
text-transform: uppercase;
|
|
93
|
+
letter-spacing: 0.08em;
|
|
94
|
+
}
|
|
95
|
+
.card .value {
|
|
96
|
+
font-size: 26px;
|
|
97
|
+
font-weight: 600;
|
|
98
|
+
}
|
|
99
|
+
.section-title {
|
|
100
|
+
margin: 8px 0;
|
|
101
|
+
font-size: 20px;
|
|
102
|
+
}
|
|
103
|
+
.filters {
|
|
104
|
+
display: flex;
|
|
105
|
+
flex-wrap: wrap;
|
|
106
|
+
gap: 8px;
|
|
107
|
+
margin-bottom: 8px;
|
|
108
|
+
}
|
|
109
|
+
button.filter {
|
|
110
|
+
border: 1px solid var(--border);
|
|
111
|
+
background: rgba(255, 255, 255, 0.9);
|
|
112
|
+
border-radius: 999px;
|
|
113
|
+
padding: 6px 12px;
|
|
114
|
+
cursor: pointer;
|
|
115
|
+
font-weight: 600;
|
|
116
|
+
}
|
|
117
|
+
button.filter.active {
|
|
118
|
+
background: var(--accent);
|
|
119
|
+
color: #fff;
|
|
120
|
+
border-color: transparent;
|
|
121
|
+
}
|
|
122
|
+
table {
|
|
123
|
+
width: 100%;
|
|
124
|
+
border-collapse: collapse;
|
|
125
|
+
font-size: 14px;
|
|
126
|
+
}
|
|
127
|
+
th, td {
|
|
128
|
+
text-align: left;
|
|
129
|
+
padding: 10px 12px;
|
|
130
|
+
border-bottom: 1px solid var(--border);
|
|
131
|
+
}
|
|
132
|
+
tr:hover {
|
|
133
|
+
background: rgba(31, 122, 140, 0.08);
|
|
134
|
+
cursor: pointer;
|
|
135
|
+
}
|
|
136
|
+
.status {
|
|
137
|
+
font-weight: 700;
|
|
138
|
+
text-transform: uppercase;
|
|
139
|
+
font-size: 12px;
|
|
140
|
+
letter-spacing: 0.08em;
|
|
141
|
+
}
|
|
142
|
+
.status.pass { color: #1b7f5a; }
|
|
143
|
+
.status.fail { color: #c0392b; }
|
|
144
|
+
.status.error { color: #c06a00; }
|
|
145
|
+
.status.skipped { color: #7b7b7b; }
|
|
146
|
+
.detail {
|
|
147
|
+
display: grid;
|
|
148
|
+
gap: 16px;
|
|
149
|
+
grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
|
|
150
|
+
}
|
|
151
|
+
.trace {
|
|
152
|
+
max-height: 340px;
|
|
153
|
+
overflow: auto;
|
|
154
|
+
background: #0f1c1f;
|
|
155
|
+
color: #f4f1ed;
|
|
156
|
+
padding: 12px;
|
|
157
|
+
border-radius: 12px;
|
|
158
|
+
font-family: "IBM Plex Mono", "Consolas", monospace;
|
|
159
|
+
font-size: 12px;
|
|
160
|
+
}
|
|
161
|
+
.trace-event {
|
|
162
|
+
border-left: 3px solid var(--accent-2);
|
|
163
|
+
padding: 8px 10px;
|
|
164
|
+
margin-bottom: 8px;
|
|
165
|
+
background: rgba(224, 122, 95, 0.08);
|
|
166
|
+
}
|
|
167
|
+
.trace-event strong {
|
|
168
|
+
display: block;
|
|
169
|
+
color: #ffd166;
|
|
170
|
+
}
|
|
171
|
+
.badge {
|
|
172
|
+
display: inline-block;
|
|
173
|
+
background: rgba(31, 122, 140, 0.15);
|
|
174
|
+
color: var(--accent);
|
|
175
|
+
padding: 4px 8px;
|
|
176
|
+
border-radius: 6px;
|
|
177
|
+
font-size: 12px;
|
|
178
|
+
margin-right: 6px;
|
|
179
|
+
}
|
|
180
|
+
@keyframes rise {
|
|
181
|
+
from { opacity: 0; transform: translateY(10px); }
|
|
182
|
+
to { opacity: 1; transform: translateY(0); }
|
|
183
|
+
}
|
|
184
|
+
@media (max-width: 720px) {
|
|
185
|
+
header { padding: 24px 16px 8px; }
|
|
186
|
+
main { padding: 0 16px 32px; }
|
|
187
|
+
table { font-size: 13px; }
|
|
188
|
+
}
|
|
189
|
+
</style>
|
|
190
|
+
</head>
|
|
191
|
+
<body>
|
|
192
|
+
<header>
|
|
193
|
+
<h1>RunLedger Report</h1>
|
|
194
|
+
<div class="subtitle" id="suite-subtitle"></div>
|
|
195
|
+
<div id="meta"></div>
|
|
196
|
+
</header>
|
|
197
|
+
<main>
|
|
198
|
+
<section class="grid" id="kpis"></section>
|
|
199
|
+
<section class="card">
|
|
200
|
+
<h2 class="section-title">Cases</h2>
|
|
201
|
+
<div class="filters" id="filters"></div>
|
|
202
|
+
<table>
|
|
203
|
+
<thead>
|
|
204
|
+
<tr>
|
|
205
|
+
<th>Case</th>
|
|
206
|
+
<th>Status</th>
|
|
207
|
+
<th>Wall (ms)</th>
|
|
208
|
+
<th>Tool Calls</th>
|
|
209
|
+
<th>Tool Errors</th>
|
|
210
|
+
</tr>
|
|
211
|
+
</thead>
|
|
212
|
+
<tbody id="case-rows"></tbody>
|
|
213
|
+
</table>
|
|
214
|
+
</section>
|
|
215
|
+
<section class="card">
|
|
216
|
+
<h2 class="section-title">Case Detail</h2>
|
|
217
|
+
<div class="detail">
|
|
218
|
+
<div>
|
|
219
|
+
<div id="case-meta"></div>
|
|
220
|
+
<div id="case-metrics"></div>
|
|
221
|
+
<div id="case-failure"></div>
|
|
222
|
+
</div>
|
|
223
|
+
<div>
|
|
224
|
+
<div class="filters" id="trace-filters"></div>
|
|
225
|
+
<div class="trace" id="trace-view"></div>
|
|
226
|
+
</div>
|
|
227
|
+
</div>
|
|
228
|
+
</section>
|
|
229
|
+
</main>
|
|
230
|
+
<script id="runledger-data" type="application/json">{{ data_json }}</script>
|
|
231
|
+
<script>
|
|
232
|
+
const data = JSON.parse(document.getElementById("runledger-data").textContent);
|
|
233
|
+
const summary = data.summary || {};
|
|
234
|
+
const cases = summary.cases || [];
|
|
235
|
+
const traces = data.traces || {};
|
|
236
|
+
|
|
237
|
+
const suiteName = summary.suite?.name || "unknown";
|
|
238
|
+
const runId = summary.run?.run_id || "n/a";
|
|
239
|
+
const mode = summary.run?.mode || summary.suite?.tool_mode || "n/a";
|
|
240
|
+
const status = summary.run?.exit_status || "n/a";
|
|
241
|
+
|
|
242
|
+
document.getElementById("suite-subtitle").textContent =
|
|
243
|
+
`${suiteName} • run ${runId}`;
|
|
244
|
+
|
|
245
|
+
const meta = document.getElementById("meta");
|
|
246
|
+
meta.innerHTML = [
|
|
247
|
+
`<span class="pill">Mode: ${mode}</span>`,
|
|
248
|
+
`<span class="pill">Status: ${status}</span>`,
|
|
249
|
+
`<span class="pill">Generated: ${summary.generated_at || "n/a"}</span>`
|
|
250
|
+
].join("");
|
|
251
|
+
|
|
252
|
+
const aggregates = summary.aggregates || {};
|
|
253
|
+
const metrics = aggregates.metrics || {};
|
|
254
|
+
const passRate = aggregates.pass_rate !== undefined ? (aggregates.pass_rate * 100).toFixed(1) + "%" : "n/a";
|
|
255
|
+
|
|
256
|
+
const kpis = [
|
|
257
|
+
{ label: "Pass Rate", value: passRate },
|
|
258
|
+
{ label: "Cases Pass", value: aggregates.cases_pass ?? "n/a" },
|
|
259
|
+
{ label: "Cases Fail", value: aggregates.cases_fail ?? "n/a" },
|
|
260
|
+
{ label: "Cases Error", value: aggregates.cases_error ?? "n/a" },
|
|
261
|
+
{ label: "p95 Wall (ms)", value: metrics.wall_ms?.p95 ?? "n/a" },
|
|
262
|
+
{ label: "Avg Wall (ms)", value: metrics.wall_ms?.mean ?? "n/a" },
|
|
263
|
+
{ label: "p95 Tool Calls", value: metrics.tool_calls?.p95 ?? "n/a" },
|
|
264
|
+
{ label: "Avg Tool Calls", value: metrics.tool_calls?.mean ?? "n/a" },
|
|
265
|
+
];
|
|
266
|
+
|
|
267
|
+
const kpiWrap = document.getElementById("kpis");
|
|
268
|
+
kpis.forEach((item, idx) => {
|
|
269
|
+
const card = document.createElement("div");
|
|
270
|
+
card.className = "card";
|
|
271
|
+
card.style.animationDelay = `${0.05 * idx}s`;
|
|
272
|
+
card.innerHTML = `<h3>${item.label}</h3><div class="value">${item.value}</div>`;
|
|
273
|
+
kpiWrap.appendChild(card);
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
const statusFilters = ["all", "pass", "fail", "error", "skipped"];
|
|
277
|
+
const filters = document.getElementById("filters");
|
|
278
|
+
let activeFilter = "all";
|
|
279
|
+
|
|
280
|
+
function renderFilters() {
|
|
281
|
+
filters.innerHTML = "";
|
|
282
|
+
statusFilters.forEach((filter) => {
|
|
283
|
+
const btn = document.createElement("button");
|
|
284
|
+
btn.className = "filter" + (filter === activeFilter ? " active" : "");
|
|
285
|
+
btn.textContent = filter.toUpperCase();
|
|
286
|
+
btn.onclick = () => {
|
|
287
|
+
activeFilter = filter;
|
|
288
|
+
renderFilters();
|
|
289
|
+
renderTable();
|
|
290
|
+
};
|
|
291
|
+
filters.appendChild(btn);
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function renderTable() {
|
|
296
|
+
const tbody = document.getElementById("case-rows");
|
|
297
|
+
tbody.innerHTML = "";
|
|
298
|
+
const filtered = cases.filter((item) => {
|
|
299
|
+
if (activeFilter === "all") return true;
|
|
300
|
+
return item.status === activeFilter;
|
|
301
|
+
});
|
|
302
|
+
filtered.forEach((item) => {
|
|
303
|
+
const row = document.createElement("tr");
|
|
304
|
+
row.innerHTML = `
|
|
305
|
+
<td>${item.id}</td>
|
|
306
|
+
<td><span class="status ${item.status}">${item.status}</span></td>
|
|
307
|
+
<td>${item.wall_ms}</td>
|
|
308
|
+
<td>${item.tool_calls}</td>
|
|
309
|
+
<td>${item.tool_errors}</td>
|
|
310
|
+
`;
|
|
311
|
+
row.onclick = () => selectCase(item.id);
|
|
312
|
+
tbody.appendChild(row);
|
|
313
|
+
});
|
|
314
|
+
if (filtered.length && !selectedCaseId) {
|
|
315
|
+
selectCase(filtered[0].id);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
const traceFilters = ["all", "task_start", "tool_call", "tool_result", "final_output", "log", "case_end"];
|
|
320
|
+
let activeTraceFilter = "all";
|
|
321
|
+
|
|
322
|
+
function renderTraceFilters() {
|
|
323
|
+
const container = document.getElementById("trace-filters");
|
|
324
|
+
container.innerHTML = "";
|
|
325
|
+
traceFilters.forEach((filter) => {
|
|
326
|
+
const btn = document.createElement("button");
|
|
327
|
+
btn.className = "filter" + (filter === activeTraceFilter ? " active" : "");
|
|
328
|
+
btn.textContent = filter.replace("_", " ").toUpperCase();
|
|
329
|
+
btn.onclick = () => {
|
|
330
|
+
activeTraceFilter = filter;
|
|
331
|
+
renderTrace();
|
|
332
|
+
renderTraceFilters();
|
|
333
|
+
};
|
|
334
|
+
container.appendChild(btn);
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
let selectedCaseId = null;
|
|
339
|
+
|
|
340
|
+
function selectCase(caseId) {
|
|
341
|
+
selectedCaseId = caseId;
|
|
342
|
+
const item = cases.find((c) => c.id === caseId);
|
|
343
|
+
if (!item) return;
|
|
344
|
+
document.getElementById("case-meta").innerHTML = [
|
|
345
|
+
`<span class="badge">Case: ${item.id}</span>`,
|
|
346
|
+
`<span class="badge">Status: ${item.status}</span>`,
|
|
347
|
+
`<span class="badge">Assertions: ${item.assertions?.failed ?? 0}/${item.assertions?.total ?? 0}</span>`
|
|
348
|
+
].join("");
|
|
349
|
+
document.getElementById("case-metrics").innerHTML = `
|
|
350
|
+
<p>Wall: <strong>${item.wall_ms}</strong> ms</p>
|
|
351
|
+
<p>Tool calls: <strong>${item.tool_calls}</strong> • errors: <strong>${item.tool_errors}</strong></p>
|
|
352
|
+
<p>Tokens in/out: <strong>${item.tokens_in ?? "n/a"}</strong> / <strong>${item.tokens_out ?? "n/a"}</strong></p>
|
|
353
|
+
<p>Cost USD: <strong>${item.cost_usd ?? "n/a"}</strong> • Steps: <strong>${item.steps ?? "n/a"}</strong></p>
|
|
354
|
+
`;
|
|
355
|
+
const failure = item.failure_reason || "None";
|
|
356
|
+
document.getElementById("case-failure").innerHTML = `<p>Failure: <strong>${failure}</strong></p>`;
|
|
357
|
+
renderTrace();
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
function renderTrace() {
|
|
361
|
+
const traceView = document.getElementById("trace-view");
|
|
362
|
+
const events = traces[selectedCaseId] || [];
|
|
363
|
+
const filtered = events.filter((event) => {
|
|
364
|
+
if (activeTraceFilter === "all") return true;
|
|
365
|
+
return event.type === activeTraceFilter;
|
|
366
|
+
});
|
|
367
|
+
if (!filtered.length) {
|
|
368
|
+
traceView.innerHTML = "<em>No trace events to display.</em>";
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
traceView.innerHTML = filtered.map((event) => {
|
|
372
|
+
const payload = JSON.stringify(event, null, 2);
|
|
373
|
+
return `
|
|
374
|
+
<div class="trace-event">
|
|
375
|
+
<strong>${event.type}</strong>
|
|
376
|
+
<pre>${payload}</pre>
|
|
377
|
+
</div>
|
|
378
|
+
`;
|
|
379
|
+
}).join("");
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
renderFilters();
|
|
383
|
+
renderTraceFilters();
|
|
384
|
+
renderTable();
|
|
385
|
+
</script>
|
|
386
|
+
</body>
|
|
387
|
+
</html>
|
|
388
|
+
"""
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _load_run_log(run_path: Path) -> dict[str, list[dict[str, Any]]]:
|
|
392
|
+
traces: dict[str, list[dict[str, Any]]] = {}
|
|
393
|
+
if not run_path.is_file():
|
|
394
|
+
return traces
|
|
395
|
+
for line in run_path.read_text(encoding="utf-8").splitlines():
|
|
396
|
+
if not line.strip():
|
|
397
|
+
continue
|
|
398
|
+
try:
|
|
399
|
+
event = json.loads(line)
|
|
400
|
+
except json.JSONDecodeError:
|
|
401
|
+
continue
|
|
402
|
+
case_id = str(event.get("case_id", "unknown"))
|
|
403
|
+
traces.setdefault(case_id, []).append(event)
|
|
404
|
+
return traces
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def write_report(
|
|
408
|
+
run_dir: Path,
|
|
409
|
+
*,
|
|
410
|
+
summary: dict[str, Any],
|
|
411
|
+
run_log_path: Path | None = None,
|
|
412
|
+
) -> Path:
|
|
413
|
+
if run_log_path is None:
|
|
414
|
+
run_log_path = run_dir / "run.jsonl"
|
|
415
|
+
traces = _load_run_log(run_log_path)
|
|
416
|
+
report_data = {
|
|
417
|
+
"summary": redact(summary),
|
|
418
|
+
"traces": redact(traces),
|
|
419
|
+
}
|
|
420
|
+
data_json = json.dumps(report_data, ensure_ascii=False).replace("</", "<\\/")
|
|
421
|
+
html = Template(_REPORT_TEMPLATE).render(data_json=Markup(data_json))
|
|
422
|
+
report_path = run_dir / "report.html"
|
|
423
|
+
report_path.write_text(html, encoding="utf-8")
|
|
424
|
+
return report_path
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Iterable
|
|
6
|
+
|
|
7
|
+
from runledger.runner.models import CaseResult
|
|
8
|
+
from runledger.util.redaction import redact
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def write_run_log(run_dir: Path, cases: Iterable[CaseResult]) -> Path:
|
|
12
|
+
run_path = run_dir / "run.jsonl"
|
|
13
|
+
run_dir.mkdir(parents=True, exist_ok=True)
|
|
14
|
+
|
|
15
|
+
with run_path.open("w", encoding="utf-8") as handle:
|
|
16
|
+
for case in cases:
|
|
17
|
+
for event in case.trace:
|
|
18
|
+
event = redact(event)
|
|
19
|
+
handle.write(
|
|
20
|
+
json.dumps(event, separators=(",", ":"), ensure_ascii=False)
|
|
21
|
+
+ "\n"
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
return run_path
|