rulelab-cli 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rulelab/__init__.py +3 -0
- rulelab/__main__.py +3 -0
- rulelab/_demo/results.json +956 -0
- rulelab/analysis.py +174 -0
- rulelab/cli.py +344 -0
- rulelab/configs.py +39 -0
- rulelab/engine.py +272 -0
- rulelab/errors.py +9 -0
- rulelab/experiment.py +195 -0
- rulelab/report.py +205 -0
- rulelab/rules.py +225 -0
- rulelab/runner.py +94 -0
- rulelab/stats.py +180 -0
- rulelab/templates/report.html.j2 +379 -0
- rulelab_cli-0.1.2.dist-info/METADATA +260 -0
- rulelab_cli-0.1.2.dist-info/RECORD +19 -0
- rulelab_cli-0.1.2.dist-info/WHEEL +4 -0
- rulelab_cli-0.1.2.dist-info/entry_points.txt +3 -0
- rulelab_cli-0.1.2.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
{% macro t(en, es) %}<span lang="en">{{ en }}</span><span lang="es">{{ es }}</span>{% endmacro %}
|
|
2
|
+
{% macro chip(v) %}<span class="chip v-{{ v }}">{{ t(verdicts[v][0], verdicts[v][1]) }}</span>{% endmacro %}
|
|
3
|
+
{% macro bar(b, label) %}
|
|
4
|
+
{% if b.ok %}
|
|
5
|
+
<svg class="ci" viewBox="0 0 {{ b.w }} 22" width="{{ b.w }}" height="22" role="img" aria-label="{{ label }}">
|
|
6
|
+
<rect x="{{ b.m0 }}" y="3" width="{{ b.m1 - b.m0 }}" height="16" class="ci-margin"/>
|
|
7
|
+
<line x1="{{ b.z }}" x2="{{ b.z }}" y1="1" y2="21" class="ci-zero"/>
|
|
8
|
+
<line x1="{{ b.x0 }}" x2="{{ b.x1 }}" y1="11" y2="11" class="ci-range"/>
|
|
9
|
+
<line x1="{{ b.x0 }}" x2="{{ b.x0 }}" y1="6" y2="16" class="ci-range"/>
|
|
10
|
+
<line x1="{{ b.x1 }}" x2="{{ b.x1 }}" y1="6" y2="16" class="ci-range"/>
|
|
11
|
+
{% if b.xe is not none %}<circle cx="{{ b.xe }}" cy="11" r="3.6" class="ci-est"/>{% endif %}
|
|
12
|
+
</svg>
|
|
13
|
+
{% else %}<span class="muted">n/a</span>{% endif %}
|
|
14
|
+
{% endmacro %}
|
|
15
|
+
<!doctype html>
|
|
16
|
+
<html lang="en" data-lang="en">
|
|
17
|
+
<head>
|
|
18
|
+
<meta charset="utf-8">
|
|
19
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
20
|
+
<title>rulelab report · {{ res.file.path }}</title>
|
|
21
|
+
<meta name="description" content="Leave-one-out ablation of {{ res.file.path }}: success and cost per rule.">
|
|
22
|
+
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'%3E%3Crect x='1' y='1' width='22' height='22' rx='6' fill='%235d51d1'/%3E%3Cpath d='M10 5.2v4.6l-3.6 6.9a1.2 1.2 0 0 0 1.1 1.8h9a1.2 1.2 0 0 0 1.1-1.8L14 9.8V5.2M8.9 5.2h6.2' fill='none' stroke='%23fff' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'/%3E%3Cpath d='M8.3 14.3h7.4' fill='none' stroke='%23fff' stroke-width='2' stroke-linecap='round'/%3E%3C/svg%3E">
|
|
23
|
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
24
|
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
25
|
+
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
|
26
|
+
<script>
|
|
27
|
+
(function () {
|
|
28
|
+
var d = document.documentElement, lang = null, theme = null;
|
|
29
|
+
try { lang = localStorage.getItem("rulelab-lang"); theme = localStorage.getItem("rulelab-theme"); } catch (e) {}
|
|
30
|
+
if (!lang) lang = (navigator.language || "en").toLowerCase().indexOf("es") === 0 ? "es" : "en";
|
|
31
|
+
d.setAttribute("data-lang", lang); d.setAttribute("lang", lang);
|
|
32
|
+
if (theme) d.setAttribute("data-theme", theme);
|
|
33
|
+
})();
|
|
34
|
+
</script>
|
|
35
|
+
<style>
|
|
36
|
+
:root {
|
|
37
|
+
--accent: #5d51d1; --accent-soft: rgba(93, 81, 209, .10); --tile: #5d51d1; --tile-fg: #ffffff;
|
|
38
|
+
--bg: #ffffff; --surface: #ffffff; --surface-2: #f7f7f9; --border: #e4e4e9;
|
|
39
|
+
--shadow: 0 1px 2px rgba(17, 17, 22, .04), 0 4px 16px rgba(17, 17, 22, .05);
|
|
40
|
+
--fg: #111116; --muted: #5c5d66;
|
|
41
|
+
--ok: #11813c; --ok-soft: rgba(17, 129, 60, .10);
|
|
42
|
+
--warn: #a45e00; --warn-soft: rgba(164, 94, 0, .10);
|
|
43
|
+
--err: #c22826; --err-soft: rgba(194, 40, 38, .09);
|
|
44
|
+
--neutral: #5c5d66; --neutral-soft: rgba(92, 93, 102, .10);
|
|
45
|
+
--sans: "Inter", ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
|
|
46
|
+
--mono: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
|
47
|
+
color-scheme: light;
|
|
48
|
+
}
|
|
49
|
+
@media (prefers-color-scheme: dark) {
|
|
50
|
+
:root:not([data-theme="light"]) {
|
|
51
|
+
--accent: #a0a2ff; --accent-soft: rgba(160, 162, 255, .14);
|
|
52
|
+
--bg: #000000; --surface: #0d0d10; --surface-2: #141418; --border: #28282e; --shadow: none;
|
|
53
|
+
--fg: #f4f5f8; --muted: #a3a4ae;
|
|
54
|
+
--ok: #5fd37f; --ok-soft: rgba(95, 211, 127, .13);
|
|
55
|
+
--warn: #f7b83d; --warn-soft: rgba(247, 184, 61, .13);
|
|
56
|
+
--err: #f66d67; --err-soft: rgba(246, 109, 103, .13);
|
|
57
|
+
--neutral: #a3a4ae; --neutral-soft: rgba(163, 164, 174, .13);
|
|
58
|
+
color-scheme: dark;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
:root[data-theme="dark"] {
|
|
62
|
+
--accent: #a0a2ff; --accent-soft: rgba(160, 162, 255, .14);
|
|
63
|
+
--bg: #000000; --surface: #0d0d10; --surface-2: #141418; --border: #28282e; --shadow: none;
|
|
64
|
+
--fg: #f4f5f8; --muted: #a3a4ae;
|
|
65
|
+
--ok: #5fd37f; --ok-soft: rgba(95, 211, 127, .13);
|
|
66
|
+
--warn: #f7b83d; --warn-soft: rgba(247, 184, 61, .13);
|
|
67
|
+
--err: #f66d67; --err-soft: rgba(246, 109, 103, .13);
|
|
68
|
+
--neutral: #a3a4ae; --neutral-soft: rgba(163, 164, 174, .13);
|
|
69
|
+
color-scheme: dark;
|
|
70
|
+
}
|
|
71
|
+
html[data-lang="es"] [lang="en"], html[data-lang="en"] [lang="es"] { display: none !important; }
|
|
72
|
+
* { box-sizing: border-box; }
|
|
73
|
+
html { -webkit-text-size-adjust: 100%; }
|
|
74
|
+
body { margin: 0; background: var(--bg); color: var(--fg); font: 15px/1.55 var(--sans); }
|
|
75
|
+
a { color: var(--accent); text-underline-offset: 2px; }
|
|
76
|
+
code, .mono, .num { font-family: var(--mono); font-feature-settings: "zero" 0; }
|
|
77
|
+
.num { font-variant-numeric: tabular-nums; white-space: nowrap; }
|
|
78
|
+
.muted { color: var(--muted); }
|
|
79
|
+
.wrap { max-width: 1080px; margin: 0 auto; padding: 0 24px; }
|
|
80
|
+
header.top { border-bottom: 1px solid var(--border); }
|
|
81
|
+
header.top .wrap { display: flex; align-items: center; gap: 12px; height: 60px; }
|
|
82
|
+
.brand { display: flex; align-items: center; gap: 10px; font-weight: 650; font-size: 17px; letter-spacing: -.01em; color: var(--fg); text-decoration: none; }
|
|
83
|
+
.brand svg { width: 28px; height: 28px; }
|
|
84
|
+
.spacer { flex: 1; }
|
|
85
|
+
.toggles { display: flex; gap: 8px; }
|
|
86
|
+
.toggle { font: 500 13px/1 var(--sans); color: var(--fg); background: var(--surface); border: 1px solid var(--border); border-radius: 8px; height: 32px; padding: 0 10px; cursor: pointer; display: inline-flex; align-items: center; gap: 6px; }
|
|
87
|
+
.toggle:hover { border-color: var(--accent); }
|
|
88
|
+
.toggle:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
|
|
89
|
+
.toggle svg { width: 16px; height: 16px; }
|
|
90
|
+
main { padding: 32px 0 56px; }
|
|
91
|
+
h1 { font-size: 28px; line-height: 1.2; letter-spacing: -.02em; margin: 0 0 6px; }
|
|
92
|
+
h2 { font-size: 19px; letter-spacing: -.01em; margin: 44px 0 6px; }
|
|
93
|
+
.lede { color: var(--muted); margin: 0 0 20px; max-width: 72ch; }
|
|
94
|
+
.meta { display: flex; flex-wrap: wrap; gap: 6px 18px; color: var(--muted); font-size: 13px; margin-bottom: 24px; }
|
|
95
|
+
.meta b { color: var(--fg); font-weight: 500; }
|
|
96
|
+
.recorded { display: flex; gap: 10px; align-items: flex-start; border: 1px solid var(--border); background: var(--accent-soft); border-radius: 10px; padding: 10px 14px; font-size: 13.5px; margin-bottom: 20px; }
|
|
97
|
+
.recorded strong { color: var(--accent); }
|
|
98
|
+
.card { background: var(--surface); border: 1px solid var(--border); border-radius: 12px; box-shadow: var(--shadow); }
|
|
99
|
+
.kpis { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; }
|
|
100
|
+
.kpi { padding: 16px 18px; }
|
|
101
|
+
.kpi .label { font-size: 12.5px; color: var(--muted); }
|
|
102
|
+
.kpi .value { font: 600 26px/1.2 var(--sans); letter-spacing: -.02em; margin: 6px 0 4px; font-variant-numeric: tabular-nums; }
|
|
103
|
+
.kpi .sub { font-size: 12.5px; color: var(--muted); }
|
|
104
|
+
.notice { margin-top: 16px; border: 1px solid var(--border); border-left: 3px solid var(--warn); border-radius: 10px; background: var(--warn-soft); padding: 12px 16px; font-size: 14px; }
|
|
105
|
+
.notice strong { color: var(--warn); }
|
|
106
|
+
.chip { display: inline-flex; align-items: center; gap: 6px; border-radius: 999px; padding: 2px 10px 2px 8px; font-size: 12.5px; font-weight: 600; white-space: nowrap; }
|
|
107
|
+
.chip::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: currentColor; }
|
|
108
|
+
.v-helps { color: var(--ok); background: var(--ok-soft); }
|
|
109
|
+
.v-hurts { color: var(--err); background: var(--err-soft); }
|
|
110
|
+
.v-no_effect { color: var(--neutral); background: var(--neutral-soft); }
|
|
111
|
+
.v-inconclusive { color: var(--warn); background: var(--warn-soft); }
|
|
112
|
+
.rules { overflow: hidden; }
|
|
113
|
+
.row { display: grid; grid-template-columns: minmax(0, 1fr) 70px 196px 128px 104px 96px; gap: 12px; align-items: center; padding: 12px 18px; border-top: 1px solid var(--border); }
|
|
114
|
+
.row.head { border-top: 0; font-size: 12px; color: var(--muted); font-weight: 500; padding-top: 10px; padding-bottom: 10px; background: var(--surface-2); }
|
|
115
|
+
.row .r-id { font: 500 12px var(--mono); color: var(--accent); margin-right: 6px; }
|
|
116
|
+
.row .r-title { font-weight: 550; }
|
|
117
|
+
.row .r-lines { font-size: 12px; color: var(--muted); }
|
|
118
|
+
.row .right { text-align: right; }
|
|
119
|
+
.effect { display: flex; flex-direction: column; gap: 2px; }
|
|
120
|
+
.effect .num { font-size: 12.5px; }
|
|
121
|
+
.ci { display: block; max-width: 100%; height: auto; }
|
|
122
|
+
.ci-margin { fill: var(--neutral-soft); }
|
|
123
|
+
.ci-zero { stroke: var(--muted); stroke-width: 1; stroke-dasharray: 2 2; }
|
|
124
|
+
.ci-range { stroke: var(--accent); stroke-width: 2; stroke-linecap: round; }
|
|
125
|
+
.ci-est { fill: var(--accent); }
|
|
126
|
+
.mlabel { display: none; }
|
|
127
|
+
.axis { display: flex; justify-content: space-between; font: 11px var(--mono); color: var(--muted); width: 168px; }
|
|
128
|
+
.file { overflow: hidden; }
|
|
129
|
+
.file-head { display: flex; align-items: center; gap: 10px; padding: 10px 16px; border-bottom: 1px solid var(--border); background: var(--surface-2); font-size: 13px; }
|
|
130
|
+
.file-head .mono { font-weight: 500; }
|
|
131
|
+
.block { padding: 0 16px; }
|
|
132
|
+
.block pre { margin: 0; padding: 4px 0; font: 13px/1.6 var(--mono); white-space: pre-wrap; word-break: break-word; }
|
|
133
|
+
.block.fixed pre { color: var(--muted); }
|
|
134
|
+
.block.rule { border-left: 3px solid var(--neutral); margin: 8px 12px; padding: 8px 12px; border-radius: 0 8px 8px 0; background: var(--surface-2); }
|
|
135
|
+
.block.rule.v-helps { border-left-color: var(--ok); }
|
|
136
|
+
.block.rule.v-hurts { border-left-color: var(--err); }
|
|
137
|
+
.block.rule.v-inconclusive { border-left-color: var(--warn); }
|
|
138
|
+
.block.rule .ann { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 12px; font-size: 12.5px; color: var(--muted); margin-bottom: 4px; background: none; }
|
|
139
|
+
.block.rule .ann .r-id { font: 600 12px var(--mono); color: var(--accent); }
|
|
140
|
+
.block.rule .chip { background: transparent; padding: 0; }
|
|
141
|
+
.block.rule pre { color: var(--fg); }
|
|
142
|
+
table { width: 100%; border-collapse: collapse; font-size: 14px; }
|
|
143
|
+
th, td { text-align: left; padding: 10px 16px; border-top: 1px solid var(--border); }
|
|
144
|
+
thead th { border-top: 0; font-size: 12px; font-weight: 500; color: var(--muted); background: var(--surface-2); }
|
|
145
|
+
td.num, th.num { text-align: right; }
|
|
146
|
+
.tablewrap { overflow-x: auto; }
|
|
147
|
+
.method { columns: 2 320px; column-gap: 32px; font-size: 14px; color: var(--muted); }
|
|
148
|
+
.method p { margin: 0 0 12px; break-inside: avoid; }
|
|
149
|
+
.method b { color: var(--fg); font-weight: 600; }
|
|
150
|
+
footer { border-top: 1px solid var(--border); padding: 20px 0 32px; font-size: 13px; color: var(--muted); }
|
|
151
|
+
@media (max-width: 900px) {
|
|
152
|
+
.kpis { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
|
153
|
+
.row { grid-template-columns: minmax(0, 1fr) auto; gap: 6px 12px; }
|
|
154
|
+
.row.head { display: none; }
|
|
155
|
+
.row > .c-title { grid-column: 1 / -1; }
|
|
156
|
+
.row > .c-effect { grid-column: 1 / 2; }
|
|
157
|
+
.row > .c-verdict { grid-column: 2 / 3; justify-self: end; align-self: start; }
|
|
158
|
+
.row > .c-tokens, .row > .c-delta, .row > .c-month { text-align: left; font-size: 13px; }
|
|
159
|
+
.row > .c-tokens { grid-column: 1 / -1; grid-row: auto; }
|
|
160
|
+
.mlabel { display: inline; color: var(--muted); font-family: var(--sans); }
|
|
161
|
+
.row .right { text-align: left; }
|
|
162
|
+
}
|
|
163
|
+
@media (max-width: 560px) {
|
|
164
|
+
.wrap { padding: 0 16px; }
|
|
165
|
+
h1 { font-size: 23px; }
|
|
166
|
+
.kpi { padding: 14px; }
|
|
167
|
+
.kpi .value { font-size: 22px; }
|
|
168
|
+
.toggle .tl { display: none; }
|
|
169
|
+
th, td { padding: 9px 10px; }
|
|
170
|
+
.block.rule { margin: 8px 6px; }
|
|
171
|
+
.block { padding: 0 10px; }
|
|
172
|
+
}
|
|
173
|
+
</style>
|
|
174
|
+
</head>
|
|
175
|
+
<body>
|
|
176
|
+
<svg width="0" height="0" style="position:absolute" aria-hidden="true" focusable="false">
|
|
177
|
+
<defs>
|
|
178
|
+
<symbol id="g-rule" viewBox="0 0 24 24">
|
|
179
|
+
<rect x="1" y="1" width="22" height="22" rx="6" fill="var(--tile)"/>
|
|
180
|
+
<path d="M10 5.2v4.6l-3.6 6.9a1.2 1.2 0 0 0 1.1 1.8h9a1.2 1.2 0 0 0 1.1-1.8L14 9.8V5.2M8.9 5.2h6.2" fill="none" stroke="var(--tile-fg)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
|
181
|
+
<path d="M8.3 14.3h7.4" fill="none" stroke="var(--tile-fg)" stroke-width="2" stroke-linecap="round"/>
|
|
182
|
+
</symbol>
|
|
183
|
+
</defs>
|
|
184
|
+
</svg>
|
|
185
|
+
<header class="top">
|
|
186
|
+
<div class="wrap">
|
|
187
|
+
<a class="brand" href="https://github.com/diegoperezg7/rulelab"><svg aria-hidden="true"><use href="#g-rule"/></svg>rulelab</a>
|
|
188
|
+
<span class="spacer"></span>
|
|
189
|
+
<div class="toggles">
|
|
190
|
+
<button class="toggle" id="lang" type="button" aria-label="Cambiar idioma / Switch language"><span lang="en">ES</span><span lang="es">EN</span></button>
|
|
191
|
+
<button class="toggle" id="theme" type="button" aria-label="Toggle theme">
|
|
192
|
+
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v2M12 20v2M4.9 4.9l1.4 1.4M17.7 17.7l1.4 1.4M2 12h2M20 12h2M4.9 19.1l1.4-1.4M17.7 6.3l1.4-1.4"/></svg>
|
|
193
|
+
<span class="tl">{{ t("Theme", "Tema") }}</span>
|
|
194
|
+
</button>
|
|
195
|
+
</div>
|
|
196
|
+
</div>
|
|
197
|
+
</header>
|
|
198
|
+
<main>
|
|
199
|
+
<div class="wrap">
|
|
200
|
+
{% if recorded %}
|
|
201
|
+
<div class="recorded" role="note">
|
|
202
|
+
<div>
|
|
203
|
+
<strong>{{ t("Recorded run", "Ejecución grabada") }}</strong> ·
|
|
204
|
+
{{ t("Real results produced with", "Resultados reales obtenidos con") }} <code>rulelab run</code>
|
|
205
|
+
{{ t("on", "el") }} {{ recorded.date }} · rulelab {{ recorded.version }} · {{ recorded.hardware }}.
|
|
206
|
+
{{ t("Nothing below is simulated.", "Nada de lo que sigue es simulado.") }}
|
|
207
|
+
</div>
|
|
208
|
+
</div>
|
|
209
|
+
{% endif %}
|
|
210
|
+
<h1>{{ t("Which rules in", "¿Qué reglas de") }} <code>{{ res.file.path }}</code> {{ t("earn their tokens?", "se ganan sus tokens?") }}</h1>
|
|
211
|
+
<p class="lede">{{ t("Each rule was removed on its own and the agent re-ran the same real tasks. A rule helps if success drops without it; it costs tokens on every model call either way.",
|
|
212
|
+
"Se quitó cada regla por separado y el agente repitió las mismas tareas reales. Una regla ayuda si el acierto baja sin ella; en cualquier caso cuesta tokens en cada llamada al modelo.") }}</p>
|
|
213
|
+
<div class="meta">
|
|
214
|
+
<span>{{ t("Model", "Modelo") }} <b class="mono">{{ res.model }}</b></span>
|
|
215
|
+
<span>{{ t("Tasks", "Tareas") }} <b>{{ res.tasks|length }}</b></span>
|
|
216
|
+
<span>{{ t("Repeats", "Repeticiones") }} <b>{{ res.repeats }}</b></span>
|
|
217
|
+
<span>{{ t("Configurations", "Configuraciones") }} <b>{{ res.configs|length }}</b></span>
|
|
218
|
+
<span>{{ t("Agent runs", "Ejecuciones") }} <b>{{ total_runs }}/{{ planned_runs }}</b></span>
|
|
219
|
+
<span>{{ t("Split by", "Partido por") }} <b>{{ t(res.file.granularity, "sección" if res.file.granularity == "section" else "viñeta") }}</b></span>
|
|
220
|
+
<span>{{ t("Spent", "Gastado") }} <b class="num">{{ res.spent_usd|usd(4) }}</b>{% if res.meta.price_source %} <span title="{{ res.meta.cost_method }}">({{ res.meta.price_in_per_mtok|plain }}/{{ res.meta.price_out_per_mtok|plain }} {{ t("USD per 1M tokens in/out", "US$ por 1M de tokens de entrada/salida") }}, {{ res.meta.price_source }})</span>{% endif %}</span>
|
|
221
|
+
{% if res.meta.engine %}<span>{{ t("Engine", "Motor") }} <b>{{ res.meta.engine }}</b></span>{% endif %}
|
|
222
|
+
</div>
|
|
223
|
+
|
|
224
|
+
<section class="kpis" aria-label="Summary">
|
|
225
|
+
<div class="card kpi">
|
|
226
|
+
<div class="label">{{ t("Success with the full file", "Acierto con el fichero completo") }}</div>
|
|
227
|
+
<div class="value">{{ full_rate[0]|pct }}</div>
|
|
228
|
+
<div class="sub num">{{ full_rate[1] }}/{{ full_rate[2] }} {{ t("runs", "ejecuciones") }}</div>
|
|
229
|
+
</div>
|
|
230
|
+
<div class="card kpi">
|
|
231
|
+
<div class="label">{{ t("Success with no file", "Acierto sin fichero") }}</div>
|
|
232
|
+
<div class="value">{{ none_rate[0]|pct if none_rate else "n/a" }}</div>
|
|
233
|
+
<div class="sub">{% if a.whole_file %}{{ t("File effect", "Efecto del fichero") }} <span class="num">{{ a.whole_file.estimate|pp }} pp [{{ a.whole_file.low|pp }}, {{ a.whole_file.high|pp }}]</span>{% endif %}</div>
|
|
234
|
+
</div>
|
|
235
|
+
<div class="card kpi">
|
|
236
|
+
<div class="label">{{ t("File size", "Tamaño del fichero") }}</div>
|
|
237
|
+
<div class="value num">≈{{ res.file.tokens|num }} <span style="font-size:14px;font-weight:500" class="muted">tok</span></div>
|
|
238
|
+
<div class="sub">{{ t("Estimated", "Estimado") }} <span class="num">≈{{ file_per_run|num }} tok/{{ t("run", "ejec.") }}</span> {{ t("if resent on each of", "si se reenvía en cada una de las") }} ~{{ a.mean_turns|num(1) }} {{ t("model calls", "llamadas al modelo") }}.{% if a.whole_file_tokens and a.whole_file_tokens.n %}
|
|
239
|
+
{{ t("Measured difference vs no file in this run:", "Diferencia medida frente a sin fichero en esta ejecución:") }} <span class="num">{{ a.whole_file_tokens.mean|signed_int }} tok/{{ t("run", "ejec.") }}</span>{% endif %}</div>
|
|
240
|
+
</div>
|
|
241
|
+
<div class="card kpi">
|
|
242
|
+
<div class="label">{{ t("Verdicts", "Veredictos") }}</div>
|
|
243
|
+
<div class="value">{{ a.rules|length }} <span style="font-size:14px;font-weight:500" class="muted">{{ t("rules", "reglas") }}</span></div>
|
|
244
|
+
<div class="sub">{% for k, n in a.verdict_counts.items() if n %}{{ n }} {{ t(verdicts[k][0], plural_es[k] if n > 1 else verdicts[k][1]) }}{% if not loop.last %} · {% endif %}{% endfor %}</div>
|
|
245
|
+
</div>
|
|
246
|
+
</section>
|
|
247
|
+
|
|
248
|
+
{% if a.floor_or_ceiling == "floor" %}
|
|
249
|
+
<div class="notice" role="note">
|
|
250
|
+
<strong>{{ t("Floor effect.", "Efecto suelo.") }}</strong>
|
|
251
|
+
{{ t("Only", "Solo pasó el") }} {{ a.overall_rate|pct }} {{ t("of all runs passed. This model rarely solves these tasks with or without the file, so success cannot show what a rule does: every verdict below is inconclusive for that reason. The token and cost columns are still real measurements. To learn about success, use a stronger model, more --max-steps or easier tasks.",
|
|
252
|
+
"de todas las ejecuciones. Este modelo casi nunca resuelve estas tareas, con o sin fichero, así que el acierto no puede mostrar qué hace cada regla: por eso todos los veredictos salen no concluyentes. Las columnas de tokens y coste sí son mediciones reales. Para saber algo del acierto, usa un modelo más fuerte, más --max-steps o tareas más fáciles.") }}
|
|
253
|
+
</div>
|
|
254
|
+
{% elif a.floor_or_ceiling == "ceiling" %}
|
|
255
|
+
<div class="notice" role="note">
|
|
256
|
+
<strong>{{ t("Ceiling effect.", "Efecto techo.") }}</strong>
|
|
257
|
+
{{ a.overall_rate|pct }} {{ t("of all runs passed. The tasks are too easy for this model to show what a rule does. The token and cost columns are still real measurements.",
|
|
258
|
+
"de las ejecuciones pasaron. Las tareas son demasiado fáciles para este modelo como para ver qué hace cada regla. Las columnas de tokens y coste sí son mediciones reales.") }}
|
|
259
|
+
</div>
|
|
260
|
+
{% endif %}
|
|
261
|
+
{% if a.low_power %}
|
|
262
|
+
<div class="notice" role="note">
|
|
263
|
+
<strong>{{ t("Small sample.", "Muestra pequeña.") }}</strong>
|
|
264
|
+
{{ t("With", "Con") }} {{ res.tasks|length }} {{ t("tasks and", "tareas y") }} {{ res.repeats }} {{ t("repeats, only large effects can be told apart from noise, so most rules come out inconclusive. That is the honest answer, not a failure: add tasks or repeats before deleting a rule on the strength of this report.",
|
|
265
|
+
"repeticiones solo se distinguen del ruido los efectos grandes, así que la mayoría de reglas salen no concluyentes. Es la respuesta honesta, no un fallo: añade tareas o repeticiones antes de borrar una regla basándote en este informe.") }}
|
|
266
|
+
</div>
|
|
267
|
+
{% endif %}
|
|
268
|
+
{% if res.status != "complete" %}
|
|
269
|
+
<div class="notice" role="note">
|
|
270
|
+
<strong>{{ t("Partial results.", "Resultados parciales.") }}</strong>
|
|
271
|
+
{% if res.status == "budget_exceeded" %}{{ t("The run stopped at the --max-cost budget.", "La ejecución se detuvo al llegar al tope de --max-cost.") }}{% else %}{{ t("The run was interrupted.", "La ejecución se interrumpió.") }}{% endif %}
|
|
272
|
+
{{ total_runs }}/{{ planned_runs }} {{ t("agent runs finished.", "ejecuciones terminadas.") }}
|
|
273
|
+
</div>
|
|
274
|
+
{% endif %}
|
|
275
|
+
|
|
276
|
+
<h2>{{ t("Rules", "Reglas") }}</h2>
|
|
277
|
+
<p class="lede">{{ t("Effect on success = success with the full file minus success without the rule, in percentage points, with a 95% bootstrap interval. The shaded band is ±", "Efecto en el acierto = acierto con el fichero completo menos acierto sin la regla, en puntos porcentuales, con intervalo bootstrap del 95 %. La banda sombreada es ±") }}{{ (a.margin * 100)|round|int }} pp: {{ t("an interval entirely inside it counts as no effect.", "un intervalo que cae entero dentro cuenta como sin efecto.") }}</p>
|
|
278
|
+
<div class="card rules">
|
|
279
|
+
<div class="row head">
|
|
280
|
+
<div>{{ t("Rule", "Regla") }}</div>
|
|
281
|
+
<div class="right">{{ t("Tokens", "Tokens") }}</div>
|
|
282
|
+
<div>{{ t("Effect on success", "Efecto en el acierto") }}</div>
|
|
283
|
+
<div>{{ t("Verdict", "Veredicto") }}</div>
|
|
284
|
+
<div class="right">{{ t("Input tok/run", "Tok entrada/ejec.") }}</div>
|
|
285
|
+
<div class="right">{{ t("Per month", "Al mes") }}</div>
|
|
286
|
+
</div>
|
|
287
|
+
{% for item in rules %}{% set r = item.r %}
|
|
288
|
+
<div class="row">
|
|
289
|
+
<div class="c-title"><span class="r-id">{{ r.id }}</span><span class="r-title">{{ r.title }}</span>
|
|
290
|
+
<div class="r-lines">{{ t("lines", "líneas") }} {{ r.start_line }}–{{ r.end_line }}</div></div>
|
|
291
|
+
<div class="c-tokens right num"><span class="mlabel">{{ t("Tokens", "Tokens") }} </span>≈{{ r.tokens }} <span class="muted">({{ (item.share * 100)|round|int }}%)</span></div>
|
|
292
|
+
<div class="c-effect effect">
|
|
293
|
+
{{ bar(item.bar, r.id) }}
|
|
294
|
+
<span class="num">{{ r.success.estimate|pp }} pp <span class="muted">[{{ r.success.low|pp }}, {{ r.success.high|pp }}]</span></span>
|
|
295
|
+
</div>
|
|
296
|
+
<div class="c-verdict">{{ chip(r.success.verdict) }}</div>
|
|
297
|
+
<div class="c-delta right num"><span class="mlabel">{{ t("Input tok/run", "Tok entrada/ejec.") }} </span>{{ r.input_tokens.mean|signed_int }}</div>
|
|
298
|
+
<div class="c-month right num"><span class="mlabel">{{ t("Per month", "Al mes") }} </span>{{ r.monthly_usd|usd }}</div>
|
|
299
|
+
</div>
|
|
300
|
+
{% endfor %}
|
|
301
|
+
</div>
|
|
302
|
+
<p class="lede" style="margin-top:12px;font-size:13px">
|
|
303
|
+
{{ t("Tokens are approximate (characters / 4). \"Input tok/run\" is the measured difference in input tokens per run with and without the rule; it also moves when the agent takes a different path. \"Per month\" is the rule's tokens × model calls per run ×",
|
|
304
|
+
"Los tokens son aproximados (caracteres / 4). «Tok entrada/ejec.» es la diferencia medida de tokens de entrada por ejecución con y sin la regla; también cambia cuando el agente toma otro camino. «Al mes» es tokens de la regla × llamadas al modelo por ejecución ×") }}
|
|
305
|
+
{{ a.sessions_per_month|num }} {{ t("sessions at", "sesiones a") }} {{ a.price_in|plain }} {{ t("USD / 1M tok", "US$ / 1M tok") }} ({{ t(a.price_label, a.price_label_es) }}){{ t(", without prompt caching.", ", sin caché de prompt.") }}
|
|
306
|
+
{{ t("The whole file:", "El fichero entero:") }} <span class="num">{{ file_monthly|usd }}</span>.
|
|
307
|
+
</p>
|
|
308
|
+
|
|
309
|
+
<h2>{{ t("Annotated file", "Fichero anotado") }}</h2>
|
|
310
|
+
<p class="lede">{{ t("Every rule as it appears in the file, with its verdict. Grey lines are kept in every configuration.", "Cada regla tal como aparece en el fichero, con su veredicto. Las líneas en gris se mantienen en todas las configuraciones.") }}</p>
|
|
311
|
+
<div class="card file">
|
|
312
|
+
<div class="file-head"><svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/><path d="M14 2v6h6"/></svg><span class="mono">{{ res.file.path }}</span><span class="muted">≈{{ res.file.tokens|num }} tok</span></div>
|
|
313
|
+
{% for b in blocks %}
|
|
314
|
+
{% if b.rule %}{% set r = b.rule %}
|
|
315
|
+
<div class="block rule v-{{ r.success.verdict }}" id="{{ r.id }}">
|
|
316
|
+
<div class="ann"><span class="r-id">{{ r.id }}</span>{{ chip(r.success.verdict) }}<span class="num">{{ r.success.estimate|pp }} pp [{{ r.success.low|pp }}, {{ r.success.high|pp }}]</span><span class="num">≈{{ r.tokens }} tok</span></div>
|
|
317
|
+
<pre>{{ b.text }}</pre>
|
|
318
|
+
</div>
|
|
319
|
+
{% else %}
|
|
320
|
+
<div class="block fixed"><pre>{{ b.text }}</pre></div>
|
|
321
|
+
{% endif %}
|
|
322
|
+
{% endfor %}
|
|
323
|
+
</div>
|
|
324
|
+
|
|
325
|
+
<h2>{{ t("Configurations", "Configuraciones") }}</h2>
|
|
326
|
+
<p class="lede">{{ t("Raw success per configuration, pooled over tasks.", "Acierto bruto por configuración, sumando todas las tareas.") }}</p>
|
|
327
|
+
<div class="card tablewrap">
|
|
328
|
+
<table>
|
|
329
|
+
<thead><tr><th>{{ t("Configuration", "Configuración") }}</th><th class="num">{{ t("Success", "Acierto") }}</th><th class="num">{{ t("Runs", "Ejec.") }}</th><th class="num">{{ t("Input tok/run", "Tok entrada/ejec.") }}</th><th class="num">{{ t("USD/run", "US$/ejec.") }}</th></tr></thead>
|
|
330
|
+
<tbody>
|
|
331
|
+
{% for c in configs %}
|
|
332
|
+
<tr>
|
|
333
|
+
<td>{% if c.id == "none" %}{{ t("No file", "Sin fichero") }}{% elif c.id == "full" %}{{ t("Full file", "Fichero completo") }}{% else %}{{ t("Without", "Sin") }} <span class="mono">{{ c.removed }}</span>{% endif %}</td>
|
|
334
|
+
<td class="num">{{ c.rate|pct }} <span class="muted">({{ c.s }}/{{ c.n }})</span></td>
|
|
335
|
+
<td class="num">{{ c.n }}/{{ c.planned }}</td>
|
|
336
|
+
<td class="num">{{ c.tokens|num }}</td>
|
|
337
|
+
<td class="num">{{ c.cost|usd(4) }}</td>
|
|
338
|
+
</tr>
|
|
339
|
+
{% endfor %}
|
|
340
|
+
</tbody>
|
|
341
|
+
</table>
|
|
342
|
+
</div>
|
|
343
|
+
{% if a.errors %}<p class="lede" style="margin-top:10px;font-size:13px">{{ a.errors }} {{ t("runs ended in an infrastructure error and are excluded from the statistics.", "ejecuciones acabaron en error de infraestructura y no cuentan en la estadística.") }}</p>{% endif %}
|
|
344
|
+
|
|
345
|
+
<h2>{{ t("Method", "Método") }}</h2>
|
|
346
|
+
<div class="method">
|
|
347
|
+
<p><b>{{ t("Leave-one-out.", "Quitar de una en una.") }}</b> {{ t("Configurations: no file, the full file, and the full file minus each rule. Every configuration runs the same tasks the same number of times, in an interleaved order so a partial run stays balanced.",
|
|
348
|
+
"Configuraciones: sin fichero, fichero completo y fichero completo sin cada regla. Todas corren las mismas tareas el mismo número de veces, en orden intercalado para que una ejecución parcial siga equilibrada.") }}</p>
|
|
349
|
+
<p><b>{{ t("Real tasks.", "Tareas reales.") }}</b> {{ t("Tasks come from commits of the repository and are scored by its own tests, run inside a Docker sandbox with no network. The model is called from the host.",
|
|
350
|
+
"Las tareas salen de commits del repositorio y se puntúan con sus propios tests, dentro de un sandbox Docker sin red. El modelo se llama desde el host.") }}</p>
|
|
351
|
+
<p><b>{{ t("Interval.", "Intervalo.") }}</b> {{ t("Two-level bootstrap: tasks are resampled with replacement and each configuration's per-task success rate is drawn from a Beta(s+½, f+½) posterior, so 3/3 vs 3/3 does not read as certainty.",
|
|
352
|
+
"Bootstrap en dos niveles: se remuestrean tareas con reemplazo y el acierto por tarea de cada configuración se saca de una posterior Beta(s+½, f+½), para que 3/3 frente a 3/3 no parezca una certeza.") }}</p>
|
|
353
|
+
<p><b>{{ t("Verdict.", "Veredicto.") }}</b> {{ t("Helps if the whole interval is above 0, hurts if below 0, no effect if it sits inside ±", "Ayuda si todo el intervalo está por encima de 0, empeora si está por debajo, sin efecto si cabe dentro de ±") }}{{ (a.margin * 100)|round|int }} pp, {{ t("inconclusive otherwise.", "y no concluyente en otro caso.") }}</p>
|
|
354
|
+
<p><b>{{ t("Why measure.", "Por qué medir.") }}</b> {{ t("Gloaguen et al. (ETH Zurich, LogicStar.ai) found that context files do not generally improve task success while raising inference cost by over 20% on average.", "Gloaguen et al. (ETH Zúrich, LogicStar.ai) vieron que los ficheros de contexto no mejoran en general el acierto y suben el coste de inferencia más de un 20 % de media.") }} <a href="https://arxiv.org/abs/2602.11988">arXiv 2602.11988</a></p>
|
|
355
|
+
</div>
|
|
356
|
+
</div>
|
|
357
|
+
</main>
|
|
358
|
+
<footer>
|
|
359
|
+
<div class="wrap">
|
|
360
|
+
{{ t("Generated by", "Generado por") }} <a href="https://github.com/diegoperezg7/rulelab">rulelab</a> {{ version }} · {{ res.created }}{% if res.meta.platform %} · {{ res.meta.platform }}{% endif %}
|
|
361
|
+
</div>
|
|
362
|
+
</footer>
|
|
363
|
+
<script>
|
|
364
|
+
(function () {
|
|
365
|
+
var d = document.documentElement;
|
|
366
|
+
function save(k, v) { try { localStorage.setItem(k, v); } catch (e) {} }
|
|
367
|
+
document.getElementById("lang").addEventListener("click", function () {
|
|
368
|
+
var next = d.getAttribute("data-lang") === "es" ? "en" : "es";
|
|
369
|
+
d.setAttribute("data-lang", next); d.setAttribute("lang", next); save("rulelab-lang", next);
|
|
370
|
+
});
|
|
371
|
+
document.getElementById("theme").addEventListener("click", function () {
|
|
372
|
+
var cur = d.getAttribute("data-theme") || (matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light");
|
|
373
|
+
var next = cur === "dark" ? "light" : "dark";
|
|
374
|
+
d.setAttribute("data-theme", next); save("rulelab-theme", next);
|
|
375
|
+
});
|
|
376
|
+
})();
|
|
377
|
+
</script>
|
|
378
|
+
</body>
|
|
379
|
+
</html>
|