testledger 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testledger/__init__.py +11 -0
- testledger/aitest/__init__.py +3 -0
- testledger/aitest/detector.py +152 -0
- testledger/cli.py +262 -0
- testledger/config.py +105 -0
- testledger/ingest/__init__.py +7 -0
- testledger/ingest/cucumber.py +79 -0
- testledger/ingest/github_actions.py +145 -0
- testledger/ingest/gitlog.py +118 -0
- testledger/ingest/jenkins.py +127 -0
- testledger/ingest/junit.py +114 -0
- testledger/ledger/__init__.py +3 -0
- testledger/ledger/engine.py +332 -0
- testledger/models.py +194 -0
- testledger/reports/__init__.py +3 -0
- testledger/reports/builder.py +224 -0
- testledger/templates/memo.html.j2 +77 -0
- testledger/templates/memo.md.j2 +78 -0
- testledger-0.1.0.dist-info/METADATA +208 -0
- testledger-0.1.0.dist-info/RECORD +24 -0
- testledger-0.1.0.dist-info/WHEEL +5 -0
- testledger-0.1.0.dist-info/entry_points.txt +2 -0
- testledger-0.1.0.dist-info/licenses/LICENSE +21 -0
- testledger-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
"""The ledger engine: books cost and value entries per test and issues verdicts.
|
|
2
|
+
|
|
3
|
+
Cost model (debits), per test over the analysis window
|
|
4
|
+
------------------------------------------------------
|
|
5
|
+
* **CI compute** = Σ duration × ci_minute_usd (every execution)
|
|
6
|
+
* **Rerun compute** = executions on attempt > 1 × duration × ci_minute_usd
|
|
7
|
+
* **Triage time** = flaky failures × triage_minutes × engineer_hourly / 60
|
|
8
|
+
* **Maintenance** = lines changed in the test's file × maintenance_minutes_per_line × engineer_hourly / 60
|
|
9
|
+
(apportioned across tests sharing the file)
|
|
10
|
+
|
|
11
|
+
Value model (credits)
|
|
12
|
+
---------------------
|
|
13
|
+
* **Regression caught** = a genuine failure (failed on first attempt, and was
|
|
14
|
+
NOT subsequently passed on a rerun of the same SHA) that was followed within
|
|
15
|
+
``FIX_WINDOW`` by a product-code commit matching the fix pattern.
|
|
16
|
+
Each such event is worth ``regression_value_usd``.
|
|
17
|
+
* **Risk coverage** = optional flat value per stable test (default 0) for
|
|
18
|
+
organisations that want to recognise a "safety net" value.
|
|
19
|
+
|
|
20
|
+
Flake detection
|
|
21
|
+
---------------
|
|
22
|
+
A flaky failure is: failed on attempt N and passed on attempt N+1 for the
|
|
23
|
+
same commit SHA. Nothing changed between the two runs, so this is unambiguous.
|
|
24
|
+
Flake rate = flaky commits / commits tested (first-attempt executions); reruns
|
|
25
|
+
are excluded from the denominator so they cannot dilute the signal they reveal.
|
|
26
|
+
(A future rule could also flag fail→pass across adjacent commits that touched
|
|
27
|
+
no related product code.)
|
|
28
|
+
|
|
29
|
+
Verdicts
|
|
30
|
+
--------
|
|
31
|
+
QUARANTINE flake_rate ≥ quarantine threshold
|
|
32
|
+
RETIRE enough executions, zero regressions caught, net < 0, cost ≥ retire_min_cost_usd
|
|
33
|
+
DEMOTE regressions caught > 0 but UI layer and cost per catch ≥ ratio × service-layer median
|
|
34
|
+
FIX regressions caught > 0 and (flake_rate ≥ fix threshold or avg duration ≥ slow threshold)
|
|
35
|
+
ASSET roi ≥ asset threshold
|
|
36
|
+
HOLD everything else, including insufficient data
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
from collections import defaultdict
|
|
42
|
+
from datetime import UTC, datetime, timedelta
|
|
43
|
+
from statistics import median
|
|
44
|
+
|
|
45
|
+
from testledger.config import Config
|
|
46
|
+
from testledger.models import (
|
|
47
|
+
ChurnRecord,
|
|
48
|
+
EntryKind,
|
|
49
|
+
FixCommit,
|
|
50
|
+
Layer,
|
|
51
|
+
Ledger,
|
|
52
|
+
LedgerEntry,
|
|
53
|
+
Outcome,
|
|
54
|
+
TestAccount,
|
|
55
|
+
TestRun,
|
|
56
|
+
Verdict,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
FIX_WINDOW = timedelta(days=3)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def classify_layer(test_id: str, file: str | None, cfg: Config) -> Layer:
|
|
63
|
+
hay = f"{test_id} {file or ''}".lower()
|
|
64
|
+
for layer, needles in (
|
|
65
|
+
(Layer.UI, cfg.layers.ui),
|
|
66
|
+
(Layer.SERVICE, cfg.layers.service),
|
|
67
|
+
(Layer.UNIT, cfg.layers.unit),
|
|
68
|
+
):
|
|
69
|
+
if any(n.lower() in hay for n in needles):
|
|
70
|
+
return layer
|
|
71
|
+
return Layer.UNKNOWN
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _detect_flakes(runs: list[TestRun]) -> dict[str, int]:
|
|
75
|
+
"""Count same-SHA fail→pass transitions per test."""
|
|
76
|
+
by_sha: dict[str, list[TestRun]] = defaultdict(list)
|
|
77
|
+
for r in runs:
|
|
78
|
+
if r.commit_sha:
|
|
79
|
+
by_sha[r.commit_sha].append(r)
|
|
80
|
+
flakes: dict[str, int] = defaultdict(int)
|
|
81
|
+
for sha_runs in by_sha.values():
|
|
82
|
+
sha_runs.sort(key=lambda r: (r.attempt, r.started_at))
|
|
83
|
+
outcomes: dict[str, list[Outcome]] = defaultdict(list)
|
|
84
|
+
for r in sha_runs:
|
|
85
|
+
for res in r.results:
|
|
86
|
+
outcomes[res.test_id].append(res.outcome)
|
|
87
|
+
for test_id, seq in outcomes.items():
|
|
88
|
+
for a, b in zip(seq, seq[1:], strict=False):
|
|
89
|
+
if a.is_failure and b == Outcome.PASSED:
|
|
90
|
+
flakes[test_id] += 1
|
|
91
|
+
return flakes
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _genuine_failures(
|
|
95
|
+
runs: list[TestRun], flakes_by_sha_test: set[tuple[str, str]]
|
|
96
|
+
) -> dict[str, list[datetime]]:
|
|
97
|
+
"""Failures on first attempt that were not later cleared by a same-SHA rerun."""
|
|
98
|
+
out: dict[str, list[datetime]] = defaultdict(list)
|
|
99
|
+
for r in runs:
|
|
100
|
+
if r.attempt != 1:
|
|
101
|
+
continue
|
|
102
|
+
for res in r.results:
|
|
103
|
+
if res.outcome.is_failure and (r.commit_sha or "", res.test_id) not in flakes_by_sha_test:
|
|
104
|
+
out[res.test_id].append(r.started_at)
|
|
105
|
+
return out
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _flaky_pairs(runs: list[TestRun]) -> set[tuple[str, str]]:
|
|
109
|
+
pairs: set[tuple[str, str]] = set()
|
|
110
|
+
by_sha: dict[str, list[TestRun]] = defaultdict(list)
|
|
111
|
+
for r in runs:
|
|
112
|
+
if r.commit_sha:
|
|
113
|
+
by_sha[r.commit_sha].append(r)
|
|
114
|
+
for sha, sha_runs in by_sha.items():
|
|
115
|
+
sha_runs.sort(key=lambda r: (r.attempt, r.started_at))
|
|
116
|
+
seen_fail: set[str] = set()
|
|
117
|
+
for r in sha_runs:
|
|
118
|
+
for res in r.results:
|
|
119
|
+
if res.outcome.is_failure:
|
|
120
|
+
seen_fail.add(res.test_id)
|
|
121
|
+
elif res.outcome == Outcome.PASSED and res.test_id in seen_fail:
|
|
122
|
+
pairs.add((sha, res.test_id))
|
|
123
|
+
return pairs
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _credit_regressions(failures: dict[str, list[datetime]], fixes: list[FixCommit]) -> dict[str, int]:
|
|
127
|
+
"""A genuine failure followed by a fix commit within FIX_WINDOW counts once per fix."""
|
|
128
|
+
credited: dict[str, int] = defaultdict(int)
|
|
129
|
+
fix_times = sorted(f.timestamp.astimezone(UTC) for f in fixes)
|
|
130
|
+
for test_id, times in failures.items():
|
|
131
|
+
used: set[datetime] = set()
|
|
132
|
+
for t in times:
|
|
133
|
+
t = t.astimezone(UTC)
|
|
134
|
+
for ft in fix_times:
|
|
135
|
+
if ft in used:
|
|
136
|
+
continue
|
|
137
|
+
if t <= ft <= t + FIX_WINDOW:
|
|
138
|
+
credited[test_id] += 1
|
|
139
|
+
used.add(ft)
|
|
140
|
+
break
|
|
141
|
+
return credited
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def build_ledger(
|
|
145
|
+
runs: list[TestRun],
|
|
146
|
+
cfg: Config,
|
|
147
|
+
*,
|
|
148
|
+
churn: dict[str, ChurnRecord] | None = None,
|
|
149
|
+
fixes: list[FixCommit] | None = None,
|
|
150
|
+
ai_signals: dict[str, dict] | None = None,
|
|
151
|
+
now: datetime | None = None,
|
|
152
|
+
) -> Ledger:
|
|
153
|
+
churn = churn or {}
|
|
154
|
+
fixes = fixes or []
|
|
155
|
+
ai_signals = ai_signals or {}
|
|
156
|
+
now = now or datetime.now(tz=UTC)
|
|
157
|
+
window_start = now - timedelta(days=cfg.window_days)
|
|
158
|
+
runs = [r for r in runs if r.started_at >= window_start]
|
|
159
|
+
rates, th = cfg.rates, cfg.thresholds
|
|
160
|
+
|
|
161
|
+
accounts: dict[str, TestAccount] = {}
|
|
162
|
+
durations: dict[str, list[float]] = defaultdict(list)
|
|
163
|
+
rerun_execs: dict[str, int] = defaultdict(int)
|
|
164
|
+
file_members: dict[str, set[str]] = defaultdict(set)
|
|
165
|
+
|
|
166
|
+
for r in runs:
|
|
167
|
+
for res in r.results:
|
|
168
|
+
acc = accounts.get(res.test_id)
|
|
169
|
+
if acc is None:
|
|
170
|
+
acc = TestAccount(
|
|
171
|
+
test_id=res.test_id,
|
|
172
|
+
name=res.name,
|
|
173
|
+
suite=res.suite,
|
|
174
|
+
file=res.file,
|
|
175
|
+
layer=classify_layer(res.test_id, res.file, cfg),
|
|
176
|
+
)
|
|
177
|
+
accounts[res.test_id] = acc
|
|
178
|
+
if res.outcome == Outcome.SKIPPED:
|
|
179
|
+
continue
|
|
180
|
+
acc.executions += 1
|
|
181
|
+
if res.outcome.is_failure:
|
|
182
|
+
acc.failures += 1
|
|
183
|
+
durations[res.test_id].append(res.duration_s)
|
|
184
|
+
if r.attempt > 1:
|
|
185
|
+
rerun_execs[res.test_id] += 1
|
|
186
|
+
if res.file:
|
|
187
|
+
file_members[res.file].add(res.test_id)
|
|
188
|
+
|
|
189
|
+
flakes = _detect_flakes(runs)
|
|
190
|
+
flaky_pairs = _flaky_pairs(runs)
|
|
191
|
+
genuine = _genuine_failures(runs, flaky_pairs)
|
|
192
|
+
credited = _credit_regressions(genuine, fixes)
|
|
193
|
+
|
|
194
|
+
for test_id, acc in accounts.items():
|
|
195
|
+
d = durations.get(test_id, [])
|
|
196
|
+
acc.avg_duration_s = round(sum(d) / len(d), 3) if d else 0.0
|
|
197
|
+
acc.rerun_passes = flakes.get(test_id, 0)
|
|
198
|
+
# Flake rate is per commit tested (first attempts), not per execution:
|
|
199
|
+
# counting reruns in the denominator would dilute the very signal reruns reveal.
|
|
200
|
+
first_attempts = acc.executions - rerun_execs.get(test_id, 0)
|
|
201
|
+
acc.flake_rate = round(acc.rerun_passes / first_attempts, 3) if first_attempts > 0 else 0.0
|
|
202
|
+
acc.regressions_caught = credited.get(test_id, 0)
|
|
203
|
+
|
|
204
|
+
total_minutes = sum(d) / 60.0
|
|
205
|
+
if total_minutes:
|
|
206
|
+
acc.entries.append(
|
|
207
|
+
LedgerEntry(
|
|
208
|
+
test_id=test_id,
|
|
209
|
+
kind=EntryKind.CI_COMPUTE,
|
|
210
|
+
amount_usd=round(total_minutes * rates.ci_minute_usd, 4),
|
|
211
|
+
quantity=round(total_minutes, 2),
|
|
212
|
+
unit="ci-minutes",
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
if rerun_execs.get(test_id):
|
|
216
|
+
rerun_minutes = rerun_execs[test_id] * acc.avg_duration_s / 60.0
|
|
217
|
+
acc.entries.append(
|
|
218
|
+
LedgerEntry(
|
|
219
|
+
test_id=test_id,
|
|
220
|
+
kind=EntryKind.RERUN_COMPUTE,
|
|
221
|
+
amount_usd=round(rerun_minutes * rates.ci_minute_usd, 4),
|
|
222
|
+
quantity=rerun_execs[test_id],
|
|
223
|
+
unit="rerun-executions",
|
|
224
|
+
)
|
|
225
|
+
)
|
|
226
|
+
if acc.rerun_passes:
|
|
227
|
+
triage_hours = acc.rerun_passes * rates.triage_minutes_per_false_alarm / 60.0
|
|
228
|
+
acc.entries.append(
|
|
229
|
+
LedgerEntry(
|
|
230
|
+
test_id=test_id,
|
|
231
|
+
kind=EntryKind.TRIAGE_TIME,
|
|
232
|
+
amount_usd=round(triage_hours * rates.engineer_hourly_usd, 2),
|
|
233
|
+
quantity=acc.rerun_passes,
|
|
234
|
+
unit="false-alarms",
|
|
235
|
+
)
|
|
236
|
+
)
|
|
237
|
+
if acc.file and acc.file in churn:
|
|
238
|
+
rec = churn[acc.file]
|
|
239
|
+
share = 1 / max(1, len(file_members.get(acc.file, {test_id})))
|
|
240
|
+
acc.churn_commits = rec.commits
|
|
241
|
+
acc.churn_lines = round(rec.lines_changed * share)
|
|
242
|
+
maint_hours = acc.churn_lines * rates.maintenance_minutes_per_line / 60.0
|
|
243
|
+
if maint_hours:
|
|
244
|
+
acc.entries.append(
|
|
245
|
+
LedgerEntry(
|
|
246
|
+
test_id=test_id,
|
|
247
|
+
kind=EntryKind.MAINTENANCE,
|
|
248
|
+
amount_usd=round(maint_hours * rates.engineer_hourly_usd, 2),
|
|
249
|
+
quantity=acc.churn_lines,
|
|
250
|
+
unit="lines-changed",
|
|
251
|
+
)
|
|
252
|
+
)
|
|
253
|
+
if acc.regressions_caught:
|
|
254
|
+
acc.entries.append(
|
|
255
|
+
LedgerEntry(
|
|
256
|
+
test_id=test_id,
|
|
257
|
+
kind=EntryKind.REGRESSION_CAUGHT,
|
|
258
|
+
amount_usd=round(acc.regressions_caught * rates.regression_value_usd, 2),
|
|
259
|
+
quantity=acc.regressions_caught,
|
|
260
|
+
unit="regressions",
|
|
261
|
+
)
|
|
262
|
+
)
|
|
263
|
+
if rates.risk_coverage_value_usd and acc.flake_rate < th.fix_flake_rate:
|
|
264
|
+
acc.entries.append(
|
|
265
|
+
LedgerEntry(
|
|
266
|
+
test_id=test_id,
|
|
267
|
+
kind=EntryKind.RISK_COVERAGE,
|
|
268
|
+
amount_usd=rates.risk_coverage_value_usd,
|
|
269
|
+
quantity=1,
|
|
270
|
+
unit="window",
|
|
271
|
+
)
|
|
272
|
+
)
|
|
273
|
+
sig = ai_signals.get(test_id) or (ai_signals.get(acc.file) if acc.file else None)
|
|
274
|
+
if sig:
|
|
275
|
+
acc.ai_generated = bool(sig.get("ai_generated", False))
|
|
276
|
+
acc.mutation_score = sig.get("mutation_score")
|
|
277
|
+
acc.assertion_density = sig.get("assertion_density")
|
|
278
|
+
|
|
279
|
+
_assign_verdicts(accounts, cfg)
|
|
280
|
+
|
|
281
|
+
notes = [
|
|
282
|
+
f"Window: {cfg.window_days} days, {len(runs)} runs analysed.",
|
|
283
|
+
f"Rates: engineer ${rates.engineer_hourly_usd}/h, CI ${rates.ci_minute_usd}/min, regression value ${rates.regression_value_usd}.",
|
|
284
|
+
f"Fix commits detected: {len(fixes)}; test files with churn: {len(churn)}.",
|
|
285
|
+
]
|
|
286
|
+
return Ledger(
|
|
287
|
+
generated_at=now,
|
|
288
|
+
window_start=window_start,
|
|
289
|
+
window_end=now,
|
|
290
|
+
runs_analysed=len(runs),
|
|
291
|
+
accounts=accounts,
|
|
292
|
+
notes=notes,
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _assign_verdicts(accounts: dict[str, TestAccount], cfg: Config) -> None:
|
|
297
|
+
th = cfg.thresholds
|
|
298
|
+
service_cost_per_catch = [
|
|
299
|
+
a.total_cost / a.regressions_caught
|
|
300
|
+
for a in accounts.values()
|
|
301
|
+
if a.layer in (Layer.SERVICE, Layer.UNIT) and a.regressions_caught > 0
|
|
302
|
+
]
|
|
303
|
+
svc_median = median(service_cost_per_catch) if service_cost_per_catch else None
|
|
304
|
+
|
|
305
|
+
for a in accounts.values():
|
|
306
|
+
if a.executions < th.min_executions_for_verdict:
|
|
307
|
+
a.verdict = Verdict.HOLD
|
|
308
|
+
continue
|
|
309
|
+
if a.flake_rate >= th.quarantine_flake_rate:
|
|
310
|
+
a.verdict = Verdict.QUARANTINE
|
|
311
|
+
continue
|
|
312
|
+
if a.regressions_caught == 0:
|
|
313
|
+
a.verdict = (
|
|
314
|
+
Verdict.RETIRE if (a.net < 0 and a.total_cost >= th.retire_min_cost_usd) else Verdict.HOLD
|
|
315
|
+
)
|
|
316
|
+
continue
|
|
317
|
+
if (
|
|
318
|
+
a.layer == Layer.UI
|
|
319
|
+
and svc_median
|
|
320
|
+
and (a.total_cost / a.regressions_caught) >= th.demote_layer_cost_ratio * svc_median
|
|
321
|
+
):
|
|
322
|
+
a.verdict = Verdict.DEMOTE
|
|
323
|
+
continue
|
|
324
|
+
if a.flake_rate >= th.fix_flake_rate or a.avg_duration_s >= th.slow_test_seconds:
|
|
325
|
+
a.verdict = Verdict.FIX
|
|
326
|
+
continue
|
|
327
|
+
roi = a.roi
|
|
328
|
+
# A test that caught a regression at (near-)zero cost is the best asset there is.
|
|
329
|
+
if a.total_value > 0 and (roi is None or roi >= th.asset_roi):
|
|
330
|
+
a.verdict = Verdict.ASSET
|
|
331
|
+
else:
|
|
332
|
+
a.verdict = Verdict.HOLD
|
testledger/models.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Core domain model.
|
|
2
|
+
|
|
3
|
+
The model is deliberately small and framework-agnostic. Every ingester
|
|
4
|
+
(JUnit, Cucumber, GitHub Actions, Jenkins, git) normalises its source into
|
|
5
|
+
``TestRun`` / ``TestResult`` objects plus optional ``ChurnRecord`` entries.
|
|
6
|
+
The ledger engine then books ``LedgerEntry`` rows against each ``TestAccount``.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from enum import StrEnum
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel, Field
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Outcome(StrEnum):
|
|
18
|
+
PASSED = "passed"
|
|
19
|
+
FAILED = "failed"
|
|
20
|
+
ERROR = "error"
|
|
21
|
+
SKIPPED = "skipped"
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def is_failure(self) -> bool:
|
|
25
|
+
return self in (Outcome.FAILED, Outcome.ERROR)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Layer(StrEnum):
|
|
29
|
+
"""Test-pyramid layer. Drives default cost multipliers and demotion advice."""
|
|
30
|
+
|
|
31
|
+
UNIT = "unit"
|
|
32
|
+
SERVICE = "service"
|
|
33
|
+
UI = "ui"
|
|
34
|
+
UNKNOWN = "unknown"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class TestResult(BaseModel):
|
|
38
|
+
"""One execution of one test case inside one run."""
|
|
39
|
+
|
|
40
|
+
test_id: str = Field(description="Stable identifier, e.g. 'com.acme.LoginTest.valid_login'")
|
|
41
|
+
name: str
|
|
42
|
+
suite: str = ""
|
|
43
|
+
file: str | None = Field(default=None, description="Source path when known (used for git churn)")
|
|
44
|
+
outcome: Outcome
|
|
45
|
+
duration_s: float = 0.0
|
|
46
|
+
message: str | None = None
|
|
47
|
+
tags: list[str] = Field(default_factory=list)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class TestRun(BaseModel):
|
|
51
|
+
"""A single CI execution (one workflow run / Jenkins build / local run)."""
|
|
52
|
+
|
|
53
|
+
run_id: str
|
|
54
|
+
started_at: datetime
|
|
55
|
+
commit_sha: str | None = None
|
|
56
|
+
branch: str | None = None
|
|
57
|
+
attempt: int = Field(default=1, description="1 = first execution; >1 means a rerun of the same SHA")
|
|
58
|
+
source: str = Field(default="junit", description="junit | cucumber | github-actions | jenkins")
|
|
59
|
+
results: list[TestResult] = Field(default_factory=list)
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def failed(self) -> int:
|
|
63
|
+
return sum(1 for r in self.results if r.outcome.is_failure)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class ChurnRecord(BaseModel):
|
|
67
|
+
"""Git activity against a test file in the analysis window."""
|
|
68
|
+
|
|
69
|
+
file: str
|
|
70
|
+
commits: int = 0
|
|
71
|
+
lines_changed: int = 0
|
|
72
|
+
authors: int = 0
|
|
73
|
+
last_touched: datetime | None = None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class FixCommit(BaseModel):
|
|
77
|
+
"""A product-code commit that looks like a bug fix (used to credit tests that caught it)."""
|
|
78
|
+
|
|
79
|
+
sha: str
|
|
80
|
+
timestamp: datetime
|
|
81
|
+
message: str
|
|
82
|
+
files: list[str] = Field(default_factory=list)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class EntryKind(StrEnum):
|
|
86
|
+
# Costs (debits)
|
|
87
|
+
CI_COMPUTE = "ci_compute"
|
|
88
|
+
RERUN_COMPUTE = "rerun_compute"
|
|
89
|
+
TRIAGE_TIME = "triage_time"
|
|
90
|
+
MAINTENANCE = "maintenance"
|
|
91
|
+
# Value (credits)
|
|
92
|
+
REGRESSION_CAUGHT = "regression_caught"
|
|
93
|
+
RISK_COVERAGE = "risk_coverage"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class LedgerEntry(BaseModel):
|
|
97
|
+
"""One booked line. Costs are positive 'debit'; value is positive 'credit'."""
|
|
98
|
+
|
|
99
|
+
test_id: str
|
|
100
|
+
kind: EntryKind
|
|
101
|
+
amount_usd: float
|
|
102
|
+
quantity: float = 0.0
|
|
103
|
+
unit: str = ""
|
|
104
|
+
note: str = ""
|
|
105
|
+
|
|
106
|
+
@property
|
|
107
|
+
def is_cost(self) -> bool:
|
|
108
|
+
return self.kind in (
|
|
109
|
+
EntryKind.CI_COMPUTE,
|
|
110
|
+
EntryKind.RERUN_COMPUTE,
|
|
111
|
+
EntryKind.TRIAGE_TIME,
|
|
112
|
+
EntryKind.MAINTENANCE,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class Verdict(StrEnum):
|
|
117
|
+
ASSET = "asset" # value clearly exceeds cost
|
|
118
|
+
HOLD = "hold" # roughly break-even or insufficient data
|
|
119
|
+
FIX = "fix" # valuable but flaky/slow — repair it
|
|
120
|
+
DEMOTE = "demote" # valuable but expensive layer — rewrite lower in the pyramid
|
|
121
|
+
RETIRE = "retire" # pure cost, no evidence of value
|
|
122
|
+
QUARANTINE = "quarantine" # flake rate so high it is actively misleading
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class TestAccount(BaseModel):
|
|
126
|
+
"""The per-test ledger account with derived statistics."""
|
|
127
|
+
|
|
128
|
+
test_id: str
|
|
129
|
+
name: str
|
|
130
|
+
suite: str = ""
|
|
131
|
+
file: str | None = None
|
|
132
|
+
layer: Layer = Layer.UNKNOWN
|
|
133
|
+
executions: int = 0
|
|
134
|
+
failures: int = 0
|
|
135
|
+
rerun_passes: int = Field(
|
|
136
|
+
default=0, description="failed on attempt N, passed on attempt N+1 for same SHA"
|
|
137
|
+
)
|
|
138
|
+
flake_rate: float = 0.0
|
|
139
|
+
avg_duration_s: float = 0.0
|
|
140
|
+
regressions_caught: int = 0
|
|
141
|
+
churn_commits: int = 0
|
|
142
|
+
churn_lines: int = 0
|
|
143
|
+
entries: list[LedgerEntry] = Field(default_factory=list)
|
|
144
|
+
verdict: Verdict = Verdict.HOLD
|
|
145
|
+
ai_generated: bool = False
|
|
146
|
+
mutation_score: float | None = None
|
|
147
|
+
assertion_density: float | None = None
|
|
148
|
+
|
|
149
|
+
@property
|
|
150
|
+
def total_cost(self) -> float:
|
|
151
|
+
return round(sum(e.amount_usd for e in self.entries if e.is_cost), 4)
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def total_value(self) -> float:
|
|
155
|
+
return round(sum(e.amount_usd for e in self.entries if not e.is_cost), 2)
|
|
156
|
+
|
|
157
|
+
@property
|
|
158
|
+
def net(self) -> float:
|
|
159
|
+
return round(self.total_value - self.total_cost, 2)
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def roi(self) -> float | None:
|
|
163
|
+
if self.total_cost <= 0:
|
|
164
|
+
return None
|
|
165
|
+
return round((self.total_value - self.total_cost) / self.total_cost, 3)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class Ledger(BaseModel):
|
|
169
|
+
"""The full book for one analysis window."""
|
|
170
|
+
|
|
171
|
+
generated_at: datetime
|
|
172
|
+
window_start: datetime
|
|
173
|
+
window_end: datetime
|
|
174
|
+
runs_analysed: int
|
|
175
|
+
accounts: dict[str, TestAccount] = Field(default_factory=dict)
|
|
176
|
+
notes: list[str] = Field(default_factory=list)
|
|
177
|
+
|
|
178
|
+
@property
|
|
179
|
+
def total_cost(self) -> float:
|
|
180
|
+
return round(sum(a.total_cost for a in self.accounts.values()), 2)
|
|
181
|
+
|
|
182
|
+
@property
|
|
183
|
+
def total_value(self) -> float:
|
|
184
|
+
return round(sum(a.total_value for a in self.accounts.values()), 2)
|
|
185
|
+
|
|
186
|
+
@property
|
|
187
|
+
def net(self) -> float:
|
|
188
|
+
return round(self.total_value - self.total_cost, 2)
|
|
189
|
+
|
|
190
|
+
def by_verdict(self, verdict: Verdict) -> list[TestAccount]:
|
|
191
|
+
return sorted(
|
|
192
|
+
(a for a in self.accounts.values() if a.verdict == verdict),
|
|
193
|
+
key=lambda a: a.net,
|
|
194
|
+
)
|