testledger 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,332 @@
1
+ """The ledger engine: books cost and value entries per test and issues verdicts.
2
+
3
+ Cost model (debits), per test over the analysis window
4
+ ------------------------------------------------------
5
+ * **CI compute** = Σ duration × ci_minute_usd (every execution)
6
+ * **Rerun compute** = executions on attempt > 1 × duration × ci_minute_usd
7
+ * **Triage time** = flaky failures × triage_minutes × engineer_hourly / 60
8
+ * **Maintenance** = lines changed in the test's file × maintenance_minutes_per_line × engineer_hourly / 60
9
+ (apportioned across tests sharing the file)
10
+
11
+ Value model (credits)
12
+ ---------------------
13
+ * **Regression caught** = a genuine failure (failed on first attempt, and was
14
+ NOT subsequently passed on a rerun of the same SHA) that was followed within
15
+ ``FIX_WINDOW`` by a product-code commit matching the fix pattern.
16
+ Each such event is worth ``regression_value_usd``.
17
+ * **Risk coverage** = optional flat value per stable test (default 0) for
18
+ organisations that want to recognise a "safety net" value.
19
+
20
+ Flake detection
21
+ ---------------
22
+ A flaky failure is: failed on attempt N and passed on attempt N+1 for the
23
+ same commit SHA. Nothing changed between the two runs, so this is unambiguous.
24
+ Flake rate = flaky commits / commits tested (first-attempt executions); reruns
25
+ are excluded from the denominator so they cannot dilute the signal they reveal.
26
+ (A future rule could also flag fail→pass across adjacent commits that touched
27
+ no related product code.)
28
+
29
+ Verdicts
30
+ --------
31
+ QUARANTINE flake_rate ≥ quarantine threshold
32
+ RETIRE enough executions, zero regressions caught, net < 0, cost ≥ retire_min_cost_usd
33
+ DEMOTE regressions caught > 0 but UI layer and cost per catch ≥ ratio × service-layer median
34
+ FIX regressions caught > 0 and (flake_rate ≥ fix threshold or avg duration ≥ slow threshold)
35
+ ASSET roi ≥ asset threshold
36
+ HOLD everything else, including insufficient data
37
+ """
38
+
39
+ from __future__ import annotations
40
+
41
+ from collections import defaultdict
42
+ from datetime import UTC, datetime, timedelta
43
+ from statistics import median
44
+
45
+ from testledger.config import Config
46
+ from testledger.models import (
47
+ ChurnRecord,
48
+ EntryKind,
49
+ FixCommit,
50
+ Layer,
51
+ Ledger,
52
+ LedgerEntry,
53
+ Outcome,
54
+ TestAccount,
55
+ TestRun,
56
+ Verdict,
57
+ )
58
+
59
+ FIX_WINDOW = timedelta(days=3)
60
+
61
+
62
+ def classify_layer(test_id: str, file: str | None, cfg: Config) -> Layer:
63
+ hay = f"{test_id} {file or ''}".lower()
64
+ for layer, needles in (
65
+ (Layer.UI, cfg.layers.ui),
66
+ (Layer.SERVICE, cfg.layers.service),
67
+ (Layer.UNIT, cfg.layers.unit),
68
+ ):
69
+ if any(n.lower() in hay for n in needles):
70
+ return layer
71
+ return Layer.UNKNOWN
72
+
73
+
74
+ def _detect_flakes(runs: list[TestRun]) -> dict[str, int]:
75
+ """Count same-SHA fail→pass transitions per test."""
76
+ by_sha: dict[str, list[TestRun]] = defaultdict(list)
77
+ for r in runs:
78
+ if r.commit_sha:
79
+ by_sha[r.commit_sha].append(r)
80
+ flakes: dict[str, int] = defaultdict(int)
81
+ for sha_runs in by_sha.values():
82
+ sha_runs.sort(key=lambda r: (r.attempt, r.started_at))
83
+ outcomes: dict[str, list[Outcome]] = defaultdict(list)
84
+ for r in sha_runs:
85
+ for res in r.results:
86
+ outcomes[res.test_id].append(res.outcome)
87
+ for test_id, seq in outcomes.items():
88
+ for a, b in zip(seq, seq[1:], strict=False):
89
+ if a.is_failure and b == Outcome.PASSED:
90
+ flakes[test_id] += 1
91
+ return flakes
92
+
93
+
94
+ def _genuine_failures(
95
+ runs: list[TestRun], flakes_by_sha_test: set[tuple[str, str]]
96
+ ) -> dict[str, list[datetime]]:
97
+ """Failures on first attempt that were not later cleared by a same-SHA rerun."""
98
+ out: dict[str, list[datetime]] = defaultdict(list)
99
+ for r in runs:
100
+ if r.attempt != 1:
101
+ continue
102
+ for res in r.results:
103
+ if res.outcome.is_failure and (r.commit_sha or "", res.test_id) not in flakes_by_sha_test:
104
+ out[res.test_id].append(r.started_at)
105
+ return out
106
+
107
+
108
+ def _flaky_pairs(runs: list[TestRun]) -> set[tuple[str, str]]:
109
+ pairs: set[tuple[str, str]] = set()
110
+ by_sha: dict[str, list[TestRun]] = defaultdict(list)
111
+ for r in runs:
112
+ if r.commit_sha:
113
+ by_sha[r.commit_sha].append(r)
114
+ for sha, sha_runs in by_sha.items():
115
+ sha_runs.sort(key=lambda r: (r.attempt, r.started_at))
116
+ seen_fail: set[str] = set()
117
+ for r in sha_runs:
118
+ for res in r.results:
119
+ if res.outcome.is_failure:
120
+ seen_fail.add(res.test_id)
121
+ elif res.outcome == Outcome.PASSED and res.test_id in seen_fail:
122
+ pairs.add((sha, res.test_id))
123
+ return pairs
124
+
125
+
126
+ def _credit_regressions(failures: dict[str, list[datetime]], fixes: list[FixCommit]) -> dict[str, int]:
127
+ """A genuine failure followed by a fix commit within FIX_WINDOW counts once per fix."""
128
+ credited: dict[str, int] = defaultdict(int)
129
+ fix_times = sorted(f.timestamp.astimezone(UTC) for f in fixes)
130
+ for test_id, times in failures.items():
131
+ used: set[datetime] = set()
132
+ for t in times:
133
+ t = t.astimezone(UTC)
134
+ for ft in fix_times:
135
+ if ft in used:
136
+ continue
137
+ if t <= ft <= t + FIX_WINDOW:
138
+ credited[test_id] += 1
139
+ used.add(ft)
140
+ break
141
+ return credited
142
+
143
+
144
+ def build_ledger(
145
+ runs: list[TestRun],
146
+ cfg: Config,
147
+ *,
148
+ churn: dict[str, ChurnRecord] | None = None,
149
+ fixes: list[FixCommit] | None = None,
150
+ ai_signals: dict[str, dict] | None = None,
151
+ now: datetime | None = None,
152
+ ) -> Ledger:
153
+ churn = churn or {}
154
+ fixes = fixes or []
155
+ ai_signals = ai_signals or {}
156
+ now = now or datetime.now(tz=UTC)
157
+ window_start = now - timedelta(days=cfg.window_days)
158
+ runs = [r for r in runs if r.started_at >= window_start]
159
+ rates, th = cfg.rates, cfg.thresholds
160
+
161
+ accounts: dict[str, TestAccount] = {}
162
+ durations: dict[str, list[float]] = defaultdict(list)
163
+ rerun_execs: dict[str, int] = defaultdict(int)
164
+ file_members: dict[str, set[str]] = defaultdict(set)
165
+
166
+ for r in runs:
167
+ for res in r.results:
168
+ acc = accounts.get(res.test_id)
169
+ if acc is None:
170
+ acc = TestAccount(
171
+ test_id=res.test_id,
172
+ name=res.name,
173
+ suite=res.suite,
174
+ file=res.file,
175
+ layer=classify_layer(res.test_id, res.file, cfg),
176
+ )
177
+ accounts[res.test_id] = acc
178
+ if res.outcome == Outcome.SKIPPED:
179
+ continue
180
+ acc.executions += 1
181
+ if res.outcome.is_failure:
182
+ acc.failures += 1
183
+ durations[res.test_id].append(res.duration_s)
184
+ if r.attempt > 1:
185
+ rerun_execs[res.test_id] += 1
186
+ if res.file:
187
+ file_members[res.file].add(res.test_id)
188
+
189
+ flakes = _detect_flakes(runs)
190
+ flaky_pairs = _flaky_pairs(runs)
191
+ genuine = _genuine_failures(runs, flaky_pairs)
192
+ credited = _credit_regressions(genuine, fixes)
193
+
194
+ for test_id, acc in accounts.items():
195
+ d = durations.get(test_id, [])
196
+ acc.avg_duration_s = round(sum(d) / len(d), 3) if d else 0.0
197
+ acc.rerun_passes = flakes.get(test_id, 0)
198
+ # Flake rate is per commit tested (first attempts), not per execution:
199
+ # counting reruns in the denominator would dilute the very signal reruns reveal.
200
+ first_attempts = acc.executions - rerun_execs.get(test_id, 0)
201
+ acc.flake_rate = round(acc.rerun_passes / first_attempts, 3) if first_attempts > 0 else 0.0
202
+ acc.regressions_caught = credited.get(test_id, 0)
203
+
204
+ total_minutes = sum(d) / 60.0
205
+ if total_minutes:
206
+ acc.entries.append(
207
+ LedgerEntry(
208
+ test_id=test_id,
209
+ kind=EntryKind.CI_COMPUTE,
210
+ amount_usd=round(total_minutes * rates.ci_minute_usd, 4),
211
+ quantity=round(total_minutes, 2),
212
+ unit="ci-minutes",
213
+ )
214
+ )
215
+ if rerun_execs.get(test_id):
216
+ rerun_minutes = rerun_execs[test_id] * acc.avg_duration_s / 60.0
217
+ acc.entries.append(
218
+ LedgerEntry(
219
+ test_id=test_id,
220
+ kind=EntryKind.RERUN_COMPUTE,
221
+ amount_usd=round(rerun_minutes * rates.ci_minute_usd, 4),
222
+ quantity=rerun_execs[test_id],
223
+ unit="rerun-executions",
224
+ )
225
+ )
226
+ if acc.rerun_passes:
227
+ triage_hours = acc.rerun_passes * rates.triage_minutes_per_false_alarm / 60.0
228
+ acc.entries.append(
229
+ LedgerEntry(
230
+ test_id=test_id,
231
+ kind=EntryKind.TRIAGE_TIME,
232
+ amount_usd=round(triage_hours * rates.engineer_hourly_usd, 2),
233
+ quantity=acc.rerun_passes,
234
+ unit="false-alarms",
235
+ )
236
+ )
237
+ if acc.file and acc.file in churn:
238
+ rec = churn[acc.file]
239
+ share = 1 / max(1, len(file_members.get(acc.file, {test_id})))
240
+ acc.churn_commits = rec.commits
241
+ acc.churn_lines = round(rec.lines_changed * share)
242
+ maint_hours = acc.churn_lines * rates.maintenance_minutes_per_line / 60.0
243
+ if maint_hours:
244
+ acc.entries.append(
245
+ LedgerEntry(
246
+ test_id=test_id,
247
+ kind=EntryKind.MAINTENANCE,
248
+ amount_usd=round(maint_hours * rates.engineer_hourly_usd, 2),
249
+ quantity=acc.churn_lines,
250
+ unit="lines-changed",
251
+ )
252
+ )
253
+ if acc.regressions_caught:
254
+ acc.entries.append(
255
+ LedgerEntry(
256
+ test_id=test_id,
257
+ kind=EntryKind.REGRESSION_CAUGHT,
258
+ amount_usd=round(acc.regressions_caught * rates.regression_value_usd, 2),
259
+ quantity=acc.regressions_caught,
260
+ unit="regressions",
261
+ )
262
+ )
263
+ if rates.risk_coverage_value_usd and acc.flake_rate < th.fix_flake_rate:
264
+ acc.entries.append(
265
+ LedgerEntry(
266
+ test_id=test_id,
267
+ kind=EntryKind.RISK_COVERAGE,
268
+ amount_usd=rates.risk_coverage_value_usd,
269
+ quantity=1,
270
+ unit="window",
271
+ )
272
+ )
273
+ sig = ai_signals.get(test_id) or (ai_signals.get(acc.file) if acc.file else None)
274
+ if sig:
275
+ acc.ai_generated = bool(sig.get("ai_generated", False))
276
+ acc.mutation_score = sig.get("mutation_score")
277
+ acc.assertion_density = sig.get("assertion_density")
278
+
279
+ _assign_verdicts(accounts, cfg)
280
+
281
+ notes = [
282
+ f"Window: {cfg.window_days} days, {len(runs)} runs analysed.",
283
+ f"Rates: engineer ${rates.engineer_hourly_usd}/h, CI ${rates.ci_minute_usd}/min, regression value ${rates.regression_value_usd}.",
284
+ f"Fix commits detected: {len(fixes)}; test files with churn: {len(churn)}.",
285
+ ]
286
+ return Ledger(
287
+ generated_at=now,
288
+ window_start=window_start,
289
+ window_end=now,
290
+ runs_analysed=len(runs),
291
+ accounts=accounts,
292
+ notes=notes,
293
+ )
294
+
295
+
296
+ def _assign_verdicts(accounts: dict[str, TestAccount], cfg: Config) -> None:
297
+ th = cfg.thresholds
298
+ service_cost_per_catch = [
299
+ a.total_cost / a.regressions_caught
300
+ for a in accounts.values()
301
+ if a.layer in (Layer.SERVICE, Layer.UNIT) and a.regressions_caught > 0
302
+ ]
303
+ svc_median = median(service_cost_per_catch) if service_cost_per_catch else None
304
+
305
+ for a in accounts.values():
306
+ if a.executions < th.min_executions_for_verdict:
307
+ a.verdict = Verdict.HOLD
308
+ continue
309
+ if a.flake_rate >= th.quarantine_flake_rate:
310
+ a.verdict = Verdict.QUARANTINE
311
+ continue
312
+ if a.regressions_caught == 0:
313
+ a.verdict = (
314
+ Verdict.RETIRE if (a.net < 0 and a.total_cost >= th.retire_min_cost_usd) else Verdict.HOLD
315
+ )
316
+ continue
317
+ if (
318
+ a.layer == Layer.UI
319
+ and svc_median
320
+ and (a.total_cost / a.regressions_caught) >= th.demote_layer_cost_ratio * svc_median
321
+ ):
322
+ a.verdict = Verdict.DEMOTE
323
+ continue
324
+ if a.flake_rate >= th.fix_flake_rate or a.avg_duration_s >= th.slow_test_seconds:
325
+ a.verdict = Verdict.FIX
326
+ continue
327
+ roi = a.roi
328
+ # A test that caught a regression at (near-)zero cost is the best asset there is.
329
+ if a.total_value > 0 and (roi is None or roi >= th.asset_roi):
330
+ a.verdict = Verdict.ASSET
331
+ else:
332
+ a.verdict = Verdict.HOLD
testledger/models.py ADDED
@@ -0,0 +1,194 @@
1
+ """Core domain model.
2
+
3
+ The model is deliberately small and framework-agnostic. Every ingester
4
+ (JUnit, Cucumber, GitHub Actions, Jenkins, git) normalises its source into
5
+ ``TestRun`` / ``TestResult`` objects plus optional ``ChurnRecord`` entries.
6
+ The ledger engine then books ``LedgerEntry`` rows against each ``TestAccount``.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from datetime import datetime
12
+ from enum import StrEnum
13
+
14
+ from pydantic import BaseModel, Field
15
+
16
+
17
+ class Outcome(StrEnum):
18
+ PASSED = "passed"
19
+ FAILED = "failed"
20
+ ERROR = "error"
21
+ SKIPPED = "skipped"
22
+
23
+ @property
24
+ def is_failure(self) -> bool:
25
+ return self in (Outcome.FAILED, Outcome.ERROR)
26
+
27
+
28
+ class Layer(StrEnum):
29
+ """Test-pyramid layer. Drives default cost multipliers and demotion advice."""
30
+
31
+ UNIT = "unit"
32
+ SERVICE = "service"
33
+ UI = "ui"
34
+ UNKNOWN = "unknown"
35
+
36
+
37
+ class TestResult(BaseModel):
38
+ """One execution of one test case inside one run."""
39
+
40
+ test_id: str = Field(description="Stable identifier, e.g. 'com.acme.LoginTest.valid_login'")
41
+ name: str
42
+ suite: str = ""
43
+ file: str | None = Field(default=None, description="Source path when known (used for git churn)")
44
+ outcome: Outcome
45
+ duration_s: float = 0.0
46
+ message: str | None = None
47
+ tags: list[str] = Field(default_factory=list)
48
+
49
+
50
+ class TestRun(BaseModel):
51
+ """A single CI execution (one workflow run / Jenkins build / local run)."""
52
+
53
+ run_id: str
54
+ started_at: datetime
55
+ commit_sha: str | None = None
56
+ branch: str | None = None
57
+ attempt: int = Field(default=1, description="1 = first execution; >1 means a rerun of the same SHA")
58
+ source: str = Field(default="junit", description="junit | cucumber | github-actions | jenkins")
59
+ results: list[TestResult] = Field(default_factory=list)
60
+
61
+ @property
62
+ def failed(self) -> int:
63
+ return sum(1 for r in self.results if r.outcome.is_failure)
64
+
65
+
66
+ class ChurnRecord(BaseModel):
67
+ """Git activity against a test file in the analysis window."""
68
+
69
+ file: str
70
+ commits: int = 0
71
+ lines_changed: int = 0
72
+ authors: int = 0
73
+ last_touched: datetime | None = None
74
+
75
+
76
+ class FixCommit(BaseModel):
77
+ """A product-code commit that looks like a bug fix (used to credit tests that caught it)."""
78
+
79
+ sha: str
80
+ timestamp: datetime
81
+ message: str
82
+ files: list[str] = Field(default_factory=list)
83
+
84
+
85
+ class EntryKind(StrEnum):
86
+ # Costs (debits)
87
+ CI_COMPUTE = "ci_compute"
88
+ RERUN_COMPUTE = "rerun_compute"
89
+ TRIAGE_TIME = "triage_time"
90
+ MAINTENANCE = "maintenance"
91
+ # Value (credits)
92
+ REGRESSION_CAUGHT = "regression_caught"
93
+ RISK_COVERAGE = "risk_coverage"
94
+
95
+
96
+ class LedgerEntry(BaseModel):
97
+ """One booked line. Costs are positive 'debit'; value is positive 'credit'."""
98
+
99
+ test_id: str
100
+ kind: EntryKind
101
+ amount_usd: float
102
+ quantity: float = 0.0
103
+ unit: str = ""
104
+ note: str = ""
105
+
106
+ @property
107
+ def is_cost(self) -> bool:
108
+ return self.kind in (
109
+ EntryKind.CI_COMPUTE,
110
+ EntryKind.RERUN_COMPUTE,
111
+ EntryKind.TRIAGE_TIME,
112
+ EntryKind.MAINTENANCE,
113
+ )
114
+
115
+
116
+ class Verdict(StrEnum):
117
+ ASSET = "asset" # value clearly exceeds cost
118
+ HOLD = "hold" # roughly break-even or insufficient data
119
+ FIX = "fix" # valuable but flaky/slow — repair it
120
+ DEMOTE = "demote" # valuable but expensive layer — rewrite lower in the pyramid
121
+ RETIRE = "retire" # pure cost, no evidence of value
122
+ QUARANTINE = "quarantine" # flake rate so high it is actively misleading
123
+
124
+
125
+ class TestAccount(BaseModel):
126
+ """The per-test ledger account with derived statistics."""
127
+
128
+ test_id: str
129
+ name: str
130
+ suite: str = ""
131
+ file: str | None = None
132
+ layer: Layer = Layer.UNKNOWN
133
+ executions: int = 0
134
+ failures: int = 0
135
+ rerun_passes: int = Field(
136
+ default=0, description="failed on attempt N, passed on attempt N+1 for same SHA"
137
+ )
138
+ flake_rate: float = 0.0
139
+ avg_duration_s: float = 0.0
140
+ regressions_caught: int = 0
141
+ churn_commits: int = 0
142
+ churn_lines: int = 0
143
+ entries: list[LedgerEntry] = Field(default_factory=list)
144
+ verdict: Verdict = Verdict.HOLD
145
+ ai_generated: bool = False
146
+ mutation_score: float | None = None
147
+ assertion_density: float | None = None
148
+
149
+ @property
150
+ def total_cost(self) -> float:
151
+ return round(sum(e.amount_usd for e in self.entries if e.is_cost), 4)
152
+
153
+ @property
154
+ def total_value(self) -> float:
155
+ return round(sum(e.amount_usd for e in self.entries if not e.is_cost), 2)
156
+
157
+ @property
158
+ def net(self) -> float:
159
+ return round(self.total_value - self.total_cost, 2)
160
+
161
+ @property
162
+ def roi(self) -> float | None:
163
+ if self.total_cost <= 0:
164
+ return None
165
+ return round((self.total_value - self.total_cost) / self.total_cost, 3)
166
+
167
+
168
+ class Ledger(BaseModel):
169
+ """The full book for one analysis window."""
170
+
171
+ generated_at: datetime
172
+ window_start: datetime
173
+ window_end: datetime
174
+ runs_analysed: int
175
+ accounts: dict[str, TestAccount] = Field(default_factory=dict)
176
+ notes: list[str] = Field(default_factory=list)
177
+
178
+ @property
179
+ def total_cost(self) -> float:
180
+ return round(sum(a.total_cost for a in self.accounts.values()), 2)
181
+
182
+ @property
183
+ def total_value(self) -> float:
184
+ return round(sum(a.total_value for a in self.accounts.values()), 2)
185
+
186
+ @property
187
+ def net(self) -> float:
188
+ return round(self.total_value - self.total_cost, 2)
189
+
190
+ def by_verdict(self, verdict: Verdict) -> list[TestAccount]:
191
+ return sorted(
192
+ (a for a in self.accounts.values() if a.verdict == verdict),
193
+ key=lambda a: a.net,
194
+ )
@@ -0,0 +1,3 @@
1
+ from testledger.reports.builder import ReportBundle, build_reports, write_reports
2
+
3
+ __all__ = ["ReportBundle", "build_reports", "write_reports"]