testledger 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,224 @@
1
+ """Report builder: turns a ``Ledger`` into executive artefacts.
2
+
3
+ * ``balance_sheet`` — assets vs. liabilities by test, by layer and by suite
4
+ * ``income_statement``— cost and value totals for the window, by cost category
5
+ * ``writeoffs`` — the action list: retire / demote / fix / quarantine
6
+ * ``ai_liabilities`` — AI-generated tests failing governance thresholds
7
+ * ``memo`` — the one-page Quality Investment Memo (Markdown + HTML)
8
+ * ``ledger.json`` — the full book for dashboards and diffing between quarters
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ from collections import defaultdict
15
+ from dataclasses import dataclass, field
16
+ from pathlib import Path
17
+
18
+ from jinja2 import Environment, PackageLoader, select_autoescape
19
+
20
+ from testledger.config import Config
21
+ from testledger.models import EntryKind, Layer, Ledger, TestAccount, Verdict
22
+
23
+ VERDICT_ACTION = {
24
+ Verdict.RETIRE: "Delete. No regression caught in window; pure carrying cost.",
25
+ Verdict.DEMOTE: "Rewrite at service/unit layer; same coverage at a fraction of the cost.",
26
+ Verdict.FIX: "Stabilise or speed up; the value is real but the signal is noisy.",
27
+ Verdict.QUARANTINE: "Remove from the merge gate immediately; failing it blocks nothing but trust.",
28
+ Verdict.ASSET: "Keep. Protect from churn; use as a template.",
29
+ Verdict.HOLD: "No action; insufficient evidence or break-even.",
30
+ }
31
+
32
+
33
+ @dataclass
34
+ class ReportBundle:
35
+ summary: dict
36
+ balance_sheet: dict
37
+ income_statement: dict
38
+ writeoffs: list[dict]
39
+ ai_liabilities: list[dict]
40
+ memo_markdown: str
41
+ memo_html: str
42
+ ledger_json: dict
43
+ assumptions: dict = field(default_factory=dict)
44
+
45
+
46
+ def _acc_row(a: TestAccount) -> dict:
47
+ return {
48
+ "test_id": a.test_id,
49
+ "suite": a.suite,
50
+ "layer": a.layer.value,
51
+ "executions": a.executions,
52
+ "failures": a.failures,
53
+ "flake_rate": a.flake_rate,
54
+ "avg_duration_s": a.avg_duration_s,
55
+ "regressions_caught": a.regressions_caught,
56
+ "cost_usd": a.total_cost,
57
+ "value_usd": a.total_value,
58
+ "net_usd": a.net,
59
+ "roi": a.roi,
60
+ "verdict": a.verdict.value,
61
+ "action": VERDICT_ACTION[a.verdict],
62
+ "ai_generated": a.ai_generated,
63
+ "mutation_score": a.mutation_score,
64
+ "assertion_density": a.assertion_density,
65
+ }
66
+
67
+
68
+ def build_reports(ledger: Ledger, cfg: Config) -> ReportBundle:
69
+ accounts = list(ledger.accounts.values())
70
+ assets = sorted((a for a in accounts if a.net > 0), key=lambda a: -a.net)
71
+ liabilities = sorted((a for a in accounts if a.net < 0), key=lambda a: a.net)
72
+
73
+ by_layer: dict[str, dict] = defaultdict(
74
+ lambda: {"tests": 0, "cost_usd": 0.0, "value_usd": 0.0, "regressions": 0}
75
+ )
76
+ by_suite: dict[str, dict] = defaultdict(
77
+ lambda: {"tests": 0, "cost_usd": 0.0, "value_usd": 0.0, "regressions": 0}
78
+ )
79
+ by_kind: dict[str, float] = defaultdict(float)
80
+ verdict_counts: dict[str, int] = defaultdict(int)
81
+ for a in accounts:
82
+ for bucket in (by_layer[a.layer.value], by_suite[a.suite or "(no suite)"]):
83
+ bucket["tests"] += 1
84
+ bucket["cost_usd"] = round(bucket["cost_usd"] + a.total_cost, 2)
85
+ bucket["value_usd"] = round(bucket["value_usd"] + a.total_value, 2)
86
+ bucket["regressions"] += a.regressions_caught
87
+ for e in a.entries:
88
+ by_kind[e.kind.value] = round(by_kind[e.kind.value] + e.amount_usd, 2)
89
+ verdict_counts[a.verdict.value] += 1
90
+
91
+ writeoffs = [
92
+ _acc_row(a)
93
+ for v in (Verdict.QUARANTINE, Verdict.RETIRE, Verdict.DEMOTE, Verdict.FIX)
94
+ for a in ledger.by_verdict(v)
95
+ ]
96
+ writeoff_savings = round(
97
+ sum(a.total_cost for a in accounts if a.verdict in (Verdict.RETIRE, Verdict.QUARANTINE)), 2
98
+ )
99
+ demote_savings = round(
100
+ sum(
101
+ a.total_cost * (1 - 1 / cfg.thresholds.demote_layer_cost_ratio)
102
+ for a in accounts
103
+ if a.verdict == Verdict.DEMOTE
104
+ ),
105
+ 2,
106
+ )
107
+
108
+ th = cfg.thresholds
109
+ ai_liabilities = [
110
+ _acc_row(a)
111
+ for a in accounts
112
+ if a.ai_generated
113
+ and (
114
+ (a.mutation_score is not None and a.mutation_score < th.ai_min_mutation_score)
115
+ or (a.assertion_density is not None and a.assertion_density < th.ai_min_assertion_density)
116
+ )
117
+ ]
118
+
119
+ summary = {
120
+ "org": cfg.org_name,
121
+ "generated_at": ledger.generated_at.isoformat(timespec="seconds"),
122
+ "window_days": cfg.window_days,
123
+ "runs_analysed": ledger.runs_analysed,
124
+ "tests": len(accounts),
125
+ "total_cost_usd": ledger.total_cost,
126
+ "total_value_usd": ledger.total_value,
127
+ "net_usd": ledger.net,
128
+ "portfolio_roi": round(ledger.net / ledger.total_cost, 3) if ledger.total_cost else None,
129
+ "regressions_caught": sum(a.regressions_caught for a in accounts),
130
+ "asset_tests": len(assets),
131
+ "liability_tests": len(liabilities),
132
+ "verdicts": dict(verdict_counts),
133
+ "annualised_writeoff_savings_usd": round(writeoff_savings * 365 / cfg.window_days, 2),
134
+ "annualised_demote_savings_usd": round(demote_savings * 365 / cfg.window_days, 2),
135
+ "ai_generated_tests": sum(1 for a in accounts if a.ai_generated),
136
+ "ai_unverified_liabilities": len(ai_liabilities),
137
+ }
138
+ balance_sheet = {
139
+ "assets": [_acc_row(a) for a in assets[:50]],
140
+ "liabilities": [_acc_row(a) for a in liabilities[:50]],
141
+ "by_layer": dict(by_layer),
142
+ "by_suite": dict(sorted(by_suite.items(), key=lambda kv: kv[1]["cost_usd"], reverse=True)[:25]),
143
+ }
144
+ income_statement = {
145
+ "costs": {
146
+ k: v
147
+ for k, v in by_kind.items()
148
+ if k
149
+ in {
150
+ e.value
151
+ for e in EntryKind
152
+ if e
153
+ in (
154
+ EntryKind.CI_COMPUTE,
155
+ EntryKind.RERUN_COMPUTE,
156
+ EntryKind.TRIAGE_TIME,
157
+ EntryKind.MAINTENANCE,
158
+ )
159
+ }
160
+ },
161
+ "value": {
162
+ k: v
163
+ for k, v in by_kind.items()
164
+ if k in (EntryKind.REGRESSION_CAUGHT.value, EntryKind.RISK_COVERAGE.value)
165
+ },
166
+ "net_usd": ledger.net,
167
+ }
168
+ assumptions = cfg.rates.model_dump() | {"thresholds": cfg.thresholds.model_dump()}
169
+
170
+ env = Environment(loader=PackageLoader("testledger", "templates"), autoescape=select_autoescape(["html"]))
171
+ ctx = {
172
+ "s": summary,
173
+ "bs": balance_sheet,
174
+ "inc": income_statement,
175
+ "writeoffs": writeoffs[:40],
176
+ "ai": ai_liabilities[:20],
177
+ "notes": ledger.notes,
178
+ "assumptions": assumptions,
179
+ "Layer": Layer,
180
+ "top_assets": balance_sheet["assets"][:10],
181
+ "top_liabilities": balance_sheet["liabilities"][:10],
182
+ }
183
+ memo_md = env.get_template("memo.md.j2").render(**ctx)
184
+ memo_html = env.get_template("memo.html.j2").render(**ctx, memo_md=memo_md)
185
+
186
+ ledger_json = {
187
+ "summary": summary,
188
+ "assumptions": assumptions,
189
+ "accounts": [
190
+ _acc_row(a) | {"entries": [e.model_dump(mode="json") for e in a.entries]} for a in accounts
191
+ ],
192
+ }
193
+ return ReportBundle(
194
+ summary,
195
+ balance_sheet,
196
+ income_statement,
197
+ writeoffs,
198
+ ai_liabilities,
199
+ memo_md,
200
+ memo_html,
201
+ ledger_json,
202
+ assumptions,
203
+ )
204
+
205
+
206
+ def write_reports(bundle: ReportBundle, out_dir: str | Path) -> dict[str, Path]:
207
+ out = Path(out_dir)
208
+ out.mkdir(parents=True, exist_ok=True)
209
+ files = {
210
+ "memo.md": bundle.memo_markdown,
211
+ "memo.html": bundle.memo_html,
212
+ "balance_sheet.json": json.dumps(bundle.balance_sheet, indent=2),
213
+ "income_statement.json": json.dumps(bundle.income_statement, indent=2),
214
+ "writeoffs.json": json.dumps(bundle.writeoffs, indent=2),
215
+ "ai_liabilities.json": json.dumps(bundle.ai_liabilities, indent=2),
216
+ "ledger.json": json.dumps(bundle.ledger_json, indent=2),
217
+ "summary.json": json.dumps(bundle.summary, indent=2),
218
+ }
219
+ written: dict[str, Path] = {}
220
+ for name, content in files.items():
221
+ p = out / name
222
+ p.write_text(content, encoding="utf-8")
223
+ written[name] = p
224
+ return written
@@ -0,0 +1,77 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <title>Quality Investment Memo — {{ s.org }}</title>
6
+ <style>
7
+ :root { --ink:#1b1f24; --muted:#5b6470; --line:#e3e6ea; --asset:#1f7a4d; --liab:#b3261e; --bg:#fafbfc; }
8
+ body { font: 15px/1.55 -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; color:var(--ink); background:var(--bg); margin:0; }
9
+ main { max-width: 980px; margin: 0 auto; padding: 40px 28px 80px; background:#fff; }
10
+ h1 { font-size: 26px; margin:0 0 4px; } h2 { font-size:18px; margin:32px 0 10px; border-bottom:1px solid var(--line); padding-bottom:6px; }
11
+ .meta { color:var(--muted); font-size:13px; margin-bottom:24px; }
12
+ .tiles { display:grid; grid-template-columns: repeat(4, 1fr); gap:12px; margin: 18px 0; }
13
+ .tile { border:1px solid var(--line); border-radius:8px; padding:14px; }
14
+ .tile .k { font-size:12px; color:var(--muted); text-transform:uppercase; letter-spacing:.04em; }
15
+ .tile .v { font-size:22px; font-weight:600; margin-top:4px; }
16
+ .pos { color:var(--asset);} .neg { color:var(--liab);}
17
+ table { border-collapse: collapse; width:100%; font-size:13.5px; margin: 8px 0 16px; }
18
+ th, td { text-align:left; padding:7px 9px; border-bottom:1px solid var(--line); vertical-align:top; }
19
+ th { color:var(--muted); font-weight:600; font-size:12px; text-transform:uppercase; letter-spacing:.03em; }
20
+ td.num, th.num { text-align:right; font-variant-numeric: tabular-nums; }
21
+ code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size:12.5px; }
22
+ .badge { display:inline-block; padding:2px 8px; border-radius:999px; font-size:11.5px; font-weight:600; }
23
+ .retire,.quarantine { background:#fde8e6; color:var(--liab);} .demote,.fix { background:#fff4d6; color:#8a5a00;} .asset { background:#e3f4ea; color:var(--asset);} .hold { background:#eef0f3; color:var(--muted);}
24
+ .foot { color:var(--muted); font-size:12.5px; margin-top:28px; }
25
+ </style>
26
+ </head>
27
+ <body><main>
28
+ <h1>Quality Investment Memo — {{ s.org }}</h1>
29
+ <div class="meta">Last {{ s.window_days }} days · generated {{ s.generated_at }} · {{ s.runs_analysed }} CI runs · {{ s.tests }} tests on the books</div>
30
+
31
+ <div class="tiles">
32
+ <div class="tile"><div class="k">Carrying cost</div><div class="v">${{ "%.0f"|format(s.total_cost_usd) }}</div></div>
33
+ <div class="tile"><div class="k">Value delivered</div><div class="v">${{ "%.0f"|format(s.total_value_usd) }}</div></div>
34
+ <div class="tile"><div class="k">Net</div><div class="v {{ 'pos' if s.net_usd >= 0 else 'neg' }}">${{ "%.0f"|format(s.net_usd) }}</div></div>
35
+ <div class="tile"><div class="k">Regressions caught</div><div class="v">{{ s.regressions_caught }}</div></div>
36
+ </div>
37
+
38
+ <h2>Recommended reallocation</h2>
39
+ <table>
40
+ <tr><th>Action</th><th class="num">Tests</th><th class="num">Annualised impact</th></tr>
41
+ <tr><td>Retire / quarantine</td><td class="num">{{ s.verdicts.get('retire',0)+s.verdicts.get('quarantine',0) }}</td><td class="num">save ≈ ${{ "%.0f"|format(s.annualised_writeoff_savings_usd) }}</td></tr>
42
+ <tr><td>Demote UI tests to lower layer</td><td class="num">{{ s.verdicts.get('demote',0) }}</td><td class="num">save ≈ ${{ "%.0f"|format(s.annualised_demote_savings_usd) }}</td></tr>
43
+ <tr><td>Fix (flaky / slow but valuable)</td><td class="num">{{ s.verdicts.get('fix',0) }}</td><td class="num">protects delivered value</td></tr>
44
+ <tr><td>Keep as assets</td><td class="num">{{ s.verdicts.get('asset',0) }}</td><td class="num">—</td></tr>
45
+ <tr><td>Hold</td><td class="num">{{ s.verdicts.get('hold',0) }}</td><td class="num">re-evaluate next quarter</td></tr>
46
+ </table>
47
+
48
+ <h2>Where the money goes</h2>
49
+ <table><tr><th>Cost category</th><th class="num">USD</th></tr>
50
+ {% for k, v in inc.costs.items() %}<tr><td>{{ k.replace('_',' ') }}</td><td class="num">{{ "%.2f"|format(v) }}</td></tr>{% endfor %}
51
+ </table>
52
+ <table><tr><th>Layer</th><th class="num">Tests</th><th class="num">Cost</th><th class="num">Value</th><th class="num">Regressions</th></tr>
53
+ {% for layer, b in bs.by_layer.items() %}<tr><td>{{ layer }}</td><td class="num">{{ b.tests }}</td><td class="num">{{ "%.2f"|format(b.cost_usd) }}</td><td class="num">{{ "%.2f"|format(b.value_usd) }}</td><td class="num">{{ b.regressions }}</td></tr>{% endfor %}
54
+ </table>
55
+
56
+ <h2>Write-off list</h2>
57
+ <table><tr><th>Test</th><th>Layer</th><th class="num">Flake</th><th class="num">Avg s</th><th class="num">Cost</th><th class="num">Value</th><th>Verdict</th><th>Action</th></tr>
58
+ {% for a in writeoffs %}<tr><td><code>{{ a.test_id }}</code></td><td>{{ a.layer }}</td><td class="num">{{ "%.0f"|format(a.flake_rate*100) }}%</td><td class="num">{{ "%.1f"|format(a.avg_duration_s) }}</td><td class="num">{{ "%.2f"|format(a.cost_usd) }}</td><td class="num">{{ "%.2f"|format(a.value_usd) }}</td><td><span class="badge {{ a.verdict }}">{{ a.verdict }}</span></td><td>{{ a.action }}</td></tr>{% endfor %}
59
+ </table>
60
+
61
+ <h2>Top assets</h2>
62
+ <table><tr><th>Test</th><th>Layer</th><th class="num">Regressions</th><th class="num">Cost</th><th class="num">Value</th></tr>
63
+ {% for a in top_assets %}<tr><td><code>{{ a.test_id }}</code></td><td>{{ a.layer }}</td><td class="num">{{ a.regressions_caught }}</td><td class="num">{{ "%.2f"|format(a.cost_usd) }}</td><td class="num">{{ "%.2f"|format(a.value_usd) }}</td></tr>{% endfor %}
64
+ </table>
65
+
66
+ {% if s.ai_generated_tests %}
67
+ <h2>AI-generated test governance</h2>
68
+ <p>{{ s.ai_generated_tests }} AI-generated tests; {{ s.ai_unverified_liabilities }} below governance thresholds (mutation ≥ {{ assumptions.thresholds.ai_min_mutation_score }}, assertion density ≥ {{ assumptions.thresholds.ai_min_assertion_density }}).</p>
69
+ <table><tr><th>Test</th><th class="num">Mutation</th><th class="num">Assertion density</th><th class="num">Cost</th></tr>
70
+ {% for a in ai %}<tr><td><code>{{ a.test_id }}</code></td><td class="num">{{ a.mutation_score if a.mutation_score is not none else "n/a" }}</td><td class="num">{{ a.assertion_density if a.assertion_density is not none else "n/a" }}</td><td class="num">{{ "%.2f"|format(a.cost_usd) }}</td></tr>{% endfor %}
71
+ </table>
72
+ {% endif %}
73
+
74
+ <h2>Assumptions</h2>
75
+ <table>{% for k, v in assumptions.items() if k != 'thresholds' %}<tr><td>{{ k.replace('_',' ') }}</td><td class="num">{{ v }}</td></tr>{% endfor %}</table>
76
+ <div class="foot">{% for n in notes %}{{ n }}<br>{% endfor %}Generated by TestLedger · github.com/surkuar/testledger</div>
77
+ </main></body></html>
@@ -0,0 +1,78 @@
1
+ # Quality Investment Memo — {{ s.org }}
2
+
3
+ **Period:** last {{ s.window_days }} days · **Generated:** {{ s.generated_at }} · **Runs analysed:** {{ s.runs_analysed }} · **Tests on the books:** {{ s.tests }}
4
+
5
+ ## 1. Bottom line
6
+
7
+ | | USD |
8
+ |---|---:|
9
+ | Carrying cost of the test estate (CI, reruns, triage, maintenance) | {{ "%.2f"|format(s.total_cost_usd) }} |
10
+ | Value delivered (regressions caught × cost of a production defect) | {{ "%.2f"|format(s.total_value_usd) }} |
11
+ | **Net** | **{{ "%.2f"|format(s.net_usd) }}** |
12
+ | Portfolio ROI | {{ s.portfolio_roi if s.portfolio_roi is not none else "n/a" }} |
13
+
14
+ {{ s.regressions_caught }} regression(s) were caught before production in the window.
15
+ {{ s.asset_tests }} tests are net assets; {{ s.liability_tests }} are net liabilities.
16
+
17
+ ## 2. Recommended reallocation
18
+
19
+ | Action | Tests | Annualised impact |
20
+ |---|---:|---:|
21
+ | Retire / quarantine | {{ s.verdicts.get('retire', 0) + s.verdicts.get('quarantine', 0) }} | save ≈ ${{ "%.0f"|format(s.annualised_writeoff_savings_usd) }} |
22
+ | Demote UI tests to service/unit layer | {{ s.verdicts.get('demote', 0) }} | save ≈ ${{ "%.0f"|format(s.annualised_demote_savings_usd) }} |
23
+ | Fix (flaky or slow but valuable) | {{ s.verdicts.get('fix', 0) }} | protects value already being delivered |
24
+ | Keep as assets | {{ s.verdicts.get('asset', 0) }} | — |
25
+ | Hold (insufficient evidence) | {{ s.verdicts.get('hold', 0) }} | re-evaluate next quarter |
26
+
27
+ ## 3. Where the money goes
28
+
29
+ | Cost category | USD |
30
+ |---|---:|
31
+ {% for k, v in inc.costs.items() -%}
32
+ | {{ k.replace('_', ' ') }} | {{ "%.2f"|format(v) }} |
33
+ {% endfor %}
34
+
35
+ | By layer | Tests | Cost | Value | Regressions |
36
+ |---|---:|---:|---:|---:|
37
+ {% for layer, b in bs.by_layer.items() -%}
38
+ | {{ layer }} | {{ b.tests }} | {{ "%.2f"|format(b.cost_usd) }} | {{ "%.2f"|format(b.value_usd) }} | {{ b.regressions }} |
39
+ {% endfor %}
40
+
41
+ ## 4. Top liabilities (act on these first)
42
+
43
+ | Test | Layer | Flake | Avg s | Cost | Value | Verdict |
44
+ |---|---|---:|---:|---:|---:|---|
45
+ {% for a in top_liabilities -%}
46
+ | `{{ a.test_id }}` | {{ a.layer }} | {{ "%.0f"|format(a.flake_rate*100) }}% | {{ "%.1f"|format(a.avg_duration_s) }} | {{ "%.2f"|format(a.cost_usd) }} | {{ "%.2f"|format(a.value_usd) }} | **{{ a.verdict }}** |
47
+ {% endfor %}
48
+
49
+ ## 5. Top assets (protect these)
50
+
51
+ | Test | Layer | Regressions caught | Cost | Value |
52
+ |---|---|---:|---:|---:|
53
+ {% for a in top_assets -%}
54
+ | `{{ a.test_id }}` | {{ a.layer }} | {{ a.regressions_caught }} | {{ "%.2f"|format(a.cost_usd) }} | {{ "%.2f"|format(a.value_usd) }} |
55
+ {% endfor %}
56
+
57
+ {% if s.ai_generated_tests %}
58
+ ## 6. AI-generated test governance
59
+
60
+ {{ s.ai_generated_tests }} tests are marked AI-generated; {{ s.ai_unverified_liabilities }} fall below the governance thresholds (mutation score ≥ {{ assumptions.thresholds.ai_min_mutation_score }}, assertion density ≥ {{ assumptions.thresholds.ai_min_assertion_density }}). These are **unverified liabilities**: they add carrying cost without evidence they would fail on a real defect.
61
+
62
+ | Test | Mutation score | Assertion density | Cost |
63
+ |---|---:|---:|---:|
64
+ {% for a in ai -%}
65
+ | `{{ a.test_id }}` | {{ a.mutation_score if a.mutation_score is not none else "n/a" }} | {{ a.assertion_density if a.assertion_density is not none else "n/a" }} | {{ "%.2f"|format(a.cost_usd) }} |
66
+ {% endfor %}
67
+ {% endif %}
68
+
69
+ ## 7. Assumptions (challenge these)
70
+
71
+ {% for k, v in assumptions.items() if k != 'thresholds' -%}
72
+ - {{ k.replace('_', ' ') }}: {{ v }}
73
+ {% endfor %}
74
+ {% for n in notes -%}
75
+ - {{ n }}
76
+ {% endfor %}
77
+
78
+ *Method: every test is booked like an account. Debits are CI minutes, rerun minutes, engineer triage time on flaky failures and maintenance churn from git history. Credits are genuine first-attempt failures followed within 3 days by a product fix commit, valued at the organisation's cost of a production defect. Full method: docs/methodology.md.*
@@ -0,0 +1,208 @@
1
+ Metadata-Version: 2.4
2
+ Name: testledger
3
+ Version: 0.1.0
4
+ Summary: Test Estate Balance Sheet: turn CI results and git history into per-test P&L and executive Quality Investment Memos.
5
+ Author-email: Suresh Babu <sureshbabu.santhakumar@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/surkuar/testledger
8
+ Project-URL: Issues, https://github.com/surkuar/testledger/issues
9
+ Project-URL: Documentation, https://github.com/surkuar/testledger/tree/main/docs
10
+ Keywords: testing,quality-engineering,ci,flaky-tests,test-roi,engineering-leadership
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Software Development :: Testing
20
+ Requires-Python: >=3.11
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: pydantic>=2.5
24
+ Requires-Dist: PyYAML>=6.0
25
+ Requires-Dist: Jinja2>=3.1
26
+ Requires-Dist: requests>=2.31
27
+ Requires-Dist: typer>=0.12
28
+ Requires-Dist: rich>=13.0
29
+ Provides-Extra: dev
30
+ Requires-Dist: pytest>=8.0; extra == "dev"
31
+ Requires-Dist: pytest-cov>=5.0; extra == "dev"
32
+ Requires-Dist: ruff>=0.5; extra == "dev"
33
+ Requires-Dist: mypy>=1.10; extra == "dev"
34
+ Requires-Dist: build>=1.2; extra == "dev"
35
+ Dynamic: license-file
36
+
37
+ # TestLedger — Test Estate Balance Sheet
38
+
39
+ [![CI](https://github.com/surkuar/testledger/actions/workflows/ci.yml/badge.svg)](https://github.com/surkuar/testledger/actions/workflows/ci.yml)
40
+ [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
41
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](pyproject.toml)
42
+ [![GitHub Action](https://img.shields.io/badge/GitHub%20Action-ready-2ea44f?logo=githubactions&logoColor=white)](action.yml)
43
+
44
+ > **Your test suite is a portfolio. TestLedger is its balance sheet.**
45
+ > Every automated test carries a cost (CI minutes, reruns, engineer triage, maintenance churn) and produces a return (regressions caught before production). TestLedger books both from the data you already have — CI results and git history — and produces a one-page **Quality Investment Memo** that a VP of Engineering or CFO can act on.
46
+
47
+ Most tooling in this space stops at "which tests are flaky". TestLedger answers the question leaders actually ask: **which tests are worth what we pay for them, and where should the next dollar of QE effort go?**
48
+
49
+ ---
50
+
51
+ ## What you get
52
+
53
+ | Artefact | Audience | What it says |
54
+ |---|---|---|
55
+ | `memo.md` / `memo.html` | VP Eng, CFO, Director of QE | Bottom line, reallocation recommendation, where the money goes, top liabilities/assets, assumptions |
56
+ | `balance_sheet.json` | QE leads | Assets vs. liabilities per test, by pyramid layer, by suite |
57
+ | `income_statement.json` | Finance / QE leads | Cost by category (CI, reruns, triage, maintenance) vs. value delivered |
58
+ | `writeoffs.json` | Team leads | The action list: **retire / demote / fix / quarantine**, each with a reason |
59
+ | `ai_liabilities.json` | QE governance | AI-generated tests that fail mutation-score or assertion-density thresholds ("test theater") |
60
+ | `ledger.json` | Dashboards | The full book — diff it quarter over quarter |
61
+ | PR comment / job summary | Engineers | Compact table posted by the GitHub Action |
62
+
63
+ **See a real one:** [sample memo (Markdown)](docs/sample-report/memo.md) · [HTML version](docs/sample-report/memo.html) · [write-off list](docs/sample-report/writeoffs.json) — generated from the built-in demo in under a second.
64
+
65
+ ## Quick start (any OS — Windows, macOS, Linux)
66
+
67
+ ```bash
68
+ pip install git+https://github.com/surkuar/testledger.git
69
+ # or, from a clone: pip install -e .
70
+
71
+ # 1. Try the built-in demo (fake repo, 12 weeks of CI history, reruns, a real regression, an AI-written test)
72
+ python examples/generate_demo.py demo-estate
73
+ cd demo-estate
74
+ testledger analyze --config testledger.yml --reports . --mutation mutation.json --out reports
75
+ testledger memo reports # print the memo
76
+ start reports/memo.html # Windows (macOS: open, Linux: xdg-open)
77
+
78
+ # 2. On your own repo
79
+ testledger init # writes testledger.yml with documented defaults
80
+ testledger analyze --config testledger.yml --reports path/to/ci-reports --out testledger-out
81
+ ```
82
+
83
+ ### Local report folder convention
84
+ TestLedger needs to know which CI runs re-executed the **same commit** — that is how flakiness is detected without any test-framework changes. For local folders, name each run directory:
85
+
86
+ ```
87
+ ci-reports/
88
+ 20260915-066b939951-attempt1/TEST-results.xml # <YYYYMMDD>[THHMM]-<sha>-attempt<N>
89
+ 20260915-066b939951-attempt2/TEST-results.xml # same SHA, attempt 2 => a rerun
90
+ 20260917-17f9d0b9e0-attempt1/cucumber.json
91
+ ```
92
+ When you use the GitHub Actions or Jenkins ingesters this metadata comes from the CI API automatically.
93
+
94
+ ## Inputs supported (v1)
95
+
96
+ | Source | How | Notes |
97
+ |---|---|---|
98
+ | JUnit XML (Surefire, Gradle, pytest, Jest, …) | `sources.junit_globs` | universal baseline |
99
+ | TestNG `testng-results.xml` | `sources.junit_globs` | auto-detected |
100
+ | Cucumber JSON (jvm / ruby / js) | `sources.cucumber_globs` | scenario = test, outline rows distinct |
101
+ | Git history | `sources.git_repo` | maintenance churn on test files; fix-commit detection for value credit |
102
+ | GitHub Actions | `sources.github` | run history + artifacts via REST API, `run_attempt` = rerun |
103
+ | Jenkins | `sources.jenkins` | build history + `testReport` JSON; same SHA in consecutive builds = rerun |
104
+ | Mutation reports | `--mutation` | PIT XML, Stryker JSON, or `{file: score}` JSON |
105
+
106
+ ## How it works (short version)
107
+
108
+ ```mermaid
109
+ flowchart LR
110
+ A[JUnit / TestNG XML] --> N[Normalise to TestRun]
111
+ B[Cucumber JSON] --> N
112
+ C[GitHub Actions API] --> N
113
+ D[Jenkins API] --> N
114
+ G[git log] --> H[Churn + fix commits]
115
+ S[Test sources] --> AI[AI provenance + assertion density]
116
+ M[Mutation report] --> AI
117
+ N --> E[Ledger engine]
118
+ H --> E
119
+ AI --> E
120
+ E --> R[Balance sheet · Income statement · Write-offs · Memo]
121
+ ```
122
+
123
+ **Debits (cost) per test:** CI minutes × rate · rerun minutes · flaky false-alarms × triage minutes × engineer rate · test-file lines changed × maintenance minutes × engineer rate.
124
+
125
+ **Credits (value) per test:** a genuine first-attempt failure (not cleared by a same-SHA rerun) followed within 3 days by a product-code commit that looks like a fix, valued at your organisation's cost of a production defect.
126
+
127
+ **Verdicts:** `quarantine` (flake ≥ 30 %) · `retire` (no catches, net negative, cost above floor) · `demote` (UI test whose cost-per-catch is ≥ 3× the service-layer median) · `fix` (valuable but flaky or slow) · `asset` · `hold`.
128
+
129
+ Every rate and threshold lives in `testledger.yml` and is printed on the memo so finance can challenge it. Full derivation: **[docs/methodology.md](docs/methodology.md)**. Architecture and flowcharts: **[docs/design.md](docs/design.md)**.
130
+
131
+ ## Use as a GitHub Action
132
+
133
+ ```yaml
134
+ # .github/workflows/testledger.yml
135
+ name: TestLedger
136
+ on:
137
+ schedule: [{ cron: "0 6 * * 1" }] # weekly memo
138
+ pull_request:
139
+ permissions:
140
+ contents: read
141
+ actions: read
142
+ pull-requests: write
143
+ jobs:
144
+ ledger:
145
+ runs-on: ubuntu-latest
146
+ steps:
147
+ - uses: actions/checkout@v4
148
+ with: { fetch-depth: 0 } # full history for churn / fix detection
149
+ - uses: surkuar/testledger@v0.1.0
150
+ with:
151
+ config: testledger.yml
152
+ reports: build/test-results # or leave empty and configure sources.github
153
+ fail-on-quarantine: "false"
154
+ ```
155
+ The action uploads all reports as an artifact, appends the memo to the job summary, and keeps one live comment on each PR.
156
+
157
+ ## Configuration
158
+
159
+ `testledger init` writes a fully commented file. The important parts:
160
+
161
+ ```yaml
162
+ org_name: Acme Commerce
163
+ window_days: 90
164
+ rates:
165
+ engineer_hourly_usd: 75 # fully loaded
166
+ ci_minute_usd: 0.008
167
+ triage_minutes_per_false_alarm: 18
168
+ maintenance_minutes_per_line: 1.5
169
+ regression_value_usd: 2000 # cost of one escaped defect — your number
170
+ thresholds:
171
+ quarantine_flake_rate: 0.30
172
+ fix_flake_rate: 0.10
173
+ slow_test_seconds: 60
174
+ demote_layer_cost_ratio: 3.0
175
+ retire_min_cost_usd: 25
176
+ ai_min_mutation_score: 0.5
177
+ ai_min_assertion_density: 0.8
178
+ sources:
179
+ junit_globs: ["**/surefire-reports/*.xml", "**/TEST-*.xml"]
180
+ cucumber_globs: ["**/cucumber*.json"]
181
+ git_repo: "."
182
+ github: { repo: "acme/shop", workflow: "ci.yml", token_env: "GITHUB_TOKEN", max_runs: 200 }
183
+ jenkins: { url: "https://ci.acme.com", job: "shop/main", user_env: "JENKINS_USER", token_env: "JENKINS_TOKEN" }
184
+ ```
185
+
186
+ ## Why this exists
187
+
188
+ Engineering leaders are routinely told to present automation "as risk-adjusted ROI" — but nothing computes it from real data. Flaky-test tools price flakes; ROI calculators take hand-typed guesses; the test-portfolio idea lives in blog posts. TestLedger closes that gap: it is the first open-source tool that treats a test estate as a managed portfolio with per-test P&L, a balance sheet, and an executive memo — framework-agnostic, CI-agnostic, and runnable in two minutes on any OS.
189
+
190
+ It was built by a QE leader for QE leaders. If you run a quality organisation and need to defend (or reallocate) a testing budget, this is the number you were missing.
191
+
192
+ ## Development
193
+
194
+ ```bash
195
+ pip install -e ".[dev]"
196
+ pytest # 25 tests, offline (CI APIs stubbed; git tests use a throwaway repo)
197
+ ruff check src tests
198
+ ```
199
+
200
+ ## Roadmap
201
+ - [ ] GitLab CI and Azure DevOps ingesters
202
+ - [ ] Code-ownership join (CODEOWNERS) so the memo is grouped by team
203
+ - [ ] Quarter-over-quarter diff (`testledger diff q3.json q4.json`)
204
+ - [ ] Optional LLM narrative for the memo (opt-in, local or API)
205
+ - [ ] Web dashboard reading `ledger.json`
206
+
207
+ ## License
208
+ MIT © Suresh Babu. See [LICENSE](LICENSE).
@@ -0,0 +1,24 @@
1
+ testledger/__init__.py,sha256=-RJbr_0GuY1t7lcry-3fnRXR5M9mhdRyA36Y67X02G8,497
2
+ testledger/cli.py,sha256=BTDHvYE39FxkofcRXAVTUeqvedeb4ORQb_fUlZ54nQM,10152
3
+ testledger/config.py,sha256=duQKUnBMXl7HPP8Qn2osgvWAhdGOlYDt0P4AR26M-YI,3982
4
+ testledger/models.py,sha256=O1asU_HhzuZDmFPcXKw9orHd8EnFZwEB0vKQydZYBMc,5672
5
+ testledger/aitest/__init__.py,sha256=HP59GBABPVglUo9-Uxkg-5Upq3g3NTcBSsMbkYMLtuY,176
6
+ testledger/aitest/detector.py,sha256=zP1zSqY23zmmk59O9JIe8Weoq3w0JV50W4Op-jny4FU,6543
7
+ testledger/ingest/__init__.py,sha256=tbqv8j2vj62nGVrEQR8_9ujcpP5qffGIzGw6ztoHxio,360
8
+ testledger/ingest/cucumber.py,sha256=W-3xEfbrVAbbvpHRLHJUMw-HJc-WoiQRf49UTwEZyV4,2785
9
+ testledger/ingest/github_actions.py,sha256=0djl4ZGki8EmrvXXp40ECcO0pGRy-zGlUbf-UI5nGOQ,5323
10
+ testledger/ingest/gitlog.py,sha256=qYrV3PvipD-LM9prW-VDuLi81sLLIZMCJsSy_CernfU,4054
11
+ testledger/ingest/jenkins.py,sha256=PsD_uAAhja1GMrj0AYVjwilWFljjfh8dIKye-Vg8o48,4458
12
+ testledger/ingest/junit.py,sha256=0T2x-jgpo6Dd62YybpLqk8cz8X-t0iEecC5FR7_dJ6I,3950
13
+ testledger/ledger/__init__.py,sha256=jKDwgoDBFSBNgXq0_FF_Xphoh4H1T29fjMAM8j8vLsw,78
14
+ testledger/ledger/engine.py,sha256=xbKeM4Phi7-fCwhM4m9e-zxqi0EM22Be2-SzN5etI6k,13092
15
+ testledger/reports/__init__.py,sha256=yxoVVoVN7nHc1BYlE8NMSGSegmJ9eM35bP2MYxRiJUY,144
16
+ testledger/reports/builder.py,sha256=kAqrXkYMQ4vRC_JbB4r0noFrP9mOoLv3QrPpreDojQQ,8339
17
+ testledger/templates/memo.html.j2,sha256=toowHzx0e2iiPEZPFpXCXuN7CFBFoZ_XwrxPrr4culg,6389
18
+ testledger/templates/memo.md.j2,sha256=ZxGm0nn2PcV5z5wrQq0LLOs01UnOmj-6-Sf1F3VNv5M,3861
19
+ testledger-0.1.0.dist-info/licenses/LICENSE,sha256=b5VjMe6maw4kFGJ5pGGYDHbGswEfLmgD1_Vs1ow9eOg,1068
20
+ testledger-0.1.0.dist-info/METADATA,sha256=V3CWH8p0hUIb7M3rMM4KZn3wrOQNyMavS7irV9Nj6Ec,9970
21
+ testledger-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
22
+ testledger-0.1.0.dist-info/entry_points.txt,sha256=xGXFB7FzwjNVQhv0odf4536yCsgd_NWo-GSQkSOmUOQ,50
23
+ testledger-0.1.0.dist-info/top_level.txt,sha256=svnnR_i_rH1QDsb80qwmcJortM5n7npbF2D3RnU2wqA,11
24
+ testledger-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ testledger = testledger.cli:app
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Suresh Babu
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ testledger