testledger 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testledger-0.1.0/LICENSE +21 -0
- testledger-0.1.0/PKG-INFO +208 -0
- testledger-0.1.0/README.md +172 -0
- testledger-0.1.0/pyproject.toml +68 -0
- testledger-0.1.0/setup.cfg +4 -0
- testledger-0.1.0/src/testledger/__init__.py +11 -0
- testledger-0.1.0/src/testledger/aitest/__init__.py +3 -0
- testledger-0.1.0/src/testledger/aitest/detector.py +152 -0
- testledger-0.1.0/src/testledger/cli.py +262 -0
- testledger-0.1.0/src/testledger/config.py +105 -0
- testledger-0.1.0/src/testledger/ingest/__init__.py +7 -0
- testledger-0.1.0/src/testledger/ingest/cucumber.py +79 -0
- testledger-0.1.0/src/testledger/ingest/github_actions.py +145 -0
- testledger-0.1.0/src/testledger/ingest/gitlog.py +118 -0
- testledger-0.1.0/src/testledger/ingest/jenkins.py +127 -0
- testledger-0.1.0/src/testledger/ingest/junit.py +114 -0
- testledger-0.1.0/src/testledger/ledger/__init__.py +3 -0
- testledger-0.1.0/src/testledger/ledger/engine.py +332 -0
- testledger-0.1.0/src/testledger/models.py +194 -0
- testledger-0.1.0/src/testledger/reports/__init__.py +3 -0
- testledger-0.1.0/src/testledger/reports/builder.py +224 -0
- testledger-0.1.0/src/testledger/templates/memo.html.j2 +77 -0
- testledger-0.1.0/src/testledger/templates/memo.md.j2 +78 -0
- testledger-0.1.0/src/testledger.egg-info/PKG-INFO +208 -0
- testledger-0.1.0/src/testledger.egg-info/SOURCES.txt +33 -0
- testledger-0.1.0/src/testledger.egg-info/dependency_links.txt +1 -0
- testledger-0.1.0/src/testledger.egg-info/entry_points.txt +2 -0
- testledger-0.1.0/src/testledger.egg-info/requires.txt +13 -0
- testledger-0.1.0/src/testledger.egg-info/top_level.txt +1 -0
- testledger-0.1.0/tests/test_aitest.py +50 -0
- testledger-0.1.0/tests/test_ci_clients.py +99 -0
- testledger-0.1.0/tests/test_engine.py +157 -0
- testledger-0.1.0/tests/test_gitlog.py +62 -0
- testledger-0.1.0/tests/test_ingest.py +41 -0
- testledger-0.1.0/tests/test_reports_and_cli.py +105 -0
testledger-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Suresh Babu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: testledger
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Test Estate Balance Sheet: turn CI results and git history into per-test P&L and executive Quality Investment Memos.
|
|
5
|
+
Author-email: Suresh Babu <sureshbabu.santhakumar@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/surkuar/testledger
|
|
8
|
+
Project-URL: Issues, https://github.com/surkuar/testledger/issues
|
|
9
|
+
Project-URL: Documentation, https://github.com/surkuar/testledger/tree/main/docs
|
|
10
|
+
Keywords: testing,quality-engineering,ci,flaky-tests,test-roi,engineering-leadership
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Software Development :: Testing
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: pydantic>=2.5
|
|
24
|
+
Requires-Dist: PyYAML>=6.0
|
|
25
|
+
Requires-Dist: Jinja2>=3.1
|
|
26
|
+
Requires-Dist: requests>=2.31
|
|
27
|
+
Requires-Dist: typer>=0.12
|
|
28
|
+
Requires-Dist: rich>=13.0
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
31
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
32
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
33
|
+
Requires-Dist: mypy>=1.10; extra == "dev"
|
|
34
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
# TestLedger — Test Estate Balance Sheet
|
|
38
|
+
|
|
39
|
+
[](https://github.com/surkuar/testledger/actions/workflows/ci.yml)
|
|
40
|
+
[](LICENSE)
|
|
41
|
+
[](pyproject.toml)
|
|
42
|
+
[](action.yml)
|
|
43
|
+
|
|
44
|
+
> **Your test suite is a portfolio. TestLedger is its balance sheet.**
|
|
45
|
+
> Every automated test carries a cost (CI minutes, reruns, engineer triage, maintenance churn) and produces a return (regressions caught before production). TestLedger books both from the data you already have — CI results and git history — and produces a one-page **Quality Investment Memo** that a VP of Engineering or CFO can act on.
|
|
46
|
+
|
|
47
|
+
Most tooling in this space stops at "which tests are flaky". TestLedger answers the question leaders actually ask: **which tests are worth what we pay for them, and where should the next dollar of QE effort go?**
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## What you get
|
|
52
|
+
|
|
53
|
+
| Artefact | Audience | What it says |
|
|
54
|
+
|---|---|---|
|
|
55
|
+
| `memo.md` / `memo.html` | VP Eng, CFO, Director of QE | Bottom line, reallocation recommendation, where the money goes, top liabilities/assets, assumptions |
|
|
56
|
+
| `balance_sheet.json` | QE leads | Assets vs. liabilities per test, by pyramid layer, by suite |
|
|
57
|
+
| `income_statement.json` | Finance / QE leads | Cost by category (CI, reruns, triage, maintenance) vs. value delivered |
|
|
58
|
+
| `writeoffs.json` | Team leads | The action list: **retire / demote / fix / quarantine**, each with a reason |
|
|
59
|
+
| `ai_liabilities.json` | QE governance | AI-generated tests that fail mutation-score or assertion-density thresholds ("test theater") |
|
|
60
|
+
| `ledger.json` | Dashboards | The full book — diff it quarter over quarter |
|
|
61
|
+
| PR comment / job summary | Engineers | Compact table posted by the GitHub Action |
|
|
62
|
+
|
|
63
|
+
**See a real one:** [sample memo (Markdown)](docs/sample-report/memo.md) · [HTML version](docs/sample-report/memo.html) · [write-off list](docs/sample-report/writeoffs.json) — generated from the built-in demo in under a second.
|
|
64
|
+
|
|
65
|
+
## Quick start (any OS — Windows, macOS, Linux)
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install git+https://github.com/surkuar/testledger.git
|
|
69
|
+
# or, from a clone: pip install -e .
|
|
70
|
+
|
|
71
|
+
# 1. Try the built-in demo (fake repo, 12 weeks of CI history, reruns, a real regression, an AI-written test)
|
|
72
|
+
python examples/generate_demo.py demo-estate
|
|
73
|
+
cd demo-estate
|
|
74
|
+
testledger analyze --config testledger.yml --reports . --mutation mutation.json --out reports
|
|
75
|
+
testledger memo reports # print the memo
|
|
76
|
+
start reports/memo.html # Windows (macOS: open, Linux: xdg-open)
|
|
77
|
+
|
|
78
|
+
# 2. On your own repo
|
|
79
|
+
testledger init # writes testledger.yml with documented defaults
|
|
80
|
+
testledger analyze --config testledger.yml --reports path/to/ci-reports --out testledger-out
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### Local report folder convention
|
|
84
|
+
TestLedger needs to know which CI runs re-executed the **same commit** — that is how flakiness is detected without any test-framework changes. For local folders, name each run directory:
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
ci-reports/
|
|
88
|
+
20260915-066b939951-attempt1/TEST-results.xml # <YYYYMMDD>[THHMM]-<sha>-attempt<N>
|
|
89
|
+
20260915-066b939951-attempt2/TEST-results.xml # same SHA, attempt 2 => a rerun
|
|
90
|
+
20260917-17f9d0b9e0-attempt1/cucumber.json
|
|
91
|
+
```
|
|
92
|
+
When you use the GitHub Actions or Jenkins ingesters this metadata comes from the CI API automatically.
|
|
93
|
+
|
|
94
|
+
## Inputs supported (v1)
|
|
95
|
+
|
|
96
|
+
| Source | How | Notes |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| JUnit XML (Surefire, Gradle, pytest, Jest, …) | `sources.junit_globs` | universal baseline |
|
|
99
|
+
| TestNG `testng-results.xml` | `sources.junit_globs` | auto-detected |
|
|
100
|
+
| Cucumber JSON (jvm / ruby / js) | `sources.cucumber_globs` | scenario = test, outline rows distinct |
|
|
101
|
+
| Git history | `sources.git_repo` | maintenance churn on test files; fix-commit detection for value credit |
|
|
102
|
+
| GitHub Actions | `sources.github` | run history + artifacts via REST API, `run_attempt` = rerun |
|
|
103
|
+
| Jenkins | `sources.jenkins` | build history + `testReport` JSON; same SHA in consecutive builds = rerun |
|
|
104
|
+
| Mutation reports | `--mutation` | PIT XML, Stryker JSON, or `{file: score}` JSON |
|
|
105
|
+
|
|
106
|
+
## How it works (short version)
|
|
107
|
+
|
|
108
|
+
```mermaid
|
|
109
|
+
flowchart LR
|
|
110
|
+
A[JUnit / TestNG XML] --> N[Normalise to TestRun]
|
|
111
|
+
B[Cucumber JSON] --> N
|
|
112
|
+
C[GitHub Actions API] --> N
|
|
113
|
+
D[Jenkins API] --> N
|
|
114
|
+
G[git log] --> H[Churn + fix commits]
|
|
115
|
+
S[Test sources] --> AI[AI provenance + assertion density]
|
|
116
|
+
M[Mutation report] --> AI
|
|
117
|
+
N --> E[Ledger engine]
|
|
118
|
+
H --> E
|
|
119
|
+
AI --> E
|
|
120
|
+
E --> R[Balance sheet · Income statement · Write-offs · Memo]
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
**Debits (cost) per test:** CI minutes × rate · rerun minutes · flaky false-alarms × triage minutes × engineer rate · test-file lines changed × maintenance minutes × engineer rate.
|
|
124
|
+
|
|
125
|
+
**Credits (value) per test:** a genuine first-attempt failure (not cleared by a same-SHA rerun) followed within 3 days by a product-code commit that looks like a fix, valued at your organisation's cost of a production defect.
|
|
126
|
+
|
|
127
|
+
**Verdicts:** `quarantine` (flake ≥ 30 %) · `retire` (no catches, net negative, cost above floor) · `demote` (UI test whose cost-per-catch is ≥ 3× the service-layer median) · `fix` (valuable but flaky or slow) · `asset` · `hold`.
|
|
128
|
+
|
|
129
|
+
Every rate and threshold lives in `testledger.yml` and is printed on the memo so finance can challenge it. Full derivation: **[docs/methodology.md](docs/methodology.md)**. Architecture and flowcharts: **[docs/design.md](docs/design.md)**.
|
|
130
|
+
|
|
131
|
+
## Use as a GitHub Action
|
|
132
|
+
|
|
133
|
+
```yaml
|
|
134
|
+
# .github/workflows/testledger.yml
|
|
135
|
+
name: TestLedger
|
|
136
|
+
on:
|
|
137
|
+
schedule: [{ cron: "0 6 * * 1" }] # weekly memo
|
|
138
|
+
pull_request:
|
|
139
|
+
permissions:
|
|
140
|
+
contents: read
|
|
141
|
+
actions: read
|
|
142
|
+
pull-requests: write
|
|
143
|
+
jobs:
|
|
144
|
+
ledger:
|
|
145
|
+
runs-on: ubuntu-latest
|
|
146
|
+
steps:
|
|
147
|
+
- uses: actions/checkout@v4
|
|
148
|
+
with: { fetch-depth: 0 } # full history for churn / fix detection
|
|
149
|
+
- uses: surkuar/testledger@v0.1.0
|
|
150
|
+
with:
|
|
151
|
+
config: testledger.yml
|
|
152
|
+
reports: build/test-results # or leave empty and configure sources.github
|
|
153
|
+
fail-on-quarantine: "false"
|
|
154
|
+
```
|
|
155
|
+
The action uploads all reports as an artifact, appends the memo to the job summary, and keeps one live comment on each PR.
|
|
156
|
+
|
|
157
|
+
## Configuration
|
|
158
|
+
|
|
159
|
+
`testledger init` writes a fully commented file. The important parts:
|
|
160
|
+
|
|
161
|
+
```yaml
|
|
162
|
+
org_name: Acme Commerce
|
|
163
|
+
window_days: 90
|
|
164
|
+
rates:
|
|
165
|
+
engineer_hourly_usd: 75 # fully loaded
|
|
166
|
+
ci_minute_usd: 0.008
|
|
167
|
+
triage_minutes_per_false_alarm: 18
|
|
168
|
+
maintenance_minutes_per_line: 1.5
|
|
169
|
+
regression_value_usd: 2000 # cost of one escaped defect — your number
|
|
170
|
+
thresholds:
|
|
171
|
+
quarantine_flake_rate: 0.30
|
|
172
|
+
fix_flake_rate: 0.10
|
|
173
|
+
slow_test_seconds: 60
|
|
174
|
+
demote_layer_cost_ratio: 3.0
|
|
175
|
+
retire_min_cost_usd: 25
|
|
176
|
+
ai_min_mutation_score: 0.5
|
|
177
|
+
ai_min_assertion_density: 0.8
|
|
178
|
+
sources:
|
|
179
|
+
junit_globs: ["**/surefire-reports/*.xml", "**/TEST-*.xml"]
|
|
180
|
+
cucumber_globs: ["**/cucumber*.json"]
|
|
181
|
+
git_repo: "."
|
|
182
|
+
github: { repo: "acme/shop", workflow: "ci.yml", token_env: "GITHUB_TOKEN", max_runs: 200 }
|
|
183
|
+
jenkins: { url: "https://ci.acme.com", job: "shop/main", user_env: "JENKINS_USER", token_env: "JENKINS_TOKEN" }
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## Why this exists
|
|
187
|
+
|
|
188
|
+
Engineering leaders are routinely told to present automation "as risk-adjusted ROI" — but nothing computes it from real data. Flaky-test tools price flakes; ROI calculators take hand-typed guesses; the test-portfolio idea lives in blog posts. TestLedger closes that gap: it is the first open-source tool that treats a test estate as a managed portfolio with per-test P&L, a balance sheet, and an executive memo — framework-agnostic, CI-agnostic, and runnable in two minutes on any OS.
|
|
189
|
+
|
|
190
|
+
It was built by a QE leader for QE leaders. If you run a quality organisation and need to defend (or reallocate) a testing budget, this is the number you were missing.
|
|
191
|
+
|
|
192
|
+
## Development
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
pip install -e ".[dev]"
|
|
196
|
+
pytest # 25 tests, offline (CI APIs stubbed; git tests use a throwaway repo)
|
|
197
|
+
ruff check src tests
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## Roadmap
|
|
201
|
+
- [ ] GitLab CI and Azure DevOps ingesters
|
|
202
|
+
- [ ] Code-ownership join (CODEOWNERS) so the memo is grouped by team
|
|
203
|
+
- [ ] Quarter-over-quarter diff (`testledger diff q3.json q4.json`)
|
|
204
|
+
- [ ] Optional LLM narrative for the memo (opt-in, local or API)
|
|
205
|
+
- [ ] Web dashboard reading `ledger.json`
|
|
206
|
+
|
|
207
|
+
## License
|
|
208
|
+
MIT © Suresh Babu. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# TestLedger — Test Estate Balance Sheet
|
|
2
|
+
|
|
3
|
+
[](https://github.com/surkuar/testledger/actions/workflows/ci.yml)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
[](pyproject.toml)
|
|
6
|
+
[](action.yml)
|
|
7
|
+
|
|
8
|
+
> **Your test suite is a portfolio. TestLedger is its balance sheet.**
|
|
9
|
+
> Every automated test carries a cost (CI minutes, reruns, engineer triage, maintenance churn) and produces a return (regressions caught before production). TestLedger books both from the data you already have — CI results and git history — and produces a one-page **Quality Investment Memo** that a VP of Engineering or CFO can act on.
|
|
10
|
+
|
|
11
|
+
Most tooling in this space stops at "which tests are flaky". TestLedger answers the question leaders actually ask: **which tests are worth what we pay for them, and where should the next dollar of QE effort go?**
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## What you get
|
|
16
|
+
|
|
17
|
+
| Artefact | Audience | What it says |
|
|
18
|
+
|---|---|---|
|
|
19
|
+
| `memo.md` / `memo.html` | VP Eng, CFO, Director of QE | Bottom line, reallocation recommendation, where the money goes, top liabilities/assets, assumptions |
|
|
20
|
+
| `balance_sheet.json` | QE leads | Assets vs. liabilities per test, by pyramid layer, by suite |
|
|
21
|
+
| `income_statement.json` | Finance / QE leads | Cost by category (CI, reruns, triage, maintenance) vs. value delivered |
|
|
22
|
+
| `writeoffs.json` | Team leads | The action list: **retire / demote / fix / quarantine**, each with a reason |
|
|
23
|
+
| `ai_liabilities.json` | QE governance | AI-generated tests that fail mutation-score or assertion-density thresholds ("test theater") |
|
|
24
|
+
| `ledger.json` | Dashboards | The full book — diff it quarter over quarter |
|
|
25
|
+
| PR comment / job summary | Engineers | Compact table posted by the GitHub Action |
|
|
26
|
+
|
|
27
|
+
**See a real one:** [sample memo (Markdown)](docs/sample-report/memo.md) · [HTML version](docs/sample-report/memo.html) · [write-off list](docs/sample-report/writeoffs.json) — generated from the built-in demo in under a second.
|
|
28
|
+
|
|
29
|
+
## Quick start (any OS — Windows, macOS, Linux)
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install git+https://github.com/surkuar/testledger.git
|
|
33
|
+
# or, from a clone: pip install -e .
|
|
34
|
+
|
|
35
|
+
# 1. Try the built-in demo (fake repo, 12 weeks of CI history, reruns, a real regression, an AI-written test)
|
|
36
|
+
python examples/generate_demo.py demo-estate
|
|
37
|
+
cd demo-estate
|
|
38
|
+
testledger analyze --config testledger.yml --reports . --mutation mutation.json --out reports
|
|
39
|
+
testledger memo reports # print the memo
|
|
40
|
+
start reports/memo.html # Windows (macOS: open, Linux: xdg-open)
|
|
41
|
+
|
|
42
|
+
# 2. On your own repo
|
|
43
|
+
testledger init # writes testledger.yml with documented defaults
|
|
44
|
+
testledger analyze --config testledger.yml --reports path/to/ci-reports --out testledger-out
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### Local report folder convention
|
|
48
|
+
TestLedger needs to know which CI runs re-executed the **same commit** — that is how flakiness is detected without any test-framework changes. For local folders, name each run directory:
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
ci-reports/
|
|
52
|
+
20260915-066b939951-attempt1/TEST-results.xml # <YYYYMMDD>[THHMM]-<sha>-attempt<N>
|
|
53
|
+
20260915-066b939951-attempt2/TEST-results.xml # same SHA, attempt 2 => a rerun
|
|
54
|
+
20260917-17f9d0b9e0-attempt1/cucumber.json
|
|
55
|
+
```
|
|
56
|
+
When you use the GitHub Actions or Jenkins ingesters this metadata comes from the CI API automatically.
|
|
57
|
+
|
|
58
|
+
## Inputs supported (v1)
|
|
59
|
+
|
|
60
|
+
| Source | How | Notes |
|
|
61
|
+
|---|---|---|
|
|
62
|
+
| JUnit XML (Surefire, Gradle, pytest, Jest, …) | `sources.junit_globs` | universal baseline |
|
|
63
|
+
| TestNG `testng-results.xml` | `sources.junit_globs` | auto-detected |
|
|
64
|
+
| Cucumber JSON (jvm / ruby / js) | `sources.cucumber_globs` | scenario = test, outline rows distinct |
|
|
65
|
+
| Git history | `sources.git_repo` | maintenance churn on test files; fix-commit detection for value credit |
|
|
66
|
+
| GitHub Actions | `sources.github` | run history + artifacts via REST API, `run_attempt` = rerun |
|
|
67
|
+
| Jenkins | `sources.jenkins` | build history + `testReport` JSON; same SHA in consecutive builds = rerun |
|
|
68
|
+
| Mutation reports | `--mutation` | PIT XML, Stryker JSON, or `{file: score}` JSON |
|
|
69
|
+
|
|
70
|
+
## How it works (short version)
|
|
71
|
+
|
|
72
|
+
```mermaid
|
|
73
|
+
flowchart LR
|
|
74
|
+
A[JUnit / TestNG XML] --> N[Normalise to TestRun]
|
|
75
|
+
B[Cucumber JSON] --> N
|
|
76
|
+
C[GitHub Actions API] --> N
|
|
77
|
+
D[Jenkins API] --> N
|
|
78
|
+
G[git log] --> H[Churn + fix commits]
|
|
79
|
+
S[Test sources] --> AI[AI provenance + assertion density]
|
|
80
|
+
M[Mutation report] --> AI
|
|
81
|
+
N --> E[Ledger engine]
|
|
82
|
+
H --> E
|
|
83
|
+
AI --> E
|
|
84
|
+
E --> R[Balance sheet · Income statement · Write-offs · Memo]
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
**Debits (cost) per test:** CI minutes × rate · rerun minutes · flaky false-alarms × triage minutes × engineer rate · test-file lines changed × maintenance minutes × engineer rate.
|
|
88
|
+
|
|
89
|
+
**Credits (value) per test:** a genuine first-attempt failure (not cleared by a same-SHA rerun) followed within 3 days by a product-code commit that looks like a fix, valued at your organisation's cost of a production defect.
|
|
90
|
+
|
|
91
|
+
**Verdicts:** `quarantine` (flake ≥ 30 %) · `retire` (no catches, net negative, cost above floor) · `demote` (UI test whose cost-per-catch is ≥ 3× the service-layer median) · `fix` (valuable but flaky or slow) · `asset` · `hold`.
|
|
92
|
+
|
|
93
|
+
Every rate and threshold lives in `testledger.yml` and is printed on the memo so finance can challenge it. Full derivation: **[docs/methodology.md](docs/methodology.md)**. Architecture and flowcharts: **[docs/design.md](docs/design.md)**.
|
|
94
|
+
|
|
95
|
+
## Use as a GitHub Action
|
|
96
|
+
|
|
97
|
+
```yaml
|
|
98
|
+
# .github/workflows/testledger.yml
|
|
99
|
+
name: TestLedger
|
|
100
|
+
on:
|
|
101
|
+
schedule: [{ cron: "0 6 * * 1" }] # weekly memo
|
|
102
|
+
pull_request:
|
|
103
|
+
permissions:
|
|
104
|
+
contents: read
|
|
105
|
+
actions: read
|
|
106
|
+
pull-requests: write
|
|
107
|
+
jobs:
|
|
108
|
+
ledger:
|
|
109
|
+
runs-on: ubuntu-latest
|
|
110
|
+
steps:
|
|
111
|
+
- uses: actions/checkout@v4
|
|
112
|
+
with: { fetch-depth: 0 } # full history for churn / fix detection
|
|
113
|
+
- uses: surkuar/testledger@v0.1.0
|
|
114
|
+
with:
|
|
115
|
+
config: testledger.yml
|
|
116
|
+
reports: build/test-results # or leave empty and configure sources.github
|
|
117
|
+
fail-on-quarantine: "false"
|
|
118
|
+
```
|
|
119
|
+
The action uploads all reports as an artifact, appends the memo to the job summary, and keeps one live comment on each PR.
|
|
120
|
+
|
|
121
|
+
## Configuration
|
|
122
|
+
|
|
123
|
+
`testledger init` writes a fully commented file. The important parts:
|
|
124
|
+
|
|
125
|
+
```yaml
|
|
126
|
+
org_name: Acme Commerce
|
|
127
|
+
window_days: 90
|
|
128
|
+
rates:
|
|
129
|
+
engineer_hourly_usd: 75 # fully loaded
|
|
130
|
+
ci_minute_usd: 0.008
|
|
131
|
+
triage_minutes_per_false_alarm: 18
|
|
132
|
+
maintenance_minutes_per_line: 1.5
|
|
133
|
+
regression_value_usd: 2000 # cost of one escaped defect — your number
|
|
134
|
+
thresholds:
|
|
135
|
+
quarantine_flake_rate: 0.30
|
|
136
|
+
fix_flake_rate: 0.10
|
|
137
|
+
slow_test_seconds: 60
|
|
138
|
+
demote_layer_cost_ratio: 3.0
|
|
139
|
+
retire_min_cost_usd: 25
|
|
140
|
+
ai_min_mutation_score: 0.5
|
|
141
|
+
ai_min_assertion_density: 0.8
|
|
142
|
+
sources:
|
|
143
|
+
junit_globs: ["**/surefire-reports/*.xml", "**/TEST-*.xml"]
|
|
144
|
+
cucumber_globs: ["**/cucumber*.json"]
|
|
145
|
+
git_repo: "."
|
|
146
|
+
github: { repo: "acme/shop", workflow: "ci.yml", token_env: "GITHUB_TOKEN", max_runs: 200 }
|
|
147
|
+
jenkins: { url: "https://ci.acme.com", job: "shop/main", user_env: "JENKINS_USER", token_env: "JENKINS_TOKEN" }
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
## Why this exists
|
|
151
|
+
|
|
152
|
+
Engineering leaders are routinely told to present automation "as risk-adjusted ROI" — but nothing computes it from real data. Flaky-test tools price flakes; ROI calculators take hand-typed guesses; the test-portfolio idea lives in blog posts. TestLedger closes that gap: it is the first open-source tool that treats a test estate as a managed portfolio with per-test P&L, a balance sheet, and an executive memo — framework-agnostic, CI-agnostic, and runnable in two minutes on any OS.
|
|
153
|
+
|
|
154
|
+
It was built by a QE leader for QE leaders. If you run a quality organisation and need to defend (or reallocate) a testing budget, this is the number you were missing.
|
|
155
|
+
|
|
156
|
+
## Development
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
pip install -e ".[dev]"
|
|
160
|
+
pytest # 25 tests, offline (CI APIs stubbed; git tests use a throwaway repo)
|
|
161
|
+
ruff check src tests
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
## Roadmap
|
|
165
|
+
- [ ] GitLab CI and Azure DevOps ingesters
|
|
166
|
+
- [ ] Code-ownership join (CODEOWNERS) so the memo is grouped by team
|
|
167
|
+
- [ ] Quarter-over-quarter diff (`testledger diff q3.json q4.json`)
|
|
168
|
+
- [ ] Optional LLM narrative for the memo (opt-in, local or API)
|
|
169
|
+
- [ ] Web dashboard reading `ledger.json`
|
|
170
|
+
|
|
171
|
+
## License
|
|
172
|
+
MIT © Suresh Babu. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "testledger"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Test Estate Balance Sheet: turn CI results and git history into per-test P&L and executive Quality Investment Memos."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Suresh Babu", email = "sureshbabu.santhakumar@gmail.com" }]
|
|
13
|
+
keywords = ["testing", "quality-engineering", "ci", "flaky-tests", "test-roi", "engineering-leadership"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Topic :: Software Development :: Testing",
|
|
24
|
+
]
|
|
25
|
+
dependencies = [
|
|
26
|
+
"pydantic>=2.5",
|
|
27
|
+
"PyYAML>=6.0",
|
|
28
|
+
"Jinja2>=3.1",
|
|
29
|
+
"requests>=2.31",
|
|
30
|
+
"typer>=0.12",
|
|
31
|
+
"rich>=13.0",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.5", "mypy>=1.10", "build>=1.2"]
|
|
36
|
+
|
|
37
|
+
[project.scripts]
|
|
38
|
+
testledger = "testledger.cli:app"
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/surkuar/testledger"
|
|
42
|
+
Issues = "https://github.com/surkuar/testledger/issues"
|
|
43
|
+
Documentation = "https://github.com/surkuar/testledger/tree/main/docs"
|
|
44
|
+
|
|
45
|
+
[tool.setuptools.packages.find]
|
|
46
|
+
where = ["src"]
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.package-data]
|
|
49
|
+
testledger = ["templates/*.j2"]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
testpaths = ["tests"]
|
|
53
|
+
addopts = "-q"
|
|
54
|
+
|
|
55
|
+
[tool.ruff]
|
|
56
|
+
line-length = 110
|
|
57
|
+
target-version = "py311"
|
|
58
|
+
|
|
59
|
+
[tool.ruff.lint]
|
|
60
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
61
|
+
ignore = ["E501"] # line length is owned by `ruff format`
|
|
62
|
+
|
|
63
|
+
[tool.mypy]
|
|
64
|
+
python_version = "3.11"
|
|
65
|
+
ignore_missing_imports = true
|
|
66
|
+
|
|
67
|
+
[tool.ruff.lint.flake8-bugbear]
|
|
68
|
+
extend-immutable-calls = ["typer.Option", "typer.Argument"]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""TestLedger — Test Estate Balance Sheet.
|
|
2
|
+
|
|
3
|
+
Treats an automated test suite as an investment portfolio: every test carries a
|
|
4
|
+
cost (CI minutes, reruns, triage, maintenance churn) and produces a return
|
|
5
|
+
(regressions caught). TestLedger ingests CI results and git history, books
|
|
6
|
+
every cost and value entry into a double-entry style ledger, and renders
|
|
7
|
+
executive-facing reports: a Balance Sheet, an Income Statement, a Write-off
|
|
8
|
+
list and a one-page Quality Investment Memo.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""AI-generated test governance ("test theater" detection).
|
|
2
|
+
|
|
3
|
+
Three signals, each cheap and language-agnostic:
|
|
4
|
+
|
|
5
|
+
1. **Provenance** — the test file contains a configured marker
|
|
6
|
+
(``@ai-generated``, ``Generated-By:``, Copilot/Claude co-author trailers)
|
|
7
|
+
or its last commit message does.
|
|
8
|
+
2. **Assertion density** — assertions per test function. Tests with density
|
|
9
|
+
< threshold are likely "smoke only" and prove little. Detects common
|
|
10
|
+
assertion idioms across Java (JUnit/TestNG/AssertJ/Hamcrest), Python,
|
|
11
|
+
JavaScript/TypeScript, Ruby (RSpec/Minitest), C#, Go.
|
|
12
|
+
3. **Mutation score** — read from PIT (Java), Stryker (JS/TS/C#),
|
|
13
|
+
mutmut/cosmic-ray (Python) reports. A high line-coverage, low
|
|
14
|
+
mutation-score test is the signature of an assertion that mirrors the
|
|
15
|
+
implementation rather than checking it.
|
|
16
|
+
|
|
17
|
+
Output is a mapping ``{file_or_test_id: {"ai_generated": bool,
|
|
18
|
+
"assertion_density": float, "mutation_score": float | None}}`` that the ledger
|
|
19
|
+
engine merges into each account. The memo lists AI-generated tests that fall
|
|
20
|
+
below the governance thresholds as **unverified liabilities**.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
import re
|
|
27
|
+
import subprocess
|
|
28
|
+
import xml.etree.ElementTree as ET
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
TEST_FUNC_PATTERNS = [
|
|
32
|
+
re.compile(r"@Test\b"), # JUnit / TestNG
|
|
33
|
+
re.compile(r"^\s*def\s+test_\w+", re.M), # pytest / unittest
|
|
34
|
+
re.compile(r"\b(it|test)\s*\(\s*['\"`]", re.M), # jest / mocha / vitest
|
|
35
|
+
re.compile(r"^\s*(it|scenario|specify)\s+['\"]", re.M), # rspec
|
|
36
|
+
re.compile(r"\[(Fact|Theory|Test|TestMethod)\]"), # xUnit / NUnit / MSTest
|
|
37
|
+
re.compile(r"^func\s+Test\w+\s*\(", re.M), # go
|
|
38
|
+
]
|
|
39
|
+
ASSERTION_PATTERNS = [
|
|
40
|
+
re.compile(r"\bassert\w*\s*\("), # assertEquals(, assertTrue(, assert(
|
|
41
|
+
re.compile(r"\bassertThat\s*\("),
|
|
42
|
+
re.compile(r"^\s*assert\s+\S", re.M), # python bare assert
|
|
43
|
+
re.compile(r"\bexpect\s*\("), # jest / rspec / chai
|
|
44
|
+
re.compile(r"\bshould\b"),
|
|
45
|
+
re.compile(r"\bAssert\.\w+\s*\("), # C#
|
|
46
|
+
re.compile(r"\bverify\s*\("), # mockito interaction verification
|
|
47
|
+
re.compile(r"\.(toBe|toEqual|toMatch|toHaveLength|toThrow|isEqualTo|hasSize|contains|isTrue|isFalse)\b"),
|
|
48
|
+
re.compile(r"\bt\.(Errorf|Fatalf|Error|Fatal)\s*\("), # go
|
|
49
|
+
]
|
|
50
|
+
SOURCE_EXT = {".java", ".kt", ".py", ".js", ".ts", ".tsx", ".jsx", ".rb", ".cs", ".go", ".scala"}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def assertion_density(text: str) -> tuple[int, int, float]:
|
|
54
|
+
tests = sum(len(p.findall(text)) for p in TEST_FUNC_PATTERNS)
|
|
55
|
+
asserts = sum(len(p.findall(text)) for p in ASSERTION_PATTERNS)
|
|
56
|
+
density = round(asserts / tests, 2) if tests else 0.0
|
|
57
|
+
return tests, asserts, density
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _last_commit_message(repo: Path, file: Path) -> str:
|
|
61
|
+
try:
|
|
62
|
+
out = subprocess.run(
|
|
63
|
+
["git", "-C", str(repo), "log", "-1", "--format=%B", "--", str(file.relative_to(repo))],
|
|
64
|
+
capture_output=True,
|
|
65
|
+
text=True,
|
|
66
|
+
encoding="utf-8",
|
|
67
|
+
errors="replace",
|
|
68
|
+
check=False,
|
|
69
|
+
)
|
|
70
|
+
return out.stdout if out.returncode == 0 else ""
|
|
71
|
+
except (OSError, ValueError):
|
|
72
|
+
return ""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def scan_sources(
|
|
76
|
+
repo: str | Path, markers: list[str], *, test_dirs: list[str] | None = None
|
|
77
|
+
) -> dict[str, dict]:
|
|
78
|
+
"""Walk test source files and compute provenance + assertion density."""
|
|
79
|
+
repo = Path(repo)
|
|
80
|
+
test_dirs = test_dirs or ["test", "tests", "src/test", "spec", "features", "e2e", "__tests__"]
|
|
81
|
+
signals: dict[str, dict] = {}
|
|
82
|
+
seen: set[Path] = set()
|
|
83
|
+
for d in test_dirs:
|
|
84
|
+
base = repo / d
|
|
85
|
+
if not base.exists():
|
|
86
|
+
continue
|
|
87
|
+
for path in base.rglob("*"):
|
|
88
|
+
if path.suffix.lower() not in SOURCE_EXT or not path.is_file() or path in seen:
|
|
89
|
+
continue
|
|
90
|
+
seen.add(path)
|
|
91
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
92
|
+
tests, asserts, density = assertion_density(text)
|
|
93
|
+
if tests == 0:
|
|
94
|
+
continue
|
|
95
|
+
commit_msg = _last_commit_message(repo, path)
|
|
96
|
+
ai = any(m in text or m in commit_msg for m in markers)
|
|
97
|
+
rel = path.relative_to(repo).as_posix()
|
|
98
|
+
signals[rel] = {
|
|
99
|
+
"ai_generated": ai,
|
|
100
|
+
"test_functions": tests,
|
|
101
|
+
"assertions": asserts,
|
|
102
|
+
"assertion_density": density,
|
|
103
|
+
"mutation_score": None,
|
|
104
|
+
}
|
|
105
|
+
return signals
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def load_mutation_report(path: str | Path) -> dict[str, float]:
|
|
109
|
+
"""Return {source_file: mutation_score} from PIT XML, Stryker JSON or mutmut JSON."""
|
|
110
|
+
p = Path(path)
|
|
111
|
+
if not p.exists():
|
|
112
|
+
return {}
|
|
113
|
+
if p.suffix.lower() == ".xml": # PIT mutations.xml
|
|
114
|
+
root = ET.parse(p).getroot()
|
|
115
|
+
totals: dict[str, list[int]] = {}
|
|
116
|
+
for m in root.iter("mutation"):
|
|
117
|
+
src = m.findtext("sourceFile") or "unknown"
|
|
118
|
+
killed = (m.get("status") or "").upper() in ("KILLED", "TIMED_OUT")
|
|
119
|
+
t = totals.setdefault(src, [0, 0])
|
|
120
|
+
t[0] += 1
|
|
121
|
+
t[1] += int(killed)
|
|
122
|
+
return {k: round(v[1] / v[0], 3) for k, v in totals.items() if v[0]}
|
|
123
|
+
data = json.loads(p.read_text(encoding="utf-8"))
|
|
124
|
+
if "files" in data: # Stryker mutation-report schema
|
|
125
|
+
out: dict[str, float] = {}
|
|
126
|
+
for file, info in data["files"].items():
|
|
127
|
+
muts = info.get("mutants", [])
|
|
128
|
+
if not muts:
|
|
129
|
+
continue
|
|
130
|
+
killed = sum(1 for m in muts if m.get("status") in ("Killed", "Timeout"))
|
|
131
|
+
out[file] = round(killed / len(muts), 3)
|
|
132
|
+
return out
|
|
133
|
+
if isinstance(data, dict) and all(isinstance(v, (int, float)) for v in data.values()):
|
|
134
|
+
return {k: float(v) for k, v in data.items()} # simple {file: score}
|
|
135
|
+
return {}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def attach_mutation_scores(signals: dict[str, dict], mutation: dict[str, float]) -> None:
|
|
139
|
+
"""Best-effort join: a test file 'FooTest.java' is matched to source 'Foo.java'."""
|
|
140
|
+
if not mutation:
|
|
141
|
+
return
|
|
142
|
+
by_stem = {Path(k).stem.lower(): v for k, v in mutation.items()}
|
|
143
|
+
for test_file, sig in signals.items():
|
|
144
|
+
stem = Path(test_file).stem.lower()
|
|
145
|
+
for suffix in ("test", "tests", "spec", "_test", "_spec", ".test", ".spec"):
|
|
146
|
+
if stem.endswith(suffix):
|
|
147
|
+
candidate = stem[: -len(suffix)].rstrip("_.")
|
|
148
|
+
if candidate in by_stem:
|
|
149
|
+
sig["mutation_score"] = by_stem[candidate]
|
|
150
|
+
break
|
|
151
|
+
if sig["mutation_score"] is None and stem.startswith("test_") and stem[5:] in by_stem:
|
|
152
|
+
sig["mutation_score"] = by_stem[stem[5:]]
|