testledger 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. testledger-0.1.0/LICENSE +21 -0
  2. testledger-0.1.0/PKG-INFO +208 -0
  3. testledger-0.1.0/README.md +172 -0
  4. testledger-0.1.0/pyproject.toml +68 -0
  5. testledger-0.1.0/setup.cfg +4 -0
  6. testledger-0.1.0/src/testledger/__init__.py +11 -0
  7. testledger-0.1.0/src/testledger/aitest/__init__.py +3 -0
  8. testledger-0.1.0/src/testledger/aitest/detector.py +152 -0
  9. testledger-0.1.0/src/testledger/cli.py +262 -0
  10. testledger-0.1.0/src/testledger/config.py +105 -0
  11. testledger-0.1.0/src/testledger/ingest/__init__.py +7 -0
  12. testledger-0.1.0/src/testledger/ingest/cucumber.py +79 -0
  13. testledger-0.1.0/src/testledger/ingest/github_actions.py +145 -0
  14. testledger-0.1.0/src/testledger/ingest/gitlog.py +118 -0
  15. testledger-0.1.0/src/testledger/ingest/jenkins.py +127 -0
  16. testledger-0.1.0/src/testledger/ingest/junit.py +114 -0
  17. testledger-0.1.0/src/testledger/ledger/__init__.py +3 -0
  18. testledger-0.1.0/src/testledger/ledger/engine.py +332 -0
  19. testledger-0.1.0/src/testledger/models.py +194 -0
  20. testledger-0.1.0/src/testledger/reports/__init__.py +3 -0
  21. testledger-0.1.0/src/testledger/reports/builder.py +224 -0
  22. testledger-0.1.0/src/testledger/templates/memo.html.j2 +77 -0
  23. testledger-0.1.0/src/testledger/templates/memo.md.j2 +78 -0
  24. testledger-0.1.0/src/testledger.egg-info/PKG-INFO +208 -0
  25. testledger-0.1.0/src/testledger.egg-info/SOURCES.txt +33 -0
  26. testledger-0.1.0/src/testledger.egg-info/dependency_links.txt +1 -0
  27. testledger-0.1.0/src/testledger.egg-info/entry_points.txt +2 -0
  28. testledger-0.1.0/src/testledger.egg-info/requires.txt +13 -0
  29. testledger-0.1.0/src/testledger.egg-info/top_level.txt +1 -0
  30. testledger-0.1.0/tests/test_aitest.py +50 -0
  31. testledger-0.1.0/tests/test_ci_clients.py +99 -0
  32. testledger-0.1.0/tests/test_engine.py +157 -0
  33. testledger-0.1.0/tests/test_gitlog.py +62 -0
  34. testledger-0.1.0/tests/test_ingest.py +41 -0
  35. testledger-0.1.0/tests/test_reports_and_cli.py +105 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Suresh Babu
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,208 @@
1
+ Metadata-Version: 2.4
2
+ Name: testledger
3
+ Version: 0.1.0
4
+ Summary: Test Estate Balance Sheet: turn CI results and git history into per-test P&L and executive Quality Investment Memos.
5
+ Author-email: Suresh Babu <sureshbabu.santhakumar@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/surkuar/testledger
8
+ Project-URL: Issues, https://github.com/surkuar/testledger/issues
9
+ Project-URL: Documentation, https://github.com/surkuar/testledger/tree/main/docs
10
+ Keywords: testing,quality-engineering,ci,flaky-tests,test-roi,engineering-leadership
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Software Development :: Testing
20
+ Requires-Python: >=3.11
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: pydantic>=2.5
24
+ Requires-Dist: PyYAML>=6.0
25
+ Requires-Dist: Jinja2>=3.1
26
+ Requires-Dist: requests>=2.31
27
+ Requires-Dist: typer>=0.12
28
+ Requires-Dist: rich>=13.0
29
+ Provides-Extra: dev
30
+ Requires-Dist: pytest>=8.0; extra == "dev"
31
+ Requires-Dist: pytest-cov>=5.0; extra == "dev"
32
+ Requires-Dist: ruff>=0.5; extra == "dev"
33
+ Requires-Dist: mypy>=1.10; extra == "dev"
34
+ Requires-Dist: build>=1.2; extra == "dev"
35
+ Dynamic: license-file
36
+
37
+ # TestLedger — Test Estate Balance Sheet
38
+
39
+ [![CI](https://github.com/surkuar/testledger/actions/workflows/ci.yml/badge.svg)](https://github.com/surkuar/testledger/actions/workflows/ci.yml)
40
+ [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
41
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](pyproject.toml)
42
+ [![GitHub Action](https://img.shields.io/badge/GitHub%20Action-ready-2ea44f?logo=githubactions&logoColor=white)](action.yml)
43
+
44
+ > **Your test suite is a portfolio. TestLedger is its balance sheet.**
45
+ > Every automated test carries a cost (CI minutes, reruns, engineer triage, maintenance churn) and produces a return (regressions caught before production). TestLedger books both from the data you already have — CI results and git history — and produces a one-page **Quality Investment Memo** that a VP of Engineering or CFO can act on.
46
+
47
+ Most tooling in this space stops at "which tests are flaky". TestLedger answers the question leaders actually ask: **which tests are worth what we pay for them, and where should the next dollar of QE effort go?**
48
+
49
+ ---
50
+
51
+ ## What you get
52
+
53
+ | Artefact | Audience | What it says |
54
+ |---|---|---|
55
+ | `memo.md` / `memo.html` | VP Eng, CFO, Director of QE | Bottom line, reallocation recommendation, where the money goes, top liabilities/assets, assumptions |
56
+ | `balance_sheet.json` | QE leads | Assets vs. liabilities per test, by pyramid layer, by suite |
57
+ | `income_statement.json` | Finance / QE leads | Cost by category (CI, reruns, triage, maintenance) vs. value delivered |
58
+ | `writeoffs.json` | Team leads | The action list: **retire / demote / fix / quarantine**, each with a reason |
59
+ | `ai_liabilities.json` | QE governance | AI-generated tests that fail mutation-score or assertion-density thresholds ("test theater") |
60
+ | `ledger.json` | Dashboards | The full book — diff it quarter over quarter |
61
+ | PR comment / job summary | Engineers | Compact table posted by the GitHub Action |
62
+
63
+ **See a real one:** [sample memo (Markdown)](docs/sample-report/memo.md) · [HTML version](docs/sample-report/memo.html) · [write-off list](docs/sample-report/writeoffs.json) — generated from the built-in demo in under a second.
64
+
65
+ ## Quick start (any OS — Windows, macOS, Linux)
66
+
67
+ ```bash
68
+ pip install git+https://github.com/surkuar/testledger.git
69
+ # or, from a clone: pip install -e .
70
+
71
+ # 1. Try the built-in demo (fake repo, 12 weeks of CI history, reruns, a real regression, an AI-written test)
72
+ python examples/generate_demo.py demo-estate
73
+ cd demo-estate
74
+ testledger analyze --config testledger.yml --reports . --mutation mutation.json --out reports
75
+ testledger memo reports # print the memo
76
+ start reports/memo.html # Windows (macOS: open, Linux: xdg-open)
77
+
78
+ # 2. On your own repo
79
+ testledger init # writes testledger.yml with documented defaults
80
+ testledger analyze --config testledger.yml --reports path/to/ci-reports --out testledger-out
81
+ ```
82
+
83
+ ### Local report folder convention
84
+ TestLedger needs to know which CI runs re-executed the **same commit** — that is how flakiness is detected without any test-framework changes. For local folders, name each run directory:
85
+
86
+ ```
87
+ ci-reports/
88
+ 20260915-066b939951-attempt1/TEST-results.xml # <YYYYMMDD>[THHMM]-<sha>-attempt<N>
89
+ 20260915-066b939951-attempt2/TEST-results.xml # same SHA, attempt 2 => a rerun
90
+ 20260917-17f9d0b9e0-attempt1/cucumber.json
91
+ ```
92
+ When you use the GitHub Actions or Jenkins ingesters this metadata comes from the CI API automatically.
93
+
94
+ ## Inputs supported (v1)
95
+
96
+ | Source | How | Notes |
97
+ |---|---|---|
98
+ | JUnit XML (Surefire, Gradle, pytest, Jest, …) | `sources.junit_globs` | universal baseline |
99
+ | TestNG `testng-results.xml` | `sources.junit_globs` | auto-detected |
100
+ | Cucumber JSON (jvm / ruby / js) | `sources.cucumber_globs` | scenario = test, outline rows distinct |
101
+ | Git history | `sources.git_repo` | maintenance churn on test files; fix-commit detection for value credit |
102
+ | GitHub Actions | `sources.github` | run history + artifacts via REST API, `run_attempt` = rerun |
103
+ | Jenkins | `sources.jenkins` | build history + `testReport` JSON; same SHA in consecutive builds = rerun |
104
+ | Mutation reports | `--mutation` | PIT XML, Stryker JSON, or `{file: score}` JSON |
105
+
106
+ ## How it works (short version)
107
+
108
+ ```mermaid
109
+ flowchart LR
110
+ A[JUnit / TestNG XML] --> N[Normalise to TestRun]
111
+ B[Cucumber JSON] --> N
112
+ C[GitHub Actions API] --> N
113
+ D[Jenkins API] --> N
114
+ G[git log] --> H[Churn + fix commits]
115
+ S[Test sources] --> AI[AI provenance + assertion density]
116
+ M[Mutation report] --> AI
117
+ N --> E[Ledger engine]
118
+ H --> E
119
+ AI --> E
120
+ E --> R[Balance sheet · Income statement · Write-offs · Memo]
121
+ ```
122
+
123
+ **Debits (cost) per test:** CI minutes × rate · rerun minutes · flaky false-alarms × triage minutes × engineer rate · test-file lines changed × maintenance minutes × engineer rate.
124
+
125
+ **Credits (value) per test:** a genuine first-attempt failure (not cleared by a same-SHA rerun) followed within 3 days by a product-code commit that looks like a fix, valued at your organisation's cost of a production defect.
126
+
127
+ **Verdicts:** `quarantine` (flake ≥ 30 %) · `retire` (no catches, net negative, cost above floor) · `demote` (UI test whose cost-per-catch is ≥ 3× the service-layer median) · `fix` (valuable but flaky or slow) · `asset` · `hold`.
128
+
129
+ Every rate and threshold lives in `testledger.yml` and is printed on the memo so finance can challenge it. Full derivation: **[docs/methodology.md](docs/methodology.md)**. Architecture and flowcharts: **[docs/design.md](docs/design.md)**.
130
+
131
+ ## Use as a GitHub Action
132
+
133
+ ```yaml
134
+ # .github/workflows/testledger.yml
135
+ name: TestLedger
136
+ on:
137
+ schedule: [{ cron: "0 6 * * 1" }] # weekly memo
138
+ pull_request:
139
+ permissions:
140
+ contents: read
141
+ actions: read
142
+ pull-requests: write
143
+ jobs:
144
+ ledger:
145
+ runs-on: ubuntu-latest
146
+ steps:
147
+ - uses: actions/checkout@v4
148
+ with: { fetch-depth: 0 } # full history for churn / fix detection
149
+ - uses: surkuar/testledger@v0.1.0
150
+ with:
151
+ config: testledger.yml
152
+ reports: build/test-results # or leave empty and configure sources.github
153
+ fail-on-quarantine: "false"
154
+ ```
155
+ The action uploads all reports as an artifact, appends the memo to the job summary, and keeps one live comment on each PR.
156
+
157
+ ## Configuration
158
+
159
+ `testledger init` writes a fully commented file. The important parts:
160
+
161
+ ```yaml
162
+ org_name: Acme Commerce
163
+ window_days: 90
164
+ rates:
165
+ engineer_hourly_usd: 75 # fully loaded
166
+ ci_minute_usd: 0.008
167
+ triage_minutes_per_false_alarm: 18
168
+ maintenance_minutes_per_line: 1.5
169
+ regression_value_usd: 2000 # cost of one escaped defect — your number
170
+ thresholds:
171
+ quarantine_flake_rate: 0.30
172
+ fix_flake_rate: 0.10
173
+ slow_test_seconds: 60
174
+ demote_layer_cost_ratio: 3.0
175
+ retire_min_cost_usd: 25
176
+ ai_min_mutation_score: 0.5
177
+ ai_min_assertion_density: 0.8
178
+ sources:
179
+ junit_globs: ["**/surefire-reports/*.xml", "**/TEST-*.xml"]
180
+ cucumber_globs: ["**/cucumber*.json"]
181
+ git_repo: "."
182
+ github: { repo: "acme/shop", workflow: "ci.yml", token_env: "GITHUB_TOKEN", max_runs: 200 }
183
+ jenkins: { url: "https://ci.acme.com", job: "shop/main", user_env: "JENKINS_USER", token_env: "JENKINS_TOKEN" }
184
+ ```
185
+
186
+ ## Why this exists
187
+
188
+ Engineering leaders are routinely told to present automation "as risk-adjusted ROI" — but nothing computes it from real data. Flaky-test tools price flakes; ROI calculators take hand-typed guesses; the test-portfolio idea lives in blog posts. TestLedger closes that gap: it is the first open-source tool that treats a test estate as a managed portfolio with per-test P&L, a balance sheet, and an executive memo — framework-agnostic, CI-agnostic, and runnable in two minutes on any OS.
189
+
190
+ It was built by a QE leader for QE leaders. If you run a quality organisation and need to defend (or reallocate) a testing budget, this is the number you were missing.
191
+
192
+ ## Development
193
+
194
+ ```bash
195
+ pip install -e ".[dev]"
196
+ pytest # 25 tests, offline (CI APIs stubbed; git tests use a throwaway repo)
197
+ ruff check src tests
198
+ ```
199
+
200
+ ## Roadmap
201
+ - [ ] GitLab CI and Azure DevOps ingesters
202
+ - [ ] Code-ownership join (CODEOWNERS) so the memo is grouped by team
203
+ - [ ] Quarter-over-quarter diff (`testledger diff q3.json q4.json`)
204
+ - [ ] Optional LLM narrative for the memo (opt-in, local or API)
205
+ - [ ] Web dashboard reading `ledger.json`
206
+
207
+ ## License
208
+ MIT © Suresh Babu. See [LICENSE](LICENSE).
@@ -0,0 +1,172 @@
1
+ # TestLedger — Test Estate Balance Sheet
2
+
3
+ [![CI](https://github.com/surkuar/testledger/actions/workflows/ci.yml/badge.svg)](https://github.com/surkuar/testledger/actions/workflows/ci.yml)
4
+ [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
5
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](pyproject.toml)
6
+ [![GitHub Action](https://img.shields.io/badge/GitHub%20Action-ready-2ea44f?logo=githubactions&logoColor=white)](action.yml)
7
+
8
+ > **Your test suite is a portfolio. TestLedger is its balance sheet.**
9
+ > Every automated test carries a cost (CI minutes, reruns, engineer triage, maintenance churn) and produces a return (regressions caught before production). TestLedger books both from the data you already have — CI results and git history — and produces a one-page **Quality Investment Memo** that a VP of Engineering or CFO can act on.
10
+
11
+ Most tooling in this space stops at "which tests are flaky". TestLedger answers the question leaders actually ask: **which tests are worth what we pay for them, and where should the next dollar of QE effort go?**
12
+
13
+ ---
14
+
15
+ ## What you get
16
+
17
+ | Artefact | Audience | What it says |
18
+ |---|---|---|
19
+ | `memo.md` / `memo.html` | VP Eng, CFO, Director of QE | Bottom line, reallocation recommendation, where the money goes, top liabilities/assets, assumptions |
20
+ | `balance_sheet.json` | QE leads | Assets vs. liabilities per test, by pyramid layer, by suite |
21
+ | `income_statement.json` | Finance / QE leads | Cost by category (CI, reruns, triage, maintenance) vs. value delivered |
22
+ | `writeoffs.json` | Team leads | The action list: **retire / demote / fix / quarantine**, each with a reason |
23
+ | `ai_liabilities.json` | QE governance | AI-generated tests that fail mutation-score or assertion-density thresholds ("test theater") |
24
+ | `ledger.json` | Dashboards | The full book — diff it quarter over quarter |
25
+ | PR comment / job summary | Engineers | Compact table posted by the GitHub Action |
26
+
27
+ **See a real one:** [sample memo (Markdown)](docs/sample-report/memo.md) · [HTML version](docs/sample-report/memo.html) · [write-off list](docs/sample-report/writeoffs.json) — generated from the built-in demo in under a second.
28
+
29
+ ## Quick start (any OS — Windows, macOS, Linux)
30
+
31
+ ```bash
32
+ pip install git+https://github.com/surkuar/testledger.git
33
+ # or, from a clone: pip install -e .
34
+
35
+ # 1. Try the built-in demo (fake repo, 12 weeks of CI history, reruns, a real regression, an AI-written test)
36
+ python examples/generate_demo.py demo-estate
37
+ cd demo-estate
38
+ testledger analyze --config testledger.yml --reports . --mutation mutation.json --out reports
39
+ testledger memo reports # print the memo
40
+ start reports/memo.html # Windows (macOS: open, Linux: xdg-open)
41
+
42
+ # 2. On your own repo
43
+ testledger init # writes testledger.yml with documented defaults
44
+ testledger analyze --config testledger.yml --reports path/to/ci-reports --out testledger-out
45
+ ```
46
+
47
+ ### Local report folder convention
48
+ TestLedger needs to know which CI runs re-executed the **same commit** — that is how flakiness is detected without any test-framework changes. For local folders, name each run directory:
49
+
50
+ ```
51
+ ci-reports/
52
+ 20260915-066b939951-attempt1/TEST-results.xml # <YYYYMMDD>[THHMM]-<sha>-attempt<N>
53
+ 20260915-066b939951-attempt2/TEST-results.xml # same SHA, attempt 2 => a rerun
54
+ 20260917-17f9d0b9e0-attempt1/cucumber.json
55
+ ```
56
+ When you use the GitHub Actions or Jenkins ingesters this metadata comes from the CI API automatically.
57
+
58
+ ## Inputs supported (v1)
59
+
60
+ | Source | How | Notes |
61
+ |---|---|---|
62
+ | JUnit XML (Surefire, Gradle, pytest, Jest, …) | `sources.junit_globs` | universal baseline |
63
+ | TestNG `testng-results.xml` | `sources.junit_globs` | auto-detected |
64
+ | Cucumber JSON (jvm / ruby / js) | `sources.cucumber_globs` | scenario = test, outline rows distinct |
65
+ | Git history | `sources.git_repo` | maintenance churn on test files; fix-commit detection for value credit |
66
+ | GitHub Actions | `sources.github` | run history + artifacts via REST API, `run_attempt` = rerun |
67
+ | Jenkins | `sources.jenkins` | build history + `testReport` JSON; same SHA in consecutive builds = rerun |
68
+ | Mutation reports | `--mutation` | PIT XML, Stryker JSON, or `{file: score}` JSON |
69
+
70
+ ## How it works (short version)
71
+
72
+ ```mermaid
73
+ flowchart LR
74
+ A[JUnit / TestNG XML] --> N[Normalise to TestRun]
75
+ B[Cucumber JSON] --> N
76
+ C[GitHub Actions API] --> N
77
+ D[Jenkins API] --> N
78
+ G[git log] --> H[Churn + fix commits]
79
+ S[Test sources] --> AI[AI provenance + assertion density]
80
+ M[Mutation report] --> AI
81
+ N --> E[Ledger engine]
82
+ H --> E
83
+ AI --> E
84
+ E --> R[Balance sheet · Income statement · Write-offs · Memo]
85
+ ```
86
+
87
+ **Debits (cost) per test:** CI minutes × rate · rerun minutes · flaky false-alarms × triage minutes × engineer rate · test-file lines changed × maintenance minutes × engineer rate.
88
+
89
+ **Credits (value) per test:** a genuine first-attempt failure (not cleared by a same-SHA rerun) followed within 3 days by a product-code commit that looks like a fix, valued at your organisation's cost of a production defect.
90
+
91
+ **Verdicts:** `quarantine` (flake ≥ 30 %) · `retire` (no catches, net negative, cost above floor) · `demote` (UI test whose cost-per-catch is ≥ 3× the service-layer median) · `fix` (valuable but flaky or slow) · `asset` · `hold`.
92
+
93
+ Every rate and threshold lives in `testledger.yml` and is printed on the memo so finance can challenge it. Full derivation: **[docs/methodology.md](docs/methodology.md)**. Architecture and flowcharts: **[docs/design.md](docs/design.md)**.
94
+
95
+ ## Use as a GitHub Action
96
+
97
+ ```yaml
98
+ # .github/workflows/testledger.yml
99
+ name: TestLedger
100
+ on:
101
+ schedule: [{ cron: "0 6 * * 1" }] # weekly memo
102
+ pull_request:
103
+ permissions:
104
+ contents: read
105
+ actions: read
106
+ pull-requests: write
107
+ jobs:
108
+ ledger:
109
+ runs-on: ubuntu-latest
110
+ steps:
111
+ - uses: actions/checkout@v4
112
+ with: { fetch-depth: 0 } # full history for churn / fix detection
113
+ - uses: surkuar/testledger@v0.1.0
114
+ with:
115
+ config: testledger.yml
116
+ reports: build/test-results # or leave empty and configure sources.github
117
+ fail-on-quarantine: "false"
118
+ ```
119
+ The action uploads all reports as an artifact, appends the memo to the job summary, and keeps one live comment on each PR.
120
+
121
+ ## Configuration
122
+
123
+ `testledger init` writes a fully commented file. The important parts:
124
+
125
+ ```yaml
126
+ org_name: Acme Commerce
127
+ window_days: 90
128
+ rates:
129
+ engineer_hourly_usd: 75 # fully loaded
130
+ ci_minute_usd: 0.008
131
+ triage_minutes_per_false_alarm: 18
132
+ maintenance_minutes_per_line: 1.5
133
+ regression_value_usd: 2000 # cost of one escaped defect — your number
134
+ thresholds:
135
+ quarantine_flake_rate: 0.30
136
+ fix_flake_rate: 0.10
137
+ slow_test_seconds: 60
138
+ demote_layer_cost_ratio: 3.0
139
+ retire_min_cost_usd: 25
140
+ ai_min_mutation_score: 0.5
141
+ ai_min_assertion_density: 0.8
142
+ sources:
143
+ junit_globs: ["**/surefire-reports/*.xml", "**/TEST-*.xml"]
144
+ cucumber_globs: ["**/cucumber*.json"]
145
+ git_repo: "."
146
+ github: { repo: "acme/shop", workflow: "ci.yml", token_env: "GITHUB_TOKEN", max_runs: 200 }
147
+ jenkins: { url: "https://ci.acme.com", job: "shop/main", user_env: "JENKINS_USER", token_env: "JENKINS_TOKEN" }
148
+ ```
149
+
150
+ ## Why this exists
151
+
152
+ Engineering leaders are routinely told to present automation "as risk-adjusted ROI" — but nothing computes it from real data. Flaky-test tools price flakes; ROI calculators take hand-typed guesses; the test-portfolio idea lives in blog posts. TestLedger closes that gap: it is the first open-source tool that treats a test estate as a managed portfolio with per-test P&L, a balance sheet, and an executive memo — framework-agnostic, CI-agnostic, and runnable in two minutes on any OS.
153
+
154
+ It was built by a QE leader for QE leaders. If you run a quality organisation and need to defend (or reallocate) a testing budget, this is the number you were missing.
155
+
156
+ ## Development
157
+
158
+ ```bash
159
+ pip install -e ".[dev]"
160
+ pytest # 25 tests, offline (CI APIs stubbed; git tests use a throwaway repo)
161
+ ruff check src tests
162
+ ```
163
+
164
+ ## Roadmap
165
+ - [ ] GitLab CI and Azure DevOps ingesters
166
+ - [ ] Code-ownership join (CODEOWNERS) so the memo is grouped by team
167
+ - [ ] Quarter-over-quarter diff (`testledger diff q3.json q4.json`)
168
+ - [ ] Optional LLM narrative for the memo (opt-in, local or API)
169
+ - [ ] Web dashboard reading `ledger.json`
170
+
171
+ ## License
172
+ MIT © Suresh Babu. See [LICENSE](LICENSE).
@@ -0,0 +1,68 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "testledger"
7
+ version = "0.1.0"
8
+ description = "Test Estate Balance Sheet: turn CI results and git history into per-test P&L and executive Quality Investment Memos."
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "Suresh Babu", email = "sureshbabu.santhakumar@gmail.com" }]
13
+ keywords = ["testing", "quality-engineering", "ci", "flaky-tests", "test-roi", "engineering-leadership"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Intended Audience :: Developers",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Operating System :: OS Independent",
19
+ "Programming Language :: Python :: 3",
20
+ "Programming Language :: Python :: 3.11",
21
+ "Programming Language :: Python :: 3.12",
22
+ "Programming Language :: Python :: 3.13",
23
+ "Topic :: Software Development :: Testing",
24
+ ]
25
+ dependencies = [
26
+ "pydantic>=2.5",
27
+ "PyYAML>=6.0",
28
+ "Jinja2>=3.1",
29
+ "requests>=2.31",
30
+ "typer>=0.12",
31
+ "rich>=13.0",
32
+ ]
33
+
34
+ [project.optional-dependencies]
35
+ dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.5", "mypy>=1.10", "build>=1.2"]
36
+
37
+ [project.scripts]
38
+ testledger = "testledger.cli:app"
39
+
40
+ [project.urls]
41
+ Homepage = "https://github.com/surkuar/testledger"
42
+ Issues = "https://github.com/surkuar/testledger/issues"
43
+ Documentation = "https://github.com/surkuar/testledger/tree/main/docs"
44
+
45
+ [tool.setuptools.packages.find]
46
+ where = ["src"]
47
+
48
+ [tool.setuptools.package-data]
49
+ testledger = ["templates/*.j2"]
50
+
51
+ [tool.pytest.ini_options]
52
+ testpaths = ["tests"]
53
+ addopts = "-q"
54
+
55
+ [tool.ruff]
56
+ line-length = 110
57
+ target-version = "py311"
58
+
59
+ [tool.ruff.lint]
60
+ select = ["E", "F", "I", "UP", "B"]
61
+ ignore = ["E501"] # line length is owned by `ruff format`
62
+
63
+ [tool.mypy]
64
+ python_version = "3.11"
65
+ ignore_missing_imports = true
66
+
67
+ [tool.ruff.lint.flake8-bugbear]
68
+ extend-immutable-calls = ["typer.Option", "typer.Argument"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,11 @@
1
+ """TestLedger — Test Estate Balance Sheet.
2
+
3
+ Treats an automated test suite as an investment portfolio: every test carries a
4
+ cost (CI minutes, reruns, triage, maintenance churn) and produces a return
5
+ (regressions caught). TestLedger ingests CI results and git history, books
6
+ every cost and value entry into a double-entry style ledger, and renders
7
+ executive-facing reports: a Balance Sheet, an Income Statement, a Write-off
8
+ list and a one-page Quality Investment Memo.
9
+ """
10
+
11
+ __version__ = "0.1.0"
@@ -0,0 +1,3 @@
1
+ from testledger.aitest.detector import attach_mutation_scores, load_mutation_report, scan_sources
2
+
3
+ __all__ = ["scan_sources", "load_mutation_report", "attach_mutation_scores"]
@@ -0,0 +1,152 @@
1
+ """AI-generated test governance ("test theater" detection).
2
+
3
+ Three signals, each cheap and language-agnostic:
4
+
5
+ 1. **Provenance** — the test file contains a configured marker
6
+ (``@ai-generated``, ``Generated-By:``, Copilot/Claude co-author trailers)
7
+ or its last commit message does.
8
+ 2. **Assertion density** — assertions per test function. Tests with density
9
+ < threshold are likely "smoke only" and prove little. Detects common
10
+ assertion idioms across Java (JUnit/TestNG/AssertJ/Hamcrest), Python,
11
+ JavaScript/TypeScript, Ruby (RSpec/Minitest), C#, Go.
12
+ 3. **Mutation score** — read from PIT (Java), Stryker (JS/TS/C#),
13
+ mutmut/cosmic-ray (Python) reports. A high line-coverage, low
14
+ mutation-score test is the signature of an assertion that mirrors the
15
+ implementation rather than checking it.
16
+
17
+ Output is a mapping ``{file_or_test_id: {"ai_generated": bool,
18
+ "assertion_density": float, "mutation_score": float | None}}`` that the ledger
19
+ engine merges into each account. The memo lists AI-generated tests that fall
20
+ below the governance thresholds as **unverified liabilities**.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ import re
27
+ import subprocess
28
+ import xml.etree.ElementTree as ET
29
+ from pathlib import Path
30
+
31
+ TEST_FUNC_PATTERNS = [
32
+ re.compile(r"@Test\b"), # JUnit / TestNG
33
+ re.compile(r"^\s*def\s+test_\w+", re.M), # pytest / unittest
34
+ re.compile(r"\b(it|test)\s*\(\s*['\"`]", re.M), # jest / mocha / vitest
35
+ re.compile(r"^\s*(it|scenario|specify)\s+['\"]", re.M), # rspec
36
+ re.compile(r"\[(Fact|Theory|Test|TestMethod)\]"), # xUnit / NUnit / MSTest
37
+ re.compile(r"^func\s+Test\w+\s*\(", re.M), # go
38
+ ]
39
+ ASSERTION_PATTERNS = [
40
+ re.compile(r"\bassert\w*\s*\("), # assertEquals(, assertTrue(, assert(
41
+ re.compile(r"\bassertThat\s*\("),
42
+ re.compile(r"^\s*assert\s+\S", re.M), # python bare assert
43
+ re.compile(r"\bexpect\s*\("), # jest / rspec / chai
44
+ re.compile(r"\bshould\b"),
45
+ re.compile(r"\bAssert\.\w+\s*\("), # C#
46
+ re.compile(r"\bverify\s*\("), # mockito interaction verification
47
+ re.compile(r"\.(toBe|toEqual|toMatch|toHaveLength|toThrow|isEqualTo|hasSize|contains|isTrue|isFalse)\b"),
48
+ re.compile(r"\bt\.(Errorf|Fatalf|Error|Fatal)\s*\("), # go
49
+ ]
50
+ SOURCE_EXT = {".java", ".kt", ".py", ".js", ".ts", ".tsx", ".jsx", ".rb", ".cs", ".go", ".scala"}
51
+
52
+
53
+ def assertion_density(text: str) -> tuple[int, int, float]:
54
+ tests = sum(len(p.findall(text)) for p in TEST_FUNC_PATTERNS)
55
+ asserts = sum(len(p.findall(text)) for p in ASSERTION_PATTERNS)
56
+ density = round(asserts / tests, 2) if tests else 0.0
57
+ return tests, asserts, density
58
+
59
+
60
+ def _last_commit_message(repo: Path, file: Path) -> str:
61
+ try:
62
+ out = subprocess.run(
63
+ ["git", "-C", str(repo), "log", "-1", "--format=%B", "--", str(file.relative_to(repo))],
64
+ capture_output=True,
65
+ text=True,
66
+ encoding="utf-8",
67
+ errors="replace",
68
+ check=False,
69
+ )
70
+ return out.stdout if out.returncode == 0 else ""
71
+ except (OSError, ValueError):
72
+ return ""
73
+
74
+
75
+ def scan_sources(
76
+ repo: str | Path, markers: list[str], *, test_dirs: list[str] | None = None
77
+ ) -> dict[str, dict]:
78
+ """Walk test source files and compute provenance + assertion density."""
79
+ repo = Path(repo)
80
+ test_dirs = test_dirs or ["test", "tests", "src/test", "spec", "features", "e2e", "__tests__"]
81
+ signals: dict[str, dict] = {}
82
+ seen: set[Path] = set()
83
+ for d in test_dirs:
84
+ base = repo / d
85
+ if not base.exists():
86
+ continue
87
+ for path in base.rglob("*"):
88
+ if path.suffix.lower() not in SOURCE_EXT or not path.is_file() or path in seen:
89
+ continue
90
+ seen.add(path)
91
+ text = path.read_text(encoding="utf-8", errors="replace")
92
+ tests, asserts, density = assertion_density(text)
93
+ if tests == 0:
94
+ continue
95
+ commit_msg = _last_commit_message(repo, path)
96
+ ai = any(m in text or m in commit_msg for m in markers)
97
+ rel = path.relative_to(repo).as_posix()
98
+ signals[rel] = {
99
+ "ai_generated": ai,
100
+ "test_functions": tests,
101
+ "assertions": asserts,
102
+ "assertion_density": density,
103
+ "mutation_score": None,
104
+ }
105
+ return signals
106
+
107
+
108
+ def load_mutation_report(path: str | Path) -> dict[str, float]:
109
+ """Return {source_file: mutation_score} from PIT XML, Stryker JSON or mutmut JSON."""
110
+ p = Path(path)
111
+ if not p.exists():
112
+ return {}
113
+ if p.suffix.lower() == ".xml": # PIT mutations.xml
114
+ root = ET.parse(p).getroot()
115
+ totals: dict[str, list[int]] = {}
116
+ for m in root.iter("mutation"):
117
+ src = m.findtext("sourceFile") or "unknown"
118
+ killed = (m.get("status") or "").upper() in ("KILLED", "TIMED_OUT")
119
+ t = totals.setdefault(src, [0, 0])
120
+ t[0] += 1
121
+ t[1] += int(killed)
122
+ return {k: round(v[1] / v[0], 3) for k, v in totals.items() if v[0]}
123
+ data = json.loads(p.read_text(encoding="utf-8"))
124
+ if "files" in data: # Stryker mutation-report schema
125
+ out: dict[str, float] = {}
126
+ for file, info in data["files"].items():
127
+ muts = info.get("mutants", [])
128
+ if not muts:
129
+ continue
130
+ killed = sum(1 for m in muts if m.get("status") in ("Killed", "Timeout"))
131
+ out[file] = round(killed / len(muts), 3)
132
+ return out
133
+ if isinstance(data, dict) and all(isinstance(v, (int, float)) for v in data.values()):
134
+ return {k: float(v) for k, v in data.items()} # simple {file: score}
135
+ return {}
136
+
137
+
138
+ def attach_mutation_scores(signals: dict[str, dict], mutation: dict[str, float]) -> None:
139
+ """Best-effort join: a test file 'FooTest.java' is matched to source 'Foo.java'."""
140
+ if not mutation:
141
+ return
142
+ by_stem = {Path(k).stem.lower(): v for k, v in mutation.items()}
143
+ for test_file, sig in signals.items():
144
+ stem = Path(test_file).stem.lower()
145
+ for suffix in ("test", "tests", "spec", "_test", "_spec", ".test", ".spec"):
146
+ if stem.endswith(suffix):
147
+ candidate = stem[: -len(suffix)].rstrip("_.")
148
+ if candidate in by_stem:
149
+ sig["mutation_score"] = by_stem[candidate]
150
+ break
151
+ if sig["mutation_score"] is None and stem.startswith("test_") and stem[5:] in by_stem:
152
+ sig["mutation_score"] = by_stem[stem[5:]]