testledger 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testledger/__init__.py +11 -0
- testledger/aitest/__init__.py +3 -0
- testledger/aitest/detector.py +152 -0
- testledger/cli.py +262 -0
- testledger/config.py +105 -0
- testledger/ingest/__init__.py +7 -0
- testledger/ingest/cucumber.py +79 -0
- testledger/ingest/github_actions.py +145 -0
- testledger/ingest/gitlog.py +118 -0
- testledger/ingest/jenkins.py +127 -0
- testledger/ingest/junit.py +114 -0
- testledger/ledger/__init__.py +3 -0
- testledger/ledger/engine.py +332 -0
- testledger/models.py +194 -0
- testledger/reports/__init__.py +3 -0
- testledger/reports/builder.py +224 -0
- testledger/templates/memo.html.j2 +77 -0
- testledger/templates/memo.md.j2 +78 -0
- testledger-0.1.0.dist-info/METADATA +208 -0
- testledger-0.1.0.dist-info/RECORD +24 -0
- testledger-0.1.0.dist-info/WHEEL +5 -0
- testledger-0.1.0.dist-info/entry_points.txt +2 -0
- testledger-0.1.0.dist-info/licenses/LICENSE +21 -0
- testledger-0.1.0.dist-info/top_level.txt +1 -0
testledger/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""TestLedger — Test Estate Balance Sheet.
|
|
2
|
+
|
|
3
|
+
Treats an automated test suite as an investment portfolio: every test carries a
|
|
4
|
+
cost (CI minutes, reruns, triage, maintenance churn) and produces a return
|
|
5
|
+
(regressions caught). TestLedger ingests CI results and git history, books
|
|
6
|
+
every cost and value entry into a double-entry style ledger, and renders
|
|
7
|
+
executive-facing reports: a Balance Sheet, an Income Statement, a Write-off
|
|
8
|
+
list and a one-page Quality Investment Memo.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""AI-generated test governance ("test theater" detection).
|
|
2
|
+
|
|
3
|
+
Three signals, each cheap and language-agnostic:
|
|
4
|
+
|
|
5
|
+
1. **Provenance** — the test file contains a configured marker
|
|
6
|
+
(``@ai-generated``, ``Generated-By:``, Copilot/Claude co-author trailers)
|
|
7
|
+
or its last commit message does.
|
|
8
|
+
2. **Assertion density** — assertions per test function. Tests with density
|
|
9
|
+
< threshold are likely "smoke only" and prove little. Detects common
|
|
10
|
+
assertion idioms across Java (JUnit/TestNG/AssertJ/Hamcrest), Python,
|
|
11
|
+
JavaScript/TypeScript, Ruby (RSpec/Minitest), C#, Go.
|
|
12
|
+
3. **Mutation score** — read from PIT (Java), Stryker (JS/TS/C#),
|
|
13
|
+
mutmut/cosmic-ray (Python) reports. A high line-coverage, low
|
|
14
|
+
mutation-score test is the signature of an assertion that mirrors the
|
|
15
|
+
implementation rather than checking it.
|
|
16
|
+
|
|
17
|
+
Output is a mapping ``{file_or_test_id: {"ai_generated": bool,
|
|
18
|
+
"assertion_density": float, "mutation_score": float | None}}`` that the ledger
|
|
19
|
+
engine merges into each account. The memo lists AI-generated tests that fall
|
|
20
|
+
below the governance thresholds as **unverified liabilities**.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
import re
|
|
27
|
+
import subprocess
|
|
28
|
+
import xml.etree.ElementTree as ET
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
TEST_FUNC_PATTERNS = [
|
|
32
|
+
re.compile(r"@Test\b"), # JUnit / TestNG
|
|
33
|
+
re.compile(r"^\s*def\s+test_\w+", re.M), # pytest / unittest
|
|
34
|
+
re.compile(r"\b(it|test)\s*\(\s*['\"`]", re.M), # jest / mocha / vitest
|
|
35
|
+
re.compile(r"^\s*(it|scenario|specify)\s+['\"]", re.M), # rspec
|
|
36
|
+
re.compile(r"\[(Fact|Theory|Test|TestMethod)\]"), # xUnit / NUnit / MSTest
|
|
37
|
+
re.compile(r"^func\s+Test\w+\s*\(", re.M), # go
|
|
38
|
+
]
|
|
39
|
+
ASSERTION_PATTERNS = [
|
|
40
|
+
re.compile(r"\bassert\w*\s*\("), # assertEquals(, assertTrue(, assert(
|
|
41
|
+
re.compile(r"\bassertThat\s*\("),
|
|
42
|
+
re.compile(r"^\s*assert\s+\S", re.M), # python bare assert
|
|
43
|
+
re.compile(r"\bexpect\s*\("), # jest / rspec / chai
|
|
44
|
+
re.compile(r"\bshould\b"),
|
|
45
|
+
re.compile(r"\bAssert\.\w+\s*\("), # C#
|
|
46
|
+
re.compile(r"\bverify\s*\("), # mockito interaction verification
|
|
47
|
+
re.compile(r"\.(toBe|toEqual|toMatch|toHaveLength|toThrow|isEqualTo|hasSize|contains|isTrue|isFalse)\b"),
|
|
48
|
+
re.compile(r"\bt\.(Errorf|Fatalf|Error|Fatal)\s*\("), # go
|
|
49
|
+
]
|
|
50
|
+
SOURCE_EXT = {".java", ".kt", ".py", ".js", ".ts", ".tsx", ".jsx", ".rb", ".cs", ".go", ".scala"}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def assertion_density(text: str) -> tuple[int, int, float]:
|
|
54
|
+
tests = sum(len(p.findall(text)) for p in TEST_FUNC_PATTERNS)
|
|
55
|
+
asserts = sum(len(p.findall(text)) for p in ASSERTION_PATTERNS)
|
|
56
|
+
density = round(asserts / tests, 2) if tests else 0.0
|
|
57
|
+
return tests, asserts, density
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _last_commit_message(repo: Path, file: Path) -> str:
|
|
61
|
+
try:
|
|
62
|
+
out = subprocess.run(
|
|
63
|
+
["git", "-C", str(repo), "log", "-1", "--format=%B", "--", str(file.relative_to(repo))],
|
|
64
|
+
capture_output=True,
|
|
65
|
+
text=True,
|
|
66
|
+
encoding="utf-8",
|
|
67
|
+
errors="replace",
|
|
68
|
+
check=False,
|
|
69
|
+
)
|
|
70
|
+
return out.stdout if out.returncode == 0 else ""
|
|
71
|
+
except (OSError, ValueError):
|
|
72
|
+
return ""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def scan_sources(
|
|
76
|
+
repo: str | Path, markers: list[str], *, test_dirs: list[str] | None = None
|
|
77
|
+
) -> dict[str, dict]:
|
|
78
|
+
"""Walk test source files and compute provenance + assertion density."""
|
|
79
|
+
repo = Path(repo)
|
|
80
|
+
test_dirs = test_dirs or ["test", "tests", "src/test", "spec", "features", "e2e", "__tests__"]
|
|
81
|
+
signals: dict[str, dict] = {}
|
|
82
|
+
seen: set[Path] = set()
|
|
83
|
+
for d in test_dirs:
|
|
84
|
+
base = repo / d
|
|
85
|
+
if not base.exists():
|
|
86
|
+
continue
|
|
87
|
+
for path in base.rglob("*"):
|
|
88
|
+
if path.suffix.lower() not in SOURCE_EXT or not path.is_file() or path in seen:
|
|
89
|
+
continue
|
|
90
|
+
seen.add(path)
|
|
91
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
92
|
+
tests, asserts, density = assertion_density(text)
|
|
93
|
+
if tests == 0:
|
|
94
|
+
continue
|
|
95
|
+
commit_msg = _last_commit_message(repo, path)
|
|
96
|
+
ai = any(m in text or m in commit_msg for m in markers)
|
|
97
|
+
rel = path.relative_to(repo).as_posix()
|
|
98
|
+
signals[rel] = {
|
|
99
|
+
"ai_generated": ai,
|
|
100
|
+
"test_functions": tests,
|
|
101
|
+
"assertions": asserts,
|
|
102
|
+
"assertion_density": density,
|
|
103
|
+
"mutation_score": None,
|
|
104
|
+
}
|
|
105
|
+
return signals
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def load_mutation_report(path: str | Path) -> dict[str, float]:
|
|
109
|
+
"""Return {source_file: mutation_score} from PIT XML, Stryker JSON or mutmut JSON."""
|
|
110
|
+
p = Path(path)
|
|
111
|
+
if not p.exists():
|
|
112
|
+
return {}
|
|
113
|
+
if p.suffix.lower() == ".xml": # PIT mutations.xml
|
|
114
|
+
root = ET.parse(p).getroot()
|
|
115
|
+
totals: dict[str, list[int]] = {}
|
|
116
|
+
for m in root.iter("mutation"):
|
|
117
|
+
src = m.findtext("sourceFile") or "unknown"
|
|
118
|
+
killed = (m.get("status") or "").upper() in ("KILLED", "TIMED_OUT")
|
|
119
|
+
t = totals.setdefault(src, [0, 0])
|
|
120
|
+
t[0] += 1
|
|
121
|
+
t[1] += int(killed)
|
|
122
|
+
return {k: round(v[1] / v[0], 3) for k, v in totals.items() if v[0]}
|
|
123
|
+
data = json.loads(p.read_text(encoding="utf-8"))
|
|
124
|
+
if "files" in data: # Stryker mutation-report schema
|
|
125
|
+
out: dict[str, float] = {}
|
|
126
|
+
for file, info in data["files"].items():
|
|
127
|
+
muts = info.get("mutants", [])
|
|
128
|
+
if not muts:
|
|
129
|
+
continue
|
|
130
|
+
killed = sum(1 for m in muts if m.get("status") in ("Killed", "Timeout"))
|
|
131
|
+
out[file] = round(killed / len(muts), 3)
|
|
132
|
+
return out
|
|
133
|
+
if isinstance(data, dict) and all(isinstance(v, (int, float)) for v in data.values()):
|
|
134
|
+
return {k: float(v) for k, v in data.items()} # simple {file: score}
|
|
135
|
+
return {}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def attach_mutation_scores(signals: dict[str, dict], mutation: dict[str, float]) -> None:
|
|
139
|
+
"""Best-effort join: a test file 'FooTest.java' is matched to source 'Foo.java'."""
|
|
140
|
+
if not mutation:
|
|
141
|
+
return
|
|
142
|
+
by_stem = {Path(k).stem.lower(): v for k, v in mutation.items()}
|
|
143
|
+
for test_file, sig in signals.items():
|
|
144
|
+
stem = Path(test_file).stem.lower()
|
|
145
|
+
for suffix in ("test", "tests", "spec", "_test", "_spec", ".test", ".spec"):
|
|
146
|
+
if stem.endswith(suffix):
|
|
147
|
+
candidate = stem[: -len(suffix)].rstrip("_.")
|
|
148
|
+
if candidate in by_stem:
|
|
149
|
+
sig["mutation_score"] = by_stem[candidate]
|
|
150
|
+
break
|
|
151
|
+
if sig["mutation_score"] is None and stem.startswith("test_") and stem[5:] in by_stem:
|
|
152
|
+
sig["mutation_score"] = by_stem[stem[5:]]
|
testledger/cli.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""TestLedger command-line interface.
|
|
2
|
+
|
|
3
|
+
testledger init # write a starter testledger.yml
|
|
4
|
+
testledger analyze --out reports/ # ingest, book, report
|
|
5
|
+
testledger memo reports/ # print the memo to the terminal
|
|
6
|
+
testledger pr-comment reports/ # emit a Markdown comment for a PR
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import glob
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
from datetime import UTC, datetime
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import typer
|
|
18
|
+
from rich.console import Console
|
|
19
|
+
from rich.table import Table
|
|
20
|
+
|
|
21
|
+
from testledger import __version__
|
|
22
|
+
from testledger.aitest import attach_mutation_scores, load_mutation_report, scan_sources
|
|
23
|
+
from testledger.config import Config
|
|
24
|
+
from testledger.ingest import collect_churn, collect_fix_commits, parse_cucumber_json, parse_junit_xml
|
|
25
|
+
from testledger.ingest.gitlog import default_since
|
|
26
|
+
from testledger.ledger import build_ledger
|
|
27
|
+
from testledger.models import TestRun
|
|
28
|
+
from testledger.reports import build_reports, write_reports
|
|
29
|
+
|
|
30
|
+
app = typer.Typer(
|
|
31
|
+
add_completion=False, help="TestLedger — Test Estate Balance Sheet.", invoke_without_command=True
|
|
32
|
+
)
|
|
33
|
+
console = Console()
|
|
34
|
+
|
|
35
|
+
RUN_META = re.compile(r"(?P<sha>[0-9a-f]{7,40})[-_](?:attempt|run|a)(?P<attempt>\d+)$", re.IGNORECASE)
|
|
36
|
+
RUN_DATE = re.compile(
|
|
37
|
+
r"^(?P<y>20\d{2})-?(?P<m>\d{2})-?(?P<d>\d{2})(?:T(?P<H>\d{2}):?(?P<M>\d{2}))?(?=[-_.]|$)"
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _meta_from_path(p: Path) -> tuple[str | None, int, datetime | None]:
|
|
42
|
+
"""Convention for local report folders: reports/<YYYYMMDD>[THHMM]-<sha>-attempt<N>/junit.xml.
|
|
43
|
+
|
|
44
|
+
Date and SHA are optional; without a date the file's mtime is used, and
|
|
45
|
+
without a SHA no rerun/flake detection is possible for that run.
|
|
46
|
+
"""
|
|
47
|
+
sha, attempt, ts = None, 1, None
|
|
48
|
+
for part in reversed(p.parts[:-1]):
|
|
49
|
+
m = RUN_META.search(part)
|
|
50
|
+
if m and sha is None:
|
|
51
|
+
sha, attempt = m.group("sha"), int(m.group("attempt"))
|
|
52
|
+
d = RUN_DATE.search(part)
|
|
53
|
+
if d and ts is None:
|
|
54
|
+
try:
|
|
55
|
+
ts = datetime(
|
|
56
|
+
int(d["y"]),
|
|
57
|
+
int(d["m"]),
|
|
58
|
+
int(d["d"]),
|
|
59
|
+
int(d["H"] or 0),
|
|
60
|
+
int(d["M"] or 0),
|
|
61
|
+
tzinfo=UTC,
|
|
62
|
+
)
|
|
63
|
+
except ValueError:
|
|
64
|
+
ts = None
|
|
65
|
+
if sha and ts:
|
|
66
|
+
break
|
|
67
|
+
return sha, attempt, ts
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _load_local_runs(cfg: Config, root: Path) -> list[TestRun]:
|
|
71
|
+
runs: list[TestRun] = []
|
|
72
|
+
for pattern in cfg.sources.junit_globs:
|
|
73
|
+
for f in glob.glob(str(root / pattern), recursive=True):
|
|
74
|
+
sha, attempt, ts = _meta_from_path(Path(f))
|
|
75
|
+
try:
|
|
76
|
+
runs.append(
|
|
77
|
+
parse_junit_xml(
|
|
78
|
+
f, commit_sha=sha, attempt=attempt, started_at=ts, run_id=str(Path(f).parent.name)
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
except Exception as exc: # noqa: BLE001
|
|
82
|
+
console.print(f"[yellow]skip {f}: {exc}[/yellow]")
|
|
83
|
+
for pattern in cfg.sources.cucumber_globs:
|
|
84
|
+
for f in glob.glob(str(root / pattern), recursive=True):
|
|
85
|
+
sha, attempt, ts = _meta_from_path(Path(f))
|
|
86
|
+
try:
|
|
87
|
+
runs.append(
|
|
88
|
+
parse_cucumber_json(
|
|
89
|
+
f, commit_sha=sha, attempt=attempt, started_at=ts, run_id=str(Path(f).parent.name)
|
|
90
|
+
)
|
|
91
|
+
)
|
|
92
|
+
except Exception as exc: # noqa: BLE001
|
|
93
|
+
console.print(f"[yellow]skip {f}: {exc}[/yellow]")
|
|
94
|
+
return runs
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _load_ci_runs(cfg: Config, since: datetime) -> list[TestRun]:
|
|
98
|
+
runs: list[TestRun] = []
|
|
99
|
+
gh = cfg.sources.github
|
|
100
|
+
if gh and gh.get("repo"):
|
|
101
|
+
from testledger.ingest.github_actions import fetch_runs
|
|
102
|
+
|
|
103
|
+
console.print(f"Fetching GitHub Actions runs for {gh['repo']} …")
|
|
104
|
+
runs += fetch_runs(
|
|
105
|
+
gh["repo"],
|
|
106
|
+
workflow=gh.get("workflow"),
|
|
107
|
+
since=since,
|
|
108
|
+
token_env=gh.get("token_env", "GITHUB_TOKEN"),
|
|
109
|
+
max_runs=int(gh.get("max_runs", 200)),
|
|
110
|
+
artifact_name_contains=gh.get("artifact_name_contains", "test"),
|
|
111
|
+
)
|
|
112
|
+
jk = cfg.sources.jenkins
|
|
113
|
+
if jk and jk.get("url") and jk.get("job"):
|
|
114
|
+
from testledger.ingest.jenkins import fetch_builds
|
|
115
|
+
|
|
116
|
+
console.print(f"Fetching Jenkins builds for {jk['job']} …")
|
|
117
|
+
runs += fetch_builds(
|
|
118
|
+
jk["url"],
|
|
119
|
+
jk["job"],
|
|
120
|
+
since=since,
|
|
121
|
+
user_env=jk.get("user_env", "JENKINS_USER"),
|
|
122
|
+
token_env=jk.get("token_env", "JENKINS_TOKEN"),
|
|
123
|
+
max_builds=int(jk.get("max_builds", 100)),
|
|
124
|
+
)
|
|
125
|
+
return runs
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@app.callback()
|
|
129
|
+
def _root(
|
|
130
|
+
ctx: typer.Context,
|
|
131
|
+
version: bool = typer.Option(False, "--version", "-V", help="Show version and exit"),
|
|
132
|
+
) -> None:
|
|
133
|
+
if version:
|
|
134
|
+
console.print(f"testledger {__version__}")
|
|
135
|
+
raise typer.Exit()
|
|
136
|
+
if ctx.invoked_subcommand is None:
|
|
137
|
+
console.print(ctx.get_help())
|
|
138
|
+
raise typer.Exit()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@app.command()
|
|
142
|
+
def init(
|
|
143
|
+
path: Path = typer.Option(Path("testledger.yml"), "--path", help="Where to write the config"),
|
|
144
|
+
) -> None:
|
|
145
|
+
"""Write a starter testledger.yml with documented defaults."""
|
|
146
|
+
if path.exists():
|
|
147
|
+
console.print(f"[red]{path} already exists[/red]")
|
|
148
|
+
raise typer.Exit(1)
|
|
149
|
+
path.write_text(Config().dump_example(), encoding="utf-8")
|
|
150
|
+
console.print(f"Wrote {path}. Edit rates and sources, then run: testledger analyze")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@app.command()
|
|
154
|
+
def analyze(
|
|
155
|
+
config: Path | None = typer.Option(None, "--config", "-c", help="testledger.yml (defaults if omitted)"),
|
|
156
|
+
reports_root: Path = typer.Option(
|
|
157
|
+
Path("."), "--reports", "-r", help="Folder searched for local JUnit/Cucumber reports"
|
|
158
|
+
),
|
|
159
|
+
out: Path = typer.Option(Path("testledger-out"), "--out", "-o", help="Output folder"),
|
|
160
|
+
mutation_report: Path | None = typer.Option(
|
|
161
|
+
None, "--mutation", help="PIT XML / Stryker JSON / {file: score} JSON"
|
|
162
|
+
),
|
|
163
|
+
skip_ci: bool = typer.Option(False, "--skip-ci", help="Do not call GitHub/Jenkins APIs"),
|
|
164
|
+
skip_git: bool = typer.Option(False, "--skip-git", help="Do not run git for churn/fix analysis"),
|
|
165
|
+
fail_on_quarantine: bool = typer.Option(
|
|
166
|
+
False, "--fail-on-quarantine", help="Exit 2 if any test is quarantine-grade"
|
|
167
|
+
),
|
|
168
|
+
) -> None:
|
|
169
|
+
"""Ingest test results + git history, book the ledger, write reports."""
|
|
170
|
+
cfg = Config.load(config) if config else Config()
|
|
171
|
+
now = datetime.now(tz=UTC)
|
|
172
|
+
since = default_since(cfg.window_days)
|
|
173
|
+
|
|
174
|
+
runs = _load_local_runs(cfg, reports_root)
|
|
175
|
+
if not skip_ci:
|
|
176
|
+
runs += _load_ci_runs(cfg, since)
|
|
177
|
+
if not runs:
|
|
178
|
+
console.print(
|
|
179
|
+
"[red]No test runs found. Point --reports at JUnit/Cucumber output or configure sources.github / sources.jenkins.[/red]"
|
|
180
|
+
)
|
|
181
|
+
raise typer.Exit(1)
|
|
182
|
+
|
|
183
|
+
churn, fixes, ai_signals = {}, [], {}
|
|
184
|
+
repo = cfg.sources.git_repo
|
|
185
|
+
if repo and not skip_git:
|
|
186
|
+
churn = collect_churn(repo, since)
|
|
187
|
+
fixes = collect_fix_commits(repo, since)
|
|
188
|
+
ai_signals = scan_sources(repo, cfg.ai_generated_markers)
|
|
189
|
+
if mutation_report:
|
|
190
|
+
attach_mutation_scores(ai_signals, load_mutation_report(mutation_report))
|
|
191
|
+
|
|
192
|
+
ledger = build_ledger(runs, cfg, churn=churn, fixes=fixes, ai_signals=ai_signals, now=now)
|
|
193
|
+
bundle = build_reports(ledger, cfg)
|
|
194
|
+
written = write_reports(bundle, out)
|
|
195
|
+
_print_summary(bundle.summary)
|
|
196
|
+
console.print(f"\nReports written to [bold]{out}[/bold]: " + ", ".join(written))
|
|
197
|
+
if fail_on_quarantine and bundle.summary["verdicts"].get("quarantine"):
|
|
198
|
+
raise typer.Exit(2)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _print_summary(s: dict) -> None:
|
|
202
|
+
t = Table(title=f"TestLedger — {s['org']} (last {s['window_days']} days)")
|
|
203
|
+
t.add_column("Metric")
|
|
204
|
+
t.add_column("Value", justify="right")
|
|
205
|
+
for k in (
|
|
206
|
+
"runs_analysed",
|
|
207
|
+
"tests",
|
|
208
|
+
"total_cost_usd",
|
|
209
|
+
"total_value_usd",
|
|
210
|
+
"net_usd",
|
|
211
|
+
"portfolio_roi",
|
|
212
|
+
"regressions_caught",
|
|
213
|
+
"annualised_writeoff_savings_usd",
|
|
214
|
+
"annualised_demote_savings_usd",
|
|
215
|
+
"ai_unverified_liabilities",
|
|
216
|
+
):
|
|
217
|
+
t.add_row(k.replace("_", " "), str(s.get(k)))
|
|
218
|
+
for v, n in sorted(s["verdicts"].items()):
|
|
219
|
+
t.add_row(f"verdict: {v}", str(n))
|
|
220
|
+
console.print(t)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@app.command()
|
|
224
|
+
def memo(out: Path = typer.Argument(Path("testledger-out"), help="Output folder from `analyze`")) -> None:
|
|
225
|
+
"""Print the Quality Investment Memo (Markdown)."""
|
|
226
|
+
p = out / "memo.md"
|
|
227
|
+
if not p.exists():
|
|
228
|
+
console.print(f"[red]{p} not found — run `testledger analyze` first[/red]")
|
|
229
|
+
raise typer.Exit(1)
|
|
230
|
+
console.print(p.read_text(encoding="utf-8"))
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@app.command("pr-comment")
|
|
234
|
+
def pr_comment(
|
|
235
|
+
out: Path = typer.Argument(Path("testledger-out")),
|
|
236
|
+
max_rows: int = typer.Option(10, help="Rows in the write-off table"),
|
|
237
|
+
) -> None:
|
|
238
|
+
"""Emit a compact Markdown comment for pull requests / Slack."""
|
|
239
|
+
s = json.loads((out / "summary.json").read_text(encoding="utf-8"))
|
|
240
|
+
w = json.loads((out / "writeoffs.json").read_text(encoding="utf-8"))[:max_rows]
|
|
241
|
+
lines = [
|
|
242
|
+
"### 📒 TestLedger — Test Estate Balance Sheet",
|
|
243
|
+
f"**Net:** ${s['net_usd']:,.0f} on ${s['total_cost_usd']:,.0f} carrying cost · ROI {s['portfolio_roi']} · {s['regressions_caught']} regressions caught in {s['window_days']} days",
|
|
244
|
+
f"Annualised savings available: **${s['annualised_writeoff_savings_usd'] + s['annualised_demote_savings_usd']:,.0f}** "
|
|
245
|
+
f"({s['verdicts'].get('retire', 0) + s['verdicts'].get('quarantine', 0)} retire/quarantine, {s['verdicts'].get('demote', 0)} demote)",
|
|
246
|
+
"",
|
|
247
|
+
"| Test | Verdict | Flake | Cost | Value |",
|
|
248
|
+
"|---|---|---:|---:|---:|",
|
|
249
|
+
]
|
|
250
|
+
for a in w:
|
|
251
|
+
lines.append(
|
|
252
|
+
f"| `{a['test_id']}` | {a['verdict']} | {a['flake_rate'] * 100:.0f}% | ${a['cost_usd']:.2f} | ${a['value_usd']:.2f} |"
|
|
253
|
+
)
|
|
254
|
+
if s.get("ai_unverified_liabilities"):
|
|
255
|
+
lines.append(
|
|
256
|
+
f"\n⚠️ {s['ai_unverified_liabilities']} AI-generated tests are below governance thresholds (see `ai_liabilities.json`)."
|
|
257
|
+
)
|
|
258
|
+
typer.echo("\n".join(lines))
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
if __name__ == "__main__":
|
|
262
|
+
app()
|
testledger/config.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Configuration: rates, thresholds and input sources, loaded from ``testledger.yml``.
|
|
2
|
+
|
|
3
|
+
Every number here is an *assumption the organisation owns*. Defaults are
|
|
4
|
+
conservative industry figures; the memo prints them so finance can challenge them.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import yaml
|
|
13
|
+
from pydantic import BaseModel, Field
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Rates(BaseModel):
|
|
17
|
+
engineer_hourly_usd: float = Field(default=75.0, description="Fully loaded engineer cost per hour")
|
|
18
|
+
ci_minute_usd: float = Field(
|
|
19
|
+
default=0.008, description="CI compute cost per minute (GitHub-hosted Linux default)"
|
|
20
|
+
)
|
|
21
|
+
triage_minutes_per_false_alarm: float = Field(
|
|
22
|
+
default=18.0, description="Avg minutes an engineer spends on a flaky failure"
|
|
23
|
+
)
|
|
24
|
+
maintenance_minutes_per_line: float = Field(
|
|
25
|
+
default=1.5, description="Avg engineer minutes per changed test line"
|
|
26
|
+
)
|
|
27
|
+
regression_value_usd: float = Field(
|
|
28
|
+
default=2000.0, description="Avg cost of a defect reaching production"
|
|
29
|
+
)
|
|
30
|
+
risk_coverage_value_usd: float = Field(
|
|
31
|
+
default=0.0, description="Optional baseline value per stable test per window"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Thresholds(BaseModel):
|
|
36
|
+
quarantine_flake_rate: float = 0.30
|
|
37
|
+
fix_flake_rate: float = 0.10
|
|
38
|
+
slow_test_seconds: float = 60.0
|
|
39
|
+
min_executions_for_verdict: int = 5
|
|
40
|
+
asset_roi: float = 0.5
|
|
41
|
+
retire_min_cost_usd: float = Field(
|
|
42
|
+
default=25.0, description="Tests cheaper than this per window are never 'retire', only 'hold'"
|
|
43
|
+
)
|
|
44
|
+
demote_layer_cost_ratio: float = 3.0
|
|
45
|
+
ai_min_mutation_score: float = 0.5
|
|
46
|
+
ai_min_assertion_density: float = 0.8
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class LayerRules(BaseModel):
|
|
50
|
+
"""Regex-free glob/substring rules that map test ids or paths to pyramid layers."""
|
|
51
|
+
|
|
52
|
+
unit: list[str] = Field(default_factory=lambda: ["unit", "/test/", "Test."])
|
|
53
|
+
service: list[str] = Field(default_factory=lambda: ["api", "service", "integration", "contract"])
|
|
54
|
+
ui: list[str] = Field(
|
|
55
|
+
default_factory=lambda: ["ui", "e2e", "selenium", "playwright", "cucumber", "features/"]
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Sources(BaseModel):
|
|
60
|
+
junit_globs: list[str] = Field(
|
|
61
|
+
default_factory=lambda: ["**/surefire-reports/*.xml", "**/junit*.xml", "**/TEST-*.xml"]
|
|
62
|
+
)
|
|
63
|
+
cucumber_globs: list[str] = Field(default_factory=lambda: ["**/cucumber*.json"])
|
|
64
|
+
git_repo: str | None = Field(
|
|
65
|
+
default=".", description="Path to the repo for churn analysis; null disables"
|
|
66
|
+
)
|
|
67
|
+
github: dict[str, Any] | None = Field(
|
|
68
|
+
default=None,
|
|
69
|
+
description="{'repo': 'owner/name', 'workflow': 'ci.yml', 'token_env': 'GITHUB_TOKEN', 'max_runs': 200}",
|
|
70
|
+
)
|
|
71
|
+
jenkins: dict[str, Any] | None = Field(
|
|
72
|
+
default=None,
|
|
73
|
+
description="{'url': 'https://ci.acme.com', 'job': 'acme-main', 'user_env': 'JENKINS_USER', 'token_env': 'JENKINS_TOKEN', 'max_builds': 100}",
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class Config(BaseModel):
|
|
78
|
+
org_name: str = "Your Organisation"
|
|
79
|
+
window_days: int = 90
|
|
80
|
+
rates: Rates = Field(default_factory=Rates)
|
|
81
|
+
thresholds: Thresholds = Field(default_factory=Thresholds)
|
|
82
|
+
layers: LayerRules = Field(default_factory=LayerRules)
|
|
83
|
+
sources: Sources = Field(default_factory=Sources)
|
|
84
|
+
ai_generated_markers: list[str] = Field(
|
|
85
|
+
default_factory=lambda: [
|
|
86
|
+
"@ai-generated",
|
|
87
|
+
"Generated-By:",
|
|
88
|
+
"Co-Authored-By: Copilot",
|
|
89
|
+
"Co-Authored-By: Claude",
|
|
90
|
+
],
|
|
91
|
+
description="Strings in test source or commit messages that mark a test as AI-generated",
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
@classmethod
|
|
95
|
+
def load(cls, path: str | Path | None) -> Config:
|
|
96
|
+
if path is None:
|
|
97
|
+
return cls()
|
|
98
|
+
p = Path(path)
|
|
99
|
+
if not p.exists():
|
|
100
|
+
raise FileNotFoundError(f"Config not found: {p}")
|
|
101
|
+
data = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
|
|
102
|
+
return cls.model_validate(data)
|
|
103
|
+
|
|
104
|
+
def dump_example(self) -> str:
|
|
105
|
+
return yaml.safe_dump(self.model_dump(mode="json"), sort_keys=False)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Ingesters normalise heterogeneous CI/test sources into ``TestRun`` objects."""
|
|
2
|
+
|
|
3
|
+
from testledger.ingest.cucumber import parse_cucumber_json
|
|
4
|
+
from testledger.ingest.gitlog import collect_churn, collect_fix_commits
|
|
5
|
+
from testledger.ingest.junit import parse_junit_xml
|
|
6
|
+
|
|
7
|
+
__all__ = ["parse_junit_xml", "parse_cucumber_json", "collect_churn", "collect_fix_commits"]
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Cucumber JSON report parser (cucumber-jvm, cucumber-ruby, cucumber-js share the schema).
|
|
2
|
+
|
|
3
|
+
A scenario is one ``TestResult``; its outcome is the worst step outcome.
|
|
4
|
+
Scenario Outline examples carry distinct ids so each example row is a test.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from datetime import UTC, datetime
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from testledger.models import Outcome, TestResult, TestRun
|
|
14
|
+
|
|
15
|
+
_STEP_RANK = {"passed": 0, "skipped": 1, "pending": 1, "undefined": 2, "failed": 3}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def parse_cucumber_json(
|
|
19
|
+
path: str | Path,
|
|
20
|
+
*,
|
|
21
|
+
run_id: str | None = None,
|
|
22
|
+
started_at: datetime | None = None,
|
|
23
|
+
commit_sha: str | None = None,
|
|
24
|
+
branch: str | None = None,
|
|
25
|
+
attempt: int = 1,
|
|
26
|
+
) -> TestRun:
|
|
27
|
+
p = Path(path)
|
|
28
|
+
features = json.loads(p.read_text(encoding="utf-8"))
|
|
29
|
+
results: list[TestResult] = []
|
|
30
|
+
|
|
31
|
+
for feature in features:
|
|
32
|
+
feature_name = feature.get("name", p.stem)
|
|
33
|
+
uri = feature.get("uri")
|
|
34
|
+
for element in feature.get("elements", []):
|
|
35
|
+
if element.get("type") == "background":
|
|
36
|
+
continue
|
|
37
|
+
worst = "passed"
|
|
38
|
+
duration_ns = 0
|
|
39
|
+
msg = None
|
|
40
|
+
for step in element.get("steps", []) + element.get("before", []) + element.get("after", []):
|
|
41
|
+
res = step.get("result", {})
|
|
42
|
+
status = res.get("status", "passed")
|
|
43
|
+
duration_ns += int(res.get("duration", 0) or 0)
|
|
44
|
+
if _STEP_RANK.get(status, 0) > _STEP_RANK.get(worst, 0):
|
|
45
|
+
worst = status
|
|
46
|
+
msg = (res.get("error_message") or "")[:300] or None
|
|
47
|
+
if worst == "failed" or worst == "undefined":
|
|
48
|
+
outcome = Outcome.FAILED
|
|
49
|
+
elif worst in ("skipped", "pending"):
|
|
50
|
+
outcome = Outcome.SKIPPED
|
|
51
|
+
else:
|
|
52
|
+
outcome = Outcome.PASSED
|
|
53
|
+
scenario_id = element.get("id") or f"{feature_name};{element.get('name')}"
|
|
54
|
+
tags = [t.get("name", "") for t in element.get("tags", [])]
|
|
55
|
+
results.append(
|
|
56
|
+
TestResult(
|
|
57
|
+
test_id=scenario_id,
|
|
58
|
+
name=element.get("name", "unnamed scenario"),
|
|
59
|
+
suite=feature_name,
|
|
60
|
+
file=uri,
|
|
61
|
+
outcome=outcome,
|
|
62
|
+
duration_s=duration_ns / 1e9,
|
|
63
|
+
message=msg,
|
|
64
|
+
tags=tags,
|
|
65
|
+
)
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
ts = started_at or datetime.fromtimestamp(p.stat().st_mtime, tz=UTC)
|
|
69
|
+
if ts.tzinfo is None:
|
|
70
|
+
ts = ts.replace(tzinfo=UTC)
|
|
71
|
+
return TestRun(
|
|
72
|
+
run_id=run_id or p.stem,
|
|
73
|
+
started_at=ts,
|
|
74
|
+
commit_sha=commit_sha,
|
|
75
|
+
branch=branch,
|
|
76
|
+
attempt=attempt,
|
|
77
|
+
source="cucumber",
|
|
78
|
+
results=results,
|
|
79
|
+
)
|