testledger 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
testledger/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ """TestLedger — Test Estate Balance Sheet.
2
+
3
+ Treats an automated test suite as an investment portfolio: every test carries a
4
+ cost (CI minutes, reruns, triage, maintenance churn) and produces a return
5
+ (regressions caught). TestLedger ingests CI results and git history, books
6
+ every cost and value entry into a double-entry style ledger, and renders
7
+ executive-facing reports: a Balance Sheet, an Income Statement, a Write-off
8
+ list and a one-page Quality Investment Memo.
9
+ """
10
+
11
+ __version__ = "0.1.0"
@@ -0,0 +1,3 @@
1
+ from testledger.aitest.detector import attach_mutation_scores, load_mutation_report, scan_sources
2
+
3
+ __all__ = ["scan_sources", "load_mutation_report", "attach_mutation_scores"]
@@ -0,0 +1,152 @@
1
+ """AI-generated test governance ("test theater" detection).
2
+
3
+ Three signals, each cheap and language-agnostic:
4
+
5
+ 1. **Provenance** — the test file contains a configured marker
6
+ (``@ai-generated``, ``Generated-By:``, Copilot/Claude co-author trailers)
7
+ or its last commit message does.
8
+ 2. **Assertion density** — assertions per test function. Tests with density
9
+ < threshold are likely "smoke only" and prove little. Detects common
10
+ assertion idioms across Java (JUnit/TestNG/AssertJ/Hamcrest), Python,
11
+ JavaScript/TypeScript, Ruby (RSpec/Minitest), C#, Go.
12
+ 3. **Mutation score** — read from PIT (Java), Stryker (JS/TS/C#),
13
+ mutmut/cosmic-ray (Python) reports. A high line-coverage, low
14
+ mutation-score test is the signature of an assertion that mirrors the
15
+ implementation rather than checking it.
16
+
17
+ Output is a mapping ``{file_or_test_id: {"ai_generated": bool,
18
+ "assertion_density": float, "mutation_score": float | None}}`` that the ledger
19
+ engine merges into each account. The memo lists AI-generated tests that fall
20
+ below the governance thresholds as **unverified liabilities**.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ import re
27
+ import subprocess
28
+ import xml.etree.ElementTree as ET
29
+ from pathlib import Path
30
+
31
+ TEST_FUNC_PATTERNS = [
32
+ re.compile(r"@Test\b"), # JUnit / TestNG
33
+ re.compile(r"^\s*def\s+test_\w+", re.M), # pytest / unittest
34
+ re.compile(r"\b(it|test)\s*\(\s*['\"`]", re.M), # jest / mocha / vitest
35
+ re.compile(r"^\s*(it|scenario|specify)\s+['\"]", re.M), # rspec
36
+ re.compile(r"\[(Fact|Theory|Test|TestMethod)\]"), # xUnit / NUnit / MSTest
37
+ re.compile(r"^func\s+Test\w+\s*\(", re.M), # go
38
+ ]
39
+ ASSERTION_PATTERNS = [
40
+ re.compile(r"\bassert\w*\s*\("), # assertEquals(, assertTrue(, assert(
41
+ re.compile(r"\bassertThat\s*\("),
42
+ re.compile(r"^\s*assert\s+\S", re.M), # python bare assert
43
+ re.compile(r"\bexpect\s*\("), # jest / rspec / chai
44
+ re.compile(r"\bshould\b"),
45
+ re.compile(r"\bAssert\.\w+\s*\("), # C#
46
+ re.compile(r"\bverify\s*\("), # mockito interaction verification
47
+ re.compile(r"\.(toBe|toEqual|toMatch|toHaveLength|toThrow|isEqualTo|hasSize|contains|isTrue|isFalse)\b"),
48
+ re.compile(r"\bt\.(Errorf|Fatalf|Error|Fatal)\s*\("), # go
49
+ ]
50
+ SOURCE_EXT = {".java", ".kt", ".py", ".js", ".ts", ".tsx", ".jsx", ".rb", ".cs", ".go", ".scala"}
51
+
52
+
53
+ def assertion_density(text: str) -> tuple[int, int, float]:
54
+ tests = sum(len(p.findall(text)) for p in TEST_FUNC_PATTERNS)
55
+ asserts = sum(len(p.findall(text)) for p in ASSERTION_PATTERNS)
56
+ density = round(asserts / tests, 2) if tests else 0.0
57
+ return tests, asserts, density
58
+
59
+
60
+ def _last_commit_message(repo: Path, file: Path) -> str:
61
+ try:
62
+ out = subprocess.run(
63
+ ["git", "-C", str(repo), "log", "-1", "--format=%B", "--", str(file.relative_to(repo))],
64
+ capture_output=True,
65
+ text=True,
66
+ encoding="utf-8",
67
+ errors="replace",
68
+ check=False,
69
+ )
70
+ return out.stdout if out.returncode == 0 else ""
71
+ except (OSError, ValueError):
72
+ return ""
73
+
74
+
75
+ def scan_sources(
76
+ repo: str | Path, markers: list[str], *, test_dirs: list[str] | None = None
77
+ ) -> dict[str, dict]:
78
+ """Walk test source files and compute provenance + assertion density."""
79
+ repo = Path(repo)
80
+ test_dirs = test_dirs or ["test", "tests", "src/test", "spec", "features", "e2e", "__tests__"]
81
+ signals: dict[str, dict] = {}
82
+ seen: set[Path] = set()
83
+ for d in test_dirs:
84
+ base = repo / d
85
+ if not base.exists():
86
+ continue
87
+ for path in base.rglob("*"):
88
+ if path.suffix.lower() not in SOURCE_EXT or not path.is_file() or path in seen:
89
+ continue
90
+ seen.add(path)
91
+ text = path.read_text(encoding="utf-8", errors="replace")
92
+ tests, asserts, density = assertion_density(text)
93
+ if tests == 0:
94
+ continue
95
+ commit_msg = _last_commit_message(repo, path)
96
+ ai = any(m in text or m in commit_msg for m in markers)
97
+ rel = path.relative_to(repo).as_posix()
98
+ signals[rel] = {
99
+ "ai_generated": ai,
100
+ "test_functions": tests,
101
+ "assertions": asserts,
102
+ "assertion_density": density,
103
+ "mutation_score": None,
104
+ }
105
+ return signals
106
+
107
+
108
+ def load_mutation_report(path: str | Path) -> dict[str, float]:
109
+ """Return {source_file: mutation_score} from PIT XML, Stryker JSON or mutmut JSON."""
110
+ p = Path(path)
111
+ if not p.exists():
112
+ return {}
113
+ if p.suffix.lower() == ".xml": # PIT mutations.xml
114
+ root = ET.parse(p).getroot()
115
+ totals: dict[str, list[int]] = {}
116
+ for m in root.iter("mutation"):
117
+ src = m.findtext("sourceFile") or "unknown"
118
+ killed = (m.get("status") or "").upper() in ("KILLED", "TIMED_OUT")
119
+ t = totals.setdefault(src, [0, 0])
120
+ t[0] += 1
121
+ t[1] += int(killed)
122
+ return {k: round(v[1] / v[0], 3) for k, v in totals.items() if v[0]}
123
+ data = json.loads(p.read_text(encoding="utf-8"))
124
+ if "files" in data: # Stryker mutation-report schema
125
+ out: dict[str, float] = {}
126
+ for file, info in data["files"].items():
127
+ muts = info.get("mutants", [])
128
+ if not muts:
129
+ continue
130
+ killed = sum(1 for m in muts if m.get("status") in ("Killed", "Timeout"))
131
+ out[file] = round(killed / len(muts), 3)
132
+ return out
133
+ if isinstance(data, dict) and all(isinstance(v, (int, float)) for v in data.values()):
134
+ return {k: float(v) for k, v in data.items()} # simple {file: score}
135
+ return {}
136
+
137
+
138
+ def attach_mutation_scores(signals: dict[str, dict], mutation: dict[str, float]) -> None:
139
+ """Best-effort join: a test file 'FooTest.java' is matched to source 'Foo.java'."""
140
+ if not mutation:
141
+ return
142
+ by_stem = {Path(k).stem.lower(): v for k, v in mutation.items()}
143
+ for test_file, sig in signals.items():
144
+ stem = Path(test_file).stem.lower()
145
+ for suffix in ("test", "tests", "spec", "_test", "_spec", ".test", ".spec"):
146
+ if stem.endswith(suffix):
147
+ candidate = stem[: -len(suffix)].rstrip("_.")
148
+ if candidate in by_stem:
149
+ sig["mutation_score"] = by_stem[candidate]
150
+ break
151
+ if sig["mutation_score"] is None and stem.startswith("test_") and stem[5:] in by_stem:
152
+ sig["mutation_score"] = by_stem[stem[5:]]
testledger/cli.py ADDED
@@ -0,0 +1,262 @@
1
+ """TestLedger command-line interface.
2
+
3
+ testledger init # write a starter testledger.yml
4
+ testledger analyze --out reports/ # ingest, book, report
5
+ testledger memo reports/ # print the memo to the terminal
6
+ testledger pr-comment reports/ # emit a Markdown comment for a PR
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import glob
12
+ import json
13
+ import re
14
+ from datetime import UTC, datetime
15
+ from pathlib import Path
16
+
17
+ import typer
18
+ from rich.console import Console
19
+ from rich.table import Table
20
+
21
+ from testledger import __version__
22
+ from testledger.aitest import attach_mutation_scores, load_mutation_report, scan_sources
23
+ from testledger.config import Config
24
+ from testledger.ingest import collect_churn, collect_fix_commits, parse_cucumber_json, parse_junit_xml
25
+ from testledger.ingest.gitlog import default_since
26
+ from testledger.ledger import build_ledger
27
+ from testledger.models import TestRun
28
+ from testledger.reports import build_reports, write_reports
29
+
30
+ app = typer.Typer(
31
+ add_completion=False, help="TestLedger — Test Estate Balance Sheet.", invoke_without_command=True
32
+ )
33
+ console = Console()
34
+
35
+ RUN_META = re.compile(r"(?P<sha>[0-9a-f]{7,40})[-_](?:attempt|run|a)(?P<attempt>\d+)$", re.IGNORECASE)
36
+ RUN_DATE = re.compile(
37
+ r"^(?P<y>20\d{2})-?(?P<m>\d{2})-?(?P<d>\d{2})(?:T(?P<H>\d{2}):?(?P<M>\d{2}))?(?=[-_.]|$)"
38
+ )
39
+
40
+
41
+ def _meta_from_path(p: Path) -> tuple[str | None, int, datetime | None]:
42
+ """Convention for local report folders: reports/<YYYYMMDD>[THHMM]-<sha>-attempt<N>/junit.xml.
43
+
44
+ Date and SHA are optional; without a date the file's mtime is used, and
45
+ without a SHA no rerun/flake detection is possible for that run.
46
+ """
47
+ sha, attempt, ts = None, 1, None
48
+ for part in reversed(p.parts[:-1]):
49
+ m = RUN_META.search(part)
50
+ if m and sha is None:
51
+ sha, attempt = m.group("sha"), int(m.group("attempt"))
52
+ d = RUN_DATE.search(part)
53
+ if d and ts is None:
54
+ try:
55
+ ts = datetime(
56
+ int(d["y"]),
57
+ int(d["m"]),
58
+ int(d["d"]),
59
+ int(d["H"] or 0),
60
+ int(d["M"] or 0),
61
+ tzinfo=UTC,
62
+ )
63
+ except ValueError:
64
+ ts = None
65
+ if sha and ts:
66
+ break
67
+ return sha, attempt, ts
68
+
69
+
70
+ def _load_local_runs(cfg: Config, root: Path) -> list[TestRun]:
71
+ runs: list[TestRun] = []
72
+ for pattern in cfg.sources.junit_globs:
73
+ for f in glob.glob(str(root / pattern), recursive=True):
74
+ sha, attempt, ts = _meta_from_path(Path(f))
75
+ try:
76
+ runs.append(
77
+ parse_junit_xml(
78
+ f, commit_sha=sha, attempt=attempt, started_at=ts, run_id=str(Path(f).parent.name)
79
+ )
80
+ )
81
+ except Exception as exc: # noqa: BLE001
82
+ console.print(f"[yellow]skip {f}: {exc}[/yellow]")
83
+ for pattern in cfg.sources.cucumber_globs:
84
+ for f in glob.glob(str(root / pattern), recursive=True):
85
+ sha, attempt, ts = _meta_from_path(Path(f))
86
+ try:
87
+ runs.append(
88
+ parse_cucumber_json(
89
+ f, commit_sha=sha, attempt=attempt, started_at=ts, run_id=str(Path(f).parent.name)
90
+ )
91
+ )
92
+ except Exception as exc: # noqa: BLE001
93
+ console.print(f"[yellow]skip {f}: {exc}[/yellow]")
94
+ return runs
95
+
96
+
97
+ def _load_ci_runs(cfg: Config, since: datetime) -> list[TestRun]:
98
+ runs: list[TestRun] = []
99
+ gh = cfg.sources.github
100
+ if gh and gh.get("repo"):
101
+ from testledger.ingest.github_actions import fetch_runs
102
+
103
+ console.print(f"Fetching GitHub Actions runs for {gh['repo']} …")
104
+ runs += fetch_runs(
105
+ gh["repo"],
106
+ workflow=gh.get("workflow"),
107
+ since=since,
108
+ token_env=gh.get("token_env", "GITHUB_TOKEN"),
109
+ max_runs=int(gh.get("max_runs", 200)),
110
+ artifact_name_contains=gh.get("artifact_name_contains", "test"),
111
+ )
112
+ jk = cfg.sources.jenkins
113
+ if jk and jk.get("url") and jk.get("job"):
114
+ from testledger.ingest.jenkins import fetch_builds
115
+
116
+ console.print(f"Fetching Jenkins builds for {jk['job']} …")
117
+ runs += fetch_builds(
118
+ jk["url"],
119
+ jk["job"],
120
+ since=since,
121
+ user_env=jk.get("user_env", "JENKINS_USER"),
122
+ token_env=jk.get("token_env", "JENKINS_TOKEN"),
123
+ max_builds=int(jk.get("max_builds", 100)),
124
+ )
125
+ return runs
126
+
127
+
128
+ @app.callback()
129
+ def _root(
130
+ ctx: typer.Context,
131
+ version: bool = typer.Option(False, "--version", "-V", help="Show version and exit"),
132
+ ) -> None:
133
+ if version:
134
+ console.print(f"testledger {__version__}")
135
+ raise typer.Exit()
136
+ if ctx.invoked_subcommand is None:
137
+ console.print(ctx.get_help())
138
+ raise typer.Exit()
139
+
140
+
141
+ @app.command()
142
+ def init(
143
+ path: Path = typer.Option(Path("testledger.yml"), "--path", help="Where to write the config"),
144
+ ) -> None:
145
+ """Write a starter testledger.yml with documented defaults."""
146
+ if path.exists():
147
+ console.print(f"[red]{path} already exists[/red]")
148
+ raise typer.Exit(1)
149
+ path.write_text(Config().dump_example(), encoding="utf-8")
150
+ console.print(f"Wrote {path}. Edit rates and sources, then run: testledger analyze")
151
+
152
+
153
+ @app.command()
154
+ def analyze(
155
+ config: Path | None = typer.Option(None, "--config", "-c", help="testledger.yml (defaults if omitted)"),
156
+ reports_root: Path = typer.Option(
157
+ Path("."), "--reports", "-r", help="Folder searched for local JUnit/Cucumber reports"
158
+ ),
159
+ out: Path = typer.Option(Path("testledger-out"), "--out", "-o", help="Output folder"),
160
+ mutation_report: Path | None = typer.Option(
161
+ None, "--mutation", help="PIT XML / Stryker JSON / {file: score} JSON"
162
+ ),
163
+ skip_ci: bool = typer.Option(False, "--skip-ci", help="Do not call GitHub/Jenkins APIs"),
164
+ skip_git: bool = typer.Option(False, "--skip-git", help="Do not run git for churn/fix analysis"),
165
+ fail_on_quarantine: bool = typer.Option(
166
+ False, "--fail-on-quarantine", help="Exit 2 if any test is quarantine-grade"
167
+ ),
168
+ ) -> None:
169
+ """Ingest test results + git history, book the ledger, write reports."""
170
+ cfg = Config.load(config) if config else Config()
171
+ now = datetime.now(tz=UTC)
172
+ since = default_since(cfg.window_days)
173
+
174
+ runs = _load_local_runs(cfg, reports_root)
175
+ if not skip_ci:
176
+ runs += _load_ci_runs(cfg, since)
177
+ if not runs:
178
+ console.print(
179
+ "[red]No test runs found. Point --reports at JUnit/Cucumber output or configure sources.github / sources.jenkins.[/red]"
180
+ )
181
+ raise typer.Exit(1)
182
+
183
+ churn, fixes, ai_signals = {}, [], {}
184
+ repo = cfg.sources.git_repo
185
+ if repo and not skip_git:
186
+ churn = collect_churn(repo, since)
187
+ fixes = collect_fix_commits(repo, since)
188
+ ai_signals = scan_sources(repo, cfg.ai_generated_markers)
189
+ if mutation_report:
190
+ attach_mutation_scores(ai_signals, load_mutation_report(mutation_report))
191
+
192
+ ledger = build_ledger(runs, cfg, churn=churn, fixes=fixes, ai_signals=ai_signals, now=now)
193
+ bundle = build_reports(ledger, cfg)
194
+ written = write_reports(bundle, out)
195
+ _print_summary(bundle.summary)
196
+ console.print(f"\nReports written to [bold]{out}[/bold]: " + ", ".join(written))
197
+ if fail_on_quarantine and bundle.summary["verdicts"].get("quarantine"):
198
+ raise typer.Exit(2)
199
+
200
+
201
+ def _print_summary(s: dict) -> None:
202
+ t = Table(title=f"TestLedger — {s['org']} (last {s['window_days']} days)")
203
+ t.add_column("Metric")
204
+ t.add_column("Value", justify="right")
205
+ for k in (
206
+ "runs_analysed",
207
+ "tests",
208
+ "total_cost_usd",
209
+ "total_value_usd",
210
+ "net_usd",
211
+ "portfolio_roi",
212
+ "regressions_caught",
213
+ "annualised_writeoff_savings_usd",
214
+ "annualised_demote_savings_usd",
215
+ "ai_unverified_liabilities",
216
+ ):
217
+ t.add_row(k.replace("_", " "), str(s.get(k)))
218
+ for v, n in sorted(s["verdicts"].items()):
219
+ t.add_row(f"verdict: {v}", str(n))
220
+ console.print(t)
221
+
222
+
223
+ @app.command()
224
+ def memo(out: Path = typer.Argument(Path("testledger-out"), help="Output folder from `analyze`")) -> None:
225
+ """Print the Quality Investment Memo (Markdown)."""
226
+ p = out / "memo.md"
227
+ if not p.exists():
228
+ console.print(f"[red]{p} not found — run `testledger analyze` first[/red]")
229
+ raise typer.Exit(1)
230
+ console.print(p.read_text(encoding="utf-8"))
231
+
232
+
233
+ @app.command("pr-comment")
234
+ def pr_comment(
235
+ out: Path = typer.Argument(Path("testledger-out")),
236
+ max_rows: int = typer.Option(10, help="Rows in the write-off table"),
237
+ ) -> None:
238
+ """Emit a compact Markdown comment for pull requests / Slack."""
239
+ s = json.loads((out / "summary.json").read_text(encoding="utf-8"))
240
+ w = json.loads((out / "writeoffs.json").read_text(encoding="utf-8"))[:max_rows]
241
+ lines = [
242
+ "### 📒 TestLedger — Test Estate Balance Sheet",
243
+ f"**Net:** ${s['net_usd']:,.0f} on ${s['total_cost_usd']:,.0f} carrying cost · ROI {s['portfolio_roi']} · {s['regressions_caught']} regressions caught in {s['window_days']} days",
244
+ f"Annualised savings available: **${s['annualised_writeoff_savings_usd'] + s['annualised_demote_savings_usd']:,.0f}** "
245
+ f"({s['verdicts'].get('retire', 0) + s['verdicts'].get('quarantine', 0)} retire/quarantine, {s['verdicts'].get('demote', 0)} demote)",
246
+ "",
247
+ "| Test | Verdict | Flake | Cost | Value |",
248
+ "|---|---|---:|---:|---:|",
249
+ ]
250
+ for a in w:
251
+ lines.append(
252
+ f"| `{a['test_id']}` | {a['verdict']} | {a['flake_rate'] * 100:.0f}% | ${a['cost_usd']:.2f} | ${a['value_usd']:.2f} |"
253
+ )
254
+ if s.get("ai_unverified_liabilities"):
255
+ lines.append(
256
+ f"\n⚠️ {s['ai_unverified_liabilities']} AI-generated tests are below governance thresholds (see `ai_liabilities.json`)."
257
+ )
258
+ typer.echo("\n".join(lines))
259
+
260
+
261
+ if __name__ == "__main__":
262
+ app()
testledger/config.py ADDED
@@ -0,0 +1,105 @@
1
+ """Configuration: rates, thresholds and input sources, loaded from ``testledger.yml``.
2
+
3
+ Every number here is an *assumption the organisation owns*. Defaults are
4
+ conservative industry figures; the memo prints them so finance can challenge them.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import yaml
13
+ from pydantic import BaseModel, Field
14
+
15
+
16
+ class Rates(BaseModel):
17
+ engineer_hourly_usd: float = Field(default=75.0, description="Fully loaded engineer cost per hour")
18
+ ci_minute_usd: float = Field(
19
+ default=0.008, description="CI compute cost per minute (GitHub-hosted Linux default)"
20
+ )
21
+ triage_minutes_per_false_alarm: float = Field(
22
+ default=18.0, description="Avg minutes an engineer spends on a flaky failure"
23
+ )
24
+ maintenance_minutes_per_line: float = Field(
25
+ default=1.5, description="Avg engineer minutes per changed test line"
26
+ )
27
+ regression_value_usd: float = Field(
28
+ default=2000.0, description="Avg cost of a defect reaching production"
29
+ )
30
+ risk_coverage_value_usd: float = Field(
31
+ default=0.0, description="Optional baseline value per stable test per window"
32
+ )
33
+
34
+
35
+ class Thresholds(BaseModel):
36
+ quarantine_flake_rate: float = 0.30
37
+ fix_flake_rate: float = 0.10
38
+ slow_test_seconds: float = 60.0
39
+ min_executions_for_verdict: int = 5
40
+ asset_roi: float = 0.5
41
+ retire_min_cost_usd: float = Field(
42
+ default=25.0, description="Tests cheaper than this per window are never 'retire', only 'hold'"
43
+ )
44
+ demote_layer_cost_ratio: float = 3.0
45
+ ai_min_mutation_score: float = 0.5
46
+ ai_min_assertion_density: float = 0.8
47
+
48
+
49
+ class LayerRules(BaseModel):
50
+ """Regex-free glob/substring rules that map test ids or paths to pyramid layers."""
51
+
52
+ unit: list[str] = Field(default_factory=lambda: ["unit", "/test/", "Test."])
53
+ service: list[str] = Field(default_factory=lambda: ["api", "service", "integration", "contract"])
54
+ ui: list[str] = Field(
55
+ default_factory=lambda: ["ui", "e2e", "selenium", "playwright", "cucumber", "features/"]
56
+ )
57
+
58
+
59
+ class Sources(BaseModel):
60
+ junit_globs: list[str] = Field(
61
+ default_factory=lambda: ["**/surefire-reports/*.xml", "**/junit*.xml", "**/TEST-*.xml"]
62
+ )
63
+ cucumber_globs: list[str] = Field(default_factory=lambda: ["**/cucumber*.json"])
64
+ git_repo: str | None = Field(
65
+ default=".", description="Path to the repo for churn analysis; null disables"
66
+ )
67
+ github: dict[str, Any] | None = Field(
68
+ default=None,
69
+ description="{'repo': 'owner/name', 'workflow': 'ci.yml', 'token_env': 'GITHUB_TOKEN', 'max_runs': 200}",
70
+ )
71
+ jenkins: dict[str, Any] | None = Field(
72
+ default=None,
73
+ description="{'url': 'https://ci.acme.com', 'job': 'acme-main', 'user_env': 'JENKINS_USER', 'token_env': 'JENKINS_TOKEN', 'max_builds': 100}",
74
+ )
75
+
76
+
77
+ class Config(BaseModel):
78
+ org_name: str = "Your Organisation"
79
+ window_days: int = 90
80
+ rates: Rates = Field(default_factory=Rates)
81
+ thresholds: Thresholds = Field(default_factory=Thresholds)
82
+ layers: LayerRules = Field(default_factory=LayerRules)
83
+ sources: Sources = Field(default_factory=Sources)
84
+ ai_generated_markers: list[str] = Field(
85
+ default_factory=lambda: [
86
+ "@ai-generated",
87
+ "Generated-By:",
88
+ "Co-Authored-By: Copilot",
89
+ "Co-Authored-By: Claude",
90
+ ],
91
+ description="Strings in test source or commit messages that mark a test as AI-generated",
92
+ )
93
+
94
+ @classmethod
95
+ def load(cls, path: str | Path | None) -> Config:
96
+ if path is None:
97
+ return cls()
98
+ p = Path(path)
99
+ if not p.exists():
100
+ raise FileNotFoundError(f"Config not found: {p}")
101
+ data = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
102
+ return cls.model_validate(data)
103
+
104
+ def dump_example(self) -> str:
105
+ return yaml.safe_dump(self.model_dump(mode="json"), sort_keys=False)
@@ -0,0 +1,7 @@
1
+ """Ingesters normalise heterogeneous CI/test sources into ``TestRun`` objects."""
2
+
3
+ from testledger.ingest.cucumber import parse_cucumber_json
4
+ from testledger.ingest.gitlog import collect_churn, collect_fix_commits
5
+ from testledger.ingest.junit import parse_junit_xml
6
+
7
+ __all__ = ["parse_junit_xml", "parse_cucumber_json", "collect_churn", "collect_fix_commits"]
@@ -0,0 +1,79 @@
1
+ """Cucumber JSON report parser (cucumber-jvm, cucumber-ruby, cucumber-js share the schema).
2
+
3
+ A scenario is one ``TestResult``; its outcome is the worst step outcome.
4
+ Scenario Outline examples carry distinct ids so each example row is a test.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ from datetime import UTC, datetime
11
+ from pathlib import Path
12
+
13
+ from testledger.models import Outcome, TestResult, TestRun
14
+
15
+ _STEP_RANK = {"passed": 0, "skipped": 1, "pending": 1, "undefined": 2, "failed": 3}
16
+
17
+
18
+ def parse_cucumber_json(
19
+ path: str | Path,
20
+ *,
21
+ run_id: str | None = None,
22
+ started_at: datetime | None = None,
23
+ commit_sha: str | None = None,
24
+ branch: str | None = None,
25
+ attempt: int = 1,
26
+ ) -> TestRun:
27
+ p = Path(path)
28
+ features = json.loads(p.read_text(encoding="utf-8"))
29
+ results: list[TestResult] = []
30
+
31
+ for feature in features:
32
+ feature_name = feature.get("name", p.stem)
33
+ uri = feature.get("uri")
34
+ for element in feature.get("elements", []):
35
+ if element.get("type") == "background":
36
+ continue
37
+ worst = "passed"
38
+ duration_ns = 0
39
+ msg = None
40
+ for step in element.get("steps", []) + element.get("before", []) + element.get("after", []):
41
+ res = step.get("result", {})
42
+ status = res.get("status", "passed")
43
+ duration_ns += int(res.get("duration", 0) or 0)
44
+ if _STEP_RANK.get(status, 0) > _STEP_RANK.get(worst, 0):
45
+ worst = status
46
+ msg = (res.get("error_message") or "")[:300] or None
47
+ if worst == "failed" or worst == "undefined":
48
+ outcome = Outcome.FAILED
49
+ elif worst in ("skipped", "pending"):
50
+ outcome = Outcome.SKIPPED
51
+ else:
52
+ outcome = Outcome.PASSED
53
+ scenario_id = element.get("id") or f"{feature_name};{element.get('name')}"
54
+ tags = [t.get("name", "") for t in element.get("tags", [])]
55
+ results.append(
56
+ TestResult(
57
+ test_id=scenario_id,
58
+ name=element.get("name", "unnamed scenario"),
59
+ suite=feature_name,
60
+ file=uri,
61
+ outcome=outcome,
62
+ duration_s=duration_ns / 1e9,
63
+ message=msg,
64
+ tags=tags,
65
+ )
66
+ )
67
+
68
+ ts = started_at or datetime.fromtimestamp(p.stat().st_mtime, tz=UTC)
69
+ if ts.tzinfo is None:
70
+ ts = ts.replace(tzinfo=UTC)
71
+ return TestRun(
72
+ run_id=run_id or p.stem,
73
+ started_at=ts,
74
+ commit_sha=commit_sha,
75
+ branch=branch,
76
+ attempt=attempt,
77
+ source="cucumber",
78
+ results=results,
79
+ )