testledger 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,145 @@
1
+ """GitHub Actions ingester.
2
+
3
+ Pulls workflow-run history through the REST API, downloads each run's test
4
+ report artifacts (JUnit XML or Cucumber JSON inside a zip), and yields
5
+ ``TestRun`` objects with real commit SHA, attempt number and timestamps.
6
+ Re-runs of the same SHA (``run_attempt`` > 1) are how flakiness is detected.
7
+
8
+ Authentication: a token in the environment variable named by
9
+ ``sources.github.token_env`` (default ``GITHUB_TOKEN``) with ``actions:read``.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import io
15
+ import os
16
+ import zipfile
17
+ from datetime import datetime
18
+ from pathlib import Path
19
+ from tempfile import TemporaryDirectory
20
+
21
+ import requests
22
+
23
+ from testledger.ingest.cucumber import parse_cucumber_json
24
+ from testledger.ingest.junit import parse_junit_xml
25
+ from testledger.models import TestRun
26
+
27
+ API = "https://api.github.com"
28
+
29
+
30
+ class GitHubActionsClient:
31
+ def __init__(self, repo: str, token: str | None = None, session: requests.Session | None = None):
32
+ self.repo = repo
33
+ self.session = session or requests.Session()
34
+ self.session.headers.update(
35
+ {
36
+ "Accept": "application/vnd.github+json",
37
+ "X-GitHub-Api-Version": "2022-11-28",
38
+ "User-Agent": "testledger",
39
+ }
40
+ )
41
+ if token:
42
+ self.session.headers["Authorization"] = f"Bearer {token}"
43
+
44
+ def list_runs(self, workflow: str | None, since: datetime, max_runs: int = 200) -> list[dict]:
45
+ url = f"{API}/repos/{self.repo}/actions/{'workflows/' + workflow + '/' if workflow else ''}runs"
46
+ runs: list[dict] = []
47
+ page = 1
48
+ while len(runs) < max_runs:
49
+ resp = self.session.get(
50
+ url,
51
+ params={"per_page": 100, "page": page, "created": f">={since.date().isoformat()}"},
52
+ timeout=30,
53
+ )
54
+ resp.raise_for_status()
55
+ batch = resp.json().get("workflow_runs", [])
56
+ if not batch:
57
+ break
58
+ runs.extend(batch)
59
+ page += 1
60
+ return runs[:max_runs]
61
+
62
+ def list_artifacts(self, run_id: int) -> list[dict]:
63
+ resp = self.session.get(f"{API}/repos/{self.repo}/actions/runs/{run_id}/artifacts", timeout=30)
64
+ resp.raise_for_status()
65
+ return resp.json().get("artifacts", [])
66
+
67
+ def download_artifact(self, artifact: dict) -> bytes:
68
+ resp = self.session.get(artifact["archive_download_url"], timeout=120)
69
+ resp.raise_for_status()
70
+ return resp.content
71
+
72
+
73
+ def _parse_reports_in_zip(
74
+ blob: bytes, *, run_id: str, started_at: datetime, sha: str, branch: str, attempt: int
75
+ ) -> TestRun | None:
76
+ merged: TestRun | None = None
77
+ with TemporaryDirectory() as tmp, zipfile.ZipFile(io.BytesIO(blob)) as zf:
78
+ zf.extractall(tmp)
79
+ for path in Path(tmp).rglob("*"):
80
+ if not path.is_file():
81
+ continue
82
+ run: TestRun | None = None
83
+ if path.suffix.lower() == ".xml":
84
+ try:
85
+ run = parse_junit_xml(
86
+ path,
87
+ run_id=run_id,
88
+ started_at=started_at,
89
+ commit_sha=sha,
90
+ branch=branch,
91
+ attempt=attempt,
92
+ )
93
+ except Exception: # noqa: BLE001 — unrelated XML in the artifact
94
+ continue
95
+ elif path.suffix.lower() == ".json" and "cucumber" in path.name.lower():
96
+ try:
97
+ run = parse_cucumber_json(
98
+ path,
99
+ run_id=run_id,
100
+ started_at=started_at,
101
+ commit_sha=sha,
102
+ branch=branch,
103
+ attempt=attempt,
104
+ )
105
+ except Exception: # noqa: BLE001
106
+ continue
107
+ if run and run.results:
108
+ if merged is None:
109
+ merged = run
110
+ else:
111
+ merged.results.extend(run.results)
112
+ return merged
113
+
114
+
115
+ def fetch_runs(
116
+ repo: str,
117
+ *,
118
+ workflow: str | None,
119
+ since: datetime,
120
+ token_env: str = "GITHUB_TOKEN",
121
+ max_runs: int = 200,
122
+ artifact_name_contains: str = "test",
123
+ client: GitHubActionsClient | None = None,
124
+ ) -> list[TestRun]:
125
+ client = client or GitHubActionsClient(repo, token=os.environ.get(token_env))
126
+ runs: list[TestRun] = []
127
+ for wr in client.list_runs(workflow, since, max_runs):
128
+ if wr.get("status") != "completed":
129
+ continue
130
+ for art in client.list_artifacts(wr["id"]):
131
+ if artifact_name_contains.lower() not in art["name"].lower() or art.get("expired"):
132
+ continue
133
+ blob = client.download_artifact(art)
134
+ run = _parse_reports_in_zip(
135
+ blob,
136
+ run_id=f"gha-{wr['id']}-{wr.get('run_attempt', 1)}",
137
+ started_at=datetime.fromisoformat(wr["run_started_at"].replace("Z", "+00:00")),
138
+ sha=wr["head_sha"],
139
+ branch=wr.get("head_branch") or "",
140
+ attempt=int(wr.get("run_attempt", 1)),
141
+ )
142
+ if run:
143
+ run.source = "github-actions"
144
+ runs.append(run)
145
+ return runs
@@ -0,0 +1,118 @@
1
+ """Git history analysis via the ``git`` CLI (works on Windows, macOS, Linux).
2
+
3
+ Two signals are extracted:
4
+ * **Churn** on test files — how often and how heavily each test file was edited.
5
+ Heavy churn is the maintenance cost of a test.
6
+ * **Fix commits** on product code — commits whose message looks like a bug fix.
7
+ A test that failed shortly *before* such a commit touched related code is
8
+ credited with having caught the regression.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import re
14
+ import shutil
15
+ import subprocess
16
+ from datetime import UTC, datetime, timedelta
17
+ from pathlib import Path
18
+
19
+ from testledger.models import ChurnRecord, FixCommit
20
+
21
+ FIX_PATTERN = re.compile(r"\b(fix|fixes|fixed|bug|hotfix|regression|defect|patch)\b", re.IGNORECASE)
22
+ TEST_PATH_PATTERN = re.compile(
23
+ r"(^|/)(test|tests|spec|specs|features|e2e|__tests__)(/|$)|_test\.|Test\.|\.spec\.|\.feature$",
24
+ re.IGNORECASE,
25
+ )
26
+
27
+
28
+ def git_available() -> bool:
29
+ return shutil.which("git") is not None
30
+
31
+
32
+ def _git(repo: Path, *args: str) -> str:
33
+ out = subprocess.run(
34
+ ["git", "-C", str(repo), *args],
35
+ capture_output=True,
36
+ text=True,
37
+ encoding="utf-8",
38
+ errors="replace",
39
+ check=False,
40
+ )
41
+ if out.returncode != 0:
42
+ raise RuntimeError(f"git {' '.join(args)} failed: {out.stderr.strip()}")
43
+ return out.stdout
44
+
45
+
46
+ def collect_churn(repo: str | Path, since: datetime) -> dict[str, ChurnRecord]:
47
+ """Per-file churn for test files touched since ``since``."""
48
+ repo = Path(repo)
49
+ if not git_available() or not (repo / ".git").exists():
50
+ return {}
51
+ log = _git(
52
+ repo,
53
+ "log",
54
+ f"--since={since.date().isoformat()}",
55
+ "--numstat",
56
+ "--diff-filter=M", # modifications only: authoring a new test is not maintenance
57
+ "--format=COMMIT%x09%H%x09%an%x09%aI",
58
+ )
59
+ records: dict[str, ChurnRecord] = {}
60
+ authors: dict[str, set[str]] = {}
61
+ current_author = ""
62
+ current_ts: datetime | None = None
63
+ for line in log.splitlines():
64
+ if line.startswith("COMMIT\t"):
65
+ _, _sha, current_author, ts = line.split("\t")
66
+ current_ts = datetime.fromisoformat(ts)
67
+ continue
68
+ parts = line.split("\t")
69
+ if len(parts) != 3:
70
+ continue
71
+ added, deleted, path = parts
72
+ if not TEST_PATH_PATTERN.search(path):
73
+ continue
74
+ rec = records.setdefault(path, ChurnRecord(file=path))
75
+ rec.commits += 1
76
+ rec.lines_changed += (int(added) if added.isdigit() else 0) + (
77
+ int(deleted) if deleted.isdigit() else 0
78
+ )
79
+ authors.setdefault(path, set()).add(current_author)
80
+ if current_ts and (rec.last_touched is None or current_ts > rec.last_touched):
81
+ rec.last_touched = current_ts
82
+ for path, rec in records.items():
83
+ rec.authors = len(authors.get(path, set()))
84
+ return records
85
+
86
+
87
+ def collect_fix_commits(repo: str | Path, since: datetime) -> list[FixCommit]:
88
+ """Product-code commits whose message indicates a bug fix."""
89
+ repo = Path(repo)
90
+ if not git_available() or not (repo / ".git").exists():
91
+ return []
92
+ log = _git(
93
+ repo,
94
+ "log",
95
+ f"--since={since.date().isoformat()}",
96
+ "--name-only",
97
+ "--format=COMMIT%x09%H%x09%aI%x09%s",
98
+ )
99
+ fixes: list[FixCommit] = []
100
+ current: FixCommit | None = None
101
+ for line in log.splitlines():
102
+ if line.startswith("COMMIT\t"):
103
+ _, sha, ts, subject = line.split("\t", 3)
104
+ current = (
105
+ FixCommit(sha=sha, timestamp=datetime.fromisoformat(ts), message=subject)
106
+ if FIX_PATTERN.search(subject)
107
+ else None
108
+ )
109
+ if current:
110
+ fixes.append(current)
111
+ continue
112
+ if current and line.strip() and not TEST_PATH_PATTERN.search(line):
113
+ current.files.append(line.strip())
114
+ return fixes
115
+
116
+
117
+ def default_since(window_days: int) -> datetime:
118
+ return datetime.now(tz=UTC) - timedelta(days=window_days)
@@ -0,0 +1,127 @@
1
+ """Jenkins ingester.
2
+
3
+ Uses the Jenkins JSON API (``/job/<name>/api/json`` and the
4
+ ``testReport`` endpoint of each build). Works with the JUnit plugin and the
5
+ Cucumber Reports plugin's JUnit export. Reruns are detected when the same
6
+ commit SHA appears in consecutive builds (``lastBuiltRevision`` in build
7
+ actions), so a "rebuild until green" shows up as rerun cost.
8
+
9
+ Authentication: basic auth with user + API token from environment variables
10
+ named by ``sources.jenkins.user_env`` / ``token_env``.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import os
16
+ from datetime import UTC, datetime
17
+
18
+ import requests
19
+
20
+ from testledger.models import Outcome, TestResult, TestRun
21
+
22
+ _STATUS = {
23
+ "PASSED": Outcome.PASSED,
24
+ "FIXED": Outcome.PASSED,
25
+ "FAILED": Outcome.FAILED,
26
+ "REGRESSION": Outcome.FAILED,
27
+ "SKIPPED": Outcome.SKIPPED,
28
+ }
29
+
30
+
31
+ class JenkinsClient:
32
+ def __init__(
33
+ self,
34
+ base_url: str,
35
+ user: str | None = None,
36
+ token: str | None = None,
37
+ session: requests.Session | None = None,
38
+ ):
39
+ self.base = base_url.rstrip("/")
40
+ self.session = session or requests.Session()
41
+ if user and token:
42
+ self.session.auth = (user, token)
43
+ self.session.headers["User-Agent"] = "testledger"
44
+
45
+ def builds(self, job: str, max_builds: int = 100) -> list[dict]:
46
+ job_path = "/job/" + "/job/".join(job.split("/"))
47
+ resp = self.session.get(
48
+ f"{self.base}{job_path}/api/json",
49
+ params={
50
+ "tree": f"builds[number,timestamp,result,actions[lastBuiltRevision[SHA1,branch[name]]]]{{0,{max_builds}}}"
51
+ },
52
+ timeout=30,
53
+ )
54
+ resp.raise_for_status()
55
+ return resp.json().get("builds", [])
56
+
57
+ def test_report(self, job: str, number: int) -> dict | None:
58
+ job_path = "/job/" + "/job/".join(job.split("/"))
59
+ resp = self.session.get(f"{self.base}{job_path}/{number}/testReport/api/json", timeout=60)
60
+ if resp.status_code == 404:
61
+ return None
62
+ resp.raise_for_status()
63
+ return resp.json()
64
+
65
+
66
+ def _sha_and_branch(build: dict) -> tuple[str | None, str | None]:
67
+ for action in build.get("actions", []):
68
+ rev = action.get("lastBuiltRevision") if isinstance(action, dict) else None
69
+ if rev:
70
+ branches = rev.get("branch", [])
71
+ return rev.get("SHA1"), (branches[0].get("name") if branches else None)
72
+ return None, None
73
+
74
+
75
+ def fetch_builds(
76
+ base_url: str,
77
+ job: str,
78
+ *,
79
+ since: datetime,
80
+ user_env: str = "JENKINS_USER",
81
+ token_env: str = "JENKINS_TOKEN",
82
+ max_builds: int = 100,
83
+ client: JenkinsClient | None = None,
84
+ ) -> list[TestRun]:
85
+ client = client or JenkinsClient(base_url, os.environ.get(user_env), os.environ.get(token_env))
86
+ runs: list[TestRun] = []
87
+ seen_sha: dict[str, int] = {}
88
+ # Jenkins returns newest first; reverse so attempts count upward in time.
89
+ for build in reversed(client.builds(job, max_builds)):
90
+ ts = datetime.fromtimestamp(build["timestamp"] / 1000, tz=UTC)
91
+ if ts < since:
92
+ continue
93
+ report = client.test_report(job, build["number"])
94
+ if not report:
95
+ continue
96
+ sha, branch = _sha_and_branch(build)
97
+ attempt = 1
98
+ if sha:
99
+ seen_sha[sha] = seen_sha.get(sha, 0) + 1
100
+ attempt = seen_sha[sha]
101
+ results: list[TestResult] = []
102
+ for suite in report.get("suites", []):
103
+ for case in suite.get("cases", []):
104
+ outcome = _STATUS.get(case.get("status", "PASSED").upper(), Outcome.PASSED)
105
+ cls = case.get("className", suite.get("name", ""))
106
+ results.append(
107
+ TestResult(
108
+ test_id=f"{cls}.{case.get('name')}",
109
+ name=case.get("name", ""),
110
+ suite=cls,
111
+ outcome=outcome,
112
+ duration_s=float(case.get("duration", 0.0)),
113
+ message=(case.get("errorDetails") or "")[:300] or None,
114
+ )
115
+ )
116
+ runs.append(
117
+ TestRun(
118
+ run_id=f"jenkins-{job.replace('/', '_')}-{build['number']}",
119
+ started_at=ts,
120
+ commit_sha=sha,
121
+ branch=branch,
122
+ attempt=attempt,
123
+ source="jenkins",
124
+ results=results,
125
+ )
126
+ )
127
+ return runs
@@ -0,0 +1,114 @@
1
+ """JUnit / TestNG / Surefire XML parser.
2
+
3
+ Handles the common dialects:
4
+ * ``<testsuites><testsuite><testcase/>`` (Surefire, Gradle, pytest, Jest)
5
+ * bare ``<testsuite>`` root
6
+ * TestNG's ``testng-results.xml`` (``<test-method>`` elements)
7
+
8
+ Run metadata (commit, attempt, timestamp) is not inside JUnit XML, so callers
9
+ pass it in, typically derived from the file path or the CI API.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import xml.etree.ElementTree as ET
15
+ from datetime import UTC, datetime
16
+ from pathlib import Path
17
+
18
+ from testledger.models import Outcome, TestResult, TestRun
19
+
20
+
21
+ def _outcome(case: ET.Element) -> tuple[Outcome, str | None]:
22
+ for tag, outcome in (("failure", Outcome.FAILED), ("error", Outcome.ERROR), ("skipped", Outcome.SKIPPED)):
23
+ node = case.find(tag)
24
+ if node is not None:
25
+ return outcome, (node.get("message") or (node.text or "").strip()[:300] or None)
26
+ return Outcome.PASSED, None
27
+
28
+
29
+ def _testng_outcome(method: ET.Element) -> tuple[Outcome, str | None]:
30
+ status = (method.get("status") or "PASS").upper()
31
+ if status == "FAIL":
32
+ exc = method.find("exception")
33
+ return Outcome.FAILED, (exc.get("class") if exc is not None else None)
34
+ if status == "SKIP":
35
+ return Outcome.SKIPPED, None
36
+ return Outcome.PASSED, None
37
+
38
+
39
+ def parse_junit_xml(
40
+ path: str | Path,
41
+ *,
42
+ run_id: str | None = None,
43
+ started_at: datetime | None = None,
44
+ commit_sha: str | None = None,
45
+ branch: str | None = None,
46
+ attempt: int = 1,
47
+ ) -> TestRun:
48
+ p = Path(path)
49
+ tree = ET.parse(p)
50
+ root = tree.getroot()
51
+ results: list[TestResult] = []
52
+
53
+ if root.tag == "testng-results":
54
+ for cls in root.iter("class"):
55
+ cls_name = cls.get("name", "UnknownClass")
56
+ for method in cls.iter("test-method"):
57
+ if method.get("is-config") == "true":
58
+ continue
59
+ name = method.get("name", method.get("signature", "unnamed").split("(")[0])
60
+ outcome, msg = _testng_outcome(method)
61
+ results.append(
62
+ TestResult(
63
+ test_id=f"{cls_name}.{name}",
64
+ name=name,
65
+ suite=cls_name,
66
+ file=None,
67
+ outcome=outcome,
68
+ duration_s=float(method.get("duration-ms", 0)) / 1000.0,
69
+ message=msg,
70
+ )
71
+ )
72
+ else:
73
+ suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite"))
74
+ for suite in suites:
75
+ suite_name = suite.get("name", p.stem)
76
+ for case in suite.findall("testcase"):
77
+ classname = case.get("classname") or suite_name
78
+ name = case.get("name", "unnamed")
79
+ outcome, msg = _outcome(case)
80
+ results.append(
81
+ TestResult(
82
+ test_id=f"{classname}.{name}",
83
+ name=name,
84
+ suite=classname,
85
+ file=case.get("file"),
86
+ outcome=outcome,
87
+ duration_s=float(case.get("time") or 0.0),
88
+ message=msg,
89
+ )
90
+ )
91
+
92
+ ts = started_at
93
+ if ts is None:
94
+ raw = root.get("timestamp") or (
95
+ root.find("testsuite").get("timestamp") if root.find("testsuite") is not None else None
96
+ )
97
+ try:
98
+ ts = datetime.fromisoformat(raw) if raw else None
99
+ except ValueError:
100
+ ts = None
101
+ if ts is None:
102
+ ts = datetime.fromtimestamp(p.stat().st_mtime, tz=UTC)
103
+ if ts.tzinfo is None:
104
+ ts = ts.replace(tzinfo=UTC)
105
+
106
+ return TestRun(
107
+ run_id=run_id or p.stem,
108
+ started_at=ts,
109
+ commit_sha=commit_sha,
110
+ branch=branch,
111
+ attempt=attempt,
112
+ source="junit",
113
+ results=results,
114
+ )
@@ -0,0 +1,3 @@
1
+ from testledger.ledger.engine import build_ledger
2
+
3
+ __all__ = ["build_ledger"]