stackvitals 0.1.0__py3-none-any.whl

stackvitals/ci/collect.py

@@ -0,0 +1,239 @@
+import re
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timedelta
+
+from stackvitals.ci.api import ActionsAPI, APIError, OutOfBudget
+from stackvitals.ci.store import DataStore
+from stackvitals.ci import coverage
+from stackvitals.ci.testreports import _empty, parse_archive
+
+PR_EVENTS = {"pull_request", "pull_request_target"}
+# New test ids kept per report, and the most test ids remembered (past that, nothing is called new).
+NEW_KEEP, KNOWN_MAX = 50, 50_000
+# Artifact names that usually hold test reports; users can pass their own pattern.
+TEST_REPORTS = (r"junit|test[-_ ]?results?|test[-_ ]?reports?|ctrf|surefire|pytest|jest|vitest|playwright|trx|go[-_ ]?test"
+                r"|xunit|nunit|mstest")
+# Coverage reports too, but not HTML coverage sites (nothing to read there).
+TEST_ARTIFACTS = TEST_REPORTS + r"|lcov|jacoco|cobertura|coverage(?![-_ ]?html)"
+
+
+def run_record(body: dict, n: int) -> dict:
+    # pull_requests on non-PR runs can list unrelated fork PRs, so it is trusted only for PR events.
+    prs = [p["number"] for p in body.get("pull_requests") or []] if body.get("event") in PR_EVENTS else []
+    rec = {"t": "run", "id": body["id"], "att": n, "wf": body.get("workflow_id"), "wfn": body.get("name"),
+            "path": body.get("path"), "ev": body.get("event"), "br": body.get("head_branch"),
+            "sha": body.get("head_sha"), "tree": (body.get("head_commit") or {}).get("tree_id"), "pr": prs,
+            "c": body.get("conclusion"), "cr": body.get("created_at"), "st": body.get("run_started_at"),
+            "up": body.get("updated_at")}
+    if body.get("event") in PR_EVENTS:
+        # Fork PR runs list no PR numbers; the head owner and branch find the PR later.
+        rec["ho"] = ((body.get("head_repository") or {}).get("owner") or {}).get("login")
+    return rec
+
+
+def job_record(j: dict, run_id: int, n: int) -> dict:
+    steps = [[s.get("number"), s.get("name"), s.get("conclusion"), s.get("started_at"), s.get("completed_at")]
+             for s in j.get("steps") or []]
+    return {"t": "job", "run": run_id, "att": n, "id": j["id"], "n": j.get("name"), "c": j.get("conclusion"),
+            "cr": j.get("created_at"), "st": j.get("started_at"), "en": j.get("completed_at"),
+            "lb": j.get("labels") or [], "steps": steps}
+
+
+def _fetch(api: ActionsAPI, run: dict, n: int) -> list[dict]:
+    body = run if n == run.get("run_attempt", 1) else api.attempt(run["id"], n)
+    return [run_record(body, n)] + [job_record(j, run["id"], n) for j in api.jobs(run["id"], n)]
+
+
+def _attempt_of(created: str | None, starts: dict[int, str]) -> int:
+    # An artifact belongs to the last attempt that had started before it was uploaded.
+    started = [att for att, st in starts.items() if st and created and st <= created]
+    return max(started) if started else min(starts or {1: None})
+
+
+def _pr_number(api: ActionsAPI, run: dict) -> int | None:
+    """The pull request a pull_request run tested: the one GitHub names, else (forks) the one with that head branch."""
+    if run.get("event") != "pull_request":
+        return None
+    named = [p.get("number") for p in run.get("pull_requests") or [] if p.get("number")]
+    if named:
+        return named[0]
+    owner = ((run.get("head_repository") or {}).get("owner") or {}).get("login")
+    if not owner or not run.get("head_branch"):
+        return None
+    try:
+        prs = api._get(f"/repos/{api.repo}/pulls", {"head": f"{owner}:{run['head_branch']}", "state": "all", "per_page": 5})
+    except APIError:
+        return None
+    same = [p for p in prs or [] if (p.get("head") or {}).get("sha") == run.get("head_sha")]
+    return (same or prs or [{}])[0].get("number")
+
+
+def _pr_files(api: ActionsAPI, number: int, pages: int = 10) -> list[dict]:
+    out = []
+    for page in range(1, pages + 1):
+        got = api._get(f"/repos/{api.repo}/pulls/{number}/files", {"per_page": 100, "page": page}) or []
+        out += got
+        if len(got) < 100:
+            break
+    return out
+
+
+def _reports(api: ActionsAPI, run: dict, starts: dict[int, str], pattern: str, max_bytes: int,
+             done_arts: set) -> list[dict]:
+    out, diff = [], {}
+    for art in api.artifacts(run["id"]):
+        if (art["id"] in done_arts or art.get("expired") or not re.search(pattern, art.get("name") or "", re.I)
+                or (art.get("size_in_bytes") or 0) > max_bytes):
+            continue
+        data = api.download(art["id"])  # a GitHub error here is retried on the next run
+        try:
+            parsed = parse_archive(data)
+        except Exception:  # a broken or hostile artifact is noted once and never read again
+            out.append({"t": "tests", "run": run["id"], "att": _attempt_of(art.get("created_at"), starts), "art": art["id"],
+                        "name": art.get("name"), **{k: v for k, v in _empty().items() if k != "all"}, "unreadable": True})
+            continue
+        cov = parsed.pop("cov", None)
+        rec = {"t": "tests", "run": run["id"], "att": _attempt_of(art.get("created_at"), starts), "art": art["id"],
+               "name": art.get("name"), **parsed}
+        if cov:
+            # Only the totals are kept; the line hits are read once, for the pull request's patch coverage.
+            rec["cov"] = {"format": cov["format"], "lines": cov["lines"], "covered": cov["covered"]}
+            # The run's pull request and its files are read once, however many coverage artifacts the matrix uploads.
+            if "pr" not in diff:
+                diff["pr"] = _pr_number(api, run)
+            number = diff["pr"]
+            if number:
+                try:
+                    if number not in diff:
+                        diff[number] = _pr_files(api, number)
+                    rec["patch"] = {"pr": number, **coverage.patch(cov, diff[number])}
+                except APIError:
+                    pass
+        out.append(rec)
+    return out
+
+
+def _distrust(rec: dict, own: bool) -> None:
+    # A fork PR writes its own reports: a forged "passed on retry" could make a real test known flaky (and quarantined).
+    if not own:
+        rec["flaky"] = []
+        rec.pop("cov", None)
+
+
+def _learn(rec: dict, ids: list[str], seen: set, known: bool, own: bool) -> None:
+    """Note a report's tests seen for the first time, and remember them up to the cap; a fork's runs teach nothing, so
+    its made-up ids can neither fill the list nor show up as new tests."""
+    if not own:
+        return
+    full = len(seen) >= KNOWN_MAX
+    new = [t for t in ids if t not in seen] if known and not full else []
+    if new:
+        rec["new"] = new[:NEW_KEEP]
+    room = KNOWN_MAX - len(seen)
+    if room > 0:
+        seen.update([t for t in ids if t not in seen][:room])
+
+
+def collect(api: ActionsAPI, store: DataStore, now: datetime, days: int = 90, exclude=(),
+            recheck_days: int = 3, workers: int = 10, log=None, tests: bool = False,
+            artifact_pattern: str = TEST_ARTIFACTS, max_artifact_bytes: int = 50_000_000) -> dict:
+    """Store every new attempt of every completed run in the last `days` days, newest day first,
+    and (with tests=True) the test reports in each run's artifacts."""
+    state = store.state()
+    scanned = state.setdefault("scanned", {})
+    scanned_tests = state.setdefault("scanned_tests", {})
+    checked = state.setdefault("arts_checked", {})
+    loaded = store.load()
+    seen = {(r["id"], r["att"]) for r in loaded["runs"]}
+    done_arts = {r["art"] for r in loaded["tests"]}
+    # Test ids seen before; the first collect that reads reports sets the baseline and names nothing as new.
+    known = store.known_tests() if tests else None
+    seen_ids = set(known or ())
+    starts: dict[int, dict[int, str]] = {}
+    for r in loaded["runs"]:
+        starts.setdefault(r["id"], {})[r["att"]] = r.get("st")
+    today = now.date()
+    stamp = today.isoformat()
+    excluded = set(exclude)
+    touched, complete = set(), True
+    pool = ThreadPoolExecutor(max_workers=max(1, workers))
+    try:
+        for back in range(days + 1):
+            day = (today - timedelta(days=back)).isoformat()
+            # Re-runs land on old runs, so the last few days are always checked again.
+            jobs_done = scanned.get(day) == "done" and back > recheck_days
+            tests_done = not tests or (scanned_tests.get(day) == "done" and back > recheck_days)
+            if jobs_done and tests_done:
+                continue
+            # GitHub-managed workflows (Dependabot, CodeQL default setup, Pages) live under "dynamic/".
+            listed = [run for run in api.runs_on_day(day)
+                      if run.get("path") not in excluded and not (run.get("path") or "").startswith("dynamic/")
+                      and run.get("status") == "completed"]
+            todo = [(run, n) for run in listed for n in range(1, (run.get("run_attempt") or 1) + 1)
+                    if (run["id"], n) not in seen]
+            futures = [pool.submit(_fetch, api, run, n) for run, n in todo]
+            stop, skipped = None, 0
+            for (run, n), fut in zip(todo, futures):
+                try:
+                    records = fut.result()
+                except OutOfBudget as exc:
+                    stop = stop or exc
+                    continue
+                except APIError:
+                    # One run GitHub can't serve right now is skipped; the day stays open for the next run.
+                    skipped += 1
+                    continue
+                # Each attempt is written as soon as it is read, under the day the run happened.
+                store.add(records, day)
+                seen.add((run["id"], n))
+                touched.add(run["id"])
+                starts.setdefault(run["id"], {})[n] = records[0].get("st")
+            if tests and not stop:
+                # Reports are read once per run and attempt count, so a later re-run's reports are read too.
+                mark = lambda run: f"{run['id']}:{run.get('run_attempt') or 1}"  # noqa: E731
+                pending = [run for run in listed if mark(run) not in checked
+                           and all((run["id"], n) in seen for n in range(1, (run.get("run_attempt") or 1) + 1))]
+                rfutures = [pool.submit(_reports, api, run, starts.get(run["id"], {}), artifact_pattern,
+                                        max_artifact_bytes, done_arts) for run in pending]
+                for run, fut in zip(pending, rfutures):
+                    try:
+                        reports = fut.result()
+                    except OutOfBudget as exc:
+                        stop = stop or exc
+                        continue
+                    except APIError:
+                        skipped += 1
+                        continue
+                    own = (run.get("head_repository") or {}).get("full_name") in (None, api.repo)
+                    for rec in reports:
+                        _learn(rec, [t for t, _ms in rec.pop("all", [])], seen_ids, known is not None, own)
+                        _distrust(rec, own)
+                    store.add(reports, day)
+                    done_arts.update(r["art"] for r in reports)
+                    checked[mark(run)] = day
+            if log:
+                log(f"{day}: {len({r['id'] for r in listed})} run(s) read, {api.requests} requests so far")
+            if stop:
+                raise stop
+            if skipped:
+                complete = False
+                scanned[day] = "partial"
+                state["last_stop"] = f"{skipped} run(s) GitHub could not serve on {day}; retried next time"
+            else:
+                scanned[day] = "done"
+                if tests:
+                    scanned_tests[day] = "done"
+    except OutOfBudget as exc:
+        complete = False
+        state["last_stop"] = str(exc)
+    finally:
+        pool.shutdown(wait=True)
+    oldest = (today - timedelta(days=days + recheck_days)).isoformat()
+    state["arts_checked"] = {k: v for k, v in checked.items() if v >= oldest}
+    state["last_collect"] = now.strftime("%Y-%m-%dT%H:%M:%SZ")
+    store.save_state(state)
+    if tests and seen_ids:
+        store.save_known_tests(seen_ids)
+    # Past days are compressed right away, so a big first backfill never sits in git uncompressed.
+    store.compress(stamp)
+    return {"runs": len(touched), "requests": api.requests, "complete": complete}

stackvitals/ci/cost.py

@@ -0,0 +1,107 @@
+"""CI cost: billed minutes and money per workflow and job at GitHub's published per-minute prices (docs.github.com,
+"Actions runner pricing", read 2026-10-29), each job rounded up to a whole minute as GitHub bills it; the minutes spent
+on attempts that failed and then passed on a re-run (what flaky jobs cost); and, for a public repo, where standard
+runners are free, what the same use would cost on a private one. Self-hosted runners count in minutes only."""
+import math
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta
+
+# USD per minute: standard runners, then larger ones by OS, architecture and cores.
+STANDARD = {"linux": 0.006, "linux arm": 0.005, "windows": 0.010, "windows arm": 0.010, "macos": 0.062}
+LARGER = {("linux", 4): 0.012, ("linux", 8): 0.022, ("linux", 16): 0.042, ("linux arm", 4): 0.008,
+          ("linux arm", 8): 0.014, ("linux arm", 16): 0.026, ("windows", 8): 0.042, ("windows", 16): 0.082,
+          ("windows arm", 8): 0.026, ("windows arm", 16): 0.050}
+CORES = re.compile(r"(\d+)[-_ ]?(core|cores|cpu|vcpu)", re.I)
+
+
+def rate(labels: list[str]) -> tuple[str, float | None]:
+    """(runner kind, USD per minute) from a job's runner labels; self-hosted runners have no price."""
+    low = [x.lower() for x in labels or []]
+    if "self-hosted" in low:
+        return "self-hosted", None
+    text = " ".join(low)
+    os_ = "windows" if "windows" in text else "macos" if "macos" in text or "mac" in text else "linux"
+    if os_ != "macos" and ("arm" in text or "aarch64" in text):
+        os_ += " arm"
+    cores = CORES.search(text)
+    if cores and os_ != "macos":
+        n = int(cores.group(1))
+        size = min((s for (o, s) in LARGER if o == os_ and s >= n), default=None)
+        if size:
+            return f"{os_} {size}-core", LARGER[(os_, size)]
+    if os_ == "macos" and ("xlarge" in text or "large" in text):
+        return "macos large", 0.077 if "xlarge" not in text else 0.102
+    return os_, STANDARD[os_]
+
+
+def _minutes(j: dict) -> int:
+    try:
+        secs = (datetime.fromisoformat(j["en"].replace("Z", "+00:00")) - datetime.fromisoformat(j["st"].replace("Z", "+00:00"))).total_seconds()
+    except (KeyError, TypeError, AttributeError, ValueError):
+        return 0
+    return max(1, math.ceil(secs / 60)) if secs > 0 else 0
+
+
+def report(data: dict, now: datetime, private: bool, days: int = 30) -> dict:
+    since = (now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    runs = {(r["id"], r["att"]): r for r in data.get("runs") or []}
+    jobs = [j for j in data.get("jobs") or [] if (j.get("st") or "") >= since and j.get("en")]
+    # A failed attempt followed by a later attempt of the same job that passed: the re-run's cost of a flake.
+    passed_later = {(j["run"], j["n"]) for j in jobs if j.get("c") == "success" and j.get("att", 1) > 1}
+    by_wf, by_job = defaultdict(lambda: [0, 0.0]), defaultdict(lambda: [0, 0.0, ""])
+    by_runner = defaultdict(int)
+    minutes = usd = private_usd = rerun_min = rerun_usd = self_hosted = 0
+    for j in jobs:
+        m = _minutes(j)
+        kind, price = rate(j.get("lb"))
+        run = runs.get((j["run"], j.get("att", 1))) or runs.get((j["run"], 1)) or {}
+        wf = run.get("wfn") or run.get("path") or "unknown workflow"
+        # Standard runners are free on public repos; larger runners never are.
+        paid = 0.0 if price is None else (m * price if private or kind not in STANDARD else 0.0)
+        minutes += m
+        usd += paid
+        private_usd += 0.0 if price is None else m * price
+        by_runner[kind] += m
+        if price is None:
+            self_hosted += m
+        by_wf[wf][0] += m
+        by_wf[wf][1] += paid
+        key = (wf, j.get("n") or "job")
+        by_job[key][0] += m
+        by_job[key][1] += paid
+        if j.get("c") in ("failure", "timed_out") and (j["run"], j.get("n")) in passed_later:
+            rerun_min += m
+            rerun_usd += paid
+    top = lambda d, label: [{label: k if isinstance(k, str) else k[1], **({"workflow": k[0]} if not isinstance(k, str) else {}),  # noqa: E731
+                             "minutes": v[0], "usd": round(v[1], 2)} for k, v in sorted(d.items(), key=lambda kv: -kv[1][0])][:20]
+    return {"days": days, "private": private, "minutes": minutes, "usd": round(usd, 2), "private_usd": round(private_usd, 2),
+            "rerun_minutes": rerun_min, "rerun_usd": round(rerun_usd, 2), "self_hosted_minutes": self_hosted,
+            "by_runner": dict(by_runner), "workflows": top(by_wf, "workflow"), "jobs": top(by_job, "job"),
+            "prices_read": "2026-10-29"}
+
+
+def flaky_tests(data: dict, tests: list[dict], private: bool, most: int = 20) -> list[dict]:
+    """What each flaky test cost: the job minutes (and money) of the attempts it flaked in, an attempt's cost split
+    evenly between the tests that flaked in it, plus the pull requests it got in the way of; costliest first."""
+    by_attempt = defaultdict(list)
+    for j in data.get("jobs") or []:
+        if j.get("c") in ("failure", "timed_out") and j.get("en"):
+            by_attempt[(j["run"], j.get("att", 1))].append(j)
+    sharing = defaultdict(int)
+    for t in tests:
+        for a in t.get("_attempts") or []:
+            sharing[tuple(a)] += 1
+    out = []
+    for t in tests:
+        minutes = paid = 0.0
+        for a in t.get("_attempts") or []:
+            for j in by_attempt.get(tuple(a), []):
+                m = _minutes(j) / sharing[tuple(a)]
+                kind, price = rate(j.get("lb"))
+                minutes += m
+                paid += 0.0 if price is None else (m * price if private or kind not in STANDARD else 0.0)
+        if minutes:
+            out.append({"id": t.get("id"), "suite": t.get("suite"), "minutes": round(minutes, 1), "usd": round(paid, 2),
+                        "flakes": len(t.get("_attempts") or []), "prs": len(t.get("prs_affected") or [])})
+    return sorted(out, key=lambda x: -x["minutes"])[:most]

stackvitals/ci/coverage.py

@@ -0,0 +1,155 @@
+"""Coverage from the reports CI already uploads: Cobertura XML (coverage.py, Istanbul, Go's converters), LCOV and JaCoCo
+XML, read safely in memory; and patch coverage, the share of a pull request's added lines that ran, from its diff."""
+import math
+import re
+
+from defusedxml import DefusedXmlException
+from defusedxml.ElementTree import ParseError, fromstring
+
+HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
+# JaCoCo always names its DTD; a DOCTYPE with no internal subset declares nothing, so it is dropped and the rest is read
+# with DTDs forbidden (entity tricks stay impossible).
+PLAIN_DOCTYPE = re.compile(rb"<!DOCTYPE[^\[>]*>")
+MISSED_KEEP = 50
+PROLOG = 4096  # a DOCTYPE only appears before the root element; scanning further was quadratic on crafted files
+ABSOLUTE = re.compile(r"^(?:/|[A-Za-z]:/)")
+
+
+def _xml(data: bytes):
+    try:
+        return fromstring(PLAIN_DOCTYPE.sub(b"", data[:PROLOG], count=1) + data[PROLOG:], forbid_dtd=True)
+    except (DefusedXmlException, ParseError, ValueError):
+        return None
+
+
+def _total(files: dict) -> tuple[int, int]:
+    lines = sum(len(hits) for hits in files.values())
+    return lines, sum(1 for hits in files.values() for h in hits.values() if h > 0)
+
+
+def _int(text, default=0) -> int:
+    try:
+        value = float(text)
+    except (TypeError, ValueError, OverflowError):
+        return default
+    return int(value) if math.isfinite(value) else default
+
+
+def _cobertura(root) -> dict:
+    files = {}
+    for cls in root.iter("class"):
+        name = cls.get("filename")
+        if not name:
+            continue
+        hits = files.setdefault(name, {})
+        for line in cls.iter("line"):
+            n = _int(line.get("number"), None)
+            if n is not None:
+                hits[n] = max(hits.get(n, 0), _int(line.get("hits")))
+    return files
+
+
+def _jacoco(root) -> dict:
+    files = {}
+    for package in root.iter("package"):
+        for source in package.iter("sourcefile"):
+            hits = files.setdefault(f"{package.get('name')}/{source.get('name')}".strip("/"), {})
+            for line in source.iter("line"):
+                n = _int(line.get("nr"), None)
+                if n is not None:
+                    hits[n] = _int(line.get("ci"))
+    return files
+
+
+def _lcov(text: str) -> dict:
+    files, cur = {}, None
+    for line in text.split("\n"):
+        line = line.rstrip("\r")
+        if line.startswith("SF:"):
+            cur = files.setdefault(line[3:].strip(), {})
+        elif line.startswith("DA:") and cur is not None:
+            parts = line[3:].split(",")
+            if len(parts) >= 2:
+                n = _int(parts[0], None)
+                if n is not None:
+                    cur[n] = max(cur.get(n, 0), _int(parts[1]))
+        elif line.startswith("end_of_record"):
+            cur = None
+    return files
+
+
+def parse(name: str, data: bytes) -> dict | None:
+    """{"format", "lines", "covered", "files": {path: {line: hits}}} for a coverage report, else None."""
+    data = data.removeprefix(b"\xef\xbb\xbf")  # a UTF-8 byte order mark
+    low, head = name.lower(), data.lstrip()[:200]
+    if low.endswith((".info", ".lcov")) or head.startswith((b"TN:", b"SF:")):
+        files, kind = _lcov(data.decode("utf-8", "replace")), "lcov"
+    elif head.startswith(b"<"):
+        if b"<coverage" not in data[:2000] and b"<report" not in data[:2000]:
+            return None
+        root = _xml(data)
+        if root is None or root.tag not in ("coverage", "report"):
+            return None
+        files, kind = (_cobertura(root), "cobertura") if root.tag == "coverage" else (_jacoco(root), "jacoco")
+    else:
+        return None
+    files = {path: hits for path, hits in files.items() if hits}
+    if not files:
+        return None
+    lines, covered = _total(files)
+    return {"format": kind, "lines": lines, "covered": covered, "files": files}
+
+
+def added_lines(patch: str | None) -> list[int]:
+    """The new file's line numbers of the lines a unified diff adds."""
+    out, cur = [], None
+    # Only a newline ends a diff line: splitlines() also splits on form feeds and the like inside a line.
+    for line in (patch or "").split("\n"):
+        line = line.removesuffix("\r")
+        found = HUNK.match(line)
+        if found:
+            cur = int(found.group(1))
+        elif cur is None or line.startswith("\\"):
+            continue
+        elif line.startswith("+"):
+            out.append(cur)
+            cur += 1
+        elif not line.startswith("-"):
+            cur += 1
+    return out
+
+
+def _match(name: str, paths: list[str]) -> str | None:
+    """The report's path for a repo file: the same path; else an absolute runner path ending with it (the shortest);
+    else the longest path it ends with (paths relative to a source folder). A deeper relative path is another file."""
+    norm = {p.replace("\\", "/"): p for p in paths}
+    if name in norm:
+        return norm[name]
+    tail = "/" + name
+    absolute = [p for p in norm if p.endswith(tail) and ABSOLUTE.match(p)]
+    if absolute:
+        return norm[min(absolute, key=len)]
+    inner = [p for p in norm if name.endswith("/" + p)]
+    return norm[max(inner, key=len)] if inner else None
+
+
+def patch(cov: dict, files: list[dict]) -> dict:
+    """{"lines", "covered", "missed": {file: [lines]}} over the added lines the report knows (lines it doesn't list, like
+    docs or blank lines, aren't code that could run)."""
+    paths = list(cov.get("files") or {})
+    lines = covered = 0
+    missed = {}
+    for f in files:
+        name = f.get("filename") or ""
+        path = _match(name, paths)
+        if not path:
+            continue
+        hits = cov["files"][path]
+        for n in added_lines(f.get("patch")):
+            if n in hits:
+                lines += 1
+                if hits[n] > 0:
+                    covered += 1
+                else:
+                    missed.setdefault(name, []).append(n)
+    return {"lines": lines, "covered": covered, "missed": {k: v[:MISSED_KEEP] for k, v in missed.items()}}