stackvitals 0.1.0__py3-none-any.whl

stackvitals/ci/kinds.py

@@ -0,0 +1,52 @@
+"""What kind of flake a failure looks like (the environment: infrastructure, network, concurrency, timing; or the test's code:
+resource leak, async wait, time, randomness, floating point, shared state, io, ordering), so people know where
+to look. Measured against labelled causes in bench/accuracy/flaky/kinds.py."""
+import re
+
+# Checked in this order.
+KINDS = (
+    # The runner, a cache or artifact store, or a service the job starts on its own machine.
+    ("infrastructure", re.compile(r"(?i)(ServerBusy|No space left on device|runner has received a shutdown signal|"
+                                  r"lost communication with the server|Cannot connect to the Docker daemon|"
+                                  r"failed to connect to the docker API|hosted runner encountered an error|out of memory|"
+                                  r"OOMKilled|exit code 137|Cannot allocate memory|heap out of memory|FinalizeArtifact|"
+                                  r"upload of artifact|artifact (upload|download) failed|"
+                                  r"(127\.0\.0\.1|localhost)\S{0,8}.{0,60}connection refused)")),
+    # ENOTFOUND matched case-sensitively: "FileNotFoundError" holds it in any case.
+    ("network", re.compile(r"(?i)((?-i:\bENOTFOUND\b)|EAI_AGAIN|ECONNRESET|ECONNREFUSED|ETIMEDOUT|socket hang up|Could not resolve host|"
+                           r"Temporary failure in name resolution|Name or service not known|Failed to fetch|"
+                           r"Failed to download(?! and build)|\b50[234] (Service Unavailable|Bad Gateway|Gateway Time-?out)|"
+                           r"HTTP (error )?5\d\d|status code 5\d\d|returned HTTP 5\d\d|TLS handshake timeout|i/o timeout|"
+                           r"Connection (timed out|reset|refused)|Unable to make request|toomanyrequests|Too Many Requests|"
+                           r"rate limit|quota exceeded|Service Unavailable|net/http: request canceled|Client\.Timeout exceeded|"
+                           r"Error response from daemon|download has timed out)")),
+    ("concurrency", re.compile(r"(?i)(DATA RACE|data race|deadlock|ConcurrentModificationException|race condition|"
+                               r"concurrent map (writes|read and map write)|already borrowed)")),
+    ("timing", re.compile(r"(?i)(timed? ?out (after|in|waiting)|timeout of \d+ ?ms exceeded|exceeded (the )?timeout|"
+                          r"test timed out|deadline exceeded|TimeoutError|Timeout - Async callback|waiting for .{0,80} timed out|"
+                          r"Expected condition failed: waiting for|Matcher did not succeed in time|"
+                          r"exceeded the maximum execution time|has timed out)")),
+    # Causes in the test's own code (Datadog's categories), checked after the environment's.
+    ("resource leak", re.compile(r"(?i)(Too many open files|EMFILE|ResourceWarning|unclosed (file|socket|transport)|"
+                                 r"address already in use|EADDRINUSE|port \d+ is already (in use|allocated)|leaked? (handle|connection))")),
+    ("async wait", re.compile(r"(?i)(not wrapped in act\(|Unable to find (an )?element|StaleElementReference|"
+                              r"ElementNotInteractable|detached from (the )?DOM|element is not attached|waitFor(Element)? .{0,40}failed|"
+                              r"Promise rejection was handled asynchronously|unhandled promise rejection)")),
+    ("time", re.compile(r"(?i)(time ?zone|\bDST\b|daylight saving|leap (year|second)|ZoneInfo|ZonedDateTime|"
+                        r"datetime\.(now|utcnow)|LocalDate|Clock skew|month end|end of (the )?month)")),
+    ("randomness", re.compile(r"(?i)(Falsifying example|random seed|seed[=: ]+\d+|Math\.random|randomly ordered|"
+                              r"--randomly-seed|--seed=\d+)")),
+    ("floating point", re.compile(r"(?i)(\d\.\d{12,}|not almost equal|assertAlmostEqual|approx\(|ULP|"
+                                  r"floating[- ]point|toBeCloseTo|within tolerance|rel_tol|abs_tol)")),
+    ("shared state", re.compile(r"(?i)(already registered|already exists|duplicate key|UNIQUE constraint failed|"
+                                r"IntegrityError|state from (a |another )?(previous|other) test|test pollution|singleton)")),
+    ("io", re.compile(r"(?i)(FileNotFoundError|ENOENT|No such file or directory|Permission denied|EACCES|"
+                      r"Directory not empty|ENOTEMPTY|file is being used by another process|EBUSY)")),
+    ("ordering", re.compile(r"(?i)(Items in the (first|second) (set|list)|Element counts were not equal|in any order|"
+                            r"Lists differ|expected order|unordered|sort(ed)? order)")),
+)
+
+
+def kind_of(text: str | None) -> str | None:
+    """The first kind whose signs the text shows, checked in order; None when it shows none."""
+    return next((name for name, rx in KINDS if rx.search(text or "")), None)

stackvitals/ci/monitors.py

@@ -0,0 +1,105 @@
+"""Test monitors in Trunk's documented shape: pass-on-retry (failed and passed on the same code) that heals after quiet
+days, failure rate and broken with separate switch-on and switch-off thresholds, and slow and timeout-inflation labels.
+Each test's status is kept in the data branch with the day it began, and every change is logged."""
+from datetime import datetime, timedelta
+from fnmatch import fnmatchcase
+
+from stackvitals.core.config import DEFAULTS
+from stackvitals.core.stats import median, percentile
+
+ACTIVE = {"flaky", "broken", "suspect"}
+
+
+def _iso(dt: datetime) -> str:
+    return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _cluster_start(events: list, quiet: timedelta) -> datetime | None:
+    # The first event of the latest unbroken stretch: flakes closer together than the heal time.
+    if not events:
+        return None
+    start = events[-1][0]
+    for i in range(len(events) - 1, 0, -1):
+        if events[i][0] - events[i - 1][0] >= quiet:
+            break
+        start = events[i - 1][0]
+    return start
+
+
+def evaluate(ev_all: list, ev_win: list, hist: list[tuple[datetime, bool]], runs_since: int, now: datetime, cfg: dict,
+             previous: dict | None = None, pass_ms=(), fail_ms=(), test_id: str | None = None) -> dict:
+    """One test's state from its flake events (oldest first), its default-branch results ([(time, failed)], oldest
+    first) and, when kept, its previous status; plus labels and a mute."""
+    m = (cfg or DEFAULTS)["ci"]["monitors"]
+    was = set((previous or {}).get("monitors") or [])
+    fails = [f for _, f in hist]
+    b = m["broken"]
+    window = fails[-int(b["window_runs"]):] if b["window_runs"] else []
+    rate_b = sum(window) / len(window) if window else 0.0
+    streak = int(b["streak"])
+    by_streak = streak > 0 and len(fails) >= streak and all(fails[-streak:])
+    broken = by_streak or (len(window) >= b["window_runs"] and rate_b >= b["activate"]) or \
+        ("broken" in was and bool(window) and rate_b >= b["resolve"])
+    p = m["pass_on_retry"]
+    quiet = timedelta(days=p["quiet_days"])
+    last = ev_all[-1][0] if ev_all else None
+    retry = len(ev_win) >= p["min_events"] or (len(ev_all) >= p["min_events"] and last is not None and now - last < quiet)
+    fr = m["failure_rate"]
+    win = fails[-int(fr["window_runs"]):] if fr["window_runs"] else []
+    rate = sum(win) / len(win) if win else None
+    by_rate = bool(fr["enabled"]) and len(win) >= fr["min_runs"] and rate is not None and \
+        (rate >= fr["activate"] or ("failure_rate" in was and rate > fr["resolve"]))
+    monitors = [name for name, on in (("broken", broken), ("pass_on_retry", retry), ("failure_rate", by_rate)) if on]
+    if broken:
+        state = "broken"
+        if by_streak:
+            i = len(fails)
+            while i > 0 and fails[i - 1]:
+                i -= 1
+            since = hist[i][0]
+        else:
+            since = next(t for t, f in hist[-len(window):] if f)
+    elif retry or by_rate:
+        state = "flaky"
+        since = _cluster_start(ev_all, quiet) if retry else next((t for t, f in hist[-len(win):] if f), None)
+    elif ev_win:
+        state, since = "suspect", ev_win[0][0]
+    elif len(ev_all) >= p["min_events"] and runs_since >= p["quiet_runs"]:
+        state, since = "fixed", last
+    else:
+        state, since = "ok", None
+    out = {"state": state, "monitors": monitors, "since": _iso(since) if since else None,
+           "failure_rate": round(rate, 4) if rate is not None else None, "labels": []}
+    if previous and previous.get("state") == state and previous.get("since"):
+        out["since"] = previous["since"]
+    pass_ms, fail_ms = [x for x in pass_ms if x is not None], [x for x in fail_ms if x is not None]
+    if pass_ms and median(pass_ms) > m["slow"]["p50_ms"]:
+        out["labels"].append("slow")
+    t = m["timeout_inflation"]
+    if pass_ms and len(fail_ms) >= t["min_failures"] and median(fail_ms) >= t["ratio"] * percentile(pass_ms, 0.95):
+        out["labels"].append("timeout_inflation")
+    today = now.date().isoformat()
+    for pattern, until in ((cfg or DEFAULTS)["ci"].get("mute") or {}).items():
+        if test_id and until >= today and fnmatchcase(test_id, pattern):
+            out["muted_until"] = until
+            break
+    return out
+
+
+def transitions(previous: dict, tests: list[dict], now: datetime) -> tuple[dict, list[dict]]:
+    """The status file to keep (tests that are flaky, broken or suspect, with the day it began) and the changes since the
+    last run, each one line for the transitions log."""
+    at = _iso(now)
+    previous = previous or {}
+    status, changes = {}, []
+    current = {t["id"]: t for t in tests}
+    for tid in sorted(set(current) | set(previous)):
+        t, prev = current.get(tid), previous.get(tid) or {}
+        state = t["state"] if t else "ok"
+        before = prev.get("state", "ok")
+        if state in ACTIVE:
+            since = prev["since"] if before == state and prev.get("since") else (t.get("since") or at)
+            status[tid] = {"state": state, "since": since, "monitors": t.get("monitors") or []}
+        if state != before:
+            changes.append({"at": at, "id": tid, "from": before, "to": state, "monitors": (t or {}).get("monitors") or []})
+    return status, changes

stackvitals/ci/owners.py

@@ -0,0 +1,116 @@
+"""Who owns a flaky test or job: CODEOWNERS (read with GitHub's rules) for the test's file, else for its workflow file.
+CODEOWNERS and GitHub's error check are read once a day; the repo's file list at most once a day, for new test ids."""
+import re
+
+from stackvitals import codeowners
+from stackvitals.ci.analyze import SHOWN
+from stackvitals.ci.api import APIError, NotFound, OutOfBudget
+
+SOURCE_EXT = {"py", "js", "jsx", "mjs", "cjs", "ts", "tsx", "rb", "go", "rs", "java", "kt", "kts", "scala", "cs", "php",
+              "swift", "ex", "exs", "c", "cc", "cpp", "m", "dart", "lua", "sh"}
+
+
+def _path_like(part: str) -> bool:
+    return "/" in part or ("." in part and part.rsplit(".", 1)[-1] in SOURCE_EXT)
+
+
+def _one(found: list[str]) -> str | None:
+    return found[0] if len(found) == 1 else None
+
+
+def test_file(tid: str, files: list[str] | None) -> str | None:
+    """The repo file a test lives in, confirmed against the repo's files: the path in a pytest-style id (or the one file
+    ending with it, for runners started in a subfolder), or the one file matching the dotted class name
+    (tests.test_calc -> tests/test_calc.py, com.acme.BarTest -> .../com/acme/BarTest.java). No guess when unsure."""
+    if not files:
+        return None
+    parts = tid.split("::")
+    if _path_like(parts[0]):
+        if parts[0] in files:
+            return parts[0]
+        return _one([f for f in files if f.endswith("/" + parts[0])])
+    if len(parts) < 2 or " " in parts[0]:
+        return None
+    names = parts[0].split("$", 1)[0].split(".")
+    for n in range(len(names), 0, -1):
+        stem = "/".join(names[:n])
+        found = [f for f in files if f.rsplit(".", 1)[0] == stem or f.rsplit(".", 1)[0].endswith("/" + stem)]
+        if found:
+            return _one(found)
+    return None
+
+
+def read_codeowners(api, ref: str | None, store, today: str) -> dict:
+    """The CODEOWNERS text GitHub uses (first of its three places), the owners its error check says it ignores, and the
+    lines it skips as invalid; read at most once a day. A failed read keeps the last answer and tries again next run."""
+    state = store.state()
+    kept = state.get("codeowners") or {}
+    answer = {"text": kept.get("text"), "unknown": set(kept.get("unknown") or []), "skip": set(kept.get("skip") or [])}
+    if kept.get("checked") == today:
+        return answer
+    text = None
+    try:
+        for path in codeowners.PATHS:
+            try:
+                text = api.file(path, ref).decode("utf-8", "replace")
+                break
+            except NotFound:
+                continue  # GitHub only moves on to the next place when a file is missing
+    except (APIError, OutOfBudget):
+        return answer
+    unknown, skip = answer["unknown"], answer["skip"]
+    if text:
+        try:
+            errors = (api._get(f"/repos/{api.repo}/codeowners/errors", {"ref": ref} if ref else None) or {}).get("errors") or []
+            unknown = {m.group(1).lower() for e in errors if e.get("kind") == "Unknown owner"
+                       and (m := re.search(r"make sure (\S+) exists", e.get("message") or ""))}
+            skip = {e["line"] for e in errors if e.get("kind") != "Unknown owner" and isinstance(e.get("line"), int)}
+        except (APIError, OutOfBudget):
+            pass  # the error check is extra: without it every owner is kept as written
+    store.save_state({**state, "codeowners": {"checked": today, "text": text, "unknown": sorted(unknown), "skip": sorted(skip)}})
+    return {"text": text, "unknown": unknown, "skip": skip}
+
+
+def test_files(api, ref: str | None, store, today: str, tids: list[str]) -> dict[str, str]:
+    """Each test's file, checked against the repo's file list (one request, at most once a day, only for new ids)."""
+    state = store.state()
+    known = dict(state.get("test_files") or {})
+    missing = [t for t in tids if t not in known]
+    if missing and state.get("tree_checked") != today:
+        try:
+            tree = api._get(f"/repos/{api.repo}/git/trees/{ref}", {"recursive": "1"}) or {}
+            files = [e["path"] for e in tree.get("tree") or [] if e.get("type") == "blob"]
+            for tid in missing:
+                if (path := test_file(tid, files)):
+                    known[tid] = path
+        except (APIError, OutOfBudget):
+            pass
+        store.save_state({**store.state(), "test_files": known, "tree_checked": today})
+    return {t: known[t] for t in tids if t in known}
+
+
+def attach_owners(result: dict, co: dict, files: dict[str, str]) -> None:
+    """Adds `owners` and `owners_from` (the path they were looked up for) to every test and job the dashboard lists."""
+    if not co.get("text"):
+        return
+    rules, unknown = codeowners.parse(co["text"], co.get("skip") or ()), co.get("unknown") or set()
+    for t in result.get("tests") or []:
+        if t["state"] in SHOWN:
+            path = files.get(t["id"])
+            owners = codeowners.owners_for(path, rules, unknown) if path else []
+            if not owners and t.get("workflow"):
+                path, owners = t["workflow"], codeowners.owners_for(t["workflow"], rules, unknown)
+            t["owners"], t["owners_from"] = owners, path
+    for j in result.get("jobs") or []:
+        if j["state"] in SHOWN and j.get("workflow"):
+            j["owners"], j["owners_from"] = codeowners.owners_for(j["workflow"], rules, unknown), j["workflow"]
+
+
+def owners_line(item: dict, mention: bool = False) -> str:
+    """Markdown naming the owners; code-formatted names notify nobody unless mentions are turned on."""
+    if not item.get("owners"):
+        return ""
+    # A path can't close the code format early and slip a live @-mention into the issue.
+    where = re.sub(r"[`\r\n]", "", item.get("owners_from") or "")
+    names = " ".join(item["owners"]) if mention else ", ".join(f"`{o}`" for o in item["owners"])
+    return f"Owners (CODEOWNERS for `{where}`): {names}"

stackvitals/ci/quarantine_runtime.py

@@ -0,0 +1,90 @@
+"""Quarantine at test time, so a flaky test stops failing the suite without wrapping the CI command.
+
+    stackvitals ci quarantine --print jest     # flags for the test runner, from the quarantine list
+    pytest $(stackvitals ci quarantine --print pytest)   # writes .stackvitals-quarantine.json, prints -p ...
+
+pytest runs a quarantined test as an expected failure (it still runs and reports, but can't fail the suite); Jest,
+Vitest, go test, Playwright, Maven Surefire and dotnet test leave it out by name. Other runners (RSpec, Gradle): wrap the
+command with `stackvitals ci quarantine -- <command>`. Only tests on the list are touched, and an empty or unreadable list touches none.
+"""
+import json
+import os
+import re
+from pathlib import Path
+
+
+def _parts(tid: str) -> tuple[str, str]:
+    # "file::class::name" (JUnit, Jest) or "pkg::TestName/sub" (Go): what's before the last part, and the last part.
+    head, _, name = tid.rpartition("::")
+    return head, name
+
+
+def runtime_args(ids: list[str] | set[str], runner: str) -> list[str]:
+    """The command-line arguments that keep quarantined tests from failing a run of `runner`."""
+    ids = sorted(ids or [])
+    if not ids:
+        return []
+    if runner == "pytest":
+        return ["-p", "stackvitals.ci.quarantine_runtime"]
+    if runner in ("jest", "vitest"):
+        names = sorted({re.escape(_parts(t)[1]) for t in ids if _parts(t)[1]})
+        flag = "--testNamePattern" if runner == "jest" else "-t"
+        # Runners match the full name (describe blocks first), so the test's own name is matched at the end.
+        return [flag, f"^(?!.*(?:{'|'.join(names)})$).*"]
+    if runner == "go":
+        # go test -skip takes one pattern per level; only top-level tests can be listed together safely.
+        tops = sorted({re.escape(_parts(t)[1]) for t in ids
+                       if re.fullmatch(r"(Test|Fuzz|Example)[A-Z0-9_]\w*", _parts(t)[1] or "")})
+        return ["-skip", f"^(?:{'|'.join(tops)})$"] if tops else []
+    if runner == "playwright":
+        # The full title ends with the test's own name; --grep-invert leaves those out.
+        names = sorted({re.escape(_parts(t)[1]) for t in ids if _parts(t)[1]})
+        return ["--grep-invert", f"(?:{'|'.join(names)})$"]
+    if runner == "maven":
+        # Surefire 2.19+: exclusion-only patterns run everything else; the class is the last part of the classname.
+        pats = sorted({f"!{_parts(t)[0].split('::')[-1].split('.')[-1]}#{_parts(t)[1]}" for t in ids if _parts(t)[0] and _parts(t)[1]})
+        return [f"-Dtest={','.join(pats)}", "-Dsurefire.failIfNoSpecifiedTests=false"] if pats else []
+    if runner == "dotnet":
+        fqns = sorted({f"{_parts(t)[0].split('::')[-1]}.{_parts(t)[1]}" for t in ids if _parts(t)[0] and _parts(t)[1]})
+        return ["--filter", "&".join(f"FullyQualifiedName!={f}" for f in fqns)] if fqns else []
+    raise ValueError(f"runner is one of pytest, jest, vitest, go, playwright, maven, dotnet, not {runner!r}")
+
+
+LIST_FILE = ".stackvitals-quarantine.json"  # written by `stackvitals ci quarantine --print pytest`
+
+
+def _quarantined() -> set[str]:
+    src = os.environ.get("STACKVITALS_QUARANTINE") or LIST_FILE
+    try:
+        items = json.loads(Path(src).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return set()
+    return {str(i) for i in items} if isinstance(items, list) else set()
+
+
+def _tokens(text: str) -> list[str]:
+    # "tests.test_a.TestCls" and "tests/test_a.py::TestCls" both read as tests, test_a, TestCls.
+    return [p for p in re.split(r"[./:\\]+", text) if p and p != "py"]
+
+
+def _matches(tid: str, nodeid: str, name: str) -> bool:
+    # Same test name, and the report's module and class end the same way as the node id's file and class (either may
+    # carry more folders in front, since the two can start from different roots).
+    head, last = _parts(tid)
+    if last != name:
+        return False
+    a, b = _tokens(head), _tokens(nodeid.rpartition("::")[0])
+    if not a or not b:
+        return not a
+    n = min(len(a), len(b))
+    return a[-n:] == b[-n:]
+
+
+def pytest_collection_modifyitems(config, items):  # pragma: no cover - exercised through pytest itself
+    import pytest
+    ids = _quarantined()
+    if not ids:
+        return
+    for item in items:
+        if any(_matches(t, item.nodeid, item.name) for t in ids):
+            item.add_marker(pytest.mark.xfail(reason="quarantined by StackVitals (known flaky)", strict=False))