stackvitals 0.1.0__py3-none-any.whl

mergescout/bench/ci.py

@@ -0,0 +1,257 @@
+import hashlib
+import math
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timedelta
+
+import httpx
+
+from mergescout.bench.prior import beta_fit
+
+API = "https://api.github.com"
+FAIL = {"failure", "timed_out"}
+OK = {"success", "neutral"}
+MIN_RUNS = 5
+PAGE = 50
+MIN_GROUP = 20
+
+
+class OutOfBudget(RuntimeError):
+    pass
+
+
+def _t(text: str | None) -> datetime | None:
+    return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
+
+
+def _pct(values: list[float], q: float) -> float:
+    # Nearest-rank percentile: always a value that was really seen.
+    s = sorted(values)
+    return s[min(len(s) - 1, max(0, math.ceil(q * len(s)) - 1))]
+
+
+def _median(values: list[float]) -> float:
+    s, n = sorted(values), len(values)
+    return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2
+
+
+def repo_stats(runs: list[dict], since: datetime | None = None, page_full: bool = False, durations: bool = False,
+               evidence: int = 0) -> dict | None:
+    """CI of one repo per push: how long until all its checks finished, and how they ended."""
+    # GitHub-managed workflows (Dependabot, CodeQL default setup) are not the repo's own CI.
+    own = [r for r in runs if not (r.get("path") or "").startswith("dynamic/")
+           and r.get("created_at") and r.get("updated_at") and (since is None or (_t(r.get("created_at")) or since) >= since)]
+    pushes = {}
+    for r in own:
+        # A tag pushed later on the same commit is its own push, not one push lasting days.
+        pushes.setdefault((r.get("head_sha") or r.get("id"), r.get("head_branch")), []).append(r)
+    if page_full and pushes:
+        # The page ends somewhere inside its oldest push, which may have more runs on the next page.
+        pushes.pop(min(pushes, key=lambda k: min(r.get("created_at") or "" for r in pushes[k])))
+    done = []
+    for group in pushes.values():
+        if any(r.get("status") != "completed" for r in group):
+            continue
+        # Runs cancelled by a newer push (or skipped by a condition) did not give an answer.
+        group = [r for r in group if r.get("conclusion") not in ("cancelled", "skipped")]
+        if group:
+            done.append(group)
+    if len(done) < MIN_RUNS:
+        return None
+    rerun = lambda g: any((r.get("run_attempt") or 1) > 1 for r in g)
+    durations_s = []
+    for g in done:
+        start, end = min(_t(r.get("created_at")) for r in g), max(_t(r.get("updated_at")) for r in g)
+        # The wait a developer sees: queue plus run, up to the last check; re-runs would add human delay.
+        if not rerun(g) and start and end and end >= start:
+            durations_s.append((end - start).total_seconds())
+    if not durations_s:
+        return None
+    n = len(done)
+    # The pushes that weren't green on the first try, newest first: [sha, branch, the failed or re-run run ids].
+    missed = sorted((g for g in done if rerun(g) or not all(r.get("conclusion") in OK for r in g)),
+                    key=lambda g: max(r.get("created_at") or "" for r in g), reverse=True)
+    proof = {"not_first_try": [[(g[0].get("head_sha") or "")[:12], g[0].get("head_branch"),
+                                sorted(r["id"] for r in g if (r.get("run_attempt") or 1) > 1 or r.get("conclusion") not in OK)]
+                               for g in missed[:evidence]], "not_first_try_total": len(missed)} if evidence else {}
+    return {**proof, "pushes": n, "median_s": round(_median(durations_s)), "p90_s": round(_pct(durations_s, 0.9)),
+            "fail_share": round(sum(any(r.get("conclusion") in FAIL for r in g) for g in done) / n, 4),
+            "first_try_share": round(sum(not rerun(g) and all(r.get("conclusion") in OK for r in g)
+                                         for g in done) / n, 4),
+            "rerun_share": round(sum(rerun(g) for g in done) / n, 4),
+            "workflows": len({r.get("path") for g in done for r in g}),
+            **({"durations": durations_s} if durations else {})}
+
+
+def todays_share(names: list[str], now: datetime, slices: int = 28) -> list[str]:
+    # A stable hash splits the repos into 28 parts, one per day, so every repo is read once every four weeks.
+    day = now.toordinal() % slices
+    return [n for n in names if int(hashlib.sha1(n.lower().encode()).hexdigest(), 16) % slices == day]
+
+
+class TokenPool:
+    """Several identities' REST limits (separate from the crawl's GraphQL points); stops before all run low."""
+
+    def __init__(self, tokens: list[str], client: httpx.Client | None = None, reserve: int = 300,
+                 sleep=time.sleep, clock=time.time, max_wait: float = 0):
+        seen = []
+        for t in tokens:
+            if t and t.strip() and t.strip() not in seen:
+                seen.append(t.strip())
+        self.tokens = {t: 5000 for t in seen}
+        self.resets = {t: 0.0 for t in seen}
+        self.client = client or httpx.Client(timeout=httpx.Timeout(60.0, connect=15.0), follow_redirects=True)
+        self.reserve, self.sleep, self.clock, self.lock, self.requests = reserve, sleep, clock, threading.Lock(), 0
+        self.max_wait, self.waited = max_wait, 0.0
+
+    def _pick(self) -> str:
+        with self.lock:
+            live = {t: left for t, left in self.tokens.items() if left > self.reserve}
+            if not self.tokens:
+                raise OutOfBudget("no token can read other repos' Actions runs")
+            if not live:
+                # Everyone is low: wait for the first hourly reset if allowed (other workers wait on the lock).
+                first = min(self.resets, key=self.resets.get)
+                wait = max(0.0, (self.resets[first] or self.clock() + 3600) - self.clock()) + 5
+                if self.waited + wait > self.max_wait:
+                    raise OutOfBudget("every token is near its hourly REST limit")
+                self.waited += wait
+                self.sleep(wait)
+                self.tokens[first] = 5000
+                live = {first: 5000}
+            self.requests += 1
+            return max(live, key=live.get)
+
+    def get(self, path: str, params: dict) -> httpx.Response | None:
+        for attempt in range(3):
+            token = self._pick()
+            try:
+                resp = self.client.get(f"{API}{path}", params=params,
+                                       headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json",
+                                                "User-Agent": "mergescout-bench"})
+            except httpx.TransportError:
+                self.sleep(2 * (attempt + 1))
+                continue
+            left, reset = resp.headers.get("x-ratelimit-remaining"), resp.headers.get("x-ratelimit-reset")
+            with self.lock:
+                if left is not None:
+                    self.tokens[token] = int(left)
+                if reset is not None:
+                    self.resets[token] = float(reset)
+            if resp.status_code == 403 and "not accessible by integration" in resp.text.lower():
+                # App and Actions tokens may be limited to their own repos: drop the token, never the repo.
+                with self.lock:
+                    self.tokens.pop(token, None)
+                    self.resets.pop(token, None)
+                continue
+            if resp.status_code in (403, 429) and left == "0":
+                continue
+            if resp.status_code in (403, 429) and ("retry-after" in resp.headers or "secondary" in resp.text.lower()):
+                self.sleep(min(float(resp.headers.get("retry-after", 60)), 300))
+                continue
+            if resp.status_code >= 500:
+                self.sleep(3 * (attempt + 1))
+                continue
+            return resp
+        return None
+
+
+def _one(pool: TokenPool, repo: dict, since: datetime) -> dict | None:
+    # A date filter makes GitHub's query 3x slower and costlier; the newest runs are filtered here instead.
+    resp = pool.get(f"/repos/{repo['name']}/actions/runs",
+                    {"event": "push", "per_page": PAGE, "exclude_pull_requests": "true"})
+    # 404: gone or renamed; 403: Actions turned off. Neither is a CI to compare.
+    if resp is None:
+        raise RuntimeError("GitHub gave no answer after retries")
+    if resp.status_code >= 400:
+        return None
+    runs = resp.json().get("workflow_runs") or []
+    stats = repo_stats(runs, since, page_full=len(runs) >= PAGE)
+    return {"repo": repo["name"], "language": repo.get("language"), "stars": repo.get("stars"), **stats} if stats else None
+
+
+def fetch_all(pool: TokenPool, repos: list[dict], now: datetime, days: int = 30, workers: int = 10, log=None,
+              read=None) -> dict:
+    """One sample per repo: push CI by default, or another reader (star history) given as read(pool, repo, now)."""
+    since = now - timedelta(days=days)
+    records, complete, day = [], True, now.strftime("%Y-%m-%d")
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        futures = [ex.submit(read, pool, r, now) if read else ex.submit(_one, pool, r, since) for r in repos]
+        for i, fut in enumerate(futures):
+            try:
+                rec = fut.result()
+            except Exception:
+                # One repo that can't be read (limits, a broken answer) never loses the rest of the day.
+                complete = False
+                continue
+            if rec:
+                records.append({**rec, "day": day})
+            if log and (i + 1) % 1000 == 0:
+                log(f"{i + 1}/{len(repos)} repos read, {len(records)} with enough data")
+    return {"records": records, "complete": complete, "requests": pool.requests}
+
+
+def tier(stars: int | None) -> str:
+    stars = stars or 0
+    return "10k+" if stars >= 10_000 else "1k-10k" if stars >= 1000 else "100-1k"
+
+
+BOOTSTRAP_RESAMPLES, BOOTSTRAP_SEED = 400, 20261021
+
+
+def bootstrap_median(values: list[float], resamples: int = BOOTSTRAP_RESAMPLES, seed: int = BOOTSTRAP_SEED) -> tuple:
+    """A 95% range for a group's median by resampling its repos; the same seed on every build, so the file recomputes."""
+    import random
+    rnd, n = random.Random(seed), len(values)
+    meds = sorted(_pct([values[rnd.randrange(n)] for _ in range(n)], 0.5) for _ in range(resamples))
+    return meds[int(0.025 * resamples)], meds[min(resamples - 1, int(0.975 * resamples))]
+
+
+def quantiles(values: list[float]) -> dict:
+    """P10-P90 of a group (nearest rank) and the 95% range of its median."""
+    low, high = bootstrap_median(values)
+    return {**{f"p{int(q * 100)}": _pct(values, q) for q in (0.1, 0.25, 0.5, 0.75, 0.9)}, "p50_low": low, "p50_high": high}
+
+
+def _summary(values: list[float]) -> dict:
+    return quantiles(values)
+
+
+def _prior(items: list[dict]) -> list[float] | None:
+    fit = beta_fit([(round(r["first_try_share"] * r["pushes"]), r["pushes"]) for r in items if r.get("pushes")])
+    return [round(x, 3) for x in fit] if fit else None
+
+
+def aggregate(records: list[dict], now: datetime, top_languages: int = 25) -> dict:
+    """Benchmarks by language and size; a group is published only with at least 20 repos."""
+    latest = {}
+    for r in sorted(records, key=lambda r: r.get("day") or ""):
+        latest[r["repo"].lower()] = r
+    recs = list(latest.values())
+    counts = {}
+    for r in recs:
+        counts[r.get("language") or "Other"] = counts.get(r.get("language") or "Other", 0) + 1
+    langs = {name for name, _ in sorted(counts.items(), key=lambda kv: -kv[1])[:top_languages]}
+    groups = {}
+    for r in recs:
+        lang = r.get("language") if r.get("language") in langs else "Other"
+        for key in (f"{lang}|{tier(r.get('stars'))}", f"{lang}|all", f"all|{tier(r.get('stars'))}", "all|all"):
+            groups.setdefault(key, []).append(r)
+    out = {}
+    for key, items in groups.items():
+        if len(items) < MIN_GROUP:
+            continue
+        out[key] = {"repos": len(items), "median_s": _summary([r["median_s"] for r in items]),
+                    "p90_s": _summary([r["p90_s"] for r in items]),
+                    "first_try_share": _summary([r["first_try_share"] for r in items]),
+                    "fail_share": _summary([r["fail_share"] for r in items]),
+                    "rerun_share": _summary([r["rerun_share"] for r in items]),
+                    # The Beta prior of the group's first-try shares, so a repo with few pushes can be pulled toward it.
+                    "first_try_prior": _prior(items)}
+    return {"version": 1, "built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "repos": len(recs),
+            "definition": "per push of the last 30 days (newest 50 push runs): time from push until all its checks finished, and how they ended; GitHub-managed workflows excluded",
+            "population": "public GitHub repos with 100+ stars that pushed in the last 30 days, a 28th of them read each day "
+                          "(each about once in 4 weeks)",
+            "gaps": "GitHub Actions push runs only: no pull request runs, no other CI systems, no private or enterprise repos",
+            "groups": out}

mergescout/bench/deploys.py

@@ -0,0 +1,115 @@
+"""How often public repos ship: two REST requests per sampled repo (its deployments, its releases). Production deployments
+count first, else stable releases (pre-releases and alpha/beta/rc tags left out); deployment days per week and per month
+over 90 days with DevLake's counting (weeks with none count as zero, lower median) and DORA's 2023 frequency tiers."""
+import re
+from collections import Counter, defaultdict
+from datetime import datetime, timedelta
+
+from mergescout.bench.ci import quantiles, tier
+
+DAYS = 90
+PAGE = 100
+MIN_GROUP = 20
+PRODUCTION = {"production", "prod", "live"}
+PRE_TAG = re.compile(r"(?i)[-.](alpha|beta|rc|canary|next|dev|pre|preview|nightly|snapshot|insiders)([-.\d]|$)")
+DEFINITION = ("per repo, the last 90 days: days with a successful-or-pending deployment to a production environment "
+              "(GitHub's Deployments API, an environment named production, prod or live, or starting with it), else days "
+              "with a published stable release (no pre-releases, no alpha/beta/rc tags); deployment days per week and per "
+              "month are DevLake's lower medians over the calendar (weeks with none count as zero); tiers are DORA 2023's "
+              "(elite 5+ days a week, high 1+ a week, medium 1+ a month, else low)")
+POPULATION = "public GitHub repos with 100+ stars that pushed in the last 30 days, a slice of them read each day"
+GAPS = ("deployments made outside GitHub (another repo, a hosting service without GitHub deployments, ArgoCD) are not seen, "
+        "nor deploy workflows without an environment; the newest 100 deployments and releases are read, so the busiest repos "
+        "are measured over the days those cover; deployment state is not read, so a failed deployment can count")
+
+
+def _first_word(env: str | None) -> str:
+    return re.split(r"[\s\-–_/:]+", (env or "").strip().lower(), maxsplit=1)[0]
+
+
+def _lower_median(values: list[int]) -> int:
+    # DevLake: the largest value whose percent rank is at most 0.5 (ties share the lowest rank).
+    v = sorted(values)
+    if not v:
+        return 0
+    if len(v) == 1:
+        return v[0]
+    return max(x for x in v if v.index(x) / (len(v) - 1) <= 0.5)
+
+
+def frequency(days: set[str], now: datetime, start: datetime) -> tuple[int, int, str]:
+    """(deployment days per week, per month, DORA 2023 tier) over the calendar days after `start` up to today."""
+    weeks, months = defaultdict(int), defaultdict(int)
+    k = 0
+    while now - timedelta(days=k) > start:
+        day = (now - timedelta(days=k)).date()
+        hit = day.isoformat() in days
+        weeks[day - timedelta(days=day.weekday())] += hit
+        months[day.replace(day=1)] += hit
+        k += 1
+    per_week, per_month = _lower_median(list(weeks.values())), _lower_median(list(months.values()))
+    level = "elite" if per_week >= 5 else "high" if per_week >= 1 else "medium" if per_month >= 1 else "low"
+    return per_week, per_month, level
+
+
+def _get(pool, path: str):
+    resp = pool.get(path, {"per_page": PAGE})
+    if resp is None:
+        raise RuntimeError("GitHub gave no answer after retries")
+    return resp.json() if resp.status_code < 400 else []
+
+
+def repo_deploys(pool, repo: dict, now: datetime, days: int = DAYS) -> dict:
+    """One repo: where its shipping is seen, on how many days, and its frequency (two requests)."""
+    since = now - timedelta(days=days)
+    stamp = since.strftime("%Y-%m-%dT%H:%M:%SZ")
+    deployments = _get(pool, f"/repos/{repo['name']}/deployments")
+    releases = _get(pool, f"/repos/{repo['name']}/releases")
+    found = [d["created_at"] for d in deployments if isinstance(d, dict) and _first_word(d.get("environment")) in PRODUCTION
+             and (d.get("created_at") or "") >= stamp]
+    source, page = "deployments api", deployments
+    if not found:
+        found = [r["published_at"] for r in releases if isinstance(r, dict) and r.get("published_at") and not r.get("draft")
+                 and not r.get("prerelease") and not PRE_TAG.search(r.get("tag_name") or "") and r["published_at"] >= stamp]
+        source, page = ("releases", releases) if found else (None, [])
+    start = since
+    if len(page) >= PAGE and found:
+        # A full page that ends inside the window: measure only the days it covers.
+        start = max(since, datetime.fromisoformat(min(found).replace("Z", "+00:00")) - timedelta(days=1))
+    per_week, per_month, level = frequency({t[:10] for t in found}, now, start) if found else (0, 0, None)
+    return {"repo": repo["name"], "language": repo.get("language"), "stars": repo.get("stars"), "source": source,
+            "deploy_days": len({t[:10] for t in found}), "days_per_week": per_week, "days_per_month": per_month, "tier": level,
+            "capped": len(page) >= PAGE, "requests": 2, "day": now.strftime("%Y-%m-%d")}
+
+
+def aggregate_deploys(records: list[dict], now: datetime, top_languages: int = 25) -> dict:
+    """Groups by language and stars (20+ repos): where shipping is seen, and how often among repos where it is."""
+    latest = {}
+    for r in sorted(records, key=lambda r: r.get("day") or ""):
+        latest[r["repo"].lower()] = r
+    recs = list(latest.values())
+    counts = Counter(r.get("language") or "Other" for r in recs)
+    langs = {name for name, _ in counts.most_common(top_languages)}
+    groups = defaultdict(list)
+    for r in recs:
+        lang = r.get("language") if r.get("language") in langs else "Other"
+        for key in (f"{lang}|{tier(r.get('stars'))}", f"{lang}|all", f"all|{tier(r.get('stars'))}", "all|all"):
+            groups[key].append(r)
+    out = {}
+    for key, items in groups.items():
+        if len(items) < MIN_GROUP:
+            continue
+        n = len(items)
+        shipping = [r for r in items if r.get("source")]
+        entry = {"repos": n, "sources": {s: round(sum(r.get("source") == s for r in items) / n, 4)
+                                         for s in ("deployments api", "releases")}}
+        entry["sources"]["none"] = round(1 - sum(entry["sources"].values()), 4)
+        if len(shipping) >= MIN_GROUP:
+            entry["days_per_week"] = {"repos": len(shipping), **quantiles([r["days_per_week"] for r in shipping])}
+            entry["days_per_month"] = {"repos": len(shipping), **quantiles([r["days_per_month"] for r in shipping])}
+            entry["tiers"] = {t: round(sum(r.get("tier") == t for r in shipping) / len(shipping), 4)
+                              for t in ("elite", "high", "medium", "low")}
+        out[key] = entry
+    return {"version": 1, "built_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "repos": len(recs),
+            "requests": sum(r.get("requests") or 0 for r in recs), "definition": DEFINITION, "population": POPULATION,
+            "gaps": GAPS, "groups": out}

mergescout/bench/matched.py

@@ -0,0 +1,22 @@
+"""Like-with-like comparisons for the benchmarks: a share compared within matched groups, so a crawl-wide gap is not
+just a difference in which repos, months or sizes each kind of pull request comes from."""
+import math
+
+
+def mh_risk_difference(groups: list[tuple[int, int, int, int]]) -> tuple[float | None, float | None]:
+    """(difference, standard error) of a share between two kinds of item compared within matched groups: each group is
+    (hits, items) of the first kind and (hits, items) of the second. Mantel-Haenszel weights; Sato's variance, which stays
+    right when most groups hold one or two items (a plain per-group variance falls to zero there)."""
+    w_sum = num = p_sum = q_sum = 0.0
+    for a, n1, c, n0 in groups:
+        total = n1 + n0
+        if not n1 or not n0:
+            continue
+        w_sum += n1 * n0 / total
+        num += (a * n0 - c * n1) / total
+        p_sum += (n1 * n1 * c - n0 * n0 * a + n1 * n0 * (n0 - n1) / 2) / (total * total)
+        q_sum += (a * (n0 - c) + c * (n1 - a)) / (2 * total)
+    if not w_sum:
+        return None, None
+    rd = num / w_sum
+    return rd, math.sqrt(max(0.0, rd * p_sum + q_sum)) / w_sum

mergescout/bench/prior.py

@@ -0,0 +1,42 @@
+"""Peer priors for the benchmarks: the Beta(a, b) that best explains many repos' (hits, tries) pairs, so a small repo's
+share can be pulled toward repos like it instead of shown raw."""
+import math
+
+
+def _golden(f, lo: float, hi: float, steps: int = 60) -> float:
+    # The x in [lo, hi] where f is largest, for f with one peak.
+    g = (math.sqrt(5) - 1) / 2
+    x1, x2 = hi - g * (hi - lo), lo + g * (hi - lo)
+    f1, f2 = f(x1), f(x2)
+    for _ in range(steps):
+        if f1 < f2:
+            lo, x1, f1 = x1, x2, f2
+            x2 = lo + g * (hi - lo)
+            f2 = f(x2)
+        else:
+            hi, x2, f2 = x2, x1, f1
+            x1 = hi - g * (hi - lo)
+            f1 = f(x1)
+    return (lo + hi) / 2
+
+
+def beta_fit(pairs: list[tuple[int, int]]) -> tuple[float, float] | None:
+    """The Beta(a, b) prior that best explains many (hits, tries) pairs (beta-binomial maximum likelihood)."""
+    from collections import Counter
+    counts = Counter((min(k, n), n) for k, n in pairs if n > 0)
+    if not counts:
+        return None
+    lbeta = lambda x, y: math.lgamma(x) + math.lgamma(y) - math.lgamma(x + y)  # noqa: E731
+
+    def loglik(mu: float, log_s: float) -> float:
+        s = math.exp(log_s)
+        a, b = mu * s, (1 - mu) * s
+        return sum(c * (lbeta(k + a, n - k + b) - lbeta(a, b)) for (k, n), c in counts.items())
+    total_k, total_n = sum(k * c for (k, n), c in counts.items()), sum(n * c for (k, n), c in counts.items())
+    mu, log_s = min(0.999, max(0.001, total_k / total_n)), math.log(10)
+    # Alternate between the mean and the strength (a + b); a few rounds settle both.
+    for _ in range(6):
+        log_s = _golden(lambda v: loglik(mu, v), math.log(0.05), math.log(1e4))
+        mu = _golden(lambda v: loglik(v, log_s), 0.001, 0.999)
+    s = math.exp(log_s)
+    return mu * s, (1 - mu) * s