stackvitals 0.1.0__py3-none-any.whl

stackvitals/portal/inventory.py

@@ -0,0 +1,264 @@
+"""The org's repos and what is known about each: settings, custom properties, branch rules, and the suite's own module
+results read from each repo's stackvitals-data branch (where the suite runs)."""
+import json
+import re
+from datetime import datetime, timedelta
+
+import httpx
+
+from stackvitals.ci.api import APIError
+from stackvitals.core.actors import INSIDER_ASSOC, People, kind_of
+from stackvitals.core.store import accept
+from stackvitals.portal import descriptor
+from stackvitals.portal.github import NA, REFUSED, read, read_file
+
+DATA_BRANCH = "stackvitals-data"
+FILES = ("readme", "license", "contributing", "code_of_conduct")
+SCORECARD_API = "https://api.scorecard.dev/projects/github.com/"
+
+
+def _repo(r: dict) -> dict:
+    return {"name": r["full_name"], "private": bool(r.get("private")), "archived": bool(r.get("archived")),
+            "fork": bool(r.get("fork")), "language": r.get("language"), "pushed": r.get("pushed_at"),
+            "default_branch": r.get("default_branch"), "topics": r.get("topics") or [], "stars": r.get("stargazers_count"),
+            "license": (r.get("license") or {}).get("key")}
+
+
+def list_repos(api, org: str, mode: str, max_pages: int = 50) -> list[dict]:
+    """Private mode: every repo the org's App is installed on. Public mode: the org's public repos (or a user's)."""
+    found = []
+    for page in range(1, max_pages + 1):
+        if mode == "private":
+            body = read(api, "/installation/repositories", {"per_page": 100, "page": page})
+            batch = [] if body is NA else body.get("repositories") or []
+        else:
+            # Local runs list what the user's own token can see; the Action's public mode lists public repos only.
+            body = read(api, f"/orgs/{org}/repos", {"type": "all" if mode == "local" else "public", "per_page": 100, "page": page})
+            if body is NA and page == 1:
+                body = read(api, f"/users/{org}/repos", {"type": "owner", "per_page": 100, "page": page})
+            batch = [] if body is NA else body
+        found += [_repo(r) for r in batch if isinstance(r, dict) and r.get("full_name")]
+        if len(batch) < 100:
+            break
+    return sorted({r["name"]: r for r in found}.values(), key=lambda r: r["name"].lower())
+
+
+def custom_properties(api, org: str) -> dict:
+    # Tier, lifecycle, system: GitHub's own repo metadata, read rather than rebuilt (one paged call for the whole org).
+    out = {}
+    for page in range(1, 51):
+        body = read(api, f"/orgs/{org}/properties/values", {"per_page": 100, "page": page})
+        if body is NA or not isinstance(body, list):
+            return out
+        for r in body:
+            out[r.get("repository_full_name")] = {p.get("property_name"): p.get("value") for p in r.get("properties") or []}
+        if len(body) < 100:
+            return out
+    return out
+
+
+def branch_rules(api, repo: str, branch: str | None) -> list[str] | None:
+    # The rules that apply to the default branch, from rulesets and branch protection alike.
+    body = read(api, f"/repos/{repo}/rules/branches/{branch}") if branch else NA
+    return None if body is NA else sorted({r.get("type") for r in body if isinstance(r, dict) and r.get("type")})
+
+
+def _json(api, repo: str, path: str) -> dict | None:
+    text = read_file(api, repo, path, DATA_BRANCH)
+    try:
+        data = json.loads(text) if text is not NA else None
+    except ValueError:
+        return None
+    return accept(data)
+
+
+def module_facts(api, repo: str, pre: dict | None = None) -> dict | None:
+    """The suite's results for this repo, or None where the suite doesn't run (no data branch)."""
+    if (not pre["data_branch"]) if pre else read(api, f"/repos/{repo}/branches/{DATA_BRANCH}") is NA:
+        return None
+    # Since Phase 11 one file holds every fact as a number; older installs keep the three files below.
+    app = None
+    if pre and pre.get("app"):
+        try:
+            app = accept(json.loads(pre["app"]))
+        except ValueError:
+            app = None
+    f = (app or _json(api, repo, "docs/app.json") or {}).get("facts")
+    if isinstance(f, dict):
+        share, score = f.get("ci_first_try"), f.get("community_score")
+        return {"ci_first_try": f"{share:.0%}" if isinstance(share, (int, float)) else None, "ci_first_try_share": share,
+                "flow": {"open": f.get("open_prs"), "stale": f.get("stale_prs"),
+                         "first_review_median_s": f.get("first_review_median_s")} if f.get("open_prs") is not None else None,
+                "community_score": f"{score}/100" if isinstance(score, (int, float)) else None,
+                # Every number the repo publishes, for the org view.
+                "facts": {k: v for k, v in f.items() if isinstance(v, (int, float)) and not isinstance(v, bool)}}
+    ci, flow, community = _json(api, repo, "docs/badge.json"), _json(api, repo, "docs/flow.json"), \
+        _json(api, repo, "docs/community-score.json")
+    pickup = ((flow or {}).get("stages") or {}).get("pickup") or {}
+    return {"ci_first_try": (ci or {}).get("message"),
+            "flow": {"open": flow.get("open"), "stale": flow.get("stale"), "first_review_median_s": pickup.get("median_s")}
+            if flow else None,
+            "community_score": (community or {}).get("message")}
+
+
+def repo_files(api, repo: dict, pre: dict | None = None) -> dict:
+    """README, licence, contributing guide and code of conduct: True, False, or None when nobody could check."""
+    if pre:
+        return pre["files"]
+    body = read(api, f"/repos/{repo['name']}/community/profile")
+    if body is not NA and isinstance(body, dict) and isinstance(body.get("files"), dict):
+        files = body["files"]
+        owner = repo["name"].split("/")[0].lower()
+        # GitHub fills gaps from the org's .github repo; the evidence says so.
+        defaults = [k for k in FILES if isinstance(files.get(k), dict)
+                    and f"/{owner}/.github/" in (files[k].get("html_url") or "").lower()]
+        return {**{k: bool(files.get(k)) or (k == "license" and bool(repo.get("license"))) for k in FILES}, "org_defaults": defaults}
+    # Private repos may refuse the profile: the README is asked for directly and the licence comes from the repo itself.
+    return {"readme": read(api, f"/repos/{repo['name']}/readme") is not NA, "license": bool(repo.get("license")),
+            "contributing": None, "code_of_conduct": None}
+
+
+def descriptor_for(api, repo: dict, pre: dict | None = None) -> dict:
+    """The repo's catalog-info.yaml as typed entities and their problems (one request), or present: False."""
+    if pre:
+        text = pre["descriptor"] if pre["descriptor"] is not None else NA
+    else:
+        text = read_file(api, repo["name"], descriptor.FILE, repo.get("default_branch"))
+    if text is NA:
+        return {"present": False}
+    entities, problems = descriptor.parse(text)
+    return {"present": True, "entities": entities, "problems": problems}
+
+
+def file_facts(api, repo: dict, rules: list[dict], pre: dict | None = None) -> dict:
+    """For portal.toml's file rules: whether each file is there and whether its patterns match (one request per file).
+    Only these yes/no answers are kept, never the file's text."""
+    out = {}
+    for path in dict.fromkeys(r["path"] for r in rules):
+        if pre and path in pre["rule_files"]:
+            text = pre["rule_files"][path] if pre["rule_files"][path] is not None else NA
+        else:
+            text = read_file(api, repo["name"], path, repo.get("default_branch"))
+        if text is NA:
+            out[path] = {"present": False}
+            continue
+        found = {"present": True}
+        for r in rules:
+            for key in ("contains", "lacks"):
+                if r["path"] == path and r.get(key):
+                    found[f"{key}:{r[key]}"] = bool(re.search(r[key], text[:descriptor.MAX_BYTES], re.M))
+        out[path] = found
+    return out
+
+
+def scorecard_checks(client: httpx.Client, name: str, previous: dict | None, now: datetime) -> dict | None:
+    """OpenSSF Scorecard single checks for a public repo, from its public API (results change weekly, so read once a week)."""
+    stamp = now.strftime("%Y-%m-%dT%H:%M:%SZ")
+    if previous and (previous.get("fetched") or "") >= (now - timedelta(days=7)).strftime("%Y-%m-%dT%H:%M:%SZ"):
+        return previous
+    try:
+        # The API matches names case-sensitively and lists them in lower case.
+        resp = client.get(SCORECARD_API + name.lower(), timeout=20)
+    except httpx.HTTPError:
+        return previous
+    if resp.status_code == 404:
+        return {"fetched": stamp, "date": None, "checks": None}
+    if resp.status_code != 200:
+        return previous
+    try:
+        body = resp.json()
+    except ValueError:
+        return previous
+    return {"fetched": stamp, "date": body.get("date"),
+            "checks": {c["name"]: c.get("score") for c in body.get("checks") or [] if isinstance(c, dict) and c.get("name")}}
+
+
+def push_ci(api, repo: str, now: datetime, days: int = 30, previous: dict | None = None) -> dict | None:
+    """First-try green and time to green for the newest 50 push runs, measured exactly like the crawl's CI benchmark,
+    for repos where CI Health doesn't run (one request; free when no run happened since the last check, which then
+    keeps the last check's numbers)."""
+    from mergescout.bench.ci import PAGE, repo_stats
+    path, params = f"/repos/{repo}/actions/runs", {"event": "push", "per_page": PAGE, "exclude_pull_requests": "true"}
+    tag = None
+    if hasattr(api, "get_if_changed"):
+        try:
+            body, tag = api.get_if_changed(path, params, (previous or {}).get("etag"))
+        except APIError as exc:
+            if not any(code in str(exc) for code in REFUSED):
+                raise
+            body = NA
+        if body is None:
+            return previous
+    else:
+        body = read(api, path, params)
+    runs = (body or {}).get("workflow_runs") if body is not NA and isinstance(body, dict) else None
+    if not runs:
+        return None
+    stats = repo_stats(runs, now - timedelta(days=days), page_full=len(runs) >= PAGE)
+    return {**{k: stats[k] for k in ("first_try_share", "pushes", "median_s")}, **({"etag": tag} if tag else {})} if stats else None
+
+
+REVIEW_QUERY = """
+query($owner:String!,$name:String!) {
+  rateLimit { cost remaining resetAt }
+  repository(owner:$owner,name:$name) {
+    pullRequests(last:50, orderBy:{field:CREATED_AT, direction:ASC}) {
+      nodes { createdAt author { __typename login } authorAssociation
+              reviews(first:10) { nodes { submittedAt author { login } authorAssociation } } }
+    }
+  }
+}"""
+MAINTAINERS = INSIDER_ASSOC
+
+
+def _review_records(nodes: list[dict]) -> list[dict]:
+    # The newest PRs as the shared role rule reads them (author, label, reviews); no merge events in this query.
+    out = []
+    for p in nodes:
+        author = p.get("author") or {}
+        out.append({"au": author.get("login"), "at": author.get("__typename"), "as": p.get("authorAssociation"),
+                    "ev": [["review", r.get("submittedAt"), (r.get("author") or {}).get("login"), None, r.get("authorAssociation"), None]
+                           for r in (p.get("reviews") or {}).get("nodes") or [] if r], "cm": []})
+    return out
+
+
+def review_stats(gql, repo: str, now: datetime, days: int = 90, min_prs: int = 10) -> dict | None:
+    """Median time to the first review by a maintainer other than the author, for people's PRs of the last 90 days among
+    the newest 50 (the crawl benchmark's definition), where Team Flow doesn't run (one GraphQL point)."""
+    from mergescout.github.graphql import GraphQLError, RateLimited
+    owner, name = repo.split("/", 1)
+    try:
+        data = gql.query(REVIEW_QUERY, {"owner": owner, "name": name}).data
+    except (GraphQLError, RateLimited):
+        return None
+    nodes = ((data.get("repository") or {}).get("pullRequests") or {}).get("nodes") or []
+    since = (now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    prs = [p for p in nodes if p and (p.get("createdAt") or "") >= since
+           and kind_of((p.get("author") or {}).get("login"), (p.get("author") or {}).get("__typename")) == "person"]
+    if len(prs) < min_prs:
+        return None
+    people = People(_review_records(prs))
+    waits, reviewers = [], {}
+    for p in prs:
+        author = (p.get("author") or {}).get("login")
+        for r in (p.get("reviews") or {}).get("nodes") or []:
+            who = (r or {}).get("author") and r["author"].get("login")
+            if who and who != author and people.is_insider(who):
+                reviewers[who] = reviewers.get(who, 0) + 1
+        first = min((r["submittedAt"] for r in (p.get("reviews") or {}).get("nodes") or []
+                     if r and r.get("submittedAt") and people.is_insider((r.get("author") or {}).get("login"))
+                     and (r.get("author") or {}).get("login") != author), default=None)
+        if first:
+            waits.append((datetime.fromisoformat(first.replace("Z", "+00:00"))
+                          - datetime.fromisoformat(p["createdAt"].replace("Z", "+00:00"))).total_seconds())
+    waits.sort()
+    mid = len(waits) // 2
+    median = (waits[mid] if len(waits) % 2 else (waits[mid - 1] + waits[mid]) / 2) if waits else None
+    return {"first_review_median_s": median, "prs": len(prs),
+            "reviewers": [w for w, _ in sorted(reviewers.items(), key=lambda kv: (-kv[1], kv[0]))][:3]}
+
+
+def branch_protected(api, repo: str, branch: str | None) -> bool | None:
+    # Classic branch protection isn't in the rules endpoint; whether it requires reviews needs an admin token.
+    body = read(api, f"/repos/{repo}/branches/{branch}") if branch else NA
+    return bool(body.get("protected")) if body is not NA and isinstance(body, dict) else None

stackvitals/portal/owners.py

@@ -0,0 +1,78 @@
+"""Who owns a repo, checked rather than just read: the CODEOWNERS file GitHub would use, GitHub's own error check, teams that
+have members, and users who are still in the org. Without a file, recent committers are offered as suggestions only."""
+from collections import Counter
+from datetime import datetime, timedelta
+
+from stackvitals import codeowners
+from stackvitals.portal.github import NA, read, read_file, status
+
+# Commit authors that are not people: GitHub's placeholder for unlinked emails, web edits, deleted accounts.
+PLACEHOLDERS = {"invalid-email-address", "web-flow", "ghost"}
+CATCH_ALL = {"*", "/*", "/", "**", "/**"}
+CODEOWNERS_PATHS = codeowners.PATHS
+MAX_USER_CHECKS = 10
+
+
+parse_codeowners = codeowners.parse
+
+
+def suggest(api, repo: str, now: datetime, days: int = 90) -> list[str]:
+    commits = read(api, f"/repos/{repo}/commits", {"since": (now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ"),
+                                                   "per_page": 100})
+    if commits is NA or not isinstance(commits, list):
+        return []
+    counts = Counter(((c.get("author") or {}).get("login") for c in commits))
+    # Machine accounts are often plain users named like bots (zaniebot, astral-docs-bot): never suggested as owners.
+    return [who for who, _ in counts.most_common()
+            if who and who.lower() not in PLACEHOLDERS and not who.lower().endswith(("bot", "[bot]", "-bot"))][:3]
+
+
+def owner_status(api, org: str, repo: dict, now: datetime, cache: dict, members_visible: bool = True,
+                 reviewers: list[str] | None = None, pre: dict | None = None) -> dict:
+    """valid / weak / none, with the reasons. Team sizes and memberships are cached for the whole run. In public mode
+    the token can't see private memberships, so users aren't checked (members_visible=False)."""
+    name, branch = repo["name"], repo.get("default_branch")
+    text = file = None
+    if pre:
+        file, text = pre["codeowners"]
+    for path in () if pre else CODEOWNERS_PATHS:
+        got = read_file(api, name, path, branch)
+        if got is not NA:
+            text, file = got, path
+            break
+    if text is None:
+        # People who review are better owner candidates than committers (machine accounts commit, e.g. charmcli).
+        return {"status": "none", "file": None, "reasons": ["no CODEOWNERS file"], "owners": [], "teams": {}, "notes": [],
+                "suggested": (reviewers or [])[:3] or suggest(api, name, now),
+                "suggested_from": "recent reviews" if reviewers else "recent commits"}
+    rules = parse_codeowners(text)
+    owners = sorted({o for _, os in rules for o in os})
+    reasons, teams, notes = [], {}, []
+    if not any(pattern in CATCH_ALL and os for pattern, os in rules):
+        reasons.append("no catch-all line (* or /): files outside the listed paths have no owner")
+    errors = read(api, f"/repos/{name}/codeowners/errors", {"ref": branch} if branch else None)
+    if errors is not NA and (errors or {}).get("errors"):
+        reasons.append(f"{len(errors['errors'])} errors in CODEOWNERS (GitHub's own check)")
+    users = 0
+    for owner in owners:
+        if owner.startswith("@") and "/" in owner:
+            team = owner[1:]
+            team_org, slug = team.split("/", 1)
+            if ("team", team) not in cache:
+                members = read(api, f"/orgs/{team_org}/teams/{slug}/members", {"per_page": 100})
+                cache[("team", team)] = NA if members is NA else len(members)
+            teams[team] = cache[("team", team)]
+            if teams[team] == 0:
+                reasons.append(f"team {team} has no members")
+        elif owner.startswith("@") and members_visible and users < MAX_USER_CHECKS:
+            user, users = owner[1:], users + 1
+            if ("member", user) not in cache:
+                cache[("member", user)] = status(api, f"/orgs/{org}/members/{user}")
+            if cache[("member", user)] == 404:
+                reasons.append(f"{user} is named as an owner but is not a member of {org}")
+    if any(v is NA for v in teams.values()):
+        notes.append("team members not visible to this token")
+    if not members_visible and any(o.startswith("@") and "/" not in o for o in owners):
+        notes.append("org membership of named users not checked (not visible to this token)")
+    return {"status": "weak" if reasons else "valid", "file": file, "reasons": reasons, "owners": owners, "notes": notes,
+            "teams": {k: (None if v is NA else v) for k, v in teams.items()}}

stackvitals/portal/pages.py

@@ -0,0 +1,153 @@
+"""Markdown pages in the portal repo, which GitHub shows only to people who can read that repo. Private repos never reach
+a page anyone else can open."""
+from stackvitals.flow.report import _md
+
+
+def publishable(repos: list[dict], destination_private: bool) -> list[dict]:
+    # The guard: a public destination gets public repos only, whatever the token could read.
+    return list(repos) if destination_private else [r for r in repos if not r.get("private")]
+
+
+def _cell(text) -> str:
+    return _md(" ".join(str(text).split())).replace("@", "@​")
+
+
+def _owners(r: dict) -> tuple[str, str]:
+    o = r.get("owners") or {}
+    why = "; ".join((o.get("reasons") or []) + (o.get("notes") or []))
+    if o.get("suggested"):
+        why += ("; " if why else "") + f"suggested from {o.get('suggested_from') or 'recent commits'}: " + ", ".join(o["suggested"])
+    return o.get("status") or "-", why
+
+
+CATALOG_ROWS = 200
+
+
+def catalog_entities(active: list[dict]) -> tuple[list[dict], list[tuple[str, str]], int]:
+    """Every entity the active repos' catalog-info.yaml files declare (with its repo), their problems, and how many
+    repos have a file."""
+    entities, problems, files = [], [], 0
+    for r in active:
+        d = r.get("descriptor") or {}
+        if d.get("present"):
+            files += 1
+            entities += [{**e, "repo": r["name"]} for e in d.get("entities") or []]
+            problems += [(r["name"], p) for p in d.get("problems") or []]
+    return entities, problems, files
+
+
+def _owner_cell(e: dict) -> str:
+    check = e.get("owner_check") or []
+    note = {"pass": "", "fail": " (not in CODEOWNERS)"}.get(check[0] if check else "", "")
+    return f"{e.get('owner') or '-'}{note}"
+
+
+def _catalog_lines(active: list[dict]) -> list[str]:
+    entities, problems, files = catalog_entities(active)
+    if not files:
+        return []
+    off = sum(1 for e in entities if (e.get("owner_check") or [""])[0] == "fail")
+    out = ["## Catalog", "", f"{len(entities)} entities from {files} of {len(active)} active repos' catalog-info.yaml files"
+           + (f"; {off} owner{'' if off == 1 else 's'} not in CODEOWNERS" if off else "") + ".", "",
+           "| Entity | Kind | Type | Lifecycle | Owner | System | Repo |", "|---|---|---|---|---|---|---|"]
+    for e in entities[:CATALOG_ROWS]:
+        out.append(f"| {_cell(e['name'])} | {e['kind']} | {_cell(e.get('type') or '-')} | {_cell(e.get('lifecycle') or '-')} | "
+                   f"{_cell(_owner_cell(e))} | {_cell(e.get('system') or '-')} | {_cell(e['repo'])} |")
+    if problems:
+        out += ["", "Problems: " + "; ".join(f"{_cell(repo)}: {_cell(p)}" for repo, p in problems[:20]) + "."]
+    return out + [""]
+
+
+def render_catalog(catalog: dict, destination_private: bool) -> str:
+    """The catalog page: owner status counts, then one row per active repo with the evidence."""
+    repos = publishable(catalog["repos"], destination_private)
+    active = [r for r in repos if not r.get("archived") and not r.get("fork")]
+    counts = {s: sum(1 for r in active if (r.get("owners") or {}).get("status") == s) for s in ("valid", "weak", "none")}
+    archived, forks = sum(1 for r in repos if r.get("archived")), sum(1 for r in repos if r.get("fork") and not r.get("archived"))
+    out = [f"# Portal: {_cell(catalog['org'])}", "",
+           f"{len(repos)} repos ({len(active)} active, {archived} archived, {forks} forks), updated {catalog['built_at'][:16]} UTC.",
+           "", f"Owners: {counts['valid']} valid, {counts['weak']} weak, {counts['none']} without a CODEOWNERS file.", ""]
+    if len(repos) < len(catalog["repos"]):
+        out += ["Private repos are left out of this page because the portal repo is public.", ""]
+    if not catalog.get("complete", True):
+        out += ["Some repos keep an earlier check (GitHub's hourly limit); the next run continues with them.", ""]
+    scored = [r for r in active if r.get("score")]
+    if scored:
+        n = {lvl: sum(1 for r in scored if r["score"]["level"] == lvl) for lvl in ("gold", "silver", "bronze", None)}
+        out += [f"Levels: {n['gold']} gold, {n['silver']} silver, {n['bronze']} bronze, {n[None]} below bronze.", ""]
+        failed = {}
+        for r in scored:
+            for res in r["score"]["results"]:
+                if res["result"] == "fail":
+                    failed[res["id"]] = failed.get(res["id"], 0) + 1
+        if failed:
+            top = sorted(failed.items(), key=lambda kv: (-kv[1], kv[0]))[:5]
+            out += ["Most failed rules: " + ", ".join(f"{_cell(k)} ({v} repo{'' if v == 1 else 's'})" for k, v in top) + ".", ""]
+    for c in catalog.get("campaigns") or []:
+        left = f"; {c['days_left']} days left (until {c['until']})" if c.get("days_left") is not None else ""
+        out += [f"## Campaign: {_cell(c['title'])}", "",
+                f"{c['passing']} of {c['targets']} repos pass ({', '.join(_cell(r) for r in c['rules'])}){left}."]
+        if c["failing"]:
+            out.append("Still failing: " + ", ".join(_cell(n) for n in c["failing"][:30])
+                       + (f" and {len(c['failing']) - 30} more" if len(c["failing"]) > 30 else "") + ".")
+        if len(c["burn_down"]) > 1:
+            out.append("Failing over time: " + " → ".join(f"{d} {n}" for d, n in c["burn_down"][-8:]) + ".")
+        out.append("")
+    out += _catalog_lines(active)
+    out += ["## Repos", "", "| Repo | Level | Owners | Why | Language | Last push | CI first try | Community | Branch rules |",
+            "|---|---|---|---|---|---|---|---|---|"]
+    for r in active:
+        status, why = _owners(r)
+        m = r.get("modules") or {}
+        rules = r.get("rules")
+        level = ((r.get("score") or {}).get("level") or "below bronze") if r.get("score") else "-"
+        when = r.get("since") or {}
+        if when.get("owners") and status != "-":
+            status = f"{status} since {when['owners']}"
+        if when.get("level") and r.get("score"):
+            level = f"{level} since {when['level']}"
+        out.append(f"| {_cell(r['name'])} | {level} | {status} | {_cell(why) or '-'} | {_cell(r.get('language') or '-')} | "
+                   f"{(r.get('pushed') or '-')[:10]} | {_cell(m.get('ci_first_try') or '-')} | {_cell(m.get('community_score') or '-')} | "
+                   f"{len(rules) if isinstance(rules, list) else 'not available'} |")
+    return "\n".join(out) + "\n"
+
+
+def render_catalog_html(catalog: dict, destination_private: bool, pages=()) -> str:
+    """The same catalog as a page in the shared header (docs/portal.html), under the same guard as the Markdown."""
+    import html
+
+    from stackvitals import shell
+    e = html.escape
+    repos = publishable(catalog["repos"], destination_private)
+    active = [r for r in repos if not r.get("archived") and not r.get("fork")]
+    counts = {s: sum(1 for r in active if (r.get("owners") or {}).get("status") == s) for s in ("valid", "weak", "none")}
+    body = [f"<p>{len(repos)} repos ({len(active)} active). Owners: {counts['valid']} valid, {counts['weak']} weak, "
+            f"{counts['none']} without a CODEOWNERS file.</p>"]
+    if len(repos) < len(catalog["repos"]):
+        body.append("<p class=mut>Private repos are left out of this page because the portal repo is public.</p>")
+    for c in catalog.get("campaigns") or []:
+        body.append(f"<h2>Campaign: {e(c['title'])}</h2><p>{c['passing']} of {c['targets']} repos pass "
+                    f"({e(', '.join(c['rules']))}).</p>")
+    entities, problems, files = catalog_entities(active)
+    if files:
+        body.append(f"<h2>Catalog</h2><p>{len(entities)} entities from {files} of {len(active)} active repos' catalog-info.yaml "
+                    "files.</p><div class=tbl><table class=find><tr><th>Entity</th><th data-filter>Kind</th><th data-filter>Type</th>"
+                    "<th data-filter>Lifecycle</th><th>Owner</th><th data-filter>System</th><th>Repo</th></tr>")
+        body += [f"<tr><td>{e(x['name'])}</td><td>{e(x['kind'])}</td><td>{e(x.get('type') or '-')}</td>"
+                 f"<td>{e(x.get('lifecycle') or '-')}</td><td>{e(_owner_cell(x))}</td><td>{e(x.get('system') or '-')}</td>"
+                 f"<td>{e(x['repo'])}</td></tr>" for x in entities[:CATALOG_ROWS]]
+        body.append("</table></div>")
+        if problems:
+            body.append("<p class=mut>Problems: " + "; ".join(f"{e(repo)}: {e(p)}" for repo, p in problems[:20]) + ".</p>")
+    body.append("<h2>Repos</h2><div class=tbl><table class=find><tr><th data-filter>Level</th><th>Repo</th>"
+                "<th data-filter>Owners</th><th>Why</th><th data-filter>Language</th><th>Last push</th><th>CI first try</th>"
+                "<th>Community</th></tr>")
+    for r in active:
+        status, why = _owners(r)
+        m = r.get("modules") or {}
+        level = ((r.get("score") or {}).get("level") or "below bronze") if r.get("score") else "-"
+        body.append(f"<tr><td>{e(level)}</td><td>{e(str(r['name']))}</td><td>{e(status)}</td><td class=mut>{e(why) or '-'}</td>"
+                    f"<td>{e(str(r.get('language') or '-'))}</td><td>{e((r.get('pushed') or '-')[:10])}</td>"
+                    f"<td>{e(str(m.get('ci_first_try') or '-'))}</td><td>{e(str(m.get('community_score') or '-'))}</td></tr>")
+    body.append("</table></div>")
+    return shell.page("Portal", catalog["org"], "portal.html", f"Updated {e(catalog['built_at'][:16])} UTC.", "".join(body), pages)

stackvitals/portal/rollup.py

@@ -0,0 +1,32 @@
+"""The org view: every repo that runs the suite, rolled up from the facts each publishes on its own data branch (counts
+summed, rates and times as medians across repos, and the repos that lead each), so a platform team sees the org's
+CI, review and delivery health on one page without reading any repo's raw history."""
+from statistics import median
+
+SUMS = ("ci_flaky", "ci_broken", "open_prs", "stale_prs", "deployments", "findings", "newcomers_waiting", "vulns_open",
+        "vulns_serious")
+MEDIANS = ("ci_first_try", "pickup_p75_h", "review_p75_h", "cycle_p75_h", "change_failure_rate", "community_score")
+# Which way is worse for each number the leaders are ranked by.
+WORSE_HIGH = {"ci_flaky", "ci_broken", "stale_prs", "findings", "newcomers_waiting", "pickup_p75_h", "review_p75_h",
+              "cycle_p75_h", "change_failure_rate", "vulns_serious"}
+LEADERS = 5
+
+
+def _num(v) -> bool:
+    return isinstance(v, (int, float)) and not isinstance(v, bool)
+
+
+def org_rollup(repos: list[dict]) -> dict:
+    live = [r for r in repos if not r.get("archived") and not r.get("fork")]
+    facts = {r["name"]: ((r.get("modules") or {}).get("facts") or {}) for r in live if (r.get("modules") or {}).get("facts")}
+    out = {"repos": len(live), "running": len(facts), "totals": {}, "medians": {}, "counts": {}, "leaders": {}}
+    for key in SUMS:
+        vals = [f[key] for f in facts.values() if _num(f.get(key))]
+        out["totals"][key] = sum(vals) if vals else None
+    for key in MEDIANS:
+        vals = [f[key] for f in facts.values() if _num(f.get(key))]
+        out["medians"][key], out["counts"][key] = (round(median(vals), 4) if vals else None), len(vals)
+    for key in WORSE_HIGH:
+        ranked = sorted(((n, f[key]) for n, f in facts.items() if _num(f.get(key)) and f[key] > 0), key=lambda x: -x[1])
+        out["leaders"][key] = [{"repo": n, "value": v} for n, v in ranked[:LEADERS]]
+    return out