stackvitals 0.1.0__py3-none-any.whl

stackvitals/cli.py

@@ -0,0 +1,111 @@
+import argparse
+import os
+import sys
+
+MODULES = {
+    "ci": "CI health: flaky and slow jobs (`stackvitals ci report owner/repo`)",
+    "flow": "Team Flow: who acts next on pull requests (`stackvitals flow report owner/repo`)",
+    "deps": "Dependency Steward: should you add or upgrade a package (`stackvitals deps check requests`)",
+    "portal": "Portal: what an org owns, who owns it, and its shape (`stackvitals portal report my-org`)",
+    "community": "Community: where newcomers drop out and how fast they get a first answer (`stackvitals community report owner/repo`)",
+    "contribute": "Contribute: open-source issues you can get merged, and how changes get merged in a repo (`stackvitals contribute brief owner/repo`)",
+}
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    parser = argparse.ArgumentParser(prog="stackvitals", description="Free, open-source developer suite.")
+    sub = parser.add_subparsers(dest="module", required=True)
+    for name, text in MODULES.items():
+        sub.add_parser(name, add_help=False, help=text)
+    sub.add_parser("action", add_help=False, help="Run inside GitHub Actions (used by the StackVitals Action)")
+    serve = sub.add_parser("serve", help="Run the MCP server on stdio (MergeScout and CI Health tools)")
+    serve.add_argument("--tools", choices=["core", "all"], help="core: the 11 core tools (default); all: every tool "
+                       "(or set STACKVITALS_MCP_TOOLS=all)")
+    sub.add_parser("record", add_help=False, help="Send a deployment or incident from any tool (`stackvitals record deployment owner/repo --id 42 --sha abc1234`)")
+    sub.add_parser("project", add_help=False, help="DORA for one project across several repos, as DevLake counts a project (`stackvitals project web org/a org/b`)")
+    sub.add_parser("delivery", add_help=False, help="Cycle time, DORA and stuck PRs (`stackvitals delivery owner/repo`)")
+    sub.add_parser("ui", add_help=False, help="Open a repo's pages, Explore included, from a local copy (`stackvitals ui owner/repo`)")
+    sub.add_parser("explain", add_help=False, help="What a headline number was counted from (`stackvitals explain owner/repo lead-time`)")
+    sub.add_parser("export", add_help=False, help="A repo's data as DevLake tables or a CTRF test report (`stackvitals export owner/repo --format devlake|ctrf`)")
+    sub.add_parser("insights", add_help=False, help="What to fix first, and how long merged PRs sat with failing checks (`stackvitals insights owner/repo`)")
+    sub.add_parser("sql", add_help=False, help="SQL over a repo's summaries (`stackvitals sql --repo owner/name \"select ...\"`)")
+    sub.add_parser("propose", add_help=False, help="A link proposing a settings change or decision that a person submits (`stackvitals propose owner/repo --set alerts.pickup_p75_hours=6`)")
+    sub.add_parser("site", add_help=False, help="Build the project site: landing page, the app and its demo (`stackvitals site build --out DIR`)")
+    sub.add_parser("connectors", add_help=False, help="Test each connector's key (`stackvitals connectors check jira slack`, or --all)")
+    sub.add_parser("editor", add_help=False, help="Mark flaky tests in your editor (`stackvitals editor vscode|cursor|vscodium`)")
+    # Module commands parse their own arguments.
+    args, rest = parser.parse_known_args(argv)
+    if args.module == "ci":
+        from stackvitals.ci.cli import main as ci_main
+        return ci_main(rest)
+    if args.module == "flow":
+        import stackvitals.flow.cli as flow_cli
+        return flow_cli.main(rest)
+    if args.module == "deps":
+        from stackvitals.deps.cli import main as deps_main
+        return deps_main(rest)
+    if args.module == "portal":
+        from stackvitals.portal.cli import main as portal_main
+        return portal_main(rest)
+    if args.module == "community":
+        from stackvitals.community.cli import main as community_main
+        return community_main(rest)
+    if args.module == "action":
+        from stackvitals.ci.cli import action_main
+        return action_main(rest)
+    if args.module == "ui":
+        from stackvitals.core.uicli import main as ui_main
+        return ui_main(rest)
+    if args.module == "explain":
+        from stackvitals.core.explaincli import main as explain_main
+        return explain_main(rest)
+    if args.module == "export":
+        from stackvitals.core.exportcli import main as export_main
+        return export_main(rest)
+    if args.module == "project":
+        from stackvitals.core.projectcli import main as project_main
+        return project_main(rest)
+    if args.module == "record":
+        from stackvitals.core.recorded import main as record_main
+        return record_main(rest)
+    if args.module == "delivery":
+        from stackvitals.core.deliverycli import main as delivery_main
+        return delivery_main(rest)
+    if args.module == "insights":
+        from stackvitals.core.insightscli import main as insights_main
+        return insights_main(rest)
+    if args.module == "propose":
+        from stackvitals.core.propose import main as propose_main
+        return propose_main(rest)
+    if args.module == "site":
+        from stackvitals.site import main as site_main
+        return site_main(rest)
+    if args.module == "editor":
+        from stackvitals.editors import main as editor_main
+        return editor_main(rest)
+    if args.module == "connectors":
+        from stackvitals.connectors.check import main as connectors_main
+        return connectors_main(rest)
+    if args.module == "sql":
+        from stackvitals.core.sqlcli import main as sql_main
+        return sql_main(rest)
+    if args.module == "serve":
+        import logging
+
+        from stackvitals.mcp_server import build_server
+        # Request lines from the HTTP library would only clutter the AI client's log.
+        logging.getLogger("httpx").setLevel(logging.WARNING)
+        # The 11 core tools by default; `--tools all` (or STACKVITALS_MCP_TOOLS=all) loads every one.
+        tools = args.tools or os.environ.get("STACKVITALS_MCP_TOOLS", "core")  # the command line wins over the environment
+        build_server(tools="all" if tools == "all" else "core").run("stdio")
+        return 0
+    if rest[:1] == ["brief"]:
+        import stackvitals.contribute.cli as brief_cli
+        return brief_cli.main(rest[1:])
+    import mergescout.cli as mergescout_cli
+    return mergescout_cli.main(rest)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

stackvitals/codeowners.py

@@ -0,0 +1,69 @@
+"""CODEOWNERS read the way GitHub reads it, shared by CI Health, Team Flow and Portal: gitignore-style patterns, the last
+matching line wins, a line without owners leaves its paths without owners."""
+import re
+from functools import lru_cache
+
+# GitHub's lookup order: the first file found is the one it uses.
+PATHS = (".github/CODEOWNERS", "CODEOWNERS", "docs/CODEOWNERS")
+
+
+def parse(text: str | None, skip=frozenset()) -> list[tuple[str, list[str]]]:
+    """(pattern, owners) per rule line; comments, blank lines, "!" lines (unsupported by GitHub) and the line numbers in
+    `skip` (lines GitHub's error check calls invalid, which GitHub skips) are left out."""
+    rules = []
+    for n, raw in enumerate((text or "").lstrip("\ufeff").splitlines(), 1):
+        line = raw.strip()
+        if n in skip:
+            continue
+        if not line or line.startswith(("#", "!")):
+            continue
+        # A space inside a path is written "\ ".
+        pattern, *rest = re.split(r"(?<!\\)\s+", line)
+        owners = []
+        for token in rest:
+            if token.startswith("#"):
+                break
+            owners.append(token)
+        rules.append((pattern.replace("\\ ", " "), owners))
+    return rules
+
+
+def _glob(segment: str) -> str:
+    return "".join("[^/]*" if c == "*" else "[^/]" if c == "?" else re.escape(c) for c in segment)
+
+
+@lru_cache(maxsize=65536)
+def _regex(pattern: str) -> re.Pattern:
+    # A slash at the start or in the middle ties the pattern to the repo root; otherwise it matches at any depth.
+    anchored = pattern.startswith("/") or "/" in pattern.strip("/")
+    segs = pattern.strip("/").split("/")
+    rx = "" if anchored else "(?:.*/)?"
+    for i, seg in enumerate(segs):
+        first, last = i == 0, i == len(segs) - 1
+        if seg == "**":
+            rx += (".*" if first else "/.*") if last else ("(?:.*/)?" if first else "/(?:.*/)?")
+            continue
+        if not first and segs[i - 1] != "**":
+            rx += "/"
+        rx += _glob(seg)
+    if segs[-1] == "**":
+        pass
+    elif pattern.endswith("/"):
+        rx += "/.*"  # a folder: everything inside it
+    elif not (segs[-1] == "*" and len(segs) > 1):
+        rx += "(?:/.*)?"  # a file, or a folder and everything inside it; "docs/*" stays one level deep
+    return re.compile(rx)
+
+
+def matches(pattern: str, path: str) -> bool:
+    return bool(_regex(pattern).fullmatch(path.lstrip("/")))
+
+
+def owners_for(path: str, rules: list[tuple[str, list[str]]], unknown=frozenset()) -> list[str]:
+    """The owners GitHub would request for this path, as written (@user, @org/team, email). Owners GitHub ignores
+    (`unknown`, lowercase, from its error check) are left out; the matching line still decides, as on GitHub."""
+    found = []
+    for pattern, owners in rules:
+        if matches(pattern, path):
+            found = owners
+    return [o for o in found if o.lower() not in unknown]

stackvitals/community/action.py

@@ -0,0 +1,140 @@
+import json
+from collections import Counter
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from stackvitals.ci import bench as ci_bench
+from stackvitals.ci.action import BRANCH, DataBranch, GitError
+from stackvitals.ci.api import ActionsAPI, APIError, OutOfBudget
+from stackvitals.community import affiliation
+from stackvitals.connectors.store import ConnectorStore
+from stackvitals.community.build import build
+from stackvitals.community.context import read_context
+from stackvitals.community.collect import _read, collect_discussions, collect_issues, collect_stars, collect_traffic
+from stackvitals.community.report import badges, render_html, render_markdown
+from stackvitals.core.store import published
+from stackvitals.flow.action import FLOW_EVENTS
+from stackvitals.flow.action import LABEL as FLOW_LABEL
+from stackvitals.flow.action import make_gql
+from stackvitals.flow.collect import collect
+from stackvitals.flow.store import FlowStore
+from stackvitals import shell
+from stackvitals.core.actors import kind_of
+from stackvitals.core.config import parse
+from stackvitals.core.delivery import apply_identity
+
+BENCH_FILE, STARS_FILE = "community-benchmarks.json", "stars-benchmarks.json"
+LABEL = "Community"
+
+
+def load_benchmarks(cache_dir: Path, token: str | None = None) -> tuple[dict | None, dict | None]:
+    # In the Action: public files only (none before launch). Locally, the user's own token may read them.
+    ok = lambda d: d if isinstance(d, dict) and isinstance(d.get("groups"), dict) else None  # noqa: E731
+    return (ci_bench.load(token, cache_dir, name=BENCH_FILE, url_env="STACKVITALS_COMMUNITY_BENCH_URL", validate=ok, default_url=None),
+            ci_bench.load(token, cache_dir, name=STARS_FILE, url_env="STACKVITALS_STARS_BENCH_URL", validate=ok, default_url=None))
+
+
+def write_community_outputs(root: Path, result: dict, repo: str) -> None:
+    docs = root / "docs"
+    docs.mkdir(parents=True, exist_ok=True)
+    (docs / "community.html").write_text(render_html(result, repo, shell.present(docs) | {"community.html"}), encoding="utf-8")
+    (docs / "community.json").write_text(json.dumps(published(result), indent=1), encoding="utf-8")
+    for name, badge in badges(result).items():
+        (docs / name).write_text(json.dumps(badge), encoding="utf-8")
+    (docs / ".nojekyll").write_text("", encoding="utf-8")
+
+
+def community_step(ws, env: dict, now: datetime, make_gql=make_gql, make_api=ActionsAPI, flow_ran: bool | None = None) -> dict:
+    """Community's part of a job: issues, star history and (with its own token) traffic into the shared checkout,
+    beside Team Flow's PR history; the page, the badges and (after the commit) the summary."""
+    repo, token = ws.repo, env.get("STACKVITALS_TOKEN") or env.get("GITHUB_TOKEN")
+    traffic_token = (env.get("STACKVITALS_TRAFFIC_TOKEN") or "").strip()
+    owner, name = repo.split("/")
+    if flow_ran is None:
+        # Team Flow ran earlier in this same job: its PRs are already in the checkout.
+        flow_ran = ws.results.get(FLOW_LABEL) is not None
+    notes = []
+    prs, gql = FlowStore(ws.path), make_gql(token)
+    # Team Flow already read the PRs this run; otherwise Community keeps the same history itself.
+    points = 0 if flow_ran else collect(gql, prs, owner, name, now)["points"]
+    issues = FlowStore(ws.path, "community/issues")
+    got = collect_issues(gql, issues, owner, name, now)
+    if got.get("refused"):
+        notes.append("Add `issues: read` to the workflow permissions to include issues.")
+    folder = ws.path / "community"
+    try:
+        collect_stars(make_api(token, repo, max_requests=110), folder, now)
+    except (APIError, OutOfBudget) as exc:
+        notes.append(f"Star history wasn't read this time ({exc}).")
+    if traffic_token:
+        try:
+            collect_traffic(make_api(traffic_token, repo, max_requests=10), folder, now)
+        except (APIError, OutOfBudget) as exc:
+            notes.append("The traffic token can't read traffic: it needs the repository permission "
+                         f"\"Administration: read\" ({str(exc)[:120]}).")
+    bench, star_bench = load_benchmarks(ws.work / "bench")
+    # The repo's [identity] settings decide who is a bot or a newcomer; the stored history stays as GitHub said.
+    settings = ws.cache.get("settings") or parse("")[0]
+    ident = settings.get("identity")
+    pr_data = apply_identity(prs.load(), ident)
+    context = _context(gql, folder, owner, name, now)
+    stated = {k: v for k, v in (settings.get("community") or {}).items() if v}
+    if stated:
+        context = {**(context or {}), "stated": stated}
+    talks = FlowStore(ws.path, "community/discussions")
+    if (context or {}).get("discussions"):
+        points += collect_discussions(gql, talks, owner, name, now)["points"]
+    # The people behind the last 180 days' merged work, busiest first, for the elephant factor.
+    since = (now - timedelta(days=180)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    authors = Counter(p["au"] for p in pr_data.get("closed") or [] if p.get("mg") and p["mg"] >= since and p.get("au")
+                      and kind_of(p["au"], p.get("at")) == "person")
+    aff = affiliation.collect(gql, [a for a, _n in authors.most_common()])
+    result = build(pr_data, apply_identity(issues.load(), ident), _read(folder / "stars.json"),
+                   _read(folder / "traffic.json"), prs.repo(), now, bench=bench, star_bench=star_bench,
+                   context=context, affiliation=aff, discussions=talks.load(),
+                   forum=ConnectorStore(ws.path, "discourse").load().get("topic"),
+                   chat=[t for n in ("slack", "discord") for t in ConnectorStore(ws.path, n).load().get("topic") or []])
+    write_community_outputs(ws.path, result, repo)
+    out = {"issues": got["issues"], "points": points + got["points"], "complete": got["complete"],
+           "waiting": result["waiting_total"], "score": (result.get("score") or {}).get("score")}
+
+    def after_commit():
+        summary = env.get("GITHUB_STEP_SUMMARY")
+        if summary:
+            note = "" if got["complete"] or got.get("refused") else "\n\nStill reading older issues (GitHub's hourly limit); the next run continues."
+            note += "".join(f"\n\n{n}" for n in notes)
+            with open(summary, "a", encoding="utf-8") as f:
+                f.write(render_markdown(result, repo) + note + "\n")
+    ws.after_commit(after_commit, LABEL)
+    return out
+
+
+def _context(gql, folder: Path, owner: str, name: str, now: datetime) -> dict | None:
+    """What the repo states about itself, read once a week (a file changes rarely)."""
+    kept = _read(folder / "context.json")
+    if kept.get("at", "") >= (now - timedelta(days=7)).strftime("%Y-%m-%d"):
+        return kept.get("facts")
+    facts = read_context(gql, owner, name)
+    if facts is None:
+        return kept.get("facts")
+    folder.mkdir(parents=True, exist_ok=True)
+    (folder / "context.json").write_text(json.dumps({"at": now.strftime("%Y-%m-%d"), "facts": facts}, indent=1),
+                                         encoding="utf-8")
+    return facts
+
+
+def run_community_action(env: dict, now: datetime | None = None, make_gql=make_gql, make_api=ActionsAPI,
+                         flow_ran: bool | None = None, workspace=None) -> dict | None:
+    """One Community run on the schedule or by hand: issues, star history and (with its own token) traffic, kept in the
+    data branch beside Team Flow's PR history, then the page, the badges and the summary."""
+    if env.get("GITHUB_EVENT_NAME") not in FLOW_EVENTS:
+        return None
+    from stackvitals.core.workspace import run_job
+    now = now or datetime.now(timezone.utc)
+
+    def step(ws):
+        return community_step(ws, env, now, make_gql, make_api, bool(flow_ran) if workspace is None else flow_ran)
+    if workspace is not None:
+        return step(workspace)
+    job = run_job(env, [(LABEL, step)], now, message=f"stackvitals: community {now:%Y-%m-%d %H:%M} UTC", catch=False)
+    return {**job["results"][LABEL], "pushed": job["outcome"] == "pushed"}

stackvitals/community/affiliation.py

@@ -0,0 +1,51 @@
+"""Which organisation the people behind the merged work say they work for, for the elephant factor (CHAOSS): the
+company on their public profile, else unknown (counted and shown, never guessed). Membership of the org that owns the
+repo isn't used: volunteer projects' orgs aren't employers. Looked up each run and kept only in memory, so no person's
+employer is written to the data branch."""
+import hashlib
+import json
+import re
+
+from mergescout.github.graphql import GraphQLError, RateLimited
+
+BATCH, MAX_PEOPLE = 25, 150
+SUFFIX = re.compile(r"\b(inc|llc|ltd|limited|corp|corporation|gmbh|co|company|plc|sa|ag|bv|oy|ab|the)\b\.?")
+NOT_A_COMPANY = {"freelance", "freelancer", "independent", "self", "self-employed", "selfemployed", "none", "n/a", "na",
+                 "student", "retired", "personal", "home", "me", "myself", "open source", "opensource", "unemployed"}
+
+
+def key(login: str) -> str:
+    return hashlib.sha1((login or "").lower().encode()).hexdigest()[:12]
+
+
+def normalize(company: str | None) -> str | None:
+    """A company field as one organisation name: "@Google LLC" and "google, inc." are google; self-employment is none."""
+    text = (company or "").strip().lower()
+    if not text:
+        return None
+    text = text.split()[0] if text.startswith("@") else text  # "@acme @other": the first one
+    text = re.split(r"[,/|;(]", text.lstrip("@"))[0]
+    text = SUFFIX.sub("", text).strip(" .-_")
+    text = re.sub(r"\s+", " ", text)
+    return None if not text or text in NOT_A_COMPANY else text
+
+
+def _query(logins: list[str]) -> str:
+    parts = [f"u{i}: user(login: {json.dumps(x)}) {{ login company }}" for i, x in enumerate(logins)]
+    return "query { rateLimit { cost remaining resetAt } " + " ".join(parts) + " }"
+
+
+def collect(gql, logins: list[str]) -> dict[str, str | None]:
+    """Each login's organisation (by hash), looked up fresh each run and never written down: a hash of a public login
+    can be reversed, so a kept file would name each person's employer. At most 150 people (the busiest first)."""
+    out = {}
+    for i in range(0, min(len(logins), MAX_PEOPLE), BATCH):
+        batch = logins[i:min(i + BATCH, MAX_PEOPLE)]
+        try:
+            # Partial: a renamed or deleted account answers null instead of failing the whole batch.
+            data = gql.query(_query(batch), {}, partial=True).data or {}
+        except (GraphQLError, RateLimited):
+            break
+        for j, login in enumerate(batch):
+            out[key(login)] = normalize((data.get(f"u{j}") or {}).get("company"))
+    return out