stackvitals 0.1.0__py3-none-any.whl

stackvitals/editors/vscode/extension.js

@@ -0,0 +1,101 @@
+// Marks flaky and broken tests in the editor, read through the user's own git from the stackvitals-data branch.
+"use strict";
+
+const vscode = require("vscode");
+const { execFile } = require("child_process");
+const path = require("path");
+const { findMarks, describe } = require("./marks");
+
+const TEST_FILE = /test|spec/i;
+const RELOAD_MS = 30 * 60 * 1000;
+const folders = new Map();
+let diagnostics;
+
+function git(cwd, args) {
+  return new Promise((resolve) => execFile("git", args, { cwd, maxBuffer: 64 << 20, windowsHide: true },
+    (err, out) => resolve(err ? null : out)));
+}
+
+function ref(folder) {
+  const given = vscode.workspace.getConfiguration("stackvitals", folder.uri).get("dataRef");
+  // A ref is passed to git, so anything that could read as an option is refused.
+  return typeof given === "string" && /^[\w.][\w./-]*$/.test(given) ? given : "origin/stackvitals-data";
+}
+
+async function load(folder) {
+  const raw = await git(folder.uri.fsPath, ["show", `${ref(folder)}:docs/data.json`]);
+  let data = null;
+  try {
+    data = raw ? JSON.parse(raw) : null;
+  } catch {
+    data = null; // a half-written or foreign file marks nothing
+  }
+  const remote = ((await git(folder.uri.fsPath, ["remote", "get-url", "origin"])) || "").trim();
+  const repo = (remote.match(/github\.com[:/]([\w.-]+\/[\w.-]+?)(?:\.git)?$/) || [])[1] || null;
+  const tests = data && Array.isArray(data.tests) ? data.tests : [];
+  folders.set(folder.uri.toString(), { tests, repo });
+  return tests.filter((t) => t && (t.state === "flaky" || t.state === "broken")).length;
+}
+
+function mark(doc) {
+  if (doc.uri.scheme !== "file") return;
+  const folder = vscode.workspace.getWorkspaceFolder(doc.uri);
+  const known = folder && folders.get(folder.uri.toString());
+  const rel = folder ? path.relative(folder.uri.fsPath, doc.uri.fsPath).split(path.sep).join("/") : "";
+  if (!known || !TEST_FILE.test(rel)) {
+    diagnostics.delete(doc.uri);
+    return;
+  }
+  diagnostics.set(doc.uri, findMarks(doc.getText(), rel, known.tests).map((m) => {
+    const range = new vscode.Range(m.line, m.col, m.line, m.col + m.length);
+    const d = new vscode.Diagnostic(range, describe(m.test), vscode.DiagnosticSeverity.Warning);
+    d.source = "StackVitals";
+    const proof = (Array.isArray(m.test.evidence) ? m.test.evidence : []).slice(-1)[0];
+    d.code = known.repo && proof && Number.isInteger(proof.run)
+      ? { value: "latest evidence", target: vscode.Uri.parse(`https://github.com/${known.repo}/actions/runs/${proof.run}/attempts/${proof.att || 1}`) }
+      : m.test.state;
+    return d;
+  }));
+}
+
+async function reload(fetch) {
+  let count = 0;
+  for (const folder of vscode.workspace.workspaceFolders || []) {
+    if (fetch) {
+      const [remote, branch] = ref(folder).split(/\/(.+)/);
+      // "+": the data branch is squashed now and then, which a plain fetch would refuse.
+      await git(folder.uri.fsPath, ["fetch", "--quiet", remote, `+${branch}:refs/remotes/${remote}/${branch}`]);
+    }
+    count += await load(folder);
+  }
+  vscode.workspace.textDocuments.forEach(mark);
+  return count;
+}
+
+function activate(context) {
+  diagnostics = vscode.languages.createDiagnosticCollection("stackvitals");
+  const timers = new Map();
+  const later = (doc) => {
+    clearTimeout(timers.get(doc.uri.toString()));
+    timers.set(doc.uri.toString(), setTimeout(() => mark(doc), 300));
+  };
+  const every = setInterval(() => reload(false), RELOAD_MS);
+  context.subscriptions.push(
+    diagnostics,
+    { dispose: () => clearInterval(every) },
+    vscode.workspace.onDidOpenTextDocument(mark),
+    vscode.workspace.onDidChangeTextDocument((e) => later(e.document)),
+    vscode.workspace.onDidCloseTextDocument((doc) => diagnostics.delete(doc.uri)),
+    vscode.workspace.onDidChangeWorkspaceFolders(() => reload(false)),
+    vscode.commands.registerCommand("stackvitals.refresh", async () => {
+      const n = await reload(true);
+      vscode.window.showInformationMessage(n ? `StackVitals: ${n} flaky or broken test${n === 1 ? "" : "s"} marked.`
+        : "StackVitals: no flaky or broken tests found on the data branch.");
+    }),
+  );
+  reload(false);
+}
+
+function deactivate() {}
+
+module.exports = { activate, deactivate };

stackvitals/editors/vscode/marks.js

@@ -0,0 +1,92 @@
+// Where the repo's flaky and broken tests are declared in one file, from the data branch's docs/data.json.
+"use strict";
+
+const MARKED = new Set(["flaky", "broken"]);
+// Test declarations by language; group 1 is the name (an identifier, or a title for JS and TS).
+const DECLS = [
+  { re: /^[ \t]*(?:async[ \t]+)?def[ \t]+(test\w*)/gmd },
+  { re: /^func[ \t]+(?:\([^)]*\)[ \t]*)?((?:Test|Benchmark|Fuzz|Example)\w*)/gmd },
+  { re: /\b(?:void|fun|Task|fn)[ \t]+(\w+)[ \t]*[(<]/gd },
+  { re: /\b(?:it|test|describe|context)(?:\.\w+)*[ \t]*\([ \t]*(?:'((?:\\.|[^'\\\n])+)'|"((?:\\.|[^"\\\n])+)"|`((?:\\.|[^`\\\n])+)`)/gd, title: true },
+];
+
+function parts(tid) {
+  // "file::Class::name", "pkg::TestX/sub" or "Ns.Class.Method": the scope it lives in and its own name.
+  const cut = tid.lastIndexOf("::");
+  let scope = cut >= 0 ? tid.slice(0, cut) : "";
+  let name = cut >= 0 ? tid.slice(cut + 2) : tid;
+  // Parameters ("test_x[1.5]") may hold dots, so only the part before them is split.
+  const bare = name.replace(/\[.*$/s, "");
+  if (!/\s/.test(bare) && bare.includes(".") && !bare.includes("(")) {
+    scope += "." + bare.slice(0, bare.lastIndexOf("."));
+    name = name.slice(bare.lastIndexOf(".") + 1);
+  }
+  return { scope, name };
+}
+
+function identifier(name) {
+  // test_x[param], method(args) and TestX/sub all live where their first part is declared.
+  return name.replace(/\[.*$/s, "").replace(/\(.*$/s, "").split("/")[0].trim();
+}
+
+function inFile(scope, rel) {
+  if (!scope) return true;
+  const file = rel.split("/").pop();
+  const stem = file.replace(/\.[^.]+$/, "");
+  if (stem && scope.includes(stem)) return true;
+  // Go names the package (its folder), not the file.
+  if (file.endsWith(".go")) {
+    const dirs = rel.split("/").slice(0, -1);
+    return dirs.length === 0 || scope.split("/").pop() === dirs[dirs.length - 1];
+  }
+  return false;
+}
+
+function findMarks(text, rel, tests) {
+  const wanted = (Array.isArray(tests) ? tests : [])
+    .filter((t) => t && typeof t.id === "string" && t.id && MARKED.has(t.state))
+    .map((t) => ({ t, ...parts(t.id) }))
+    .filter((w) => w.name && inFile(w.scope, rel));
+  if (!wanted.length) return [];
+  const starts = [0];
+  for (let i = 0; i < text.length; i++) if (text[i] === "\n") starts.push(i + 1);
+  const at = (offset) => {
+    let lo = 0, hi = starts.length - 1;
+    while (lo < hi) {
+      const mid = (lo + hi + 1) >> 1;
+      if (starts[mid] <= offset) lo = mid; else hi = mid - 1;
+    }
+    return { line: lo, col: offset - starts[lo] };
+  };
+  const out = [];
+  for (const decl of DECLS) {
+    decl.re.lastIndex = 0;
+    for (const m of text.matchAll(decl.re)) {
+      const g = [1, 2, 3].find((k) => m[k] !== undefined);
+      const found = m[g];
+      const hits = wanted.filter((w) => decl.title
+        ? w.name === found || [" ", " > ", " › "].some((sep) => w.name.endsWith(sep + found))
+        : identifier(w.name) === found);
+      if (!hits.length) continue;
+      // Parametrised cases share one declaration; the one that flaked most speaks for it.
+      hits.sort((a, b) => (b.t.flake_events || 0) - (a.t.flake_events || 0));
+      out.push({ ...at(m.indices[g][0]), length: found.length, test: hits[0].t, variants: hits.length });
+    }
+  }
+  return out.sort((a, b) => a.line - b.line || a.col - b.col);
+}
+
+function describe(t) {
+  const n = (v) => (Number.isFinite(v) ? v : "?");
+  if (t.state === "broken") {
+    return `Broken (StackVitals): failed in ${n(t.failures)} of the last ${n(t.runs)} runs.`
+      + (t.message ? ` Failure: ${String(t.message).slice(0, 200)}` : "");
+  }
+  const last = typeof t.last_flake === "string" ? `, last on ${t.last_flake.slice(0, 10)}` : "";
+  return `Flaky (StackVitals): failed and then passed with no code change ${n(t.flake_events)} times in ${n(t.runs)} runs`
+    + ` over the last 30 days${last}.`
+    + (t.flake ? ` Looks like a ${t.flake} flake.` : "")
+    + (t.message ? ` Usual failure: ${String(t.message).slice(0, 200)}` : "");
+}
+
+module.exports = { findMarks, describe };

stackvitals/editors/vscode/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "stackvitals-flaky-tests",
+  "displayName": "StackVitals flaky tests",
+  "description": "Marks the tests your CI history shows are flaky or broken, read from the repo's stackvitals-data branch.",
+  "version": "0.1.0",
+  "publisher": "stackvitals",
+  "license": "Apache-2.0",
+  "engines": { "vscode": "^1.80.0" },
+  "main": "./extension.js",
+  "activationEvents": ["onStartupFinished"],
+  "contributes": {
+    "commands": [
+      { "command": "stackvitals.refresh", "title": "StackVitals: Refresh flaky tests" }
+    ],
+    "configuration": {
+      "title": "StackVitals",
+      "properties": {
+        "stackvitals.dataRef": {
+          "type": "string",
+          "default": "origin/stackvitals-data",
+          "description": "The git ref holding StackVitals' data branch; the refresh command fetches it."
+        }
+      }
+    }
+  }
+}

stackvitals/flow/__init__.py

@@ -0,0 +1 @@
+"""Team Flow: what is blocking a repo's pull requests, who acts next, and how its review flow compares."""

stackvitals/flow/action.py

@@ -0,0 +1,178 @@
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from mergescout.github.graphql import GitHubGraphQL
+from stackvitals.ci.action import BRANCH, DataBranch, GitError
+from stackvitals.ci.api import ActionsAPI, APIError, OutOfBudget
+from stackvitals.core.actors import People
+from stackvitals.core.config import parse
+from stackvitals.core.delivery import apply_identity
+from stackvitals.core.store import published
+from stackvitals.core.summary import write_pr_summary
+from stackvitals.flow.report import _md
+from stackvitals.flow import peers
+from stackvitals.flow.build import build
+from stackvitals.flow.collect import collect
+from stackvitals.flow.preflight import Habits, collect_asks, gaps
+from stackvitals.flow.report import render_html, render_markdown
+from stackvitals.flow.store import FlowStore
+from stackvitals import shell
+
+# Team Flow runs on the schedule and manual runs (the user's choice: every 3 hours), when the settings file is pushed and
+# when someone makes a decision on GitHub (so a change applies at once), never on each CI run.
+# Comment commands run light (Requests and CI Health), so a burst of them finishes fast.
+FLOW_EVENTS = {"schedule", "workflow_dispatch", "push", "issues"}
+LABEL = "Team Flow"
+
+
+def make_gql(token: str) -> GitHubGraphQL:
+    # Inside Actions nothing waits for the hourly reset; the next scheduled run continues.
+    return GitHubGraphQL(token, max_wait=0, reserve=50)
+
+
+def write_flow_outputs(root: Path, result: dict, repo: str) -> None:
+    docs = root / "docs"
+    docs.mkdir(parents=True, exist_ok=True)
+    # The public page and file never carry people's answer times.
+    public = {k: v for k, v in result.items() if k != "responders"}
+    (docs / "flow.html").write_text(render_html(public, repo, shell.present(docs) | {"flow.html"}), encoding="utf-8")
+    (docs / "flow.json").write_text(json.dumps(published(public), indent=1), encoding="utf-8")
+    (docs / ".nojekyll").write_text("", encoding="utf-8")
+
+
+PREFLIGHT_MARKER = "<!-- stackvitals:preflight -->"
+
+
+def _quiet(text: str) -> str:
+    # Escaped, and with @ and www. broken by a zero-width space, so a note can never ping people or become a link.
+    return _md(text).replace("@", "@​").replace("www.", "www​.")
+
+
+def _preflight_targets(data: dict, since: str, done: set[int], now: datetime) -> list[tuple[int, str]]:
+    """New PRs from outside contributors that differ from what merged PRs here usually include, with the note to post."""
+    habits_index = Habits(data["open"] + data["closed"], now)
+    people = People(data["open"] + data["closed"])
+    out = []
+    for p in data["open"]:
+        if p["cr"] <= since or p["n"] in done or p.get("draft") or not people.is_outside(p):
+            continue
+        habits = habits_index.norms(p.get("paths") or [])
+        found = gaps(p, habits)
+        if found:
+            lines = "\n".join(f"- {_quiet(g['why'])}{'' if g['what'] == 'size' else '; this PR has none.'}" for g in found)
+            out.append((p["n"], f"{PREFLIGHT_MARKER}\n### Before review: how this PR compares with merged PRs here\n{lines}\n\n"
+                                f"<sub>From this repo's own history ({habits['merged_prs']} merged PRs). The maintainers turned "
+                                f"this note on; it is information, not a rule.</sub>"))
+    return out
+
+
+def flow_step(ws, env: dict, now: datetime, months: int = 12, make_gql=make_gql, make_api=ActionsAPI,
+              preflight_comments: bool = False) -> dict:
+    """Team Flow's part of a job: PRs changed since the last run into the shared checkout, the page, and (after the
+    commit) the pre-flight notes and the summary."""
+    repo, token = ws.repo, env.get("STACKVITALS_TOKEN") or env.get("GITHUB_TOKEN")
+    owner, name = repo.split("/")
+    store = FlowStore(ws.path)
+    previous_run = store.state().get("last_collect")
+    gql = make_gql(token)
+    stats = collect(gql, store, owner, name, now, months=months)
+    data = store.load()
+    ws.cache["flow"] = data
+    # Review comments of closed PRs, a few hundred per run (about 6 points per 20 PRs).
+    collect_asks(gql, store, owner, name, data["closed"], cap=500)
+    from stackvitals.flow import survival
+    # How much of each squash-merged PR's code is still there 30 and 90 days on (git blame, about 12 points a run).
+    survival.collect(gql, store.dir, owner, name, data["closed"], now)
+    # The repo's [identity] settings (bots, people, old logins) shape every number; the stored history stays as GitHub said.
+    seen = apply_identity(data, (ws.cache.get("settings") or parse("")[0]).get("identity"))
+    # Pull requests the team left out (a label or /stackvitals exclude) leave every number; the page says how many.
+    from stackvitals.core.decisions import visible
+    seen = visible(ws.path, seen, now.date())
+    # Comparisons come from a public file only (none before launch): no extra request otherwise.
+    repo_meta = store.repo()
+    # Public pages never name the person behind an area unless the repo asks for it.
+    names = bool(repo_meta.get("private")) or bool(((ws.cache.get("settings") or {}).get("identity") or {}).get("knowledge_names"))
+    result = build(seen, repo_meta, now, owner=owner, asks=store.asks(), bench=peers.load(None, ws.work / "bench"), names=names)
+    result["excluded"] = seen.get("excluded", 0)
+    to_comment = []
+    if preflight_comments and previous_run:
+        # Decided and remembered before the commit, so overlapping runs never comment twice.
+        done = store.state().get("preflight_commented", [])
+        to_comment = _preflight_targets(seen, previous_run, set(done), now)
+        if to_comment:
+            store.save_state({**store.state(), "preflight_commented": (done + [n for n, _ in to_comment])[-500:]})
+    # Team automation (on by default): planned and remembered before the commit, sent after it.
+    from stackvitals.flow.automate import plan, send
+    switches = (ws.cache.get("settings") or parse("")[0]).get("automation") or {}
+    acts, asked = plan(seen["open"], result["queue"], switches, store.state().get("automation") or {})
+    if acts:
+        store.save_state({**store.state(), "automation": asked})
+    # PR rules as code ([[rules]]): planned and remembered the same way, and replayed on past PRs for the app.
+    rule_list = (ws.cache.get("settings") or parse("")[0]).get("rules") or []
+    if rule_list:
+        from stackvitals.flow import rules as pr_rules
+        ctx = pr_rules.context(seen, rule_list)
+        rule_acts, rule_state, matched = pr_rules.plan(seen["open"], rule_list, ctx, store.state().get("rules") or {})
+        if rule_acts:
+            store.save_state({**store.state(), "rules": rule_state})
+            acts = acts + rule_acts
+        merged = sorted((p for p in seen["closed"] if p.get("mg")), key=lambda p: p["mg"])[-200:]
+        replayed = pr_rules.replay(rule_list, merged, ctx)
+        result["rules"] = [{"name": r["name"], "dry_run": r.get("dry_run", False), "open": matched[r["name"]],
+                            "replay": replayed[r["name"]]} for r in rule_list]
+    write_flow_outputs(ws.path, result, repo)
+    # Dependency-update hygiene from the same pull requests (open updates by age, security ones past 48 h, a quiet bot).
+    from stackvitals.core.decisions import Store as Decisions
+    from stackvitals.core.store import write_if_changed
+    from stackvitals.deps.hygiene import hygiene
+    deps = hygiene(seen, Decisions(ws.path).state(now.date()), now)
+    if deps is not None:
+        write_if_changed(ws.path / "docs" / "deps.json", published(deps))
+    write_pr_summary(ws.path, seen, now, repo)
+    out = {"prs": stats["prs"], "points": stats["points"], "complete": stats["complete"], "queue": len(result["queue"])}
+
+    def after_commit():
+        notes = []
+        if acts:
+            done = send(make_api(token, repo, max_requests=len(acts) + 5), acts)
+            notes.append(f"Team automation: {done['review']} reviewer request(s), {done['label']} label(s) added, "
+                         f"{done['unlabel']} removed, {done['comment']} rule comment(s). Turn any part off under [automation] "
+                         "in .github/stackvitals.toml (review_requests, size_labels, docs_label), or change [[rules]].")
+            if done["refused"]:
+                notes.append("Some were refused (" + "; ".join(done["refused"][:2]) + "): they need `pull-requests: write`.")
+        if to_comment:
+            api = make_api(token, repo, max_requests=len(to_comment) + 5)
+            for n, body in to_comment:
+                try:
+                    api.send("POST", f"/repos/{repo}/issues/{n}/comments", {"body": body})
+                except (APIError, OutOfBudget) as exc:
+                    notes.append(f"Could not post the pre-flight note on #{n} ({exc}); it needs `pull-requests: write`.")
+        summary = env.get("GITHUB_STEP_SUMMARY")
+        if summary:
+            note = "" if stats["complete"] else "\n\nStill reading older pull requests (GitHub's hourly limit); the next run continues."
+            note += "".join(f"\n\n{n}" for n in notes)
+            if stats["no_checks"]:
+                note += "\n\nAdd `checks: read` and `statuses: read` to the workflow permissions to see which PRs have failing checks."
+            if stats.get("no_deployments"):
+                note += "\n\nAdd `deployments: read` to the workflow permissions to measure delivery from deployments."
+            with open(summary, "a", encoding="utf-8") as f:
+                f.write(render_markdown(result, repo) + note + "\n")
+    ws.after_commit(after_commit, LABEL)
+    return out
+
+
+def run_flow_action(env: dict, now: datetime | None = None, months: int = 12, make_gql=make_gql, make_api=ActionsAPI,
+                    preflight_comments: bool = False, workspace=None) -> dict | None:
+    """One Team Flow run: read PRs changed since the last run, keep them in the data branch, rebuild the page."""
+    if env.get("GITHUB_EVENT_NAME") not in FLOW_EVENTS:
+        return None
+    from stackvitals.core.workspace import run_job
+    now = now or datetime.now(timezone.utc)
+
+    def step(ws):
+        return flow_step(ws, env, now, months, make_gql, make_api, preflight_comments)
+    if workspace is not None:
+        return step(workspace)
+    job = run_job(env, [(LABEL, step)], now, message=f"stackvitals: team flow {now:%Y-%m-%d %H:%M} UTC", catch=False)
+    return {**job["results"][LABEL], "pushed": job["outcome"] == "pushed"}