stackvitals 0.1.3__tar.gz → 0.1.4__tar.gz

stackvitals-0.1.4/.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,56 @@
+name: Bug report
+description: Something in StackVitals is wrong or broken
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for reporting. A wrong verdict or number is a bug too, and the most useful kind.
+        For a security problem, don't open an issue: use the **Security** tab's private reporting instead.
+  - type: textarea
+    id: what
+    attributes:
+      label: What happened
+      description: What you did, what you expected, and what happened instead.
+    validations:
+      required: true
+  - type: dropdown
+    id: where
+    attributes:
+      label: Where
+      options:
+        - The Action (workflow run, PR comment, verdict check)
+        - The web app (dashboard)
+        - The command line (stackvitals ...)
+        - The MCP server (an AI agent)
+        - Setup (stackvitals init or the app's Setup page)
+        - The docs
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: StackVitals version
+      description: "`pip show stackvitals` shows it. For the Action, say if you use `@v0` or a pinned tag."
+      placeholder: "0.1.4"
+    validations:
+      required: true
+  - type: dropdown
+    id: visibility
+    attributes:
+      label: Repo
+      options:
+        - Public
+        - Private
+        - Not about a repo
+  - type: input
+    id: link
+    attributes:
+      label: A link we can look at
+      description: The workflow run, pull request or app page, if the repo is public. Leave out anything private.
+  - type: textarea
+    id: logs
+    attributes:
+      label: Job summary or error output
+      description: Paste the relevant part. Remove tokens, private names and anything you don't want public.
+      render: text

stackvitals-0.1.4/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Ask a question
+    url: https://github.com/stackvitals/stackvitals/discussions/categories/q-a
+    about: How to set something up or read a number. Discussions are the best place for questions.
+  - name: Report a security problem
+    url: https://github.com/stackvitals/stackvitals/security/advisories/new
+    about: Privately, through GitHub's reporting form. Please don't open a public issue.

stackvitals-0.1.4/.github/ISSUE_TEMPLATE/feature.yml

@@ -0,0 +1,39 @@
+name: Feature request
+description: Something StackVitals should do, or do differently
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: The problem
+      description: What you're trying to find out or get done, and what makes it hard today.
+    validations:
+      required: true
+  - type: textarea
+    id: idea
+    attributes:
+      label: What would help
+      description: What you'd like StackVitals to show or do. A rough idea is fine.
+    validations:
+      required: true
+  - type: dropdown
+    id: module
+    attributes:
+      label: Area
+      options:
+        - CI Health
+        - Team Flow and Pull Requests
+        - Delivery and DORA
+        - Insights
+        - Dependencies
+        - Community
+        - Portal
+        - The web app
+        - The command line or MCP server
+        - Setup and integrations
+        - Something else
+  - type: textarea
+    id: else
+    attributes:
+      label: Anything else
+      description: Tools you use for this today, links, or examples.

stackvitals-0.1.4/.github/pull_request_template.md

@@ -0,0 +1,15 @@
+## What this changes
+
+<!-- What changes for someone using StackVitals, in a sentence or two. Link the issue if there is one. -->
+
+## How it was checked
+
+- [ ] Tests pass: `uv run --extra dev pytest tests -q`
+- [ ] If the web app changed: `cd web && npm run build && npm test`, and the rebuilt `src/stackvitals/web/` is committed
+- [ ] If a rule behind a verdict or number changed: its entry in `bench/accuracy/registry.json` and the regenerated
+      docs (`python bench/accuracy/summary.py`)
+- [ ] If it reaches a new host or needs a new permission: said so below
+
+## Anything reviewers should know
+
+<!-- Trade-offs, follow-ups, or what you weren't sure about. -->

stackvitals-0.1.4/.github/workflows/pages.yml

@@ -0,0 +1,44 @@
+name: Demo site
+# Publishes the landing page and the app's example-data demo to GitHub Pages. The site is static and demo-only: its
+# pages may fetch nothing but their own files, so there is no API or backend behind it to call or overload.
+on:
+  release:
+    types: [published]
+  push:
+    branches: [main]
+    paths: ["src/stackvitals/web/**", "src/stackvitals/site.py"]
+  workflow_dispatch:
+permissions: {}
+concurrency:
+  group: pages
+  cancel-in-progress: false
+jobs:
+  build:
+    if: github.repository == 'stackvitals/stackvitals'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+      - name: Build the demo-only site
+        run: uv run --no-dev stackvitals site build --out _site --demo-only
+      - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
+        with:
+          path: _site
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      pages: write
+      id-token: write   # Pages checks the deployment with GitHub's OIDC token
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1

{stackvitals-0.1.3 → stackvitals-0.1.4}/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.1.4 (2026-10-31)
+
+- PR verdicts catch more failures a change causes (from user feedback):
+  - a failure whose flakes in the last 30 days all came from this same pull request reads **New**, not likely flaky: a
+    race the change added fails now and then and passes on a re-run;
+  - a likely flake that fails on this pull request far more often than on the default branch (2 or more failures, under
+    a 5% chance at main's rate) reads **New**: the change made it worse;
+  - a likely flake shows its failure rate on the default branch; a first flaky-looking failure of a check that passed
+    every recent default-branch run says so;
+  - when a failure blamed on the change, or one of those checks, passes on a re-run of the same commit, the comment
+    says a green re-run doesn't clear it, instead of "CI is passing now".
+  These rules are new and not yet measured on labelled failures; [rules.md](docs/rules.md) says so.
+- A public demo site, https://stackvitals.github.io/stackvitals/, built with `stackvitals site build --demo-only`: it
+  fetches nothing outside itself, and its setup links point at the README. The landing page uses the app's font.
+- `glama.json`, so the Glama listing of the MCP server is maintained by the project.
+
 ## 0.1.3 (2026-10-31)
 
 - The app's About page and the project site say Apache 2.0 (they said MIT); the site shows the cube logo and the

{stackvitals-0.1.3 → stackvitals-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: stackvitals
-Version: 0.1.3
+Version: 0.1.4
 Summary: Free, open-source developer suite: CI health, team flow, dependencies, community, and MergeScout for contributors.
 Project-URL: Homepage, https://github.com/stackvitals/stackvitals
 Project-URL: Documentation, https://github.com/stackvitals/stackvitals/tree/main/docs

{stackvitals-0.1.3 → stackvitals-0.1.4}/bench/accuracy/registry.json

@@ -140,6 +140,34 @@
    "strength": "early observation",
    "strength_why": "2 held-out matches in 150 failures"
   },
+  {
+   "id": "flaky.only_this_pr",
+   "card": "flaky",
+   "version": 1,
+   "rule": "A failure that would read likely flaky, flaked before or maybe flaky, whose flakes in the last 30 days all came from this same pull request (none on the default branch or other pull requests), reads new: a race or order dependence the change added fails now and then and passes on a re-run.",
+   "tuned_on": "none: a public suggestion (2026-10-31) with the failure it describes written as tests",
+   "held_out": "not yet replayed on the labelled pull-request failures",
+   "pass_bar": "none set yet; it only moves a call toward the change and never excuses a failure",
+   "bar_set": "2026-10-31",
+   "result": "untested on labels: the replay needs the held-out reads again",
+   "revisions": [],
+   "strength": "early observation",
+   "strength_why": "not yet measured on labelled failures"
+  },
+  {
+   "id": "flaky.worse_on_pr",
+   "card": "flaky",
+   "version": 1,
+   "rule": "A likely flake that failed 2 or more times on this pull request's runs, where that many failures would come from its default-branch rate (smoothed by one failure and one pass, over 10 or more runs) less than 5% of the time, reads new: the change made a flaky check fail more often.",
+   "tuned_on": "none: a public suggestion (2026-10-31) with the failure it describes written as tests",
+   "held_out": "not yet measured; the plan is regressions seeded into repos of the stackvitals organisation",
+   "pass_bar": "none set yet; it only moves a call toward the change and never excuses a failure",
+   "bar_set": "2026-10-31",
+   "result": "untested on labels",
+   "revisions": [],
+   "strength": "early observation",
+   "strength_why": "not yet measured on seeded or labelled failures"
+  },
   {
    "id": "flaky.pr_details",
    "card": "flaky",

{stackvitals-0.1.3 → stackvitals-0.1.4}/docs/accuracy.md

@@ -17,7 +17,7 @@ page is built from those cards' result files by `bench/accuracy/summary.py`; the
 - No person has audited the labels yet.
 
 **How sure.** A rule is an *early observation* when its check rests on few cases; its numbers are a direction,
-not a settled rate: `flaky.already_failing` (1 of 6 held-out matches with the same error); `flaky.maybe_flaky` (2 held-out matches in 150 failures); `flaky.pr_details` (24 matches in 4 repos, one repo gave most); `dora.incident_link` (15 reports named the release that broke). A rule's *bar* is the accuracy it had to reach before it shipped; for the flaky-test rules it is that at most 5% of the failures it calls flaky were really caused by the change (*masking*: a real failure hidden as a flake). A rule whose bar *failed* is kept only where failing costs nothing it promised: `flaky.known_flaky` (too few calls to show 5%: the label says "likely", the advice says to check the error first, and the verdict check waits for a green re-run). The rest are validated: checked against their bar, on held-out data where the rule has one ([rules.md](rules.md) says which).
+not a settled rate: `flaky.already_failing` (1 of 6 held-out matches with the same error); `flaky.maybe_flaky` (2 held-out matches in 150 failures); `flaky.only_this_pr` (not yet measured on labelled failures); `flaky.worse_on_pr` (not yet measured on seeded or labelled failures); `flaky.pr_details` (24 matches in 4 repos, one repo gave most); `dora.incident_link` (15 reports named the release that broke). A rule's *bar* is the accuracy it had to reach before it shipped; for the flaky-test rules it is that at most 5% of the failures it calls flaky were really caused by the change (*masking*: a real failure hidden as a flake). A rule whose bar *failed* is kept only where failing costs nothing it promised: `flaky.known_flaky` (too few calls to show 5%: the label says "likely", the advice says to check the error first, and the verdict check waits for a green re-run). The rest are validated: checked against their bar, on held-out data where the rule has one ([rules.md](rules.md) says which).
 
 **Reproduce a number.** `python bench/accuracy/reproduce.py list` names the command behind every result file;
 `reproduce.py check <card>` re-scores it and says whether the numbers match. Scoring reads the evidence each

{stackvitals-0.1.3 → stackvitals-0.1.4}/docs/limitations.md

@@ -29,6 +29,10 @@ Numbers come from the accuracy cards ([accuracy.md](accuracy.md)).
   readable ones ([features card](../bench/accuracy/features/README.md)).
 - Jobs with rare flakes look like "never flaked" in a short window. Status precision (re-running pinned commits) is
   not measured.
+- A change that makes an already-flaky test fail more often reads "new" once it has failed 2 or more times on the pull
+  request, far above its rate on the default branch. Its first failure still reads "likely flaky" (one run can't tell a
+  5% flake from a 40% one), and a re-run can pass; the row shows the rate on the default branch to compare against.
+  How often this happens isn't measured yet.
 - No verdict lets a failure through, and none says "safe to re-run". Likely flaky, outside failure, already failing,
   maybe flaky and PR details all still block the verdict check until CI is green. GitLab job logs aren't read, so
   there a job that flaked before is never called a likely flake.

{stackvitals-0.1.3 → stackvitals-0.1.4}/docs/modules.md

@@ -84,14 +84,14 @@ never = ["tests/test_billing.py::test_charge"]
 
 | Verdict | When | What to do |
 |---|---|---|
-| Likely flaky | The job flaked 3 or more times lately, once in the step that failed now; that step's log shows a flake's sign (an outside service, the runner or a wait failing with no failing test) and names no file the pull request changed. The row shows that evidence. | Check the error, then re-run it. |
+| Likely flaky | The job flaked 3 or more times lately, once in the step that failed now; that step's log shows a flake's sign (an outside service, the runner or a wait failing with no failing test) and names no file the pull request changed. The row shows that evidence, and how often it failed on the default branch: a change can make a flaky test fail far more often. | Check the error, then re-run it. |
 | Maybe yours | The job flaked before, but its failing step names a file the pull request changed. | Look at your change first. |
 | Outside service or runner | The failing step's log says an outside service or the runner failed. | Re-run it. |
 | Flaked before, needs a look | The job flaked before, but too rarely, elsewhere or with no flake's sign to rule out the change. | Look before re-running. |
 | Already failing on main | The default branch fails with the same error (checked in the log: the same step alone isn't enough, since on labelled data 4 of 20 such failures were the change's own). | Not caused by this pull request. |
-| Maybe flaky | It never flaked, but failed the way flaky checks do (a wait running out, a benchmark gate, a live service). | Re-run it once to tell. |
+| Maybe flaky | It never flaked, but failed the way flaky checks do (a wait running out, a benchmark gate, a live service). When it passed all of 20 or more recent default-branch runs, the row says so. | Re-run it once to tell. If the re-run passes, the comment still asks for a look before merging. |
 | PR details | A check on the pull request itself (a changelog label, a description checkbox, a title prefix) in a workflow that runs again when the PR is edited; 23 of 24 labelled matches were this. | Fix the PR, not the code. |
-| New | None of the above. | Likely from this change. |
+| New | None of the above; or every flake in the last 30 days came from this same pull request (a race or order dependence it added fails now and then and passes on a re-run); or a likely flake failed on this pull request far more often than on the default branch (2 or more failures, under a 5% chance at main's rate). | Likely from this change. If a re-run of the same commit passes, the comment says a green re-run doesn't clear it. |
 
 A flaky failure also says what kind it looks like: the environment (timing, network, infrastructure, concurrency) or, when none of those shows, the test's own code (a resource leak, an async wait, time and time zones, randomness, floating point, state shared between tests, files, ordering), and a flaky job or test says when it began: its first flake, how many runs before it were clean, whether that could be chance, and the commits that reached the default branch the day before, worded only as surely as the history allows. Those commits are context, not a culprit: on 5 clear starts in 45 public repos they touched tests or the workflow 9 of 12 times, about as often as random commits there (8 of 12).
 

{stackvitals-0.1.3 → stackvitals-0.1.4}/docs/rules.md

@@ -68,6 +68,28 @@ A failure with no flake history whose failing step (or test message) shows a wai
 - **How sure:** early observation (2 held-out matches in 150 failures)
 - **Changed 2026-10-23:** was track record over all 24 signature matches; why: the product says maybe flaky only for never-flaked failures (a whole-release review); effect: the track record counts never-flaked matches: 1 of 6 (0 of 4 tuning, 1 of 2 held out)
 
+## `flaky.only_this_pr` (version 1)
+
+A failure that would read likely flaky, flaked before or maybe flaky, whose flakes in the last 30 days all came from this same pull request (none on the default branch or other pull requests), reads new: a race or order dependence the change added fails now and then and passes on a re-run.
+
+- **Card:** [bench/accuracy/flaky](../bench/accuracy/flaky/README.md)
+- **Tuned on:** none: a public suggestion (2026-10-31) with the failure it describes written as tests
+- **Checked on:** not yet replayed on the labelled pull-request failures
+- **Bar:** none set yet; it only moves a call toward the change and never excuses a failure (set 2026-10-31)
+- **Result:** untested on labels: the replay needs the held-out reads again
+- **How sure:** early observation (not yet measured on labelled failures)
+
+## `flaky.worse_on_pr` (version 1)
+
+A likely flake that failed 2 or more times on this pull request's runs, where that many failures would come from its default-branch rate (smoothed by one failure and one pass, over 10 or more runs) less than 5% of the time, reads new: the change made a flaky check fail more often.
+
+- **Card:** [bench/accuracy/flaky](../bench/accuracy/flaky/README.md)
+- **Tuned on:** none: a public suggestion (2026-10-31) with the failure it describes written as tests
+- **Checked on:** not yet measured; the plan is regressions seeded into repos of the stackvitals organisation
+- **Bar:** none set yet; it only moves a call toward the change and never excuses a failure (set 2026-10-31)
+- **Result:** untested on labels
+- **How sure:** early observation (not yet measured on seeded or labelled failures)
+
 ## `flaky.pr_details` (version 1)
 
 A pull-request failure in a workflow that also runs when the PR's title, description or labels change, with no failing test in the step, whose error lines are about the PR (the pull request, a PR title or label, a label to add or missing, a checkbox, checklist or template left unchecked or incomplete), reads PR details: fix the PR, not the code; it still blocks.

stackvitals-0.1.4/glama.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "https://glama.ai/mcp/schemas/server.json",
+  "maintainers": ["kaushik-yadav"]
+}

{stackvitals-0.1.3 → stackvitals-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "stackvitals"
-version = "0.1.3"
+version = "0.1.4"
 description = "Free, open-source developer suite: CI health, team flow, dependencies, community, and MergeScout for contributors."
 readme = "README.md"
 requires-python = ">=3.11"

{stackvitals-0.1.3 → stackvitals-0.1.4}/server.json

@@ -3,12 +3,12 @@
   "name": "io.github.stackvitals/stackvitals",
   "title": "StackVitals",
   "description": "Known flaky tests, failing and slow CI, review flow, delivery and dependency risk from kept history.",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "packages": [
     {
       "registryType": "pypi",
       "identifier": "stackvitals",
-      "version": "0.1.3",
+      "version": "0.1.4",
       "runtimeHint": "uvx",
       "transport": {
         "type": "stdio"

stackvitals-0.1.4/src/mergescout/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.4"

{stackvitals-0.1.3 → stackvitals-0.1.4}/src/stackvitals/__init__.py

@@ -1,2 +1,2 @@
 """StackVitals: a free, open-source developer suite. MergeScout (package `mergescout`) is its Contribute module."""
-__version__ = "0.1.3"
+__version__ = "0.1.4"

{stackvitals-0.1.3 → stackvitals-0.1.4}/src/stackvitals/ci/action.py

@@ -15,7 +15,7 @@ from stackvitals.ci.automation import (alert_message, apply_decisions, new_flake
                                        should_rerun, sync_issues)
 from stackvitals.ci.collect import PR_EVENTS, collect
 from stackvitals.ci.feedback import (MARKER, classify, find_pr, our_comment, quarantine_note, quarantined_failures,
-                                     remembered, render_comment, upsert_comment, verdict_payload, blamed_sha)
+                                     remembered, render_comment, upsert_comment, verdict_payload, blamed_sha, green_note)
 from stackvitals.ci.owners import attach_owners, read_codeowners, test_files
 from stackvitals.ci.report import detail_name, render_detail, render_html, render_markdown
 from stackvitals.ci.store import DataStore
@@ -339,7 +339,7 @@ def _pr_feedback(recent: list[dict], latest: dict, payload: dict | None, api, da
             existing = [c for c in api._get(f"/repos/{repo}/issues/{pr}/comments", {"per_page": 100}) or []
                         if MARKER in (c.get("body") or "")]
             if existing:
-                outcomes.append(upsert_comment(api, pr, f"{MARKER}\n### StackVitals: CI is passing now\nThe latest run passed."))
+                outcomes.append(upsert_comment(api, pr, green_note(existing[0].get("body"), newest), existing[0]))
     if verdict:
         # A merge queue tests its own commit, so a required verdict must be posted there too.
         queued = {r["sha"] for r in recent if r.get("ev") == "merge_group" and r.get("sha")}

{stackvitals-0.1.3 → stackvitals-0.1.4}/src/stackvitals/ci/analyze.py

@@ -72,6 +72,13 @@ def _pushes(runs: dict, final_att: dict, default_branch: str | None) -> list:
                   and r.get("ev") == "push" and default_branch is not None and r.get("br") == default_branch)
 
 
+def _only_pr(prs: list) -> int | None:
+    """The one pull request every flake came from (each flake's pull request numbers), else None."""
+    found = {tuple(p) for p in prs}
+    only = found.pop() if len(found) == 1 else ()
+    return only[0] if len(only) == 1 else None
+
+
 def _state(ev_all: list, ev_win: list, runs_since: int, broken: bool, now: datetime) -> str:
     if broken:
         return "broken"
@@ -177,7 +184,10 @@ def _tests(data: dict, runs: dict, final_att: dict, since: datetime, now: dateti
         status = monitors.evaluate(ev_all, ev_win, hist, runs_since, now, cfg, (previous or {}).get(tid),
                                    pass_ms.get(tid, ()), fail_ms.get(tid, ()), tid)
         lo, hi = wilson(len(ev_win), len(win))
+        main_win = [r for r in default if when[id(r)] >= since]
         out.append({"id": tid, "suite": name, "runs": len(win),
+                    # Its record on the default branch, so a first failure on a pull request can be checked against it.
+                    "main_runs": len(main_win), "main_failures": sum(tid in failed_ids[id(r)] for r in main_win),
                     "workflow": runs[(mine[-1]["run"], mine[-1]["att"])].get("path") if mine else None,
                     "failures": fail_win[tid],
                     "flake_events": len(ev_win), "flake_rate": round(len(ev_win) / len(win), 4) if win else 0.0,
@@ -190,6 +200,8 @@ def _tests(data: dict, runs: dict, final_att: dict, since: datetime, now: dateti
                     # What kind of flake its usual failure looks like (timing, network, ...), only once it has flaked.
                     "flake": kind_of(messages[tid].most_common(1)[0][0]) if messages[tid] and ev_all else None,
                     "prs_affected": sorted({p for e in ev_win for p in e[2]}),
+                    # Flakes from one pull request alone are that change's (a race it added), not a known flake.
+                    "only_pr": _only_pr([e[2] for e in ev_win]),
                     # Which report (an OS or version in a matrix) the window's flakes came from.
                     "variants": dict(Counter(e[3].get("variant") for e in ev_win if e[3].get("variant")).most_common(5)),
                     # The newest runs that prove it, so anyone can open them.
@@ -474,15 +486,18 @@ def analyze(data: dict, now: datetime, window_days: int = 30, default_branch: st
                 slower = {"before_s": a, "after_s": b, "pct": round((b / a - 1) * 100)}
         lo, hi = wilson(len(ev_win), len(in_window))
         steps = Counter(e[2] for e in ev_win)
+        main_win = [j for r, j in in_window if default_branch is None or r.get("br") == default_branch]
         out_jobs.append({
             "workflow": path, "job": name, "runs": len(in_window),
+            "main_runs": len(main_win), "main_failures": sum(j["c"] in FAIL for j in main_win),
             "failures": sum(1 for _, j in in_window if j["c"] in FAIL),
             "flake_events": len(ev_win), "flake_rate": round(len(ev_win) / len(in_window), 4) if in_window else 0.0,
             "rate_low": round(lo, 4), "rate_high": round(hi, 4), "state": state,
             "last_flake": last_event.strftime("%Y-%m-%dT%H:%M:%SZ") if last_event else None,
             "top_step": steps.most_common(1)[0][0] if steps else None, "flake_steps": sorted(steps),
             "retry_minutes": round(sum(e[3] for e in ev_win), 1),
-            "prs_affected": sorted({p for e in ev_win for p in e[4]}), "evidence": [e[5] for e in ev_all[-5:]],
+            "prs_affected": sorted({p for e in ev_win for p in e[4]}), "only_pr": _only_pr([e[4] for e in ev_win]),
+            "evidence": [e[5] for e in ev_all[-5:]],
             "median_s": _median([s for _, s in durations]), "p95_s": _pct([s for _, s in durations], 0.95),
             "slower": slower, "trend": _trend((_t(r["cr"]), s) for r, s in durations),
             # Each flake is placed at its run's start, so the run that flaked first counts after it, not before.

{stackvitals-0.1.3 → stackvitals-0.1.4}/src/stackvitals/ci/feedback.py

@@ -2,6 +2,7 @@ import json
 import re
 from collections import defaultdict
 from importlib import resources
+from math import comb
 from datetime import timedelta
 
 from stackvitals.ci.analyze import FAIL, _downstream, _failed_step
@@ -157,6 +158,11 @@ class Index:
             self.jobs[(j["run"], j["att"])].append(j)
         for t in data.get("tests") or []:
             self.reports[(t["run"], t["att"])].append(t)
+        # Every attempt of each pull request's runs, to compare how often a check fails there with its rate on main.
+        self.runs, self.pr_attempts = runs, defaultdict(set)
+        for key, r in runs.items():
+            for p in r.get("pr") or ():
+                self.pr_attempts[p].add(key)
         on_main = lambda r: r is not None and r.get("ev") == "push" and default_branch and r.get("br") == default_branch  # noqa: E731
         self.main_jobs, self.main_tests = defaultdict(list), defaultdict(list)
         for (rid, att), js in self.jobs.items():
@@ -304,6 +310,71 @@ def _sharpen(verdict: str, text: str | None, changed: list | None, name: str = "
     return verdict, sign, []
 
 
+def _only_here(entry: dict | None, run: dict, verdict: str) -> str | None:
+    """A flaky-looking verdict whose flakes all came from this pull request: the change's own (a race it added), so it
+    reads "new". Returns the detail then, else None."""
+    if verdict not in ("known flaky", "flaked before", "maybe flaky") or not entry or not entry.get("flake_events"):
+        return None
+    if entry.get("only_pr") is None or entry["only_pr"] not in (run.get("pr") or ()):
+        return None
+    times = "once" if entry["flake_events"] == 1 else f"{entry['flake_events']} times"
+    return (f"flaked {times} in the last 30 days, all of them only on this pull request and nowhere else: likely a race "
+            "or order dependence this change added; look at the change before re-running")
+
+
+# Main runs a check must have passed, all of them, for a first flaky-looking failure on a pull request to be worth a look.
+STABLE_MAIN = 20
+# A likely flake reads new when this pull request's failures would come from its main rate less than 5% of the time,
+# with at least 2 failures and 10 main runs to judge by (one failure can't tell a 5% flake from a 40% one).
+WORSE_P, WORSE_MIN_FAILS, WORSE_MIN_MAIN = 0.05, 2, 10
+
+
+def _pr_record(index: Index, run: dict, path: str | None = None, job: str | None = None, suite: str | None = None,
+               test: str | None = None) -> tuple[int, int]:
+    """(failed, ran): how many attempts of this pull request's runs failed this job or test, out of those that ran it."""
+    keys = set().union(*(index.pr_attempts.get(p, set()) for p in run.get("pr") or ()))
+    failed = ran = 0
+    for key in keys:
+        if test:
+            reps = [r for r in index.reports.get(key, ()) if r["name"] == suite]
+            if reps:
+                ran += 1
+                failed += any(f[0] == test for r in reps for f in r["failed"])
+        elif index.runs[key].get("path") == path:
+            js = [j for j in index.jobs.get(key, ()) if j["n"] == job and (j["c"] in FAIL or j["c"] == "success")]
+            if js:
+                ran += 1
+                failed += any(j["c"] in FAIL for j in js)
+    return failed, ran
+
+
+def _tail(k: int, n: int, p: float) -> float:
+    """The chance of k or more failures in n runs at failure rate p."""
+    return sum(comb(n, i) * p ** i * (1 - p) ** (n - i) for i in range(k, n + 1))
+
+
+def _main_record(item: dict, entry: dict | None, here: tuple[int, int] | None = None) -> dict:
+    """The check's record on the default branch beside a flaky-looking verdict: a likely flake shows how often it fails
+    there; a "maybe flaky" check that passed every recent run there says so (still maybe flaky: never-flaked timeouts were
+    mostly real flakes on the labels), and a green re-run of it is flagged later."""
+    n = (entry or {}).get("main_runs") or 0
+    if item["verdict"] == "known flaky" and n:
+        # A change can make a flaky check fail far more often: its rate on main lets a reviewer compare.
+        bad = (entry or {}).get("main_failures") or 0
+        k, ran = here or (0, 0)
+        # The main rate is smoothed (one failure and one pass added) so a clean main never makes one failure damning.
+        if k >= WORSE_MIN_FAILS and n >= WORSE_MIN_MAIN and _tail(k, ran, (bad + 1) / (n + 2)) < WORSE_P:
+            return item | {"verdict": "new", "detail": f"flaky, but it failed {k} of {ran} runs on this pull request against "
+                           f"{bad} of {n} ({bad / n:.0%}) on the default branch in the last 30 days: this change likely "
+                           "made it fail more often; look at the change before re-running"}
+        return item | {"detail": item["detail"] + f"; on the default branch it failed {bad} of {n} runs ({bad / n:.0%}) "
+                       "in the last 30 days: if it fails here more often than that, this change may have made it worse"}
+    if item["verdict"] != "maybe flaky" or n < STABLE_MAIN or (entry or {}).get("main_failures"):
+        return item
+    return item | {"stable_on_main": n, "detail": item["detail"] + f"; it passed all {n} runs on the default branch in the "
+                   "last 30 days, so if a re-run passes, look for a race this change added"}
+
+
 def _detail(entry: dict | None, verdict: str, message: str | None = None, step: str | None = None,
             kind: str = "job", hint: str | None = None, sign: str | None = None, named: list | None = None,
             changed: list | None = None) -> str:
@@ -371,9 +442,13 @@ def classify_run(data: dict, result: dict, run_id: int, logs: dict | None = None
             verdict, sign, named = _sharpen(_test_verdict(entry), msg, changed, name=tid)
             hint = flaky_signature(msg) if verdict == "new" else None
             verdict = "maybe flaky" if hint else verdict
+            here = _only_here(entry, run, verdict)
+            verdict = "new" if here else verdict
             item = _with_kind({"kind": "test", "name": tid, "verdict": verdict,
-                               "detail": _detail(entry, verdict, msg, kind="test", hint=hint, sign=sign, named=named,
-                                                 changed=changed)}, msg)
+                               "detail": here or _detail(entry, verdict, msg, kind="test", hint=hint, sign=sign, named=named,
+                                                         changed=changed)}, msg)
+            item = _main_record(item, entry, _pr_record(index, run, suite=(entry or {}).get("suite"), test=tid)
+                                if item["verdict"] == "known flaky" else None)
             before = on_main.get(tid) if verdict in ("new", "flaked before", "maybe flaky", "maybe yours") else None
             if before and _same_test_failure(f, before):
                 item |= {"verdict": "already failing", "detail": f"failed the same way on the default branch {before['ago']} "
@@ -425,8 +500,12 @@ def classify_run(data: dict, result: dict, run_id: int, logs: dict | None = None
             continue
         hint = flaky_signature(mine) if verdict == "new" else None
         verdict = "maybe flaky" if hint else verdict
-        item = _with_kind({**base, "verdict": verdict, "detail": _detail(entry, verdict, step=failing_step, hint=hint, sign=sign,
-                                                                         named=named, changed=changed)}, mine)
+        here = _only_here(entry, run, verdict)
+        verdict = "new" if here else verdict
+        item = _with_kind({**base, "verdict": verdict, "detail": here or _detail(entry, verdict, step=failing_step, hint=hint,
+                                                                                 sign=sign, named=named, changed=changed)}, mine)
+        item = _main_record(item, entry, _pr_record(index, run, path=run.get("path"), job=j["n"])
+                            if item["verdict"] == "known flaky" else None)
         if before:
             item["detail"] += (f"; the default branch failed in this step {before['ago']} before this (run {before['run']})"
                                + (", with a different error" if mine and theirs else ""))
@@ -527,6 +606,16 @@ def blamed_sha(body: str | None) -> str | None:
     return m[1] if m else None
 
 
+def green_note(body: str | None, sha: str) -> str:
+    """Our comment once CI turns green: a warning, without blocking, when the failure it replaces on this same commit
+    looked like the change's own or failed a check main always passes (a race a change adds passes on a re-run)."""
+    if blamed_sha(body) != sha:
+        return f"{MARKER}\n### StackVitals: CI is passing now\nThe latest run passed."
+    return (f"{MARKER}\n### StackVitals: a re-run turned this green\nA failure on this commit looked like the change's own, or "
+            "failed a check that passes on the default branch every time, and passed when run again. That is how a race or "
+            "order dependence a change adds usually shows up: look at it before merging.\n<!-- stackvitals:blamed " + sha + " -->")
+
+
 def render_comment(items: list[dict], repo: str, sha: str | None = None) -> str:
     known = sum(i["verdict"] == "known flaky" for i in items)
     outside = sum(i["verdict"] == "outside failure" for i in items)
@@ -557,7 +646,7 @@ def render_comment(items: list[dict], repo: str, sha: str | None = None) -> str:
     if views:
         # A dash only ever sits inside a JSON string here, so escaping it keeps any "-->" from closing the note.
         folded.append("<!-- stackvitals:triage " + json.dumps(views, ensure_ascii=True).replace("-", "\\u002d") + " -->")
-    if sha and re.fullmatch(r"[0-9a-f]{7,40}", sha) and any(i["verdict"] in BLAMED for i in items):
+    if sha and re.fullmatch(r"[0-9a-f]{7,40}", sha) and any(i["verdict"] in BLAMED or i.get("stable_on_main") for i in items):
         folded.append(f"<!-- stackvitals:blamed {sha} -->")
     return "\n".join([MARKER, "### StackVitals: about this CI failure", head, "",
                       "| Verdict | What failed | Why |", "|---|---|---|", *rows, *more, "", *folded,

{stackvitals-0.1.3 → stackvitals-0.1.4}/src/stackvitals/editors/vscode/package.json

@@ -2,7 +2,7 @@
   "name": "stackvitals-flaky-tests",
   "displayName": "StackVitals flaky tests",
   "description": "Marks the tests your CI history shows are flaky or broken, read from the repo's stackvitals-data branch.",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "publisher": "stackvitals",
   "license": "Apache-2.0",
   "engines": { "vscode": "^1.80.0" },