stackvitals 0.1.0__py3-none-any.whl

stackvitals/core/alerts.py

@@ -0,0 +1,224 @@
+"""Alerts on delivery numbers over the last 28 days: a heads-up at 80% of a threshold and a miss past it, each sent once
+per change of state (and once on recovery), "would have fired N times" replayed over stored history, and payloads for
+Slack, Discord, Microsoft Teams (Workflows) and plain JSON webhooks."""
+from datetime import datetime, timedelta
+
+import httpx
+
+from stackvitals.core import chat
+from stackvitals.core.stats import percentile, quantile_range, wilson
+from stackvitals.core.time import iso
+
+WINDOW_DAYS = 28
+HEADS_UP = 0.8
+MIN_N = 10
+LEVELS = ("ok", "heads_up", "miss")
+NAMES = {"pickup_p75_hours": "pickup time P75 (hours)", "review_p75_hours": "review time P75 (hours)",
+         "cycle_p75_hours": "cycle time to merge P75 (hours)", "change_failure_rate": "change failure rate"}
+PHASE_OF = {"pickup_p75_hours": "pickup", "review_p75_hours": "review", "cycle_p75_hours": "cycle_to_merge"}
+
+
+def thresholds(cfg: dict) -> dict:
+    """The thresholds in force: every metric set above 0."""
+    return {k: v for k, v in cfg["alerts"].items() if k in NAMES and v}
+
+
+def series(rows: list[tuple], deps: list[dict], incidents: list[dict], now: datetime, days: int = 90,
+           window: int = WINDOW_DAYS) -> dict[str, list[tuple]]:
+    """Each metric for each of the last `days` days, over the `window` days before it: [(day, value, n, low)], oldest
+    first; `low` is the bottom of the value's likely range, so a miss can wait until the whole range is past the limit."""
+    out = {k: [] for k in NAMES}
+    failed = {i["deployment"] for i in incidents if i.get("deployment")}
+    for k in range(days, -1, -1):
+        end = now - timedelta(days=k)
+        lo, hi = iso(end - timedelta(days=window)), iso(end)
+        merged = [ph for p, ph, _dep in rows if p.get("mg") and lo < p["mg"] <= hi]
+        for metric, phase in PHASE_OF.items():
+            values = [ph[phase] for ph in merged if ph.get(phase) is not None]
+            rng = quantile_range(values, 0.75)
+            out[metric].append((end.date().isoformat(), percentile(values, 0.75) if values else None, len(values),
+                                rng[0] if rng else None))
+        shipped = [d for d in deps if lo < d["finished"] <= hi]
+        bad = sum(1 for d in shipped if d["id"] in failed)
+        rate = bad / len(shipped) if shipped else None
+        out["change_failure_rate"].append((end.date().isoformat(), rate, len(shipped),
+                                           round(wilson(bad, len(shipped))[0], 4) if shipped else None))
+    return out
+
+
+def _level(value: float | None, n: int, limit: float, low: float | None = None) -> str:
+    if value is None or n < MIN_N:
+        return "not enough data"
+    if value > limit:
+        # Over, but the likely range still reaches under the limit: a heads-up until the evidence settles it.
+        return "miss" if low is None or low > limit else "heads_up"
+    return "heads_up" if value >= HEADS_UP * limit else "ok"
+
+
+def evaluate(values: dict[str, tuple], limits: dict, previous: dict) -> tuple[dict, list[dict]]:
+    """(new state, events) from today's (value, n) per metric: an event when a level rises, and "recovered" when a
+    metric that missed is back to ok."""
+    state, events = {}, []
+    for metric, limit in limits.items():
+        value, n, *rest = values.get(metric, (None, 0))
+        before = previous.get(metric) or {}
+        last = before.get("verdict") or "ok"  # the last real verdict: a quiet week without data changes nothing
+        level = _level(value, n, limit, rest[0] if rest else None)
+        # An open miss stays a miss while the value stays over the limit, however its range moves.
+        if level == "heads_up" and before.get("open") and value is not None and value > limit:
+            level = "miss"
+        now = {"level": level, "verdict": last if level == "not enough data" else level, "open": before.get("open", False),
+               "value": value, "n": n}
+        if level in LEVELS[1:] and LEVELS.index(level) > LEVELS.index(last):
+            events.append({"metric": metric, "level": level, "value": value, "limit": limit, "n": n})
+            now["open"] = now["open"] or level == "miss"
+        elif level == "ok" and before.get("open"):
+            events.append({"metric": metric, "level": "recovered", "value": value, "limit": limit, "n": n})
+            now["open"] = False
+        state[metric] = now
+    return state, events
+
+
+def replay(history: dict[str, list[tuple]], limits: dict) -> dict[str, int]:
+    """How many alerts each threshold would have sent over the stored days."""
+    fired = {m: 0 for m in limits}
+    state = {}
+    for k in range(max((len(v) for v in history.values()), default=0)):
+        today = {m: tuple(history[m][k][1:]) for m in limits if m in history and k < len(history[m])}
+        state, events = evaluate(today, limits, state)
+        for e in events:
+            fired[e["metric"]] += 1
+    return fired
+
+
+def _fmt(metric: str, value) -> str:
+    return f"{value:.0%}" if metric == "change_failure_rate" else f"{value:.1f}"
+
+
+def message(events: list[dict], repo: str, links=()) -> dict:
+    words = {"heads_up": "heads-up: near its limit", "miss": "over its limit", "recovered": "back under its limit"}
+    unsure = "heads-up: past its limit, but its likely range still reaches under it"
+    lines = [f"- **{NAMES[e['metric']]}** {unsure if e['level'] == 'heads_up' and e['value'] > e['limit'] else words[e['level']]}: "
+             f"{_fmt(e['metric'], e['value'])} against {_fmt(e['metric'], e['limit'])} (n={e['n']})." for e in events]
+    return chat.message(f"StackVitals alerts for {repo} (last {WINDOW_DAYS} days)", lines, links,
+                        {"repo": repo, "alerts": events})
+
+
+def send(url: str, events: list[dict], repo: str, client: httpx.Client | None = None, links=()) -> str | None:
+    """Post the alerts in the chat's own shape; trouble comes back as text for the job summary."""
+    return chat.post(url, message(events, repo, links), client, "the alert webhook")
+
+
+STATE = "alerts/state.json"
+ISSUE_TITLE = "StackVitals: delivery alerts"
+ISSUE_LABEL = "stackvitals"
+
+
+def _issue_body(state: dict, limits: dict) -> str:
+    words = {"ok": "ok", "heads_up": "near its limit", "miss": "over its limit", "not enough data": "not enough data"}
+    lines = [f"Delivery numbers over the last {WINDOW_DAYS} days against this repo's thresholds:", ""]
+    for metric, limit in limits.items():
+        s = state[metric]
+        value = "-" if s.get("value") is None else _fmt(metric, s["value"])
+        lines.append(f"- {NAMES[metric]}: {value} against {_fmt(metric, limit)} (n={s.get('n', 0)}): {words[s['level']]}")
+    lines += ["", "Change the thresholds under [alerts] in .github/stackvitals.toml (0 turns one off). This issue is kept "
+              "up to date and closed when every number is back under its limit."]
+    return "\n".join(lines)
+
+
+def _sync_issue(api, state: dict, limits: dict) -> None:
+    found = [i for i in api._get(f"/repos/{api.repo}/issues", {"labels": ISSUE_LABEL, "state": "open", "per_page": 100}) or []
+             if i.get("title") == ISSUE_TITLE and "pull_request" not in i]
+    bad = any(state[m]["verdict"] in ("heads_up", "miss") for m in limits)
+    body = _issue_body(state, limits)
+    if found and not bad:
+        api.send("PATCH", f"/repos/{api.repo}/issues/{found[0]['number']}", {"body": body, "state": "closed",
+                                                                          "state_reason": "completed"})
+    elif found:
+        api.send("PATCH", f"/repos/{api.repo}/issues/{found[0]['number']}", {"body": body})
+    elif bad:
+        api.send("POST", f"/repos/{api.repo}/issues", {"title": ISSUE_TITLE, "body": body, "labels": [ISSUE_LABEL]})
+
+
+def alerts_step(ws, env: dict, now: datetime, make_api=None, client: httpx.Client | None = None) -> dict | None:
+    """The Action's alerts after Team Flow (the schedule): today's numbers against the thresholds, sent once per change of
+    state; the stored state keeps levels only, so a run where nothing crossed a line commits nothing."""
+    import json
+
+    from stackvitals.ci.store import DataStore
+    from stackvitals.core.config import parse
+    from stackvitals.core.delivery import apply_identity, shipped
+    from stackvitals.flow.store import FlowStore
+    if ws.results.get("Team Flow") is None:
+        return None
+    cfg = ws.cache.get("settings") or parse("")[0]
+    limits = thresholds(cfg)
+    if not limits:
+        return None
+    store = FlowStore(ws.path)
+    from stackvitals.core.decisions import visible
+    flow = visible(ws.path, apply_identity(ws.cache.get("flow") or store.load(), cfg.get("identity")), now.date())
+    ci = ws.cache.get("ci") or (DataStore(ws.path).load() if (ws.path / "ci" / "days").is_dir() else {"runs": [], "jobs": []})
+    issues = []
+    if (ws.path / "community" / "issues").is_dir():
+        got = FlowStore(ws.path, "community/issues").load()
+        issues = got["open"] + got["closed"]
+    info = store.repo()
+    deps, _source, rows = shipped(flow, info, ci, cfg)
+    from stackvitals.connectors import active, public
+    from stackvitals.connectors import load as load_tracker
+    from stackvitals.core.delivery import incidents_for
+    tracker = ws.cache.get("tracker") or load_tracker(ws.path, redact=public(ws), names=active(env, cfg))
+    incidents = incidents_for(flow["closed"] + flow["open"], issues, deps, cfg, tracker, info.get("recorded"))[0]
+    history = series(rows, deps, incidents, now)
+    # Whole days only for the app's "would have fired" preview, so the published file changes at most once a day.
+    from stackvitals.core.store import published, write_if_changed
+    daily = series(rows, deps, incidents, now.replace(hour=0, minute=0, second=0, microsecond=0))
+    (ws.path / "docs").mkdir(parents=True, exist_ok=True)
+    write_if_changed(ws.path / "docs" / "alerts.json", published({
+        "window_days": WINDOW_DAYS, "heads_up": HEADS_UP, "min_n": MIN_N, "limits": limits, "names": NAMES,
+        "series": {m: [[d, None if v is None else round(v, 3), n, None if low is None else round(low, 3)]
+                       for d, v, n, low in points] for m, points in daily.items()}}))
+    today = {m: tuple(history[m][-1][1:]) for m in limits}
+    path = ws.path / STATE
+    try:
+        old = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        old = {}
+    state, events = evaluate(today, limits, old.get("metrics") or {})
+    fired = replay(history, limits) if old.get("limits") != limits else None
+    notes, channels = [], cfg["alerts"]["channels"]
+    if events and "webhook" in channels and env.get("STACKVITALS_WEBHOOK"):
+        app = chat.app_url(ws.repo, ws.cache.get("repo_info"), "delivery")
+        links = [("Open Delivery", app)] if app else []
+        links.append(("Alerts issue", f"https://github.com/{ws.repo}/issues?q=is%3Aissue+label%3Astackvitals+alerts"))
+        failed = send(env["STACKVITALS_WEBHOOK"], events, ws.repo, client=client, links=links)
+        if failed:
+            notes.append(failed)
+    if (events or fired is not None) and "issue" in channels:
+        if make_api is None:
+            from stackvitals.ci.api import ActionsAPI as make_api
+        try:
+            _sync_issue(make_api(env.get("STACKVITALS_TOKEN"), ws.repo, max_requests=10), state, limits)
+        except Exception as exc:  # noqa: BLE001 - a missing permission must not stop the job
+            notes.append(f"Could not update the alerts issue ({type(exc).__name__}); it needs `issues: write`.")
+    keep = {"limits": limits, "metrics": {m: {"verdict": s["verdict"], "open": s["open"]} for m, s in state.items()}}
+    if keep != old:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(keep, indent=1, sort_keys=True), encoding="utf-8")
+
+    def after_commit():
+        summary = env.get("GITHUB_STEP_SUMMARY")
+        if not summary or not (events or fired is not None or notes):
+            return
+        lines = []
+        if events:
+            lines.append(chat.text(message(events, ws.repo)))
+        if fired is not None:
+            lines.append("With these thresholds, alerts would have fired " + ", ".join(
+                f"{n} times for {NAMES[m]}" for m, n in fired.items()) + f" over the last 90 days (n of each day's {WINDOW_DAYS}-day window shown with every alert).")
+        lines += notes
+        with open(summary, "a", encoding="utf-8") as f:
+            f.write("### StackVitals alerts\n" + "\n\n".join(lines) + "\n")
+    ws.after_commit(after_commit, "Alerts")
+    return {"events": len(events), "levels": {m: s["level"] for m, s in state.items()}, "would_have_fired": fired}

stackvitals/core/appdata.py

@@ -0,0 +1,112 @@
+"""The files the StackVitals app reads, written beside the pages at the end of every job: app.json (what ran and when,
+links, the key facts), settings.json (every setting with its default and current value), decisions.json (what people
+decided, with the log) and, for public repos, the summary tables, so the app and Explore read everything from Pages
+itself rather than raw.githubusercontent.com, which GitHub throttles. Files are rewritten only when they change."""
+import json
+import shutil
+from pathlib import Path
+
+from stackvitals.core import schema
+from stackvitals.core.config import FILE, parse
+from stackvitals.core.decisions import UNDO, Store
+from stackvitals.core.store import published, write_if_changed
+
+# Module, its published file and its page; Portal keeps its catalog outside docs/ (copied in for the app).
+MODULES = (("ci", "data.json", "ci.html"), ("flow", "flow.json", "flow.html"), ("delivery", "delivery.json", "delivery.html"),
+           ("insights", "insights.json", "insights.html"), ("community", "community.json", "community.html"),
+           ("portal", "portal.json", "portal.html"), ("alerts", "alerts.json", None), ("deps", "deps.json", None),
+           ("supply", "supply.json", None), ("vulns", "vulns.json", None), ("security", "security.json", None))
+LOG_ROWS = 200
+
+
+def _read(path: Path):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+
+def facts(docs: dict) -> dict:
+    """The few numbers a home page, the digest or an org view lead with; None where a module didn't run."""
+    data, flow, delivery = docs.get("ci") or {}, docs.get("flow") or {}, docs.get("delivery") or {}
+    items = (data.get("tests") or []) + (data.get("jobs") or [])
+    phase = lambda name: ((delivery.get("phases") or {}).get(name) or {}).get("p75_h")
+    found = (docs.get("insights") or {}).get("findings") or []
+    return {
+        "ci_first_try": (data.get("pushes") or {}).get("first_try_share") if data else None,
+        "ci_flaky": sum(1 for i in items if i.get("state") == "flaky") if data else None,
+        "ci_broken": sum(1 for i in items if i.get("state") == "broken") if data else None,
+        "open_prs": flow.get("open"), "stale_prs": flow.get("stale"),
+        "pickup_p75_h": phase("pickup"), "review_p75_h": phase("review"), "cycle_p75_h": phase("cycle_to_merge"),
+        "deployments": (delivery.get("dora") or {}).get("deployments"),
+        "change_failure_rate": (delivery.get("dora") or {}).get("change_failure_rate"),
+        "newcomers_waiting": (docs.get("community") or {}).get("waiting_total"),
+        "findings": sum(1 for f in found if not f.get("decision")) if docs.get("insights") else None,
+        "first_review_median_s": ((flow.get("stages") or {}).get("pickup") or {}).get("median_s"),
+        "community_score": ((docs.get("community") or {}).get("score") or {}).get("score"),
+        "vulns_open": len((docs.get("vulns") or {}).get("open") or []) if docs.get("vulns") else None,
+        "vulns_serious": sum(1 for v in (docs.get("vulns") or {}).get("open") or []
+                             if v.get("malware") or v.get("severity") in ("critical", "high")) if docs.get("vulns") else None,
+    }
+
+
+def write_app_data(ws) -> None:
+    root, now = ws.path, ws.now
+    docs_dir = root / "docs"
+    if not docs_dir.is_dir():
+        return
+    info = ws.cache.get("repo_info") or {}
+    last = _read(docs_dir / "app.json") or {}
+    branch = info.get("default_branch") or last.get("default_branch") or "main"
+    # Without the repo's details, the last known visibility holds; never known means private (publish nothing).
+    private = bool(info["private"]) if "private" in info else last.get("visibility") != "public"
+    if private and info.get("has_pages") and (ws.env.get("STACKVITALS_PAGES_MODE") or "branch") != "artifact":
+        # Pages from a private repo are public on most plans: say so on every run until it's changed.
+        ws.notes.append("This repo is private but has GitHub Pages turned on; unless your plan keeps Pages private, anyone "
+                        "with the address can read the app. Turn Pages off and set pages: artifact, or use stackvitals ui.")
+    catalog = _read(root / "portal" / "catalog.json")
+    if catalog is not None:
+        # The app's copy is packed (each rule named once) and compact: 10,000 repos would be 24 MB as written.
+        from stackvitals.portal.scorecard import pack
+        write_if_changed(docs_dir / "portal.json", published(pack(catalog)), compact=True)
+    docs = {name: _read(docs_dir / file) for name, file, _page in MODULES}
+    workflow = (ws.env.get("GITHUB_WORKFLOW_REF") or "").split("@")[0].rsplit("/", 1)[-1] or None
+    base = f"https://github.com/{ws.repo}"
+    state, log = Store(root).state(now.date()), Store(root).log()
+    write_if_changed(docs_dir / "app.json", published({
+        "repo": ws.repo, "default_branch": branch, "visibility": "private" if private else "public",
+        "modules": {name: {"updated": (docs[name] or {}).get("generated") or (docs[name] or {}).get("built_at"), "file": file,
+                           **({"page": page} if page and (docs_dir / page).exists() else {})}
+                    for name, file, page in MODULES if docs[name] is not None},
+        "pages": sorted(p.name for p in docs_dir.glob("*.html")),
+        "facts": {**facts(docs), "decisions": sum(1 for v in state.values() if v.get("kind") != "setting" and v.get("verb") not in UNDO)},
+        "links": {"repo": base, "workflow": workflow,
+                  "actions": f"{base}/actions/workflows/{workflow}" if workflow else f"{base}/actions",
+                  "settings_file": f"{base}/blob/{branch}/{FILE}", "edit_settings": f"{base}/edit/{branch}/{FILE}",
+                  "new_issue": f"{base}/issues/new"},
+        "summary": "summary/" if not private else None,
+    }))
+    cfg = ws.cache.get("settings") or parse("")[0]
+    write_if_changed(docs_dir / "settings.json", published({
+        "file": FILE, "settings": schema.to_json(), "values": cfg, "notes": list(ws.cache.get("settings_notes") or [])}))
+    write_if_changed(docs_dir / "decisions.json", published({"state": state, "log": list(reversed(log))[:LOG_ROWS]}))
+    _tables(root, private)
+
+
+def _tables(root: Path, private: bool) -> None:
+    """Public repos: the summary tables next to the pages (git keeps one copy of identical files). Private: kept out."""
+    src, dst = root / "summary", root / "docs" / "summary"
+    if private or not src.is_dir():
+        if dst.exists():
+            shutil.rmtree(dst)
+        return
+    dst.mkdir(parents=True, exist_ok=True)
+    names = {p.name for p in src.iterdir() if p.is_file()}
+    for p in dst.iterdir():
+        if p.is_file() and p.name not in names:
+            p.unlink()
+    for name in names:
+        data = (src / name).read_bytes()
+        target = dst / name
+        if not target.exists() or target.read_bytes() != data:
+            target.write_bytes(data)

stackvitals/core/bucket.py

@@ -0,0 +1,190 @@
+"""Optional storage outside git: the same files the stackvitals-data branch holds, kept in an S3-compatible bucket (AWS S3,
+Cloudflare R2, MinIO and others that support conditional writes). For very busy repos and big orgs, where many runs
+writing one branch collide. Files are stored by their content hash, and one manifest maps paths to hashes; the manifest
+is replaced only if nobody replaced it since this run read it (If-Match), so overlapping runs converge as with git.
+Requests are signed with AWS Signature V4; no SDK is needed."""
+import hashlib
+import hmac
+import json
+import os
+import random
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import quote, urlparse
+
+MANIFEST = "manifest.json"
+EMPTY = hashlib.sha256(b"").hexdigest()
+
+
+class BucketError(RuntimeError):
+    pass
+
+
+def _sha(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def sign(method: str, url: str, headers: dict, payload_sha: str, key_id: str, secret: str, region: str,
+         now: datetime, service: str = "s3") -> dict:
+    """The headers of a request signed with AWS Signature Version 4 (path-style URL)."""
+    u = urlparse(url)
+    stamp, day = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d")
+    all_headers = {**{k.lower(): str(v).strip() for k, v in headers.items()}, "host": u.netloc,
+                   "x-amz-date": stamp, "x-amz-content-sha256": payload_sha}
+    names = sorted(all_headers)
+    query = "&".join(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}" for k, v in
+                     sorted(tuple(p.split("=", 1)) if "=" in p else (p, "") for p in u.query.split("&") if p))
+    # The URL's path is already percent-encoded (BucketStore._url); S3 signs it as sent, never encoded twice.
+    canonical = "\n".join([method, u.path or "/", query,
+                           "".join(f"{n}:{all_headers[n]}\n" for n in names), ";".join(names), payload_sha])
+    scope = f"{day}/{region}/{service}/aws4_request"
+    to_sign = "\n".join(["AWS4-HMAC-SHA256", stamp, scope, _sha(canonical.encode())])
+    key = f"AWS4{secret}".encode()
+    for part in (day, region, service, "aws4_request"):
+        key = hmac.new(key, part.encode(), hashlib.sha256).digest()
+    signature = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest()
+    return {**{k: v for k, v in all_headers.items() if k != "host"},
+            "authorization": f"AWS4-HMAC-SHA256 Credential={key_id}/{scope}, SignedHeaders={';'.join(names)}, Signature={signature}"}
+
+
+class BucketStore:
+    """The data branch's interface (open, dirty, commit_push, squash) over a bucket prefix."""
+
+    def __init__(self, url: str, path: Path, endpoint: str, key_id: str, secret: str, region: str = "auto", client=None):
+        u = urlparse(url)
+        if u.scheme != "s3" or not u.netloc:
+            raise BucketError("storage must look like s3://bucket/prefix")
+        self.bucket, self.prefix = u.netloc, u.path.strip("/")
+        self.endpoint = endpoint.rstrip("/")
+        self.key_id, self.secret, self.region = key_id, secret, region or "auto"
+        self.path = Path(path)
+        self.client = client
+        self.etag = None
+        self.files: dict[str, str] = {}
+
+    # -- requests ------------------------------------------------------------------------------------------------
+    def _url(self, key: str) -> str:
+        return f"{self.endpoint}/{self.bucket}/{quote(key, safe='/-_.~')}"
+
+    def _key(self, name: str) -> str:
+        return f"{self.prefix}/{name}" if self.prefix else name
+
+    def _send(self, method: str, key: str, body: bytes = b"", headers: dict | None = None):
+        if self.client is None:
+            import httpx
+            self.client = httpx.Client(timeout=60)
+        url = self._url(key)
+        signed = sign(method, url, headers or {}, _sha(body) if body else EMPTY, self.key_id, self.secret, self.region,
+                      datetime.now(timezone.utc))
+        for attempt in range(3):
+            try:
+                r = self.client.request(method, url, content=body or None, headers=signed)
+            except Exception as exc:  # noqa: BLE001 - a dropped connection is retried, then said
+                if attempt == 2:
+                    raise BucketError(f"{method} {key}: {type(exc).__name__}") from exc
+                time.sleep(1 + attempt)
+                continue
+            if r.status_code >= 500 and attempt < 2:
+                time.sleep(1 + attempt)
+                continue
+            return r
+        return r
+
+    # -- the data branch's interface ------------------------------------------------------------------------------
+    def open(self) -> None:
+        """Download every file the manifest lists into `path`, remembering the manifest's ETag."""
+        import shutil
+        for attempt in range(3):
+            if self.path.exists():
+                shutil.rmtree(self.path)
+            self.path.mkdir(parents=True)
+            r = self._send("GET", self._key(MANIFEST))
+            if r.status_code == 404:
+                self.etag, self.files = None, {}
+                return
+            if r.status_code == 403:
+                raise BucketError("reading the manifest: HTTP 403. The key must be able to read, write, delete and list "
+                                  "the prefix; AWS answers a missing file with 403 when listing isn't allowed")
+            if r.status_code != 200:
+                raise BucketError(f"reading the manifest: HTTP {r.status_code}")
+            self.etag = r.headers.get("etag")
+            self.files = json.loads(r.content).get("files") or {}
+            gone = False
+            for name, digest in self.files.items():
+                obj = self._send("GET", self._key(f"objects/{digest}"))
+                if obj.status_code == 404:
+                    gone = True  # another run swapped the manifest and cleaned up while this one read: start over
+                    break
+                if obj.status_code != 200 or _sha(obj.content) != digest:
+                    raise BucketError(f"reading {name}: HTTP {obj.status_code}")
+                target = self.path / name
+                target.parent.mkdir(parents=True, exist_ok=True)
+                target.write_bytes(obj.content)
+            if not gone:
+                return
+            time.sleep(backoff(attempt))
+        raise BucketError("the bucket kept changing while it was read; the next run will catch up")
+
+    def _current(self) -> dict[str, tuple[str, bytes]]:
+        out = {}
+        for p in self.path.rglob("*"):
+            if p.is_file():
+                data = p.read_bytes()
+                out[p.relative_to(self.path).as_posix()] = (_sha(data), data)
+        return out
+
+    def dirty(self) -> bool:
+        return {k: v[0] for k, v in self._current().items()} != self.files
+
+    def commit_push(self, message: str) -> str:
+        """Upload new contents, then swap the manifest only if nobody swapped it since open(): "pushed", "unchanged",
+        or "rejected" (the caller runs its steps again on the fresh data)."""
+        now = self._current()
+        files = {k: v[0] for k, v in now.items()}
+        if files == self.files:
+            return "unchanged"
+        for name, (digest, data) in now.items():
+            if self.files.get(name) != digest and digest not in set(self.files.values()):
+                r = self._send("PUT", self._key(f"objects/{digest}"), data)
+                if r.status_code not in (200, 201):
+                    raise BucketError(f"writing {name}: HTTP {r.status_code}")
+        doc = json.dumps({"files": files, "message": message, "at": datetime.now(timezone.utc).isoformat()},
+                         sort_keys=True).encode()
+        cond = {"if-match": self.etag} if self.etag else {"if-none-match": "*"}
+        r = self._send("PUT", self._key(MANIFEST), doc, cond)
+        if r.status_code in (409, 412):
+            # Another run won. What this attempt uploaded stays: objects are named by content, so a run still working
+            # may have uploaded the same file and be about to publish it. A few spare objects cost less than a lost one.
+            return "rejected"
+        if r.status_code not in (200, 201):
+            raise BucketError(f"writing the manifest: HTTP {r.status_code}")
+        old = set(self.files.values()) - set(files.values())
+        self.etag, self.files = r.headers.get("etag"), files
+        # Contents no file points to any more go; a run that read the old manifest only ever reads before this.
+        for digest in old:
+            self._send("DELETE", self._key(f"objects/{digest}"))
+        return "pushed"
+
+    def squash(self, message: str) -> str:
+        return "unchanged"  # a bucket keeps no history to squash
+
+
+def from_env(env: dict, path: Path, client=None) -> BucketStore | None:
+    """The bucket store when STACKVITALS_STORAGE names one (s3://bucket/prefix), else None (the git branch)."""
+    url = (env.get("STACKVITALS_STORAGE") or "").strip()
+    if not url or url == "git":
+        return None
+    missing = [k for k in ("STACKVITALS_S3_ENDPOINT", "STACKVITALS_S3_KEY_ID", "STACKVITALS_S3_SECRET") if not env.get(k)]
+    if missing:
+        raise BucketError(f"storage {url} needs {', '.join(missing)}")
+    return BucketStore(url, path, env["STACKVITALS_S3_ENDPOINT"], env["STACKVITALS_S3_KEY_ID"], env["STACKVITALS_S3_SECRET"],
+                       env.get("STACKVITALS_S3_REGION") or "auto", client)
+
+
+def backoff(attempt: int) -> float:
+    # Two runs that collided shouldn't retry in step: a random wait that grows with each attempt.
+    # Tests replay collisions on purpose (pytest names the running test in this variable) and skip the wait.
+    if os.environ.get("STACKVITALS_NO_BACKOFF") == "1" or os.environ.get("PYTEST_CURRENT_TEST"):
+        return 0
+    return random.uniform(1, 4) * (attempt + 1)

stackvitals/core/bugs.py

@@ -0,0 +1,43 @@
+"""The bug backlog week by week (DevLake's weekly bug retro): bugs opened and closed each week, how many stay open, and
+how old the open ones are. From GitHub issues with a bug label alone, plus Jira and Linear bugs when connected."""
+from datetime import datetime, timedelta
+
+from stackvitals.connectors.base import std_type
+
+WEEKS = 12
+OLDEST = 10
+
+
+def _t(s: str) -> datetime:
+    return datetime.fromisoformat(s.replace("Z", "+00:00"))
+
+
+def _bugs(issues: list[dict], tracked: list[dict]) -> list[dict]:
+    out = [{"src": "github", "id": i["n"], "created": i["cr"], "closed": i.get("cl")}
+           for i in issues or [] if i.get("cr") and any(std_type(x) == "BUG" for x in i.get("lb") or [])]
+    out += [{"src": i.get("src"), "id": i.get("key") or i.get("id"), "created": i["created"],
+             "closed": i.get("resolved") if i.get("cat") == "DONE" else None, "url": i.get("url")}
+            for i in tracked or [] if i.get("type") == "BUG" and i.get("created") and not i.get("canceled")]
+    return out
+
+
+def backlog(issues: list[dict], tracked: list[dict], now: datetime, weeks: int = WEEKS) -> dict | None:
+    """{"weeks": [{"week", "opened", "closed", "open"}], "open", "median_open_days", "oldest", "sources"}, or None."""
+    bugs = _bugs(issues, tracked)
+    if not bugs:
+        return None
+    monday = (now - timedelta(days=now.weekday())).replace(hour=0, minute=0, second=0, microsecond=0)
+    rows = []
+    for k in range(weeks - 1, -1, -1):
+        start, end = monday - timedelta(weeks=k), monday - timedelta(weeks=k - 1)
+        rows.append({"week": start.strftime("%Y-%m-%d"),
+                     "opened": sum(start <= _t(b["created"]) < end for b in bugs),
+                     "closed": sum(bool(b["closed"]) and start <= _t(b["closed"]) < end for b in bugs),
+                     "open": sum(_t(b["created"]) < min(end, now) and (not b["closed"] or _t(b["closed"]) >= min(end, now))
+                                 for b in bugs)})
+    still = sorted((b for b in bugs if not b["closed"]), key=lambda b: b["created"])
+    ages = sorted((now - _t(b["created"])).days for b in still)
+    return {"weeks": rows, "open": len(still), "median_open_days": ages[len(ages) // 2] if ages else None,
+            "oldest": [{"src": b["src"], "id": b["id"], "days": (now - _t(b["created"])).days, **({"url": b["url"]} if b.get("url") else {})}
+                       for b in still[:OLDEST]],
+            "sources": sorted({b["src"] for b in bugs})}

stackvitals/core/calendar.py

@@ -0,0 +1,34 @@
+"""A team's working calendar ([calendar] weekends and holidays): hours between two times with the non-working days
+(whole UTC days) left out, so a review phase isn't charged a weekend nobody worked."""
+from datetime import date, datetime, timedelta, timezone
+
+DAYS = ("monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday")
+
+
+def _t(text: str) -> datetime:
+    return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(timezone.utc)
+
+
+def is_set(cal: dict | None) -> bool:
+    return bool(cal and (cal.get("weekends") or cal.get("holidays")))
+
+
+def elapsed_hours(a: str | None, b: str | None, cal: dict | None) -> float | None:
+    """Hours from a to b, without the weekends and holidays in `cal` (None: plain clock hours)."""
+    if not a or not b:
+        return None
+    start, end = _t(a), _t(b)
+    total = (end - start).total_seconds() / 3600
+    if not is_set(cal) or end <= start:
+        return round(total, 4)
+    off_days = {DAYS.index(d) for d in cal.get("weekends") or []}
+    holidays = {date.fromisoformat(h) for h in cal.get("holidays") or []}
+    day = start.date()
+    while day <= end.date():
+        if day.weekday() in off_days or day in holidays:
+            lo = max(start, datetime(day.year, day.month, day.day, tzinfo=timezone.utc))
+            hi = min(end, datetime(day.year, day.month, day.day, tzinfo=timezone.utc) + timedelta(days=1))
+            if hi > lo:
+                total -= (hi - lo).total_seconds() / 3600
+        day += timedelta(days=1)
+    return round(max(total, 0.0), 4)