stackvitals 0.1.0__py3-none-any.whl

stackvitals/connectors/linear.py

@@ -0,0 +1,140 @@
+"""Linear: issues of the chosen teams, their state and cycle changes, and the teams' cycles, read with a personal API key
+over GraphQL. A pass reads everything updated in a fixed span; a capped pass carries on from its cursor next run."""
+from datetime import datetime, timedelta
+
+import httpx
+
+from stackvitals.connectors.base import BACKFILL_DAYS, PAGES, ConnectorError, std_type, utc
+from stackvitals.core.time import iso
+
+NAME, TOKEN_ENV = "linear", "STACKVITALS_LINEAR_TOKEN"
+URL = "https://api.linear.app/graphql"
+# What is asked for: 2 adds the assignee and 3 its name, so answers kept before are read again, not rebuilt.
+ASKED = 3
+CATEGORY = {"triage": "TODO", "backlog": "TODO", "unstarted": "TODO", "started": "IN_PROGRESS", "completed": "DONE",
+            "canceled": "DONE"}
+PRIORITY = {0: None, 1: "Urgent", 2: "High", 3: "Medium", 4: "Low"}
+ISSUES = """query Issues($filter: IssueFilter, $after: String) {
+  issues(first: 50, after: $after, orderBy: updatedAt, includeArchived: true, filter: $filter) {
+    nodes { id identifier title url priority estimate createdAt updatedAt startedAt completedAt canceledAt
+      state { name type } team { key } labels(first: 10) { nodes { name } } cycle { id } project { id name } assignee { id email name }
+      history(first: 30) { nodes { createdAt fromState { name } toState { name } fromCycleId toCycleId } } }
+    pageInfo { hasNextPage endCursor } } }"""
+CYCLES = """query Cycles($filter: CycleFilter, $after: String) {
+  cycles(first: 50, after: $after, filter: $filter) {
+    nodes { id number name startsAt endsAt completedAt team { key } }
+    pageInfo { hasNextPage endCursor } } }"""
+
+__all__ = ["ConnectorError", "configured", "pull"]
+
+
+def configured(env: dict, cfg: dict) -> bool:
+    return bool((env.get(TOKEN_ENV) or "").strip())
+
+
+def _post(client: httpx.Client, key: str, query: str, variables: dict) -> dict:
+    try:
+        resp = client.post(URL, json={"query": query, "variables": variables},
+                           headers={"Authorization": key, "Content-Type": "application/json"})
+    except httpx.HTTPError as exc:
+        raise ConnectorError(f"Linear could not be reached ({type(exc).__name__}); the next run tries again.") from None
+    try:
+        body = resp.json()
+    except ValueError:
+        body = {}
+    body = body if isinstance(body, dict) else {}
+    errors = body.get("errors") or []
+    codes = {str((e.get("extensions") or {}).get("code") or "") for e in errors}
+    if resp.status_code in (401, 403) or "AUTHENTICATION_ERROR" in codes:
+        raise ConnectorError(f"Linear refused the key ({resp.status_code}): check the LINEAR_API_KEY secret.")
+    if resp.status_code == 429 or "RATELIMITED" in codes:
+        raise ConnectorError("Linear's hourly limit for this key is used up; the next run carries on.")
+    if errors or resp.status_code != 200:
+        said = "; ".join(str(e.get("message") or "") for e in errors)[:200]
+        raise ConnectorError(f"Linear could not run the query ({said or resp.status_code}); check teams under [connectors.linear].")
+    return body.get("data") or {}
+
+
+def _issue_records(n: dict, incident_labels: set[str]) -> list[dict]:
+    state = n.get("state") or {}
+    labels = [x.get("name") or "" for x in (n.get("labels") or {}).get("nodes") or []]
+    low = {x.lower() for x in labels}
+    incident = bool(low & incident_labels)
+    iid = f"linear:{n['id']}"
+    history = (n.get("history") or {}).get("nodes") or []
+    # The cycles an issue was ever in: its current one and every one its history moved it into or out of.
+    cycles = {c for c in [(n.get("cycle") or {}).get("id"), *(h.get(k) for h in history for k in ("fromCycleId", "toCycleId"))] if c}
+    out = [{"t": "issue", "src": NAME, "id": iid, "key": n["identifier"], "title": n.get("title"),
+            "type": "INCIDENT" if incident else next((std_type(x) for x in labels if std_type(x) != "TASK"), "TASK"),
+            "original_type": None, "status": state.get("name"), "cat": CATEGORY.get(state.get("type"), "TODO"),
+            "created": utc(n.get("createdAt")), "updated": utc(n.get("updatedAt")),
+            "resolved": utc(n.get("completedAt") or n.get("canceledAt")),
+            "points": n.get("estimate") if isinstance(n.get("estimate"), (int, float)) else None,
+            "priority": PRIORITY.get(n.get("priority")), "labels": labels,
+            "sprints": sorted(f"linear:cycle:{c}" for c in cycles), "incident": incident,
+            "canceled": state.get("type") == "canceled", "url": n.get("url"),
+            # Who it is assigned to, for [identity] aliases to name the GitHub login; never kept on a public repo.
+            **({"assignee": f"linear:{n['assignee']['id']}", "assignee_email": n["assignee"].get("email"),
+                "assignee_name": n["assignee"].get("name")}
+               if (n.get("assignee") or {}).get("id") else {}),
+            # A Linear project stands for the epic: what a body of work belongs to.
+            **({"epic": f"linear:project:{n['project']['id']}", "epic_title": n["project"].get("name")}
+               if (n.get("project") or {}).get("id") else {})}]
+    for k, h in enumerate(history):
+        if h.get("fromState") or h.get("toState"):
+            out.append({"t": "change", "src": NAME, "id": f"{iid}:{h.get('createdAt')}:{k}:status", "issue": iid,
+                        "field": "status", "from": (h.get("fromState") or {}).get("name"),
+                        "to": (h.get("toState") or {}).get("name"), "at": utc(h.get("createdAt"))})
+        if h.get("fromCycleId") or h.get("toCycleId"):
+            out.append({"t": "change", "src": NAME, "id": f"{iid}:{h.get('createdAt')}:{k}:cycle", "issue": iid,
+                        "field": "sprint", "from": [f"linear:cycle:{h['fromCycleId']}"] if h.get("fromCycleId") else [],
+                        "to": [f"linear:cycle:{h['toCycleId']}"] if h.get("toCycleId") else [], "at": utc(h.get("createdAt"))})
+    return out
+
+
+def _cycle_record(c: dict, now: datetime) -> dict:
+    start = utc(c.get("startsAt"))
+    state = "CLOSED" if c.get("completedAt") else "ACTIVE" if start and start <= iso(now) else "FUTURE"
+    return {"t": "sprint", "src": NAME, "id": f"linear:cycle:{c['id']}", "name": c.get("name") or f"Cycle {c.get('number')}",
+            "state": state, "start": start, "end": utc(c.get("endsAt")), "complete": utc(c.get("completedAt"))}
+
+
+def pull(store, env: dict, cfg: dict, now: datetime, client: httpx.Client | None = None, pages: int = PAGES) -> dict:
+    c = cfg["connectors"]["linear"]
+    key = env.get(TOKEN_ENV, "").strip()
+    teams = {"team": {"key": {"in": list(c["teams"])}}} if c["teams"] else {}
+    labels = {x.lower() for x in c["incident_labels"]}
+    own = client is None
+    client = client or httpx.Client(timeout=30)
+    try:
+        state = store.state()
+        since = state.get("since") or iso(now - timedelta(days=BACKFILL_DAYS))
+        until = state.get("until") or iso(now)
+        cursor, day = state.get("cursor"), now.date().isoformat()
+        # Every cycle that ended inside the window or is still to come, page by page.
+        after, floor = None, iso(now - timedelta(days=BACKFILL_DAYS))
+        for _ in range(pages):
+            cycles = _post(client, key, CYCLES, {"filter": {**teams, "endsAt": {"gte": floor}}, "after": after}).get("cycles") or {}
+            store.add([_cycle_record(x, now) for x in cycles.get("nodes") or []], day)
+            info = cycles.get("pageInfo") or {}
+            after = info.get("endCursor") if info.get("hasNextPage") else None
+            if after is None:
+                break
+        read, complete = 0, False
+        for _ in range(pages):
+            found = _post(client, key, ISSUES, {"filter": {"updatedAt": {"gte": since, "lte": until}, **teams},
+                                                "after": cursor}).get("issues") or {}
+            nodes = found.get("nodes") or []
+            store.add([r for n in nodes for r in _issue_records(n, labels)], day)
+            read += len(nodes)
+            info = found.get("pageInfo") or {}
+            cursor = info.get("endCursor") if info.get("hasNextPage") else None
+            if cursor is None:
+                complete = True
+                break
+        # A finished pass moves on to whatever changed after it began; a capped one keeps its span and cursor.
+        store.save_state({"since": until} if complete else {"since": since, "until": until, "cursor": cursor})
+        return {"read": read, "incidents": 0, "complete": complete}
+    finally:
+        if own:
+            client.close()

stackvitals/connectors/pagerduty.py

@@ -0,0 +1,96 @@
+"""PagerDuty: incidents of the chosen services and urgencies, read with a REST API key (v2 List incidents). Incidents are
+found by when they began, so each pull reads the last two weeks again to catch the ones resolved since."""
+from datetime import datetime, timedelta
+
+import httpx
+
+from stackvitals.connectors.base import BACKFILL_DAYS, PAGES, ConnectorError, request, utc
+from stackvitals.core.time import iso, parse
+
+NAME, TOKEN_ENV = "pagerduty", "STACKVITALS_PAGERDUTY_TOKEN"
+URL = "https://api.pagerduty.com/incidents"
+PAGE = 100
+REREAD = timedelta(days=14)
+CHUNK = timedelta(days=90)  # PagerDuty takes at most six months per query
+CATEGORY = {"triggered": "TODO", "acknowledged": "IN_PROGRESS", "resolved": "DONE"}
+
+__all__ = ["ConnectorError", "configured", "extract", "pull"]
+
+
+def configured(env: dict, cfg: dict) -> bool:
+    return bool((env.get(TOKEN_ENV) or "").strip())
+
+
+def scope(cfg: dict) -> str | None:
+    """Said on the Integrations page while every service's incidents count for this repo."""
+    if not cfg["connectors"]["pagerduty"]["services"]:
+        return "reads every service in the account; list this repo's under services in [connectors.pagerduty] to count only those"
+    return None
+
+
+def _get(client: httpx.Client, key: str, params: list, sleep=None) -> dict:
+    return request(client, "GET", URL, "PagerDuty", refused="check the PAGERDUTY_API_KEY secret",
+                   bad=lambda r: f"PagerDuty answered {r.status_code}; check services and urgencies under [connectors.pagerduty].",
+                   params=params, headers={"Authorization": f"Token token={key}",
+                                           "Accept": "application/vnd.pagerduty+json;version=2"},
+                   **({"sleep": sleep} if sleep else {}))
+
+
+def _record(i: dict) -> dict:
+    status = i.get("status")
+    resolved = utc(i.get("resolved_at") or (i.get("last_status_change_at") if status == "resolved" else None))
+    service = (i.get("service") or {}).get("summary")
+    return {"t": "issue", "src": NAME, "id": f"pagerduty:{i['id']}", "key": f"PD-{i.get('incident_number')}",
+            "title": i.get("title"), "type": "INCIDENT", "original_type": None, "status": status,
+            "cat": CATEGORY.get(status, "TODO"), "created": utc(i.get("created_at")), "updated": utc(i.get("updated_at")),
+            "resolved": resolved if status == "resolved" else None, "points": None, "priority": i.get("urgency"),
+            "labels": [x for x in (i.get("urgency"), service) if x], "sprints": [], "incident": True, "canceled": False,
+            "url": i.get("html_url")}
+
+
+def extract(kind: str, ctx: dict, body: dict) -> list[dict]:
+    """The records in one kept answer (a page of incidents)."""
+    return [_record(i) for i in body.get("incidents") or []]
+
+
+def pull(store, env: dict, cfg: dict, now: datetime, client: httpx.Client | None = None, pages: int = PAGES) -> dict:
+    c = cfg["connectors"]["pagerduty"]
+    key = env.get(TOKEN_ENV, "").strip()
+    own = client is None
+    client = client or httpx.Client(timeout=30)
+    try:
+        state = store.state()
+        floor = now - timedelta(days=BACKFILL_DAYS)
+        if state.get("since"):
+            # A capped pass carries on from the last incident it read; only a finished one reads two weeks again.
+            since = parse(state["since"])
+            start = max(since if state.get("catching_up") else since - REREAD, floor)
+        else:
+            start = floor
+        read, used, day = 0, 0, now.date().isoformat()
+        while start < now and used < pages:
+            end, last = min(start + CHUNK, now), None
+            for offset in range(0, PAGE * (pages - used), PAGE):
+                params = [("since", iso(start)), ("until", iso(end)), ("time_zone", "UTC"), ("sort_by", "created_at:asc"),
+                          ("limit", PAGE), ("offset", offset), *(("service_ids[]", s) for s in c["services"]),
+                          *(("urgencies[]", u) for u in c["urgencies"])]
+                body = _get(client, key, params)
+                used += 1
+                found = body.get("incidents") or []
+                store.keep("incidents", {}, body, day)
+                store.add(extract("incidents", {}, body), day)
+                read += len(found)
+                last = max([last or "", *(utc(i.get("created_at")) or "" for i in found)]) or last
+                if not body.get("more"):
+                    break
+            else:
+                # Capped inside a chunk: the next run starts at the newest incident read, always a little further on.
+                start = max(parse(last), start + timedelta(seconds=1)) if last else start
+                break
+            start = end
+        complete = start >= now
+        store.save_state({"since": iso(now)} if complete else {"since": iso(start), "catching_up": True})
+        return {"read": read, "incidents": read, "complete": complete}
+    finally:
+        if own:
+            client.close()

stackvitals/connectors/slack.py

@@ -0,0 +1,110 @@
+"""Slack: support channels read with a bot token the user installs (channels:history, groups:history): each top-level
+message, when anyone first replied in its thread and when the team did (the user ids under `team`). The first pull reads
+180 days, later ones the last 30 (where new replies land); what was read is kept even when a later call fails. Only
+times and flags are kept: no text and no user ids."""
+import time
+from datetime import datetime, timedelta, timezone
+
+import httpx
+
+from stackvitals.connectors.base import BACKFILL_DAYS, PAGES, ConnectorError, json_of
+from stackvitals.core import identity
+
+NAME, TOKEN_ENV = "slack", "STACKVITALS_SLACK_TOKEN"
+API = "https://slack.com/api"
+THREADS = 150  # thread reads a run; the next run carries on
+RECENT_DAYS = 30
+MAX_WAIT = 60
+
+__all__ = ["ConnectorError", "configured", "pull"]
+
+
+def configured(env: dict, cfg: dict) -> bool:
+    return bool((env.get(TOKEN_ENV) or "").strip())
+
+
+def _iso(ts: str | None) -> str | None:
+    return datetime.fromtimestamp(float(ts), timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") if ts else None
+
+
+def _call(client: httpx.Client, key: str, method: str, sleep, **params) -> dict:
+    for attempt in (1, 2):
+        try:
+            resp = client.get(f"{API}/{method}", params=params, headers={"Authorization": f"Bearer {key}"})
+        except httpx.HTTPError as exc:
+            raise ConnectorError(f"Slack could not be reached ({type(exc).__name__}); the next run tries again.") from None
+        if resp.status_code == 429:
+            wait = float(resp.headers.get("retry-after", MAX_WAIT))
+            if attempt == 1 and wait <= MAX_WAIT:
+                sleep(wait)  # Slack says how long; one wait, then the run carries on next time
+                continue
+            raise ConnectorError("Slack's rate limit was reached; what was read is kept and the next run carries on.")
+        body = json_of(resp, "Slack")
+        if not body.get("ok"):
+            raise ConnectorError(f"Slack refused {method} ({body.get('error')}): the bot needs channels:history and to be "
+                                 "in each channel under [connectors.slack] channels.")
+        return body
+    raise ConnectorError("Slack's rate limit was reached; the next run carries on.")
+
+
+def _people(messages: list[dict]) -> list[dict]:
+    return [m for m in messages if m.get("user") and not m.get("bot_id") and m.get("subtype") in (None, "thread_broadcast")]
+
+
+def pull(store, env: dict, cfg: dict, now: datetime, client: httpx.Client | None = None, pages: int = PAGES,
+         sleep=time.sleep) -> dict:
+    c = cfg["connectors"][NAME]
+    # Team members are the ids listed here, plus those [identity] aliases map to a team member.
+    key, team = env.get(TOKEN_ENV, "").strip(), set(c["team"]) | identity.tool_ids(cfg.get("identity"), NAME)
+    state = store.state()
+    days = RECENT_DAYS if state.get("backfilled") else BACKFILL_DAYS
+    oldest = f"{(now - timedelta(days=days)).timestamp():.6f}"
+    known = {r["id"]: r for r in store.load().get("topic") or []}
+    own = client is None
+    client = client or httpx.Client(timeout=30)
+    fresh, threads, complete, failed = [], 0, True, []
+    try:
+        for channel in c["channels"]:
+            try:
+                cursor = None
+                for _ in range(pages):
+                    body = _call(client, key, "conversations.history", sleep, channel=channel, oldest=oldest, limit=200,
+                                 **({"cursor": cursor} if cursor else {}))
+                    for m in _people(body.get("messages") or []):
+                        if m.get("thread_ts") not in (None, m["ts"]):
+                            continue  # a reply shown in the channel
+                        rid = f"{NAME}:{channel}:{m['ts']}"
+                        stored = known.get(rid)
+                        if stored and (stored["first_team_reply"] or not m.get("reply_count")):
+                            continue
+                        replies = []
+                        if m.get("reply_count"):
+                            if threads >= THREADS:
+                                complete = False
+                                continue
+                            threads += 1
+                            got = _call(client, key, "conversations.replies", sleep, channel=channel, ts=m["ts"])
+                            replies = [r for r in _people(got.get("messages") or []) if r["ts"] != m["ts"] and r["user"] != m["user"]]
+                        team_reply = next((r for r in replies if r["user"] in team), None)
+                        fresh.append({"t": "topic", "src": NAME, "id": rid, "created": _iso(m["ts"]), "by_team": m["user"] in team,
+                                      "first_reply": _iso(replies[0]["ts"]) if replies else None,
+                                      "first_team_reply": _iso(team_reply["ts"]) if team_reply else None,
+                                      "solved": None, "closed": False})
+                    cursor = (body.get("response_metadata") or {}).get("next_cursor")
+                    if not body.get("has_more") or not cursor:
+                        break
+                else:
+                    complete = False
+            except ConnectorError as exc:
+                # One channel's trouble (the bot not in it, a limit) is said; the other channels go on.
+                failed.append(f"{channel}: {exc}")
+                complete = False
+    finally:
+        if own:
+            client.close()
+        store.add(fresh, now.date().isoformat())
+    if len(failed) == len(c["channels"]) and failed:
+        raise ConnectorError(failed[0])
+    if complete:
+        store.save_state({**store.state(), "backfilled": True})
+    return {"read": len(fresh), "complete": complete, **({"note": "; ".join(failed[:2])} if failed else {})}

stackvitals/connectors/sonar.py

@@ -0,0 +1,77 @@
+"""SonarQube or SonarCloud code quality (DevLake's SonarQube plugin): the project's quality gate and its measures (bugs,
+vulnerabilities, code smells, coverage, duplication, ratings), read once a day and kept, so the trend shows beside the
+delivery numbers. SonarCloud's public projects need no key; a private project or your own SonarQube takes a token."""
+import httpx
+
+from stackvitals.connectors.base import ConnectorError, json_of
+
+NAME, TOKEN_ENV = "sonar", "STACKVITALS_SONAR_TOKEN"
+METRICS = ("alert_status", "bugs", "vulnerabilities", "code_smells", "security_hotspots", "coverage", "duplicated_lines_density",
+           "ncloc", "reliability_rating", "security_rating", "sqale_rating")
+KEEP_DAYS = 400
+
+__all__ = ["ConnectorError", "configured", "pull", "summary"]
+
+
+def configured(env: dict, cfg: dict) -> bool:
+    return bool(cfg["connectors"]["sonar"].get("project"))
+
+
+def _get(client: httpx.Client, url: str, params: dict, token: str) -> dict:
+    headers = {"Authorization": f"Bearer {token}"} if token else {}
+    try:
+        resp = client.get(url, params=params, headers=headers)
+    except httpx.HTTPError as exc:
+        raise ConnectorError(f"Sonar could not be reached ({type(exc).__name__}); the next run tries again.") from None
+    if resp.status_code in (401, 403):
+        raise ConnectorError("Sonar refused the request: a private project needs the SONAR_TOKEN secret.")
+    if resp.status_code == 404:
+        raise ConnectorError("Sonar has no project with that key: check project under [connectors.sonar].")
+    if resp.status_code != 200:
+        raise ConnectorError(f"Sonar answered {resp.status_code}; the next run tries again.")
+    return json_of(resp, "Sonar")
+
+
+def _number(value):
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return value
+
+
+def pull(store, env: dict, cfg: dict, now, client: httpx.Client | None = None) -> dict:
+    c = cfg["connectors"]["sonar"]
+    base, project, token = (c.get("url") or "https://sonarcloud.io").rstrip("/"), c["project"], (env.get(TOKEN_ENV) or "").strip()
+    own = client is None
+    client = client or httpx.Client(timeout=30)
+    try:
+        got = _get(client, f"{base}/api/measures/component", {"component": project, "metricKeys": ",".join(METRICS)}, token)
+        gate = _get(client, f"{base}/api/qualitygates/project_status", {"projectKey": project}, token)
+    finally:
+        if own:
+            client.close()
+    measures = {m["metric"]: _number(m.get("value")) for m in (got.get("component") or {}).get("measures") or [] if "value" in m}
+    status = (gate.get("projectStatus") or {})
+    day = now.date().isoformat()
+    row = {"t": "quality", "id": f"sonar:{day}", "day": day, "project": project, "measures": measures,
+           "gate": status.get("status"),
+           "failing": [x.get("metricKey") for x in status.get("conditions") or [] if x.get("status") == "ERROR"]}
+    store.add([row], day)
+    return {"read": 1, "complete": True}
+
+
+def summary(root, now) -> dict | None:
+    """The newest reading, the one about 30 days before, and each day's for the charts; None before the first pull."""
+    from datetime import timedelta
+
+    from stackvitals.connectors.store import ConnectorStore
+    rows = sorted((r for r in ConnectorStore(root, NAME).load().get("quality") or [] if r.get("day")), key=lambda r: r["day"])
+    if not rows:
+        return None
+    newest = rows[-1]
+    month = (now - timedelta(days=30)).date().isoformat()
+    before = next((r for r in reversed(rows) if r["day"] <= month), None)
+    return {"project": newest.get("project"), "day": newest["day"], "gate": newest.get("gate"), "failing": newest.get("failing") or [],
+            "measures": newest.get("measures") or {}, "month_ago": (before or {}).get("measures"),
+            "days": [{"day": r["day"], **{k: (r.get("measures") or {}).get(k) for k in ("bugs", "vulnerabilities", "code_smells", "coverage")}}
+                     for r in rows[-180:]]}

stackvitals/connectors/store.py

@@ -0,0 +1,120 @@
+"""What a connector pulled, in the data branch under connectors/<name>/: one NDJSON file per pull day (compressed once the
+day is over), the newest copy of each record winning, and a small state file to resume from. A private repo also keeps
+each answer as the service sent it (raw/), so a better reading of it rebuilds the records without asking again."""
+import gzip
+import json
+import os
+import shutil
+from pathlib import Path
+
+# Words a public repo must never hold: its data branch is as public as its pages.
+REDACT = {"issue": ("title", "url", "epic_title", "assignee", "assignee_email", "assignee_name", "team"), "sprint": ("name",)}
+
+
+def _clean(r: dict) -> dict:
+    hidden = REDACT.get(r.get("t"), ())
+    return {k: v for k, v in r.items() if k not in hidden} if hidden else r
+
+
+class ConnectorStore:
+    def __init__(self, root: Path, name: str, redact: bool = False):
+        self.dir = Path(root) / "connectors" / name
+        self.days = self.dir / "days"
+        self.rawdir = self.dir / "raw"
+        self.redact = redact
+        self._known = None  # the stored copy of each record, read on the first add
+
+    def add(self, records: list[dict], day: str) -> None:
+        if self._known is None:
+            self._known = {(r.get("t"), r.get("id")): r for r in self._lines()}
+        fresh = []
+        for r in records:
+            r = _clean(r) if self.redact else r
+            # A record read again unchanged (an overlap, a cycle every issue names) is not written again.
+            if self._known.get((r.get("t"), r.get("id"))) != r:
+                self._known[(r.get("t"), r.get("id"))] = r
+                fresh.append(r)
+        if not fresh:
+            return
+        self.days.mkdir(parents=True, exist_ok=True)
+        with (self.days / f"{day}.ndjson").open("a", encoding="utf-8", newline="\n") as f:
+            for r in fresh:
+                f.write(json.dumps(r, separators=(",", ":"), ensure_ascii=False) + "\n")
+
+    def keep(self, kind: str, ctx: dict, body, day: str) -> None:
+        """One answer as the service sent it; never on a public repo, whose data branch anyone can read."""
+        if self.redact:
+            return
+        self.rawdir.mkdir(parents=True, exist_ok=True)
+        with (self.rawdir / f"{day}.ndjson").open("a", encoding="utf-8", newline="\n") as f:
+            f.write(json.dumps({"k": kind, "c": ctx, "b": body}, separators=(",", ":"), ensure_ascii=False) + "\n")
+
+    def raw(self):
+        """The kept answers, oldest first: {"k": kind, "c": what the reading needs, "b": the body}."""
+        yield from self._lines(self.rawdir)
+
+    def clear_records(self) -> None:
+        """The records only; the kept answers and the state stay, for a rebuild from them."""
+        shutil.rmtree(self.days, ignore_errors=True)
+        self._known = None
+
+    def _lines(self, folder: Path | None = None):
+        folder = folder or self.days
+        if not folder.is_dir():
+            return
+        for path in sorted(folder.iterdir()):
+            if path.name.endswith(".ndjson.gz"):
+                text = gzip.decompress(path.read_bytes()).decode("utf-8")
+            elif path.name.endswith(".ndjson"):
+                text = path.read_text(encoding="utf-8")
+            else:
+                continue
+            # Only "\n" ends a record: splitlines() also splits on U+2028 and the like, which titles may hold.
+            for line in text.split("\n"):
+                if line.strip():
+                    try:
+                        yield json.loads(line)
+                    except ValueError:
+                        continue  # a damaged line (a cut-off write) never stops every later run
+
+    def load(self) -> dict[str, list[dict]]:
+        """{"issue": [...], "change": [...], "sprint": [...]}, later records winning by id."""
+        found = {}
+        for r in self._lines():
+            found[(r.get("t"), r.get("id"))] = _clean(r) if self.redact else r
+        out = {"issue": [], "change": [], "sprint": []}
+        for (kind, _id), r in found.items():
+            out.setdefault(kind, []).append(r)
+        return out
+
+    def state(self) -> dict:
+        try:
+            return json.loads((self.dir / "state.json").read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return {}
+
+    def save_state(self, state: dict) -> None:
+        text = json.dumps(state, indent=1, sort_keys=True)
+        path = self.dir / "state.json"
+        if path.is_file() and path.read_text(encoding="utf-8") == text:
+            return
+        self.dir.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(text, encoding="utf-8", newline="\n")
+        os.replace(tmp, path)
+
+    def reset(self) -> None:
+        """Everything pulled so far is dropped, so the next pull reads from the start."""
+        shutil.rmtree(self.dir, ignore_errors=True)
+        self._known = None
+
+    def compress(self, today: str) -> None:
+        for folder in (self.days, self.rawdir):
+            if not folder.is_dir():
+                continue
+            for path in sorted(folder.glob("*.ndjson")):
+                if path.name.split(".")[0] < today:
+                    gz = path.with_name(path.name + ".gz")
+                    old = gzip.decompress(gz.read_bytes()) if gz.exists() else b""
+                    gz.write_bytes(gzip.compress(old + path.read_bytes(), mtime=0))
+                    path.unlink()

stackvitals/connectors/tracker.py

@@ -0,0 +1,108 @@
+"""What tracker issues add to delivery: PRs linked to issues by key, lead and cycle time by issue type, sprint scope and
+carry-over, and the incidents a tracker holds."""
+import re
+from collections import defaultdict
+from datetime import datetime
+
+from stackvitals.core.stats import percentile
+from stackvitals.core.time import iso, parse
+
+KEY = re.compile(r"(?<![A-Za-z0-9])([A-Za-z][A-Za-z0-9]{1,9})-(\d{1,7})(?![0-9])")
+# A status no current issue holds is guessed from its name.
+BUSY = re.compile(r"(?i)progress|review|doing|started|develop|testing|qa\b")
+SPRINTS_SHOWN = 6
+
+
+def pr_links(prs: list[dict], issues: list[dict]) -> list[tuple[int, str]]:
+    """(PR number, issue key) for every known issue key in a PR's title or head branch (any case: feature/web-12-x)."""
+    known = {i["key"].upper(): i["key"] for i in issues}
+    out = []
+    for p in prs:
+        found = []
+        for text in (p.get("ti") or "", p.get("head") or ""):
+            for prefix, number in KEY.findall(text):
+                key = known.get(f"{prefix.upper()}-{number}")
+                if key and key not in found:
+                    found.append(key)
+        out += [(p["n"], key) for key in found]
+    return out
+
+
+def _hours(a: str | None, b: str | None) -> float | None:
+    if not a or not b or b < a:
+        return None
+    return round((parse(b) - parse(a)).total_seconds() / 3600, 2)
+
+
+def _stats(values: list[float], prefix: str, min_n: int = 10) -> dict:
+    # Under ten issues the times are left out (the count stays): a P75 of three says little.
+    v = [x for x in values if x is not None]
+    return {f"{prefix}_n": len(v), f"{prefix}_p50_h": percentile(v, 0.5) if len(v) >= min_n else None,
+            f"{prefix}_p75_h": percentile(v, 0.75) if len(v) >= min_n else None}
+
+
+def lead_times(issues: list[dict], changes: list[dict], start: datetime, end: datetime, min_n: int = 10) -> dict:
+    """{type: {n, lead_p50_h, lead_p75_h, cycle_n, cycle_p50_h, cycle_p75_h}} over issues done in the range: lead time from
+    created, cycle time from the first move to an in-progress status; canceled ones are left out."""
+    s, e = iso(start), iso(end)
+    category = {i.get("status"): i.get("cat") for i in issues}
+    started = {}
+    for c in sorted((c for c in changes if c.get("field") == "status"), key=lambda c: c.get("at") or ""):
+        busy = category.get(c.get("to")) == "IN_PROGRESS" or (c.get("to") not in category and BUSY.search(c.get("to") or ""))
+        if busy:
+            started.setdefault(c["issue"], c["at"])
+    groups = defaultdict(list)
+    for i in issues:
+        if i.get("cat") == "DONE" and not i.get("canceled") and i.get("resolved") and s <= i["resolved"] <= e:
+            groups[i.get("type") or "TASK"].append(i)
+    out = {}
+    for kind in sorted(set(groups) | {"REQUIREMENT", "BUG"}):
+        rows = groups.get(kind, [])
+        lead = _stats([_hours(i.get("created"), i["resolved"]) for i in rows], "lead", min_n)
+        cycle = _stats([_hours(started.get(i["id"]), i["resolved"]) for i in rows], "cycle", min_n)
+        out[kind] = {"n": len(rows), **{k: v for k, v in lead.items() if k != "lead_n"}, **cycle}
+    return out
+
+
+def incidents(issues: list[dict]) -> list[dict]:
+    """A tracker's incidents in the shape Delivery's own incidents take, with a link to each."""
+    return [{"source": i["src"], "number": i["key"], "created": i["created"],
+             "resolved": i.get("resolved") if i.get("cat") == "DONE" else None, "url": i.get("url"),
+             **({"severity": i["severity"]} if i.get("severity") else {})}
+            for i in issues if i.get("incident") and i.get("created")]
+
+
+def planning(issues: list[dict], changes: list[dict], sprints: list[dict], now: datetime, last: int = SPRINTS_SHOWN,
+             since: datetime | None = None) -> list[dict]:
+    """The last sprints or cycles, newest last: scope, done by the sprint's end, carried over (open when it closed), added
+    after it started, and story points when the team uses them. Sprints that began before `since` (what the connector
+    reads back to) are left out, since only some of their issues were read."""
+    members = defaultdict(list)
+    for i in issues:
+        for sid in i.get("sprints") or []:
+            members[sid].append(i)
+    added = {}
+    for c in changes:
+        if c.get("field") == "sprint" and c.get("at"):
+            for sid in set(c.get("to") or []) - set(c.get("from") or []):
+                key = (c["issue"], sid)
+                added[key] = min(added.get(key, c["at"]), c["at"])
+    floor = iso(since) if since else ""
+    shown = sorted((s for s in sprints if s.get("start") and s["start"] >= floor and s.get("state") in ("ACTIVE", "CLOSED")),
+                   key=lambda s: s["start"])
+    rows = []
+    for s in shown[-last:]:
+        scope = members.get(s["id"], [])
+        end = s.get("complete") or (iso(now) if s["state"] == "ACTIVE" else s.get("end")) or iso(now)
+        done = [i for i in scope if i.get("cat") == "DONE" and not i.get("canceled") and (i.get("resolved") or "9") <= end]
+        carried = [i for i in scope if s["state"] == "CLOSED" and i not in done and not i.get("canceled")]
+        late = [i for i in scope if (i["id"], s["id"]) in added and added[(i["id"], s["id"])] > s["start"]]
+        pointed = any(i.get("points") is not None for i in scope)
+        points = (lambda xs: sum(i.get("points") or 0 for i in xs)) if pointed else (lambda xs: None)
+        # A public repo keeps no sprint names: the dates name it instead.
+        name = s.get("name") or f"Sprint ending {(s.get('end') or '')[:10] or 'later'}"
+        rows.append({"id": s["id"], "name": name, "state": s["state"], "start": s["start"],
+                     "end": s.get("complete") or s.get("end"), "scope": len(scope), "done": len(done),
+                     "carried_over": len(carried), "added_after_start": len(late), "points_scope": points(scope),
+                     "points_done": points(done)})
+    return rows