stackvitals 0.1.0__py3-none-any.whl

stackvitals/connectors/__init__.py

@@ -0,0 +1,244 @@
+"""Connectors to outside services, each read with one optional key from the repo's secrets: the Action's Connectors step,
+the status the Integrations page shows, and everything pulled so far for Delivery to join."""
+import hashlib
+import json
+from datetime import date, datetime, timedelta
+from pathlib import Path
+
+from stackvitals.connectors import ai_spend, copilot, discord, discourse, incident_io, jira, linear, pagerduty, slack, sonar
+from stackvitals.connectors.base import BACKFILL_DAYS, ConnectorError
+from stackvitals.connectors.store import ConnectorStore
+
+CONNECTORS = {"jira": jira, "linear": linear, "pagerduty": pagerduty, "incident_io": incident_io, "copilot": copilot,
+              "discourse": discourse, "slack": slack, "discord": discord, "ai_spend": ai_spend, "sonar": sonar}
+NAMES = {"jira": "Jira", "linear": "Linear", "pagerduty": "PagerDuty", "incident_io": "incident.io", "copilot": "Copilot",
+         "discourse": "Discourse", "slack": "Slack", "discord": "Discord", "ai_spend": "AI spend",
+         "sonar": "SonarQube"}
+TRACKERS = ("jira", "linear", "pagerduty", "incident_io")
+COPILOT_WINDOW = 28
+WARN_DAYS = 14
+
+
+def public(ws) -> bool:
+    """Unless the repo is known to be private, its data branch is readable by anyone: no titles, links or names."""
+    return not (ws.cache.get("repo_info") or {}).get("private")
+
+
+def active(env: dict, cfg: dict) -> list[str]:
+    """The trackers and pager tools with a key in this job: one whose key was removed stops counting."""
+    return [name for name in TRACKERS if name in CONNECTORS and CONNECTORS[name].configured(env, cfg)]
+
+
+def load(root: Path, redact: bool = False, names=TRACKERS) -> dict | None:
+    """Everything the named connectors pulled: {"sources", "issues", "changes", "sprints"}, or None before the first pull."""
+    out = {"sources": [], "issues": [], "changes": [], "sprints": []}
+    for name in (n for n in TRACKERS if n in names):
+        got = ConnectorStore(root, name, redact).load()
+        got = getattr(CONNECTORS.get(name), "finish", lambda g: g)(got)
+        if got["issue"] or got["sprint"]:
+            out["sources"].append(name)
+            out["issues"] += got["issue"]
+            out["changes"] += got["change"]
+            out["sprints"] += got["sprint"]
+    return out if out["sources"] else None
+
+
+# What a record holds; a new field (2: an issue's epic or project, 3: its assignee, 4: its name) reads everything again once.
+RECORDS = 4
+
+
+def fingerprint(cfg: dict, name: str, redact: bool, records: int | None = None) -> str:
+    """What decides which records a connector keeps: its read settings (not the key's expiry or write-back), whether
+    titles are hidden, and what a record holds (records=0 leaves that out: what decides which answers are read)."""
+    kept = {k: v for k, v in cfg["connectors"][name].items() if k not in ("key_expires", "write_back", "done_transition", "done_state")}
+    shape = RECORDS if records is None else records
+    # ASKED counts what a connector asks its service for: kept answers from before a new field can't be rebuilt from.
+    asked = getattr(CONNECTORS.get(name), "ASKED", 1)
+    return hashlib.sha1(json.dumps([kept, redact, *([shape] if shape else []), *([asked] if asked > 1 else [])],
+                                   sort_keys=True).encode()).hexdigest()[:12]
+
+
+# What the framework keeps in a connector's state across a rewind (connectors_step's own keys).
+KEEP = ("scope", "reads", "raw_full")
+
+
+def rewind(root: Path, name: str, since: str | None, now: datetime) -> str:
+    """Make a connector's next pull read again from `since` (a date; None or older than the window means the whole
+    window), keeping its records; a sentence saying what will happen."""
+    if name not in CONNECTORS:
+        raise ConnectorError(f"{name!r} has nothing to read again; use one of {', '.join(CONNECTORS)}.")
+    floor = (now - timedelta(days=BACKFILL_DAYS)).date()
+    day = date.fromisoformat(since) if since else None
+    if day and day > now.date():
+        raise ConnectorError(f"{since} is in the future; pick a past date to read {NAMES[name]} again from.")
+    store = ConnectorStore(root, name)
+    kept = {k: v for k, v in store.state().items() if k in KEEP}
+    whole = day is None or day <= floor or name in ("slack", "discord", "discourse")
+    if name == "copilot":
+        new = {} if whole else {"through": (day - timedelta(days=1)).isoformat()}
+    elif whole:
+        new = {}
+    else:
+        start = f"{day.isoformat()}T00:00:00Z"
+        new = {"since": start, **({"incident_since": start} if name == "jira" else {})}
+    store.save_state({**kept, **new})
+    return f"{NAMES[name]} reads again from {'180 days back' if whole else day.isoformat()}."
+
+
+def reprocess(root: Path, name: str, day: str | None = None) -> int:
+    """Rebuild a connector's records from the answers it kept, with no call to the service; the records written."""
+    mod, store = CONNECTORS[name], ConnectorStore(root, name)
+    kept = list(store.raw())
+    if not kept or not hasattr(mod, "extract"):
+        raise ConnectorError(f"{NAMES[name]} kept no answers to rebuild from; it reads them again instead.")
+    store.clear_records()
+    day = day or date.today().isoformat()
+    written = 0
+    for entry in kept:
+        records = mod.extract(entry["k"], entry["c"], entry["b"])
+        store.add(records, day)
+        written += len(records)
+    return written
+
+
+def copilot_summary(root: Path, now: datetime) -> dict | None:
+    """The last 28 days of the org's Copilot reports and this repo's: active users, suggestions accepted, Copilot's pull
+    requests; None before the first report."""
+    got = ConnectorStore(root, "copilot").load()
+    days = sorted(got.get("org_day") or [], key=lambda d: d["day"])
+    if not days:
+        return None
+    since = (now.date() - timedelta(days=COPILOT_WINDOW)).isoformat()
+    recent = [d for d in days if d["day"] >= since] or days[-1:]
+    repo = [d for d in got.get("repo_day") or [] if d["day"] >= since]
+    total = lambda rows, k: sum((r.get("pull_requests") or {}).get(k) or 0 for r in rows)  # noqa: E731
+    gen = sum(d.get("code_generation_activity_count") or 0 for d in recent)
+    acc = sum(d.get("code_acceptance_activity_count") or 0 for d in recent)
+    latest = recent[-1]
+    return {"org": latest.get("org"), "days_kept": len(days), "first_day": days[0]["day"], "last_day": latest["day"],
+            "monthly_active_users": latest.get("monthly_active_users"), "monthly_active_agent_users": latest.get("monthly_active_agent_users"),
+            "acceptance": round(acc / gen, 4) if gen else None,
+            "org_prs_by_copilot": total(recent, "total_created_by_copilot"),
+            "org_prs_merged_by_copilot": total(recent, "total_merged_created_by_copilot"),
+            "repo_prs_by_copilot": total(repo, "total_created_by_copilot"),
+            "repo_prs_merged_by_copilot": total(repo, "total_merged_created_by_copilot"), "window_days": COPILOT_WINDOW}
+
+
+def _write_back(ws, name: str, store, env: dict, cfg: dict, now: datetime, client=None) -> dict:
+    """Opt-in: comment on (and move) the Jira or Linear issues that newly merged pull requests name. The pairs are picked
+    and recorded before the data branch is saved and sent after it, so a retried push never sends them twice."""
+    import base64
+
+    import httpx
+
+    from stackvitals.connectors import jira, linear, writeback
+    from stackvitals.flow.store import FlowStore
+    c = cfg["connectors"][name]
+    prs = (ws.cache.get("flow") or FlowStore(ws.path).load()).get("closed") or []
+    pairs = writeback.plan(store, prs, now.strftime("%Y-%m-%dT%H:%M:%SZ"))
+    if not pairs:
+        return {"planned": 0}
+
+    def after_commit():
+        http = client or (httpx.Client(base_url=c["url"], timeout=30, follow_redirects=False) if name == "jira"
+                          else httpx.Client(timeout=30))
+        try:
+            if name == "jira":
+                email = (env.get(jira.EMAIL_ENV) or "").strip() or c["email"]
+                auth = "Basic " + base64.b64encode(f"{email}:{env.get(jira.TOKEN_ENV, '').strip()}".encode()).decode()
+                got = writeback.send(pairs, ws.repo, lambda i, text: writeback.jira_comment(http, auth, i["key"], text),
+                                     lambda i: writeback.jira_move(http, auth, i["key"], c["done_transition"]))
+            else:
+                key = env.get(linear.TOKEN_ENV, "").strip()
+                got = writeback.send(pairs, ws.repo, lambda i, text: writeback.linear_comment(http, key, i["id"], text),
+                                     lambda i: writeback.linear_move(http, key, i["id"], i["key"], c["done_state"]))
+        finally:
+            if client is None:
+                http.close()
+        lines = [f"{NAMES[name]} write-back: {got['wrote']} comment(s), {got['moved']} issue(s) moved."]
+        if got["refused"]:
+            lines.append(f"Writing stopped ({got['refused']}); the rest of this run's {len(pairs)} aren't retried. "
+                         "Give the key permission to comment, or set write_back = false.")
+        lines += [f"Not moved: {u}" for u in got["unmoved"][:3]]
+        summary = env.get("GITHUB_STEP_SUMMARY")
+        if summary:
+            with open(summary, "a", encoding="utf-8") as f:
+                f.write("\n\n".join(lines) + "\n")
+        ws.cache.setdefault("write_back", {})[name] = got
+    ws.after_commit(after_commit, f"{NAMES[name]} write-back")
+    return {"planned": len(pairs)}
+
+
+def connectors_step(ws, env: dict, now: datetime, clients: dict | None = None) -> dict | None:
+    """Pull every connector with a key; a refused key or a missing setting is said on the Integrations page and in the job
+    summary, and what was pulled before stays."""
+    from stackvitals.core.config import parse
+    from stackvitals.core.store import published, write_if_changed
+    cfg = ws.cache.get("settings") or parse("")[0]
+    try:
+        old = json.loads((ws.path / "docs" / "connectors.json").read_text(encoding="utf-8")).get("connectors") or {}
+    except (OSError, ValueError):
+        old = {}
+    status, results = {}, {}
+    for name, mod in CONNECTORS.items():
+        if not mod.configured(env, cfg):
+            continue
+        store, before = ConnectorStore(ws.path, name, redact=public(ws)), old.get(name) or {}
+        scope, reads = fingerprint(cfg, name, public(ws)), fingerprint(cfg, name, public(ws), records=0)
+        state = store.state()
+        if state.get("scope", scope) != scope and state.get("reads") == reads and state.get("raw_full") \
+                and hasattr(mod, "extract"):
+            # Only what a record holds changed, and every answer was kept: rebuild from them instead of asking again.
+            reprocess(ws.path, name, now.date().isoformat())
+            store = ConnectorStore(ws.path, name, redact=public(ws))
+            ws.notes.append(f"{NAMES[name]}: what StackVitals keeps changed, so its records were rebuilt from the answers "
+                            "it kept, with no new reads.")
+        elif state.get("scope", scope) != scope:
+            # Narrower settings must drop what they now leave out, wider ones read back; a repo gone public loses titles.
+            store.reset()
+            ws.notes.append(f"{NAMES[name]}: its settings, the repo's visibility or what StackVitals keeps changed, so it "
+                            "reads everything again "
+                            f"({BACKFILL_DAYS} days back).")
+        expires = cfg["connectors"][name].get("key_expires") or None
+        entry = {"name": NAMES[name], "as_of": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "key_expires": expires,
+                 "scope": getattr(mod, "scope", lambda cfg: None)(cfg),
+                 "expiring": bool(expires and date.fromisoformat(expires) - now.date() <= timedelta(days=WARN_DAYS)),
+                 **({"check": before["check"]} if before.get("check") else {})}  # the last Test connection result stays
+        # Every answer is kept from the first read on (private repos only), or a rebuild would miss the older ones.
+        full = not public(ws) and hasattr(mod, "extract") and (store.state().get("raw_full") or not store.days.is_dir())
+        try:
+            got = mod.pull(store, env, cfg, now, client=(clients or {}).get(name))
+            if cfg["connectors"][name].get("write_back"):
+                entry["write_back"] = _write_back(ws, name, store, env, cfg, now, (clients or {}).get(name))
+            elif store.state().get("write_since"):
+                store.save_state({k: v for k, v in store.state().items() if k != "write_since"})
+            store.save_state({**{k: v for k, v in store.state().items() if k != "raw_full"}, "scope": scope, "reads": reads,
+                              **({"raw_full": True} if full else {})})
+            store.compress(now.date().isoformat())
+            rows = {k: len(v) for k, v in store.load().items()}
+            entry.update(error=None, rows=rows, complete=got["complete"])
+            results[name] = {"read": got["read"], "complete": got["complete"], "error": None}
+            if got.get("note"):
+                ws.notes.append(f"{NAMES[name]}: {got['note']}")
+        except Exception as exc:  # a bug or an answer no one expected stops this connector, never the whole job
+            said = str(exc) if isinstance(exc, ConnectorError) else \
+                f"stopped on an unexpected {type(exc).__name__}; what was pulled before stays, and the next run tries again."
+            entry.update(error=said, rows=before.get("rows") or {}, complete=before.get("complete"))
+            results[name] = {"read": 0, "complete": False, "error": said}
+            ws.notes.append(f"{NAMES[name]}: {said}")
+        if entry["expiring"]:
+            ws.notes.append(f"{NAMES[name]}: the key expires on {expires}; create a new one and update the secret.")
+        status[name] = entry
+    if not status:
+        return None
+    # Tested services that aren't pulled (chat webhooks, the bucket, model triage) keep their last check.
+    status.update({n: {"name": e.get("name"), "check": e["check"]} for n, e in old.items()
+                   if n not in CONNECTORS and isinstance(e, dict) and e.get("check")})
+    docs = ws.path / "docs"
+    docs.mkdir(parents=True, exist_ok=True)
+    write_if_changed(docs / "connectors.json", published({"connectors": status}))
+    ws.cache["tracker"] = load(ws.path, redact=public(ws), names=active(env, cfg))
+    return results
+
+
+__all__ = ["CONNECTORS", "ConnectorError", "active", "connectors_step", "copilot_summary", "load", "reprocess"]

stackvitals/connectors/ai_spend.py

@@ -0,0 +1,190 @@
+"""AI spend: what the organisation paid Anthropic and OpenAI each day (their cost reports) and Cursor this billing cycle
+(plus its daily agent requests and accepted lines), each read with one optional admin key, so Delivery can set the
+money beside the AI-involved pull requests merged in the same weeks."""
+import base64
+from datetime import datetime, timedelta, timezone
+from decimal import ROUND_HALF_UP, Decimal
+
+import httpx
+
+from stackvitals.connectors.base import ConnectorError, request
+from stackvitals.core.time import iso, parse
+
+NAME = "ai_spend"
+KEYS = {"anthropic": "STACKVITALS_ANTHROPIC_ADMIN_KEY", "openai": "STACKVITALS_OPENAI_ADMIN_KEY",
+        "cursor": "STACKVITALS_CURSOR_ADMIN_KEY"}
+TOKEN_ENV = KEYS["anthropic"]  # the first one; check.py names all three
+WINDOW = 90       # days read back on the first pull
+REREAD = 2        # cost reports fill in late: the last two days are read again
+CURSOR_DAYS = 30  # Cursor's daily usage takes at most 30 days per call
+ANTHROPIC = "https://api.anthropic.com/v1/organizations/cost_report"
+CLAUDE_CODE = "https://api.anthropic.com/v1/organizations/usage_report/claude_code"
+OPENAI = "https://api.openai.com/v1/organization/costs"
+CURSOR = "https://api.cursor.com"
+PAGES = 10
+
+__all__ = ["ConnectorError", "configured", "pull", "summary"]
+
+
+def configured(env: dict, cfg: dict) -> bool:
+    return any((env.get(k) or "").strip() for k in KEYS.values())
+
+
+def _usd(amount) -> float:
+    """Money to the cent, rounded half up as an invoice is (decimal, so $124.455 is $124.46, never $124.45)."""
+    return float(Decimal(str(amount)).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP))
+
+
+def _day(ts) -> str:
+    return datetime.fromtimestamp(ts / (1000 if ts > 10**11 else 1), timezone.utc).date().isoformat()
+
+
+def _anthropic(client, key: str, start: datetime, now: datetime) -> list[dict]:
+    out, page = [], None
+    for _ in range(PAGES):
+        body = request(client, "GET", ANTHROPIC, "Anthropic", refused="check the ANTHROPIC_ADMIN_KEY secret (an Admin API key)",
+                       params={"starting_at": iso(start), "ending_at": iso(now), "bucket_width": "1d", "limit": 31,
+                               **({"page": page} if page else {})},
+                       headers={"x-api-key": key, "anthropic-version": "2023-06-01", "user-agent": "stackvitals"})
+        for b in body.get("data") or []:
+            # Amounts are cents as decimal strings ("123.45" is $1.23).
+            cents = sum((Decimal(str(r.get("amount") or 0)) for r in b.get("results") or []), Decimal(0))
+            out.append({"t": "day", "id": f"anthropic:{b['starting_at'][:10]}", "src": "anthropic",
+                        "day": b["starting_at"][:10], "usd": _usd(cents / 100)})
+        page = body.get("next_page")
+        if not body.get("has_more") or not page:
+            break
+    return out
+
+
+def _openai(client, key: str, start: datetime, now: datetime) -> list[dict]:
+    out, page = [], None
+    for _ in range(PAGES):
+        body = request(client, "GET", OPENAI, "OpenAI", refused="check the OPENAI_ADMIN_KEY secret (an admin key)",
+                       params={"start_time": int(start.timestamp()), "end_time": int(now.timestamp()), "bucket_width": "1d",
+                               "limit": 31, **({"page": page} if page else {})},
+                       headers={"Authorization": f"Bearer {key}"})
+        for b in body.get("data") or []:
+            usd = sum((Decimal(str((r.get("amount") or {}).get("value") or 0)) for r in b.get("results") or []), Decimal(0))
+            day = _day(b["start_time"])
+            out.append({"t": "day", "id": f"openai:{day}", "src": "openai", "day": day, "usd": _usd(usd)})
+        page = body.get("next_page")
+        if not body.get("has_more") or not page:
+            break
+    return out
+
+
+def _claude_code(client, key: str, start: datetime, now: datetime) -> list[dict]:
+    """Claude Code's daily analytics for the organisation (one call per day): sessions, lines, commits, PRs and tool
+    proposals accepted; its estimated cost is already in Anthropic's cost report, so it is kept apart, never added."""
+    out, day = [], start.date()
+    while day < now.date():
+        totals = {"sessions": 0, "lines_added": 0, "lines_removed": 0, "commits": 0, "prs": 0, "accepted": 0, "rejected": 0,
+                  "users": 0, "est_cents": Decimal(0)}
+        page = None
+        for _ in range(PAGES):
+            body = request(client, "GET", CLAUDE_CODE, "Anthropic", refused="check the ANTHROPIC_ADMIN_KEY secret (an Admin API key)",
+                           params={"starting_at": day.isoformat(), "limit": 1000, **({"page": page} if page else {})},
+                           headers={"x-api-key": key, "anthropic-version": "2023-06-01", "user-agent": "stackvitals"})
+            for r in body.get("data") or []:
+                core = r.get("core_metrics") or {}
+                totals["users"] += 1
+                totals["sessions"] += core.get("num_sessions") or 0
+                totals["lines_added"] += (core.get("lines_of_code") or {}).get("added") or 0
+                totals["lines_removed"] += (core.get("lines_of_code") or {}).get("removed") or 0
+                totals["commits"] += core.get("commits_by_claude_code") or 0
+                totals["prs"] += core.get("pull_requests_by_claude_code") or 0
+                for tool in (r.get("tool_actions") or {}).values():
+                    totals["accepted"] += (tool or {}).get("accepted") or 0
+                    totals["rejected"] += (tool or {}).get("rejected") or 0
+                totals["est_cents"] += sum((Decimal(str((m.get("estimated_cost") or {}).get("amount") or 0))
+                                            for m in r.get("model_breakdown") or []), Decimal(0))
+            page = body.get("next_page")
+            if not body.get("has_more") or not page:
+                break
+        if totals["users"]:
+            cents = totals.pop("est_cents")
+            out.append({"t": "day", "id": f"claude_code:{day.isoformat()}", "src": "claude_code", "day": day.isoformat(),
+                        **totals, "est_usd": _usd(cents / 100)})
+        day += timedelta(days=1)
+    return out
+
+
+def _cursor(client, key: str, now: datetime) -> list[dict]:
+    auth = {"Authorization": "Basic " + base64.b64encode(f"{key}:".encode()).decode()}
+    refused = "check the CURSOR_ADMIN_KEY secret (a Cursor admin API key)"
+    spend = request(client, "POST", f"{CURSOR}/teams/spend", "Cursor", refused=refused, json={}, headers=auth)
+    members = spend.get("teamMemberSpend") or []
+    start = _day(spend["subscriptionCycleStart"]) if spend.get("subscriptionCycleStart") else None
+    out = [{"t": "cycle", "id": f"cursor:{start}", "src": "cursor", "start": start, "members": len(members),
+            # All of the cycle's usage, included and on-demand, in dollars.
+            "usd": _usd(Decimal(sum(m.get("overallSpendCents") or m.get("spendCents") or 0 for m in members)) / 100)}] if start else []
+    end = int(now.timestamp() * 1000)
+    use = request(client, "POST", f"{CURSOR}/teams/daily-usage-data", "Cursor", refused=refused, headers=auth,
+                  json={"startDate": end - CURSOR_DAYS * 86_400_000, "endDate": end})
+    days = {}
+    for r in use.get("data") or []:
+        d = days.setdefault(_day(r["date"]), {"agent_requests": 0, "accepted_lines": 0})
+        d["agent_requests"] += r.get("agentRequests") or 0
+        d["accepted_lines"] += r.get("acceptedLinesAdded") or 0
+    out += [{"t": "day", "id": f"cursor:{day}", "src": "cursor", "day": day, **v} for day, v in sorted(days.items())]
+    return out
+
+
+def pull(store, env: dict, cfg: dict, now: datetime, client: httpx.Client | None = None) -> dict:
+    state = store.state()
+    floor = now - timedelta(days=WINDOW)
+    start = max(parse(state["since"]) - timedelta(days=REREAD), floor) if state.get("since") else floor
+    start = start.replace(hour=0, minute=0, second=0, microsecond=0)
+    own = client is None
+    client = client or httpx.Client(timeout=30)
+    read, day = 0, now.date().isoformat()
+    try:
+        for name, fetch in (("anthropic", lambda k: _anthropic(client, k, start, now) + _claude_code(client, k, start, now)),
+                            ("openai", lambda k: _openai(client, k, start, now)),
+                            ("cursor", lambda k: _cursor(client, k, now))):
+            key = (env.get(KEYS[name]) or "").strip()
+            if key:
+                rows = fetch(key)
+                store.add(rows, day)
+                read += len(rows)
+        store.save_state({"since": iso(now)})
+        return {"read": read, "complete": True}
+    finally:
+        if own:
+            client.close()
+
+
+def summary(root, now: datetime, ai_prs: int, window_days: int = 30) -> dict | None:
+    """Spend per provider over the last `window_days` (Cursor: its current billing cycle), the total, and the total over
+    the AI-involved pull requests merged in the same days; None before the first pull."""
+    from stackvitals.connectors.store import ConnectorStore
+    got = ConnectorStore(root, NAME).load()
+    if not got.get("day") and not got.get("cycle"):
+        return None
+    since = (now - timedelta(days=window_days)).date().isoformat()
+    providers = {}
+    for r in got.get("day") or []:
+        if r.get("usd") is not None and r["day"] >= since:
+            providers[r["src"]] = _usd(Decimal(str(providers.get(r["src"], 0))) + Decimal(str(r["usd"])))
+    cycles = sorted((r for r in got.get("cycle") or [] if r.get("start")), key=lambda r: r["start"])
+    if cycles and cycles[-1]["start"] >= since:
+        providers[cycles[-1]["src"]] = cycles[-1]["usd"]
+    total = _usd(sum((Decimal(str(v)) for v in providers.values()), Decimal(0)))
+    use = [r for r in got.get("day") or [] if r["src"] == "cursor" and r["day"] >= since]
+    cc = [r for r in got.get("day") or [] if r["src"] == "claude_code" and r["day"] >= since]
+    # The coding tools side by side over the same days: what each was used for, next to the money.
+    tools = {}
+    if cc:
+        acted = sum(r.get("accepted", 0) + r.get("rejected", 0) for r in cc)
+        tools["claude_code"] = {"sessions": sum(r.get("sessions", 0) for r in cc), "lines_added": sum(r.get("lines_added", 0) for r in cc),
+                                "commits": sum(r.get("commits", 0) for r in cc), "prs": sum(r.get("prs", 0) for r in cc),
+                                "acceptance": round(sum(r.get("accepted", 0) for r in cc) / acted, 4) if acted else None,
+                                "est_usd": _usd(sum((Decimal(str(r.get("est_usd") or 0)) for r in cc), Decimal(0)))}
+    if use:
+        tools["cursor"] = {"agent_requests": sum(r.get("agent_requests") or 0 for r in use),
+                           "lines_added": sum(r.get("accepted_lines") or 0 for r in use)}
+    return {"window_days": window_days, "providers": providers, "total_usd": total, "ai_prs": ai_prs,
+            "per_ai_pr_usd": _usd(Decimal(str(total)) / ai_prs) if ai_prs else None,
+            "cursor_agent_requests": sum(r.get("agent_requests") or 0 for r in use) if use else None,
+            **({"tools": tools} if tools else {})}

stackvitals/connectors/base.py

@@ -0,0 +1,82 @@
+"""What every connector shares: one way to ask a service (retries, rate limits, plain errors), its error, dates in UTC, and
+DevLake's issue types and status categories."""
+import time
+from datetime import datetime, timezone
+
+from stackvitals.core.time import iso
+
+BACKFILL_DAYS = 180
+PAGES = 20  # the most pages one run reads per connector; the next run carries on
+TRIES = 3
+LONGEST_WAIT = 60  # a rate limit asking for longer stops this run; the next one carries on
+
+
+class ConnectorError(Exception):
+    """A plain sentence for the job summary and the Integrations page; never holds a key."""
+
+
+def json_of(resp, service: str, kinds: tuple = (dict,)):
+    """The answer's JSON; a page that is not (a maintenance or login page) is said plainly instead of failing the job."""
+    try:
+        out = resp.json()
+    except ValueError:
+        out = None
+    if not isinstance(out, kinds):
+        raise ConnectorError(f"{service} answered with a page that is not JSON ({resp.status_code}); the next run tries again.")
+    return out
+
+
+def request(client, method: str, url: str, service: str, *, refused: str, bad=None, kinds: tuple = (dict,),
+            sleep=time.sleep, **kw):
+    """The JSON a service answers. A dropped connection or a server error is tried again, a short rate-limit wait is waited
+    out, and anything else is a plain ConnectorError: `refused` says what to check on 401/403, `bad(resp)` may word others."""
+    import httpx
+    for attempt in range(TRIES):
+        last = attempt == TRIES - 1
+        try:
+            resp = client.request(method, url, **kw)
+        except httpx.HTTPError as exc:
+            if last:
+                raise ConnectorError(f"{service} could not be reached ({type(exc).__name__}); the next run tries again.") from None
+            sleep(2 * (attempt + 1))
+            continue
+        if resp.status_code == 429:
+            try:
+                wait = float(resp.headers.get("retry-after", 5))
+            except ValueError:
+                wait = 5.0
+            if last or wait > LONGEST_WAIT:
+                raise ConnectorError(f"{service}'s rate limit was reached; the next run carries on.")
+            sleep(wait)
+            continue
+        if resp.status_code >= 500 and not last:
+            sleep(2 * (attempt + 1))
+            continue
+        if resp.status_code in (401, 403):
+            raise ConnectorError(f"{service} refused the key ({resp.status_code}): {refused}.")
+        if resp.status_code != 200:
+            said = bad(resp) if bad else None
+            raise ConnectorError(said or f"{service} answered {resp.status_code}; the next run tries again.")
+        return json_of(resp, service, kinds)
+
+
+def utc(text: str | None) -> str | None:
+    """Any ISO time (Jira's +0200 offsets, Linear's milliseconds) as UTC with a Z."""
+    if not text:
+        return None
+    try:
+        return iso(datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(timezone.utc))
+    except ValueError:
+        return None
+
+
+def std_type(name: str | None) -> str:
+    """DevLake's standard issue types from a tracker's own type or label names."""
+    low = (name or "").lower()
+    if "incident" in low:
+        return "INCIDENT"
+    if "bug" in low or "defect" in low:
+        return "BUG"
+    if any(w in low for w in ("story", "feature", "epic", "requirement", "improvement")):
+        return "REQUIREMENT"
+    return "TASK"