stackvitals 0.1.0__py3-none-any.whl

mergescout/discover/sweep.py

@@ -0,0 +1,286 @@
+import hashlib
+import os
+import sqlite3
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from mergescout.github.graphql import GraphQLError, RateLimited, TokenRejected, too_heavy
+from mergescout.labels import OPPORTUNITY_LABELS
+from mergescout.timeutil import parse_time
+
+# GitHub search never returns more than this many results for one query.
+MAX_RESULTS = 1000
+FMT = "%Y-%m-%dT%H:%M:%SZ"
+STAR_BOUNDS = [10, 20, 50, 100, 1000, 10000]
+
+COUNT_QUERY = """
+query($q: String!) {
+  rateLimit { cost remaining resetAt }
+  search(query: $q, type: REPOSITORY, first: 1) { repositoryCount }
+}
+"""
+
+REPO_QUERY = """
+query($q: String!, $cursor: String, $n: Int!, $labels: [String!]) {
+  rateLimit { cost remaining resetAt }
+  search(query: $q, type: REPOSITORY, first: $n, after: $cursor) {
+    repositoryCount
+    pageInfo { hasNextPage endCursor }
+    nodes { ... on Repository {
+      databaseId nameWithOwner stargazerCount forkCount createdAt pushedAt
+      primaryLanguage { name } hasIssuesEnabled licenseInfo { spdxId }
+      issues(states: OPEN) { totalCount }
+      opp: issues(states: OPEN, labels: $labels) { totalCount }
+    } }
+  }
+}
+"""
+
+SCHEMA = pa.schema([
+    ("repo_id", pa.int64()),
+    ("name", pa.string()),
+    ("stars", pa.int32()),
+    ("forks", pa.int32()),
+    ("created_at", pa.timestamp("us", tz="UTC")),
+    ("pushed_at", pa.timestamp("us", tz="UTC")),
+    ("language", pa.string()),
+    ("has_issues", pa.bool_()),
+    ("license", pa.string()),
+    ("open_issues", pa.int32()),
+    ("open_prs", pa.int32()),
+    ("opportunity_issues", pa.int32()),
+])
+
+
+@dataclass(frozen=True)
+class Slice:
+    stars_lo: int
+    stars_hi: int | None  # None means no upper limit
+    created_lo: datetime
+    created_hi: datetime
+    pushed_lo: datetime | None = None  # only repos pushed since then (active repos)
+
+    def query(self) -> str:
+        stars = f">={self.stars_lo}" if self.stars_hi is None else f"{self.stars_lo}..{self.stars_hi}"
+        span = f"{self.created_lo.strftime(FMT)}..{self.created_hi.strftime(FMT)}"
+        pushed = f" pushed:>={self.pushed_lo:%Y-%m-%d}" if self.pushed_lo else ""
+        return f"stars:{stars}{pushed} created:{span} fork:false archived:false"
+
+    def split(self) -> list["Slice"]:
+        # Split by stars while the range has more than one value, then by creation time.
+        if self.stars_hi is not None and self.stars_hi > self.stars_lo:
+            mid = (self.stars_lo + self.stars_hi) // 2
+            return [Slice(self.stars_lo, mid, self.created_lo, self.created_hi, self.pushed_lo),
+                    Slice(mid + 1, self.stars_hi, self.created_lo, self.created_hi, self.pushed_lo)]
+        if self.created_hi - self.created_lo <= timedelta(minutes=1):
+            return []
+        mid = (self.created_lo + (self.created_hi - self.created_lo) / 2).replace(microsecond=0)
+        return [Slice(self.stars_lo, self.stars_hi, self.created_lo, mid, self.pushed_lo),
+                Slice(self.stars_lo, self.stars_hi, mid + timedelta(seconds=1), self.created_hi, self.pushed_lo)]
+
+
+def initial_slices(min_stars: int, start: datetime, end: datetime, pushed_lo: datetime | None = None) -> list[Slice]:
+    bounds = [b for b in STAR_BOUNDS if b > min_stars]
+    lows = [min_stars] + bounds
+    highs = [b - 1 for b in bounds] + [None]
+    return [Slice(lo, hi, start, end, pushed_lo) for lo, hi in zip(lows, highs)]
+
+
+def _row(n: dict) -> dict:
+    return {
+        "repo_id": n.get("databaseId"),
+        "name": n["nameWithOwner"],
+        "stars": n.get("stargazerCount"),
+        "forks": n.get("forkCount"),
+        "created_at": parse_time(n.get("createdAt")),
+        "pushed_at": parse_time(n.get("pushedAt")),
+        "language": (n.get("primaryLanguage") or {}).get("name"),
+        "has_issues": n.get("hasIssuesEnabled"),
+        "license": (n.get("licenseInfo") or {}).get("spdxId"),
+        "open_issues": (n.get("issues") or {}).get("totalCount"),
+        "open_prs": (n.get("pullRequests") or {}).get("totalCount"),
+        "opportunity_issues": (n.get("opp") or {}).get("totalCount"),
+    }
+
+
+def fetch_slice(gql, sl: Slice, page: int = 50) -> tuple[list[dict] | None, int, int]:
+    # Returns (rows, total count, points spent); rows is None when the slice must be split first.
+    probe = gql.query(COUNT_QUERY, {"q": sl.query()})
+    cost = probe.cost
+    count = probe.data["search"]["repositoryCount"]
+    if count > MAX_RESULTS and sl.split():
+        return None, count, cost
+    rows: dict[int | str, dict] = {}
+    cursor = None
+    while count:
+        try:
+            result = gql.query(REPO_QUERY, {"q": sl.query(), "cursor": cursor, "n": page,
+                                            "labels": list(OPPORTUNITY_LABELS)})
+        except GraphQLError as exc:
+            # Big repos make pages heavy and slow; retry the same page with fewer repos.
+            if too_heavy(exc) and page > 10:
+                page = max(10, page // 2)
+                continue
+            raise
+        cost += result.cost
+        found = result.data["search"]
+        for n in found["nodes"]:
+            if n and n.get("nameWithOwner"):
+                row = _row(n)
+                rows[row["repo_id"] or row["name"]] = row
+        if not found["nodes"] or not found["pageInfo"]["hasNextPage"] or not found["pageInfo"]["endCursor"]:
+            break
+        if len(rows) >= MAX_RESULTS:
+            break
+        cursor = found["pageInfo"]["endCursor"]
+    return list(rows.values()), count, cost
+
+
+class SweepState:
+    def __init__(self, path: Path, resume: bool = False):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self.lock = threading.Lock()
+        self.db = sqlite3.connect(path, check_same_thread=False)
+        with self.db:
+            self.db.execute(
+                "CREATE TABLE IF NOT EXISTS slices (key TEXT PRIMARY KEY, stars_lo INTEGER, stars_hi INTEGER, "
+                "created_lo TEXT, created_hi TEXT, status TEXT, rows INTEGER, cost INTEGER, error TEXT)"
+            )
+            self.db.execute("CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT)")
+            if resume:
+                # Anything interrupted or failed last time gets another go.
+                self.db.execute("UPDATE slices SET status = 'queued' WHERE status IN ('running', 'failed')")
+
+    def setting(self, key: str, value: str | None = None) -> str | None:
+        with self.lock, self.db:
+            if value is not None:
+                self.db.execute("INSERT OR REPLACE INTO settings VALUES (?, ?)", (key, value))
+            row = self.db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
+        return row[0] if row else None
+
+    def seed(self, slices: list[Slice]) -> None:
+        with self.lock, self.db:
+            if self.db.execute("SELECT COUNT(*) FROM slices").fetchone()[0] == 0:
+                self._insert(slices)
+
+    def _insert(self, slices: list[Slice]) -> None:
+        self.db.executemany(
+            "INSERT OR IGNORE INTO slices VALUES (?, ?, ?, ?, ?, 'queued', NULL, NULL, NULL)",
+            [(s.query(), s.stars_lo, s.stars_hi, s.created_lo.strftime(FMT), s.created_hi.strftime(FMT))
+             for s in slices],
+        )
+
+    def claim(self) -> Slice | None:
+        pushed = parse_time(self.setting("pushed_after"))
+        with self.lock, self.db:
+            row = self.db.execute(
+                "SELECT key, stars_lo, stars_hi, created_lo, created_hi FROM slices "
+                "WHERE status = 'queued' ORDER BY stars_lo DESC LIMIT 1").fetchone()
+            if row is None:
+                return None
+            self.db.execute("UPDATE slices SET status = 'running' WHERE key = ?", (row[0],))
+        return Slice(row[1], row[2], parse_time(row[3]), parse_time(row[4]), pushed)
+
+    def done(self, sl: Slice, rows: int, cost: int) -> None:
+        with self.lock, self.db:
+            self.db.execute("UPDATE slices SET status = 'done', rows = ?, cost = ? WHERE key = ?",
+                            (rows, cost, sl.query()))
+
+    def split(self, sl: Slice, children: list[Slice], cost: int) -> None:
+        with self.lock, self.db:
+            self._insert(children)
+            self.db.execute("UPDATE slices SET status = 'split', cost = ? WHERE key = ?", (cost, sl.query()))
+
+    def release(self, sl: Slice) -> None:
+        with self.lock, self.db:
+            self.db.execute("UPDATE slices SET status = 'queued' WHERE key = ?", (sl.query(),))
+
+    def fail(self, sl: Slice, error: str) -> None:
+        with self.lock, self.db:
+            self.db.execute("UPDATE slices SET status = 'failed', error = ? WHERE key = ?", (error[:300], sl.query()))
+
+    def active(self) -> int:
+        with self.lock:
+            return self.db.execute(
+                "SELECT COUNT(*) FROM slices WHERE status IN ('queued', 'running')").fetchone()[0]
+
+    def counts(self) -> dict:
+        with self.lock:
+            return dict(self.db.execute("SELECT status, COUNT(*) FROM slices GROUP BY status").fetchall())
+
+    def totals(self) -> tuple[int, int]:
+        with self.lock:
+            rows, cost = self.db.execute("SELECT COALESCE(SUM(rows), 0), COALESCE(SUM(cost), 0) FROM slices").fetchone()
+        return rows, cost
+
+
+def _write(rows: list[dict], out: Path) -> None:
+    tmp = out.with_name(out.name + ".part")
+    pq.write_table(pa.Table.from_pylist(rows, schema=SCHEMA), tmp, compression="zstd")
+    os.replace(tmp, out)
+
+
+def run_sweep(make_gql, state: SweepState, data_dir: Path, workers: int = 4,
+              log: Callable[..., None] = print) -> bool:
+    # make_gql is one client factory (used by every worker) or a list with one factory per worker.
+    # Returns True when slices are left because an identity hit its limit; the next run resumes them.
+    out_dir = data_dir / "basics"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    factories = make_gql if isinstance(make_gql, list) else [make_gql] * workers
+
+    in_flight = [0]
+    flight_lock = threading.Lock()
+    limited = threading.Event()
+
+    def worker(factory) -> None:
+        gql = factory()
+        while True:
+            sl = state.claim()
+            if sl is None:
+                with flight_lock:
+                    busy = in_flight[0]
+                if busy == 0:
+                    return
+                time.sleep(0.5)
+                continue
+            with flight_lock:
+                in_flight[0] += 1
+            try:
+                rows, count, cost = fetch_slice(gql, sl)
+                if rows is None:
+                    state.split(sl, sl.split(), cost)
+                else:
+                    if rows:
+                        _write(rows, out_dir / f"{hashlib.sha1(sl.query().encode()).hexdigest()[:16]}.parquet")
+                    state.done(sl, len(rows), cost)
+                    if len(rows) < min(count, MAX_RESULTS):
+                        log(f"warning: {sl.query()} returned {len(rows)} of {count} repos")
+            except (RateLimited, TokenRejected) as exc:
+                # This worker's identity is out for now; the slice goes back for the others or the next run.
+                state.release(sl)
+                limited.set()
+                log(f"worker stops: {exc}")
+                return  # the finally below still marks this slice as no longer in flight
+            except Exception as exc:
+                # Anything that breaks after the fetch must not leave the slice stuck as running.
+                state.fail(sl, f"{type(exc).__name__}: {exc}")
+                log(f"failed {sl.query()}: {exc}")
+            finally:
+                with flight_lock:
+                    in_flight[0] -= 1
+            total_rows, total_cost = state.totals()
+            log(f"{datetime.now(timezone.utc):%H:%M:%S} repos={total_rows:,} points={total_cost:,} {state.counts()}")
+
+    threads = [threading.Thread(target=worker, args=(f,), daemon=True) for f in factories]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+    return limited.is_set() and state.counts().get("queued", 0) > 0

mergescout/github/graphql.py

@@ -0,0 +1,202 @@
+import os
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+import httpx
+
+from mergescout.timeutil import parse_time
+
+
+class GraphQLError(RuntimeError):
+    pass
+
+
+class NotFound(GraphQLError):
+    pass
+
+
+class RateLimited(GraphQLError):
+    pass
+
+
+class TokenRejected(GraphQLError):
+    # A token that stopped working (expired App token, revoked token): that identity steps aside.
+    pass
+
+
+class Gate:
+    # Shared by every client on one token: when GitHub says "wait", all of them wait, not just the one told.
+    def __init__(self, clock: Callable[[], float] = time.monotonic):
+        self.clock = clock
+        self.lock = threading.Lock()
+        self.until = 0.0
+        self.stats = {"requests": 0, "timeouts": 0, "rate_waits": 0, "waited_s": 0.0}
+
+    def note(self, key: str, amount: float = 1) -> None:
+        with self.lock:
+            self.stats[key] += amount
+
+    def summary(self) -> str:
+        s = self.stats
+        return (f"requests={s['requests']} timeouts={s['timeouts']} rate-limit waits={s['rate_waits']} "
+                f"waited={s['waited_s']:.0f}s")
+
+    def hold(self, seconds: float) -> None:
+        with self.lock:
+            self.until = max(self.until, self.clock() + seconds)
+
+    def remaining(self) -> float:
+        with self.lock:
+            return max(0.0, self.until - self.clock())
+
+
+def too_heavy(exc: Exception) -> bool:
+    # GitHub's ways of saying a page was too big or too slow; callers retry with a smaller page.
+    msg = str(exc).lower()
+    return any(s in msg for s in ("resource limits", "timeout", "timed out", "kept failing (http 502",
+                                  "something went wrong", "not json"))
+
+
+@dataclass
+class GraphQLResult:
+    data: dict
+    cost: int
+    remaining: int
+    reset_at: str
+
+
+class GitHubGraphQL:
+    # GitHub Enterprise Server's endpoint when Actions names one.
+    URL = os.environ.get("GITHUB_GRAPHQL_URL") or "https://api.github.com/graphql"
+    RETRY_STATUSES = (500, 502, 503, 504)
+    TIMEOUT_STATUSES = (502, 504)
+
+    def __init__(self, token: str, *, client: httpx.Client | None = None, max_retries: int = 5,
+                 reserve: int = 200, sleep: Callable[[float], None] = time.sleep,
+                 now: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
+                 max_wait: float | Callable[[], float] | None = None,
+                 timeout_retries: int = 1, gate: Gate | None = None):
+        self.client = client or httpx.Client(timeout=httpx.Timeout(60.0, connect=15.0))
+        # Tokens pasted into settings often carry a stray space or line break.
+        self.headers = {"Authorization": f"Bearer {token.strip()}", "User-Agent": "mergescout"}
+        self.max_retries = max_retries
+        self.reserve = reserve
+        self.sleep = sleep
+        self.now = now
+        # A crawl on paid runner minutes stops instead of sleeping until the hourly reset.
+        self.max_wait = max_wait
+        self.timeout_retries = timeout_retries
+        self.last_remaining: int | None = None
+        self.gate = gate or Gate()
+
+    def query(self, query: str, variables: dict | None = None, *, partial: bool = False) -> GraphQLResult:
+        timeouts = 0
+        for attempt in range(self.max_retries + 1):
+            shared = self.gate.remaining()
+            if shared > 0:
+                self._pause(shared)
+            self.gate.note("requests")
+            try:
+                resp = self.client.post(self.URL, json={"query": query, "variables": variables or {}},
+                                        headers=self.headers)
+            except httpx.LocalProtocolError as exc:
+                # The request itself is malformed (usually a bad character in the token); retrying cannot help.
+                raise GraphQLError(f"GitHub request could not be sent ({exc}); check the token for spaces or "
+                                   f"line breaks") from exc
+            except httpx.TransportError as exc:
+                # Wi-Fi drops, DNS hiccups and timeouts are retried like server errors.
+                if attempt == self.max_retries:
+                    raise GraphQLError(f"GitHub unreachable (network error: {exc}) after {attempt} retries") from exc
+                self.sleep(min(60, 2 ** attempt * 2))
+                continue
+            if resp.status_code == 401:
+                raise TokenRejected("GitHub rejected the token (401). Check GITHUB_TOKEN.")
+            if resp.status_code in self.TIMEOUT_STATUSES:
+                self.gate.note("timeouts")
+                # GitHub stops queries after 10 s; retry once, then let the caller ask for a smaller page.
+                if timeouts >= self.timeout_retries:
+                    raise GraphQLError(f"GitHub timed out (HTTP {resp.status_code}); the query may be too heavy")
+                timeouts += 1
+            if resp.status_code in self.RETRY_STATUSES or self._is_rate_limited(resp):
+                if attempt == self.max_retries:
+                    raise GraphQLError(f"GitHub kept failing (HTTP {resp.status_code}) after {attempt} retries")
+                wait = self._wait_for(resp, attempt)
+                if self._is_rate_limited(resp):
+                    self.gate.hold(wait)
+                    self.gate.note("rate_waits")
+                self._pause(wait)
+                continue
+            if resp.status_code != 200:
+                raise GraphQLError(f"GitHub returned HTTP {resp.status_code}: {resp.text[:200]}")
+            try:
+                body = resp.json()
+            except ValueError as exc:
+                raise GraphQLError(f"GitHub returned a body that is not JSON: {resp.text[:200]}") from exc
+            errors = body.get("errors") or []
+            data = body.get("data") or {}
+            if errors:
+                types = {e.get("type") for e in errors}
+                if "RATE_LIMITED" in types and attempt < self.max_retries:
+                    wait = self._seconds_until((data.get("rateLimit") or {}).get("resetAt"), fallback=60)
+                    self.gate.hold(wait)
+                    self.gate.note("rate_waits")
+                    self._pause(wait)
+                    continue
+                top = [e for e in errors if e.get("type") == "NOT_FOUND" and e.get("path") == ["repository"]]
+                if top and (partial or len(top) == len(errors)):
+                    raise NotFound(top[0].get("message", "not found"))
+                # With partial, items that no longer exist come back as null instead of failing the whole page.
+                if not (partial and all(e.get("type") == "NOT_FOUND" for e in errors)):
+                    raise GraphQLError("; ".join(sorted({e.get("message", "unknown error") for e in errors})))
+            rate = data.get("rateLimit")
+            if rate is None:
+                raise ValueError("Every query must request rateLimit { cost remaining resetAt }")
+            result = GraphQLResult(data, rate["cost"], rate["remaining"], rate["resetAt"])
+            self.last_remaining = result.remaining
+            if result.remaining < self.reserve:
+                wait = self._seconds_until(result.reset_at, fallback=0)
+                limit = self._wait_limit()
+                if limit is None or wait <= limit:
+                    self.gate.hold(wait)
+                    self.sleep(wait)
+            return result
+        raise GraphQLError("unreachable")
+
+    @staticmethod
+    def _is_rate_limited(resp: httpx.Response) -> bool:
+        if resp.status_code == 429:
+            return True
+        return resp.status_code == 403 and (
+            "rate limit" in resp.text.lower() or resp.headers.get("x-ratelimit-remaining") == "0")
+
+    def _wait_for(self, resp: httpx.Response, attempt: int) -> float:
+        retry_after = resp.headers.get("retry-after")
+        if retry_after and retry_after.isdigit():
+            return int(retry_after)
+        if resp.headers.get("x-ratelimit-remaining") == "0" and resp.headers.get("x-ratelimit-reset", "").isdigit():
+            reset = datetime.fromtimestamp(int(resp.headers["x-ratelimit-reset"]), tz=timezone.utc)
+            return max(0.0, (reset - self.now()).total_seconds() + 5)
+        if self._is_rate_limited(resp):
+            # GitHub asks for at least a minute after a secondary limit.
+            return 60
+        return min(60, 2 ** attempt * 2)
+
+    def _wait_limit(self) -> float | None:
+        # max_wait may be a function, so a crawl can sit out any wait that still fits in its time window.
+        return self.max_wait() if callable(self.max_wait) else self.max_wait
+
+    def _pause(self, seconds: float) -> None:
+        limit = self._wait_limit()
+        if limit is not None and seconds > limit:
+            raise RateLimited(f"GitHub asks to wait {int(seconds)} s for the rate limit (allowed {int(limit)} s)")
+        self.gate.note("waited_s", seconds)
+        self.sleep(seconds)
+
+    def _seconds_until(self, reset_at: str | None, fallback: float) -> float:
+        reset = parse_time(reset_at)
+        if reset is None:
+            return fallback
+        return max(0.0, (reset - self.now()).total_seconds() + 5)

mergescout/github/identities.py

@@ -0,0 +1,35 @@
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from mergescout.github.graphql import Gate, GitHubGraphQL
+
+# Hourly points relative to a personal token; GitHub reported 5,000 for all three on 2026-10-15.
+SHARES = {"personal": 1.0, "app": 1.0, "actions": 1.0}
+
+
+@dataclass
+class Identity:
+    name: str
+    make_gql: Callable
+    gate: Gate
+    share: float = 1.0
+
+
+def slots(in_flight: int, share: float) -> int:
+    return max(1, round(in_flight * share))
+
+
+def identities_from_env(primary_token: str, max_wait: float | None, client=GitHubGraphQL) -> list[Identity]:
+    # Each identity has its own limits, so each gets its own shared wait; the same token is never used twice.
+    found = [("personal", primary_token), ("app", os.environ.get("MS_APP_TOKEN")),
+             ("actions", os.environ.get("MS_ACTIONS_TOKEN"))]
+    out, seen = [], set()
+    for name, token in found:
+        token = (token or "").strip()
+        if not token or token in seen:
+            continue
+        seen.add(token)
+        gate = Gate()
+        out.append(Identity(name, lambda t=token, g=gate: client(t, max_wait=max_wait, gate=g), gate, SHARES[name]))
+    return out

mergescout/github/releases.py

@@ -0,0 +1,137 @@
+import os
+import time
+from pathlib import Path
+
+import httpx
+
+API = "https://api.github.com"
+UPLOADS = "https://uploads.github.com"
+
+
+class ReleaseError(RuntimeError):
+    pass
+
+
+class Releases:
+    # Release assets in our own repo are the durable store for crawl data and state backups.
+    def __init__(self, token: str | None, repo: str, client: httpx.Client | None = None, sleep=time.sleep, clock=time.time,
+                 max_wait: float = 600):  # the crawl job has 70 min: 52 crawling, a few to push
+        self.repo, self.sleep, self.clock, self.max_wait = repo, sleep, clock, max_wait
+        self.client = client or httpx.Client(timeout=httpx.Timeout(300.0, connect=15.0), follow_redirects=True)
+        self.headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28",
+                        "User-Agent": "mergescout"}
+        # Public data downloads need no token; private repos and uploads do.
+        if token and token.strip():
+            self.headers["Authorization"] = f"Bearer {token.strip()}"
+
+    @staticmethod
+    def _check(resp: httpx.Response, what: str) -> httpx.Response:
+        if resp.status_code >= 400:
+            raise ReleaseError(f"{what} failed: HTTP {resp.status_code} {resp.text[:200]}")
+        return resp
+
+    def _limited_wait(self, resp: httpx.Response) -> float | None:
+        # Seconds until the account's hourly limit resets, when that is why the request failed and the wait is short.
+        if resp.status_code not in (403, 429) or resp.headers.get("x-ratelimit-remaining") != "0":
+            return None
+        wait = float(resp.headers.get("x-ratelimit-reset") or 0) - self.clock() + 5
+        return wait if 0 < wait <= self.max_wait else None
+
+    def get(self, tag: str) -> dict | None:
+        resp = self.client.get(f"{API}/repos/{self.repo}/releases/tags/{tag}", headers=self.headers)
+        if resp.status_code == 404:
+            return None
+        return self._check(resp, f"reading release {tag}").json()
+
+    def ensure(self, tag: str) -> dict:
+        found = self.get(tag)
+        if found:
+            return found
+        body = {"tag_name": tag, "name": tag, "body": "Automated MergeScout data store.", "prerelease": True,
+                "make_latest": "false"}
+        resp = self.client.post(f"{API}/repos/{self.repo}/releases", headers=self.headers, json=body)
+        return self._check(resp, f"creating release {tag}").json()
+
+    def assets(self, release: dict) -> list[dict]:
+        out, page = [], 1
+        while True:
+            resp = self.client.get(f"{API}/repos/{self.repo}/releases/{release['id']}/assets", headers=self.headers,
+                                   params={"per_page": 100, "page": page})
+            batch = self._check(resp, "listing release assets").json()
+            out.extend(batch)
+            if len(batch) < 100:
+                return out
+            page += 1
+
+    def upload(self, tag: str, path: Path, name: str | None = None) -> dict:
+        release = self.ensure(tag)
+        name = name or path.name
+        existing = {a["name"]: a for a in self.assets(release)}
+        # A file being replaced is uploaded under a temporary name first, so a failed upload keeps the old copy.
+        target = f"{name}.new" if name in existing else name
+        if target in existing:
+            self.delete_asset(existing[target])
+        headers = {**self.headers, "Content-Type": "application/octet-stream",
+                   "Content-Length": str(path.stat().st_size)}
+        for attempt in range(2):
+            with path.open("rb") as f:
+                resp = self.client.post(f"{UPLOADS}/repos/{self.repo}/releases/{release['id']}/assets",
+                                        params={"name": target}, headers=headers, content=f)
+            # The hour's work is in this file: wait for the reset once rather than lose it (real failure 2026-10-20).
+            wait = self._limited_wait(resp) if attempt == 0 else None
+            if wait is None:
+                break
+            self.sleep(wait)
+        uploaded = self._check(resp, f"uploading {name}").json()
+        if target == name:
+            return uploaded
+        self.delete_asset(existing[name])
+        resp = self.client.patch(f"{API}/repos/{self.repo}/releases/assets/{uploaded['id']}", headers=self.headers,
+                                 json={"name": name})
+        return self._check(resp, f"renaming {target}").json()
+
+    def list_releases(self) -> list[dict]:
+        out, page = [], 1
+        while True:
+            resp = self.client.get(f"{API}/repos/{self.repo}/releases", headers=self.headers,
+                                   params={"per_page": 100, "page": page})
+            batch = self._check(resp, "listing releases").json()
+            out.extend(batch)
+            if len(batch) < 100:
+                return out
+            page += 1
+
+    def delete_asset(self, asset: dict) -> None:
+        self._check(self.client.delete(f"{API}/repos/{self.repo}/releases/assets/{asset['id']}", headers=self.headers),
+                    f"deleting {asset.get('name', asset['id'])}")
+
+    def delete_release(self, release: dict) -> None:
+        self._check(self.client.delete(f"{API}/repos/{self.repo}/releases/{release['id']}", headers=self.headers),
+                    f"deleting release {release['tag_name']}")
+        # The tag stays behind otherwise; a missing tag is fine.
+        resp = self.client.delete(f"{API}/repos/{self.repo}/git/refs/tags/{release['tag_name']}", headers=self.headers)
+        if resp.status_code not in (204, 404, 422):
+            self._check(resp, f"deleting tag {release['tag_name']}")
+
+    def download(self, tag: str, name: str, dest: Path) -> bool:
+        release = self.get(tag)
+        asset = next((a for a in self.assets(release) if a["name"] == name), None) if release else None
+        if asset is None:
+            return False
+        self.download_asset(asset, dest)
+        return True
+
+    def download_asset(self, asset: dict, dest: Path) -> None:
+        name = asset.get("name", asset["id"])
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        part = dest.with_name(dest.name + ".part")
+        headers = {**self.headers, "Accept": "application/octet-stream"}
+        with self.client.stream("GET", f"{API}/repos/{self.repo}/releases/assets/{asset['id']}",
+                                headers=headers) as resp:
+            if resp.status_code >= 400:
+                resp.read()
+                self._check(resp, f"downloading {name}")
+            with part.open("wb") as f:
+                for chunk in resp.iter_bytes():
+                    f.write(chunk)
+        os.replace(part, dest)