stackvitals 0.1.0__py3-none-any.whl

mergescout/crawl/runner.py

@@ -0,0 +1,176 @@
+import threading
+import time
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from mergescout.crawl.fetch import cutoff, fetch_repo
+from mergescout.crawl.outcomes import apply_outcomes, plan_outcomes
+from mergescout.crawl.state import CRAWL_VERSION, CrawlState
+from mergescout.crawl.writer import BatchWriter
+from mergescout.github.graphql import Gate, RateLimited, TokenRejected
+from mergescout.github.identities import Identity, slots
+from mergescout.labels import has_opportunity
+from mergescout.timeutil import parse_time
+
+
+def is_hot(issues: list[dict]) -> bool:
+    return any(has_opportunity(i.get("labels") or []) for i in issues if i.get("state") == "OPEN")
+
+
+def changed_since(repo: dict, now: datetime, full_every_days: int) -> datetime | None:
+    # Refreshes ask only for changes since the last crawl (two hours of overlap); a full crawl comes back monthly.
+    last, full = parse_time(repo.get("last_done_at")), parse_time(repo.get("last_full_at"))
+    outdated = (repo.get("crawl_version") or CRAWL_VERSION) < CRAWL_VERSION
+    if last is None or full is None or outdated or now - full >= timedelta(days=full_every_days):
+        return None
+    return last - timedelta(hours=2)
+
+
+class _Lane:
+    # One identity's share of the run: its own parallel slots, points left and reason for stepping aside.
+    def __init__(self, identity: Identity, in_flight: int):
+        self.name, self.make_gql, self.gate = identity.name, identity.make_gql, identity.gate
+        self.max_slots = slots(in_flight, identity.share)
+        self.slots = self.lowest = self.max_slots
+        self.busy, self.repos, self.points = 0, 0, 0
+        self.remaining: int | None = None
+        self.out: str | None = None
+        self.seen_waits, self.calm = 0, 0
+
+    def pace(self, log) -> None:
+        # Halve parallel requests when GitHub asked this identity to slow down; add one back after 20 calm repos.
+        waits = self.gate.stats["rate_waits"]
+        if waits > self.seen_waits:
+            self.seen_waits, self.calm = waits, 0
+            if self.slots > 1:
+                self.slots = max(1, self.slots // 2)
+                self.lowest = min(self.lowest, self.slots)
+                log(f"{self.name}: GitHub asked to slow down; parallel requests now {self.slots}")
+        else:
+            self.calm += 1
+            if self.calm >= 20 and self.slots < self.max_slots:
+                self.slots, self.calm = self.slots + 1, 0
+
+
+def run_crawl(state: CrawlState, make_gql, data_dir: Path, run_id: str, *, minutes: float, in_flight: int = 12,
+              reserve: int = 300, limit: int | None = None, flush_every: int = 250, hot_days: int = 1,
+              cold_days: int = 7, issue_days: int = 90, full_every_days: int = 30,
+              now=lambda: datetime.now(timezone.utc), log=print,
+              clock=time.monotonic) -> dict:
+    # make_gql is one client factory, or a list of identities that share the queue, each with its own limits.
+    identities = make_gql if isinstance(make_gql, list) else [Identity("default", make_gql, Gate())]
+    lanes = [_Lane(i, in_flight) for i in identities]
+    state.start_run(run_id, "crawl")
+    writer = BatchWriter(data_dir, run_id)
+    local = threading.local()
+    started = clock()
+    stats = {"repos": 0, "points": 0, "failed": 0, "gone": 0}
+    pending: list[tuple] = []
+    stop = None
+
+    def work(lane: _Lane, repo):
+        # Runs in a pool thread; each thread keeps one client per identity. Never raises.
+        clients = local.__dict__.setdefault("clients", {})
+        if lane.name not in clients:
+            clients[lane.name] = lane.make_gql()
+        gql = clients[lane.name]
+        try:
+            since = changed_since(repo, now(), full_every_days)
+            track = state.open_issue_numbers(repo["repo_id"]) if since is None else []
+            bundle = fetch_repo(gql, repo["repo_id"], repo["name"], now(), issue_days=issue_days, track=track,
+                                changed_since=since)
+            return repo, bundle, None, getattr(gql, "last_remaining", None)
+        except Exception as exc:
+            return repo, None, exc, getattr(gql, "last_remaining", None)
+
+    def flush():
+        # Repos count as done only once their rows are safely on disk.
+        writer.flush()
+        for repo_id, points, prs, issues, days, keep, forget, full in pending:
+            apply_outcomes(state, repo_id, keep, forget)
+            state.done(repo_id, run_id, points, prs, issues, refresh_days=days, full=full)
+        pending.clear()
+
+    with ThreadPoolExecutor(max_workers=sum(lane.max_slots for lane in lanes)) as pool:
+        futures: dict = {}
+        while True:
+            if stop is None:
+                if (clock() - started) / 60 >= minutes:
+                    stop = "time"
+                elif limit is not None and stats["repos"] + len(futures) >= limit:
+                    stop = "limit"
+                elif all(lane.out for lane in lanes):
+                    stop = "budget" if all(lane.out == "budget" for lane in lanes) else "rate_limit"
+            if stop is None:
+                claimed = False
+                for lane in lanes:
+                    room = lane.slots - lane.busy
+                    if limit is not None:
+                        room = min(room, limit - stats["repos"] - len(futures))
+                    if lane.out or room <= 0:
+                        continue
+                    for repo in state.claim(room, run_id):
+                        futures[pool.submit(work, lane, repo)] = lane
+                        lane.busy += 1
+                        claimed = True
+                if not claimed and not futures:
+                    stop = "empty"
+            if not futures:
+                break
+            finished, _ = wait(list(futures), return_when=FIRST_COMPLETED)
+            for f in finished:
+                lane = futures.pop(f)
+                lane.busy -= 1
+                repo, bundle, exc, left = f.result()
+                lane.pace(log)
+                if left is not None:
+                    lane.remaining = left
+                    if left < reserve and lane.out is None:
+                        lane.out = "budget"
+                        log(f"{lane.name}: points nearly used up ({left} left); it stops taking repos")
+                if isinstance(exc, (RateLimited, TokenRejected)):
+                    state.release(repo["repo_id"])
+                    if lane.out not in ("rate_limit", "rejected"):
+                        lane.out = "rejected" if isinstance(exc, TokenRejected) else "rate_limit"
+                        why = "its token was rejected" if lane.out == "rejected" else "GitHub's rate limit"
+                        log(f"{lane.name}: stops on {why} ({exc})")
+                elif exc is not None:
+                    state.fail(repo["repo_id"], run_id, f"{type(exc).__name__}: {exc}")
+                    stats["failed"] += 1
+                    log(f"failed {repo['name']}: {exc}")
+                elif bundle.status == "gone":
+                    state.gone(repo["repo_id"], run_id, bundle.error or "gone")
+                    stats["gone"] += 1
+                else:
+                    if bundle.meta:
+                        writer.add("repo_meta", [bundle.meta])
+                    writer.add("prs", bundle.prs)
+                    writer.add("issues", bundle.issues)
+                    writer.add("comments", bundle.comments)
+                    writer.add("closed_issues", bundle.closed_issues)
+                    today = now()
+                    events, keep, forget = plan_outcomes(state, repo["repo_id"], bundle.issues, today.date().isoformat(),
+                                                         cutoff(today, issue_days), bundle.deleted_issues)
+                    writer.add("issue_events", events)
+                    hot = is_hot(bundle.issues) or state.has_hot_tracked(repo["repo_id"])
+                    days = hot_days if hot else cold_days
+                    pending.append((repo["repo_id"], bundle.points, len(bundle.prs), len(bundle.issues), days, keep,
+                                    forget, bundle.full))
+                    stats["repos"] += 1
+                    stats["points"] += bundle.points
+                    lane.repos += 1
+                    lane.points += bundle.points
+                    if len(pending) >= flush_every:
+                        flush()
+    flush()
+    reason = stop or "empty"
+    state.finish_run(run_id, stats["points"], stats["repos"], reason)
+    per = {lane.name: {"repos": lane.repos, "points": lane.points, "left": lane.remaining, "out": lane.out,
+                       "slots": lane.slots, "lowest_slots": lane.lowest} for lane in lanes}
+    log(f"run {run_id}: repos={stats['repos']} points={stats['points']} failed={stats['failed']} "
+        f"gone={stats['gone']} reason={reason} queue={state.counts()}")
+    for name, s in per.items():
+        log(f"  {name}: repos={s['repos']} points={s['points']} left={s['left']} slots={s['slots']} "
+            f"(lowest {s['lowest_slots']}) stopped={s['out'] or 'no'}")
+    return {**stats, "reason": reason, "identities": per}

mergescout/crawl/seed.py

@@ -0,0 +1,77 @@
+import math
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import pyarrow.compute as pc
+import pyarrow.dataset as ds
+
+from mergescout.crawl.state import CrawlState
+from mergescout.discover.sweep import SCHEMA
+
+
+def active_repos_from_sweep(data_dir: Path, min_stars: int, active_days: int, now: datetime) -> list[dict]:
+    base = data_dir / "discover" / "basics"
+    if not base.exists() or not any(base.glob("*.parquet")):
+        return []
+    # The fixed schema lets older sweep files without newer columns load as nulls.
+    table = ds.dataset(str(base), format="parquet", schema=SCHEMA).to_table(
+        filter=(pc.field("stars") >= min_stars) & (pc.field("pushed_at") >= now - timedelta(days=active_days)))
+    best: dict[int, dict] = {}
+    for r in table.to_pylist():
+        # A repo seen in two slices (stars moved mid-sweep) keeps the higher count.
+        if r["repo_id"] is not None and (r["repo_id"] not in best or r["stars"] > best[r["repo_id"]]["stars"]):
+            best[r["repo_id"]] = r
+    return sorted(best.values(), key=lambda r: -r["stars"])
+
+
+def expected_cost(repo: dict) -> int:
+    # Points a first crawl will likely cost: settings, PR pages (busier with stars), 2 points per 50 open issues.
+    stars = repo.get("stars") or 0
+    pr_pages = 6 if stars >= 10000 else 3 if stars >= 1000 else 1
+    issue_pages = max(1, math.ceil(min(repo.get("open_issues") or 0, 200) / 50))
+    return 1 + pr_pages + 2 * issue_pages
+
+
+def priority(repo: dict) -> int:
+    # Beginner issues per expected point, so the daily budget covers the most candidate issues first.
+    beginner = min(repo.get("opportunity_issues") or 0, 20)
+    if beginner:
+        value = beginner
+    elif repo.get("has_issues") and (repo.get("open_issues") or 0) > 0:
+        value = 0.5
+    else:
+        value = 0.1
+    return round(1000 * value / expected_cost(repo))
+
+
+# Watched repos (a list of owner/name lines) are crawled before everything else, so their history stays current.
+WATCHED = 1_000_000
+
+
+def read_watch(path: Path | None) -> set[str]:
+    if not path or not Path(path).exists():
+        return set()
+    lines = Path(path).read_text(encoding="utf-8").splitlines()
+    return {x.strip().lower() for x in lines if x.strip() and not x.strip().startswith("#")}
+
+
+def apply_watch(rows: list[dict], watch: set[str]) -> int:
+    hits = 0
+    for r in rows:
+        if r["name"].lower() in watch:
+            r["priority"], hits = WATCHED, hits + 1
+    return hits
+
+
+def seed_queue(state: CrawlState, data_dir: Path, now: datetime, min_stars: int = 100, active_days: int = 90,
+               log=print, watch: set[str] | None = None) -> dict:
+    active = active_repos_from_sweep(data_dir, min_stars, active_days, now)
+    rows = [{"repo_id": r["repo_id"], "name": r["name"], "stars": r["stars"],
+             "pushed_at": r["pushed_at"].isoformat() if r["pushed_at"] else None, "priority": priority(r)}
+            for r in active]
+    watched = apply_watch(rows, watch or set())
+    added = state.upsert_repos(rows)
+    out = {"active": len(active), "opportunity_repos": sum(1 for r in active if r.get("opportunity_issues")),
+           "added": added, "watched": watched}
+    log(f"seed: {out}")
+    return out

mergescout/crawl/state.py

@@ -0,0 +1,169 @@
+import sqlite3
+import threading
+from collections.abc import Callable
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from mergescout.labels import has_opportunity
+
+# Bump when full crawls start collecting new fields: repos fully crawled by an older version are re-crawled in full.
+CRAWL_VERSION = 2
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS repos (
+  repo_id INTEGER PRIMARY KEY, name TEXT NOT NULL, stars INTEGER, pushed_at TEXT, priority INTEGER DEFAULT 0,
+  status TEXT NOT NULL DEFAULT 'queued', attempts INTEGER DEFAULT 0, run_id TEXT, claimed_at TEXT,
+  last_done_at TEXT, next_due TEXT, last_points INTEGER, last_prs INTEGER, last_issues INTEGER, error TEXT,
+  last_full_at TEXT, crawl_version INTEGER DEFAULT 1);
+CREATE INDEX IF NOT EXISTS repos_claim ON repos(status, priority DESC, stars DESC);
+CREATE TABLE IF NOT EXISTS runs (run_id TEXT PRIMARY KEY, kind TEXT, started_at TEXT, finished_at TEXT,
+  points INTEGER, repos INTEGER, reason TEXT);
+CREATE TABLE IF NOT EXISTS issue_state (repo_id INTEGER, number INTEGER, observed_at TEXT, state TEXT,
+  assignees INTEGER, linked_open_prs INTEGER, linked_merged_prs INTEGER, comments INTEGER, labels TEXT,
+  updated_at TEXT, PRIMARY KEY (repo_id, number));
+"""
+
+
+class CrawlState:
+    def __init__(self, path: Path, now: Callable[[], datetime] = lambda: datetime.now(timezone.utc)):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self.now = now
+        self.lock = threading.Lock()
+        self.db = sqlite3.connect(path, check_same_thread=False)
+        self.db.row_factory = sqlite3.Row
+        with self.db:
+            self.db.executescript(SCHEMA)
+            # State files saved before refreshes became incremental: their crawls were all full ones.
+            columns = {r[1] for r in self.db.execute("PRAGMA table_info(repos)")}
+            if "last_full_at" not in columns:
+                self.db.execute("ALTER TABLE repos ADD COLUMN last_full_at TEXT")
+                self.db.execute("UPDATE repos SET last_full_at = last_done_at WHERE last_done_at IS NOT NULL")
+            if "crawl_version" not in columns:
+                self.db.execute("ALTER TABLE repos ADD COLUMN crawl_version INTEGER DEFAULT 1")
+
+    def _ts(self) -> str:
+        return self.now().isoformat(timespec="seconds")
+
+    def upsert_repos(self, rows: list[dict]) -> int:
+        # New repos are queued; known repos get fresher metadata and the latest seed's priority.
+        added = 0
+        with self.lock, self.db:
+            for r in rows:
+                known = self.db.execute("SELECT 1 FROM repos WHERE repo_id = ?", (r["repo_id"],)).fetchone()
+                if known:
+                    self.db.execute("UPDATE repos SET name = ?, stars = ?, pushed_at = ?, priority = ? "
+                                    "WHERE repo_id = ?",
+                                    (r["name"], r.get("stars"), r.get("pushed_at"), r.get("priority", 0), r["repo_id"]))
+                else:
+                    self.db.execute("INSERT INTO repos (repo_id, name, stars, pushed_at, priority) VALUES (?, ?, ?, ?, ?)",
+                                    (r["repo_id"], r["name"], r.get("stars"), r.get("pushed_at"), r.get("priority", 0)))
+                    added += 1
+        return added
+
+    def claim(self, n: int, run_id: str) -> list[dict]:
+        now = self._ts()
+        with self.lock, self.db:
+            rows = self.db.execute(
+                "SELECT repo_id, name, stars, priority, last_done_at, last_full_at, crawl_version FROM repos "
+                "WHERE status = 'queued' OR (status = 'done' AND ((next_due IS NOT NULL AND next_due <= ?) "
+                "OR crawl_version < ?)) ORDER BY priority DESC, stars DESC LIMIT ?",
+                (now, CRAWL_VERSION, n)).fetchall()
+            for r in rows:
+                self.db.execute("UPDATE repos SET status = 'running', run_id = ?, claimed_at = ?, "
+                                "attempts = attempts + 1 WHERE repo_id = ?", (run_id, now, r["repo_id"]))
+        return [dict(r) for r in rows]
+
+    def done(self, repo_id: int, run_id: str, points: int, prs: int, issues: int, refresh_days: int = 7,
+             full: bool = True) -> None:
+        due = (self.now() + timedelta(days=refresh_days)).isoformat(timespec="seconds")
+        now = self._ts()
+        with self.lock, self.db:
+            self.db.execute("UPDATE repos SET status = 'done', last_done_at = ?, next_due = ?, last_points = ?, "
+                            "last_prs = ?, last_issues = ?, error = NULL, attempts = 0, "
+                            "last_full_at = CASE WHEN ? THEN ? ELSE last_full_at END, "
+                            "crawl_version = CASE WHEN ? THEN ? ELSE crawl_version END WHERE repo_id = ?",
+                            (now, due, points, prs, issues, full, now, full, CRAWL_VERSION, repo_id))
+
+    def fail(self, repo_id: int, run_id: str, error: str) -> None:
+        with self.lock, self.db:
+            self.db.execute("UPDATE repos SET status = 'failed', error = ? WHERE repo_id = ?", (error[:300], repo_id))
+
+    def gone(self, repo_id: int, run_id: str, reason: str) -> None:
+        with self.lock, self.db:
+            self.db.execute("UPDATE repos SET status = 'gone', error = ? WHERE repo_id = ?", (reason[:300], repo_id))
+
+    def release(self, repo_id: int) -> None:
+        # Back to the queue without counting an attempt (the run stopped, the repo did not fail).
+        with self.lock, self.db:
+            self.db.execute("UPDATE repos SET status = 'queued', attempts = MAX(attempts - 1, 0) WHERE repo_id = ?",
+                            (repo_id,))
+
+    def requeue_stale(self, older_than_minutes: int = 120) -> int:
+        cutoff = (self.now() - timedelta(minutes=older_than_minutes)).isoformat(timespec="seconds")
+        with self.lock, self.db:
+            cur = self.db.execute("UPDATE repos SET status = 'queued' WHERE status = 'running' AND claimed_at < ?",
+                                  (cutoff,))
+        return cur.rowcount
+
+    def requeue_failed(self, max_attempts: int = 3) -> int:
+        with self.lock, self.db:
+            cur = self.db.execute("UPDATE repos SET status = 'queued' WHERE status = 'failed' AND attempts < ?",
+                                  (max_attempts,))
+        return cur.rowcount
+
+    def start_run(self, run_id: str, kind: str) -> None:
+        with self.lock, self.db:
+            self.db.execute("INSERT OR IGNORE INTO runs (run_id, kind, started_at) VALUES (?, ?, ?)",
+                            (run_id, kind, self._ts()))
+
+    def finish_run(self, run_id: str, points: int, repos: int, reason: str) -> None:
+        with self.lock, self.db:
+            self.db.execute("UPDATE runs SET finished_at = ?, points = ?, repos = ?, reason = ? WHERE run_id = ?",
+                            (self._ts(), points, repos, reason, run_id))
+
+    def due_count(self) -> int:
+        # Same rule as claim(): queued, due for a refresh, or crawled by an older crawler.
+        with self.lock:
+            return self.db.execute(
+                "SELECT COUNT(*) FROM repos WHERE status = 'queued' OR (status = 'done' AND "
+                "((next_due IS NOT NULL AND next_due <= ?) OR crawl_version < ?))",
+                (self._ts(), CRAWL_VERSION)).fetchone()[0]
+
+    def counts(self) -> dict:
+        with self.lock:
+            return dict(self.db.execute("SELECT status, COUNT(*) FROM repos GROUP BY status").fetchall())
+
+    def log_issue_state(self, rows: list[dict], observed_at: str) -> None:
+        # Only the latest state per tracked issue lives here; the history is in the Parquet batches.
+        with self.lock, self.db:
+            self.db.executemany(
+                "INSERT OR REPLACE INTO issue_state (repo_id, number, observed_at, state, assignees, linked_open_prs, "
+                "linked_merged_prs, comments, labels, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                [(r["repo_id"], r["number"], observed_at, r["state"], r["assignees"], r["linked_open_prs"],
+                  r["linked_merged_prs"], r["comments"], r["labels"], r.get("updated_at")) for r in rows])
+
+    def latest_issue_state(self, repo_id: int) -> list[dict]:
+        with self.lock:
+            rows = self.db.execute("SELECT * FROM issue_state WHERE repo_id = ? ORDER BY number", (repo_id,)).fetchall()
+        return [dict(r) for r in rows]
+
+    def open_issue_numbers(self, repo_id: int) -> list[int]:
+        with self.lock:
+            rows = self.db.execute("SELECT number FROM issue_state WHERE repo_id = ? AND state = 'OPEN' ORDER BY number",
+                                   (repo_id,)).fetchall()
+        return [r[0] for r in rows]
+
+    def has_hot_tracked(self, repo_id: int) -> bool:
+        # A refresh only sees changed issues, so hotness also comes from the beginner issues already tracked.
+        with self.lock:
+            rows = self.db.execute("SELECT labels FROM issue_state WHERE repo_id = ? AND state = 'OPEN'",
+                                   (repo_id,)).fetchall()
+        return any(has_opportunity((r[0] or "").split("|")) for r in rows)
+
+    def forget_issues(self, repo_id: int, numbers: list[int]) -> None:
+        with self.lock, self.db:
+            self.db.executemany("DELETE FROM issue_state WHERE repo_id = ? AND number = ?",
+                                [(repo_id, n) for n in numbers])
+
+    def close(self) -> None:
+        self.db.close()

mergescout/crawl/writer.py

@@ -0,0 +1,87 @@
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+TS = pa.timestamp("us", tz="UTC")
+
+# Fixed schemas so every batch file of a table can be read together later.
+SCHEMAS = {
+    "repo_meta": pa.schema([
+        ("repo_id", pa.int64()), ("name", pa.string()), ("stars", pa.int32()), ("forks", pa.int32()),
+        ("pushed_at", TS), ("language", pa.string()), ("archived", pa.bool_()), ("has_issues", pa.bool_()),
+        ("has_discussions", pa.bool_()), ("has_projects", pa.bool_()), ("license", pa.string()),
+        ("default_branch", pa.string()), ("contributing_sha", pa.string()), ("ai_policy_sha", pa.string()),
+        ("pr_template_sha", pa.string()), ("readme_excerpt", pa.string()), ("last_release_at", TS),
+        ("releases_12m", pa.int32()), ("open_prs", pa.int32()), ("open_issues", pa.int32()),
+        ("contributing_excerpt", pa.string()), ("label_names", pa.list_(pa.string())),
+        ("owner_logins", pa.list_(pa.string())),
+        ("run_id", pa.string()), ("fetched_at", TS)]),
+    "prs": pa.schema([
+        ("repo_id", pa.int64()), ("number", pa.int32()), ("created_at", TS), ("merged_at", TS), ("closed_at", TS),
+        ("state", pa.string()), ("author_login", pa.string()), ("author_type", pa.string()),
+        ("author_association", pa.string()), ("merged_by", pa.string()), ("head_ref", pa.string()),
+        ("additions", pa.int32()), ("deletions", pa.int32()), ("changed_files", pa.int32()),
+        ("first_review_at", TS), ("first_review_by", pa.string()), ("first_review_association", pa.string()),
+        ("first_human_review_at", TS), ("first_human_review_by", pa.string()),
+        ("first_human_review_association", pa.string()),
+        ("closing_issue_numbers", pa.list_(pa.int32())), ("ai_tag", pa.string()),
+        ("title", pa.string()), ("ci_state", pa.string()), ("commit_count", pa.int32()), ("run_id", pa.string()), ("fetched_at", TS)]),
+    "issues": pa.schema([
+        ("repo_id", pa.int64()), ("number", pa.int32()), ("title", pa.string()), ("body_excerpt", pa.string()),
+        ("state", pa.string()), ("state_reason", pa.string()), ("created_at", TS), ("updated_at", TS),
+        ("closed_at", TS), ("author_login", pa.string()), ("author_association", pa.string()),
+        ("labels", pa.list_(pa.string())),
+        ("assignees", pa.int32()), ("linked_open_prs", pa.int32()), ("linked_merged_prs", pa.int32()),
+        ("comments_total", pa.int32()), ("linked_closed_prs", pa.int32()), ("fix_pr", pa.int32()),
+        ("fix_pr_state", pa.string()), ("fix_author", pa.string()), ("fix_association", pa.string()), ("run_id", pa.string()), ("fetched_at", TS)]),
+    "comments": pa.schema([
+        ("repo_id", pa.int64()), ("number", pa.int32()), ("comment_id", pa.int64()), ("created_at", TS),
+        ("author_login", pa.string()), ("author_type", pa.string()), ("author_association", pa.string()),
+        ("body_excerpt", pa.string()), ("run_id", pa.string()), ("fetched_at", TS)]),
+    "issue_events": pa.schema([
+        ("repo_id", pa.int64()), ("number", pa.int32()), ("event", pa.string()), ("detail", pa.string()),
+        ("observed_at", pa.string()), ("run_id", pa.string()), ("fetched_at", TS)]),
+    "closed_issues": pa.schema([
+        ("repo_id", pa.int64()), ("number", pa.int32()), ("created_at", TS), ("closed_at", TS),
+        ("state_reason", pa.string()), ("author_login", pa.string()), ("author_association", pa.string()),
+        ("labels", pa.list_(pa.string())), ("fix_pr", pa.int32()), ("fix_pr_state", pa.string()),
+        ("fix_author", pa.string()), ("fix_association", pa.string()), ("run_id", pa.string()),
+        ("fetched_at", TS)]),
+}
+
+
+class BatchWriter:
+    def __init__(self, data_dir: Path, run_id: str, now=lambda: datetime.now(timezone.utc)):
+        self.base = data_dir / "crawl"
+        self.run_id = run_id
+        self.now = now
+        self.rows: dict[str, list[dict]] = {}
+        self.seq = 0
+
+    def add(self, table: str, rows: list[dict]) -> None:
+        if table not in SCHEMAS:
+            raise KeyError(f"unknown table {table}")
+        if rows:
+            # Every row says when it was fetched, so later layers can keep the newest copy.
+            stamp = self.now()
+            self.rows.setdefault(table, []).extend({**r, "run_id": self.run_id, "fetched_at": stamp} for r in rows)
+
+    def flush(self) -> dict[str, Path]:
+        # Each flush is a new file per table; a file is complete or absent (written as .part, then renamed).
+        out = {}
+        for table, rows in self.rows.items():
+            schema = SCHEMAS[table]
+            path = self.base / table / f"run={self.run_id}-{self.seq:03d}.parquet"
+            path.parent.mkdir(parents=True, exist_ok=True)
+            part = path.with_name(path.name + ".part")
+            cols = {f.name: [r.get(f.name) for r in rows] for f in schema}
+            pq.write_table(pa.table(cols, schema=schema), part, compression="zstd")
+            os.replace(part, path)
+            out[table] = path
+        if out:
+            self.seq += 1
+        self.rows = {}
+        return out

mergescout/discover/__main__.py

@@ -0,0 +1,58 @@
+import argparse
+import sys
+from datetime import datetime, timedelta, timezone
+
+from mergescout.config import load_config
+from mergescout.discover.sweep import SweepState, initial_slices, run_sweep
+from mergescout.github.graphql import GitHubGraphQL
+from mergescout.github.identities import identities_from_env, slots
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m mergescout.discover", description="List public repos at scale.")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    sw = sub.add_parser("sweep", help="List every repo above a star count (safe to stop and re-run)")
+    sw.add_argument("--min-stars", type=int, default=10)
+    sw.add_argument("--workers", type=int, default=4)
+    sw.add_argument("--active-days", type=int, default=None, help="Only repos pushed in this many days")
+    sw.add_argument("--max-wait", type=float, default=None,
+                    help="Pause instead of waiting longer than this many seconds for the rate limit")
+    sub.add_parser("status", help="Show sweep progress")
+    args = parser.parse_args(argv)
+
+    cfg = load_config()
+    db_path = cfg.data_dir / "discover" / "sweep.sqlite"
+    if args.cmd == "status":
+        state = SweepState(db_path)
+        rows, cost = state.totals()
+        print(f"repos={rows:,} points={cost:,} slices={state.counts()}")
+        return 0
+    token = cfg.require_token()
+    state = SweepState(db_path, resume=True)
+    saved = state.setting("min_stars")
+    if saved is not None and int(saved) != args.min_stars:
+        print(f"This sweep was started with --min-stars {saved}; delete {db_path} to start over.", file=sys.stderr)
+        return 2
+    state.setting("min_stars", str(args.min_stars))
+    now = datetime.now(timezone.utc).replace(microsecond=0)
+    # The activity cutoff is fixed when a sweep starts, so a resumed sweep keeps the same slices.
+    if args.active_days and state.setting("pushed_after") is None and state.active() == 0 and not state.counts():
+        state.setting("pushed_after", (now - timedelta(days=args.active_days)).strftime("%Y-%m-%dT%H:%M:%SZ"))
+    pushed = state.setting("pushed_after")
+    state.seed(initial_slices(args.min_stars, datetime(2007, 1, 1, tzinfo=timezone.utc), now,
+                              datetime.fromisoformat(pushed.replace("Z", "+00:00")) if pushed else None))
+    # Every identity (personal token, App, workflow token) gets workers in proportion to its hourly points.
+    identities = identities_from_env(token, args.max_wait, client=GitHubGraphQL)
+    factories = [i.make_gql for i in identities for _ in range(slots(args.workers, i.share))]
+    paused = run_sweep(factories, state, cfg.data_dir / "discover", log=lambda msg: print(msg, flush=True))
+    rows, cost = state.totals()
+    counts = state.counts()
+    for i in identities:
+        print(f"github {i.name}: {i.gate.summary()}", flush=True)
+    word = "paused (rate limit; run again to resume)" if paused else "finished"
+    print(f"{word}: repos={rows:,} points={cost:,} slices={counts}", file=sys.stderr)
+    return 1 if counts.get("failed") else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())