stackvitals 0.1.0__py3-none-any.whl

mergescout/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

mergescout/archive/__main__.py

@@ -0,0 +1,30 @@
+import argparse
+import sys
+
+from mergescout.archive.convert import convert_hour
+from mergescout.archive.manifest import Manifest
+from mergescout.config import load_config
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m mergescout.archive",
+                                     description="Convert GH Archive hours into slim Parquet files.")
+    parser.add_argument("hours", nargs="+", help="UTC hour keys like 2026-10-01-15")
+    args = parser.parse_args(argv)
+    cfg = load_config()
+    manifest = Manifest(cfg.data_dir / "manifest.sqlite")
+    failed = 0
+    for key in args.hours:
+        try:
+            r = convert_hour(key, cfg.data_dir, manifest)
+            print(f"{r.hour_key}: {r.status} in={r.events_in} kept={r.events_kept} "
+                  f"bad={r.bad_lines} bytes={r.bytes_out}")
+        except Exception as exc:
+            failed += 1
+            print(f"{key}: failed ({exc})", file=sys.stderr)
+    manifest.close()
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mergescout/archive/convert.py

@@ -0,0 +1,83 @@
+import hashlib
+import os
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from mergescout.archive.events import SCHEMA, slim_event
+from mergescout.archive.manifest import Manifest
+from mergescout.archive.stream import fetch_chunks, hour_url, iter_events, normalize_hour_key, parse_hour_key
+
+
+@dataclass
+class HourResult:
+    hour_key: str
+    status: str
+    events_in: int = 0
+    events_kept: int = 0
+    bad_lines: int = 0
+    bytes_out: int = 0
+    path: Path | None = None
+
+
+def hour_path(data_dir: Path, key: str) -> Path:
+    day, hour = parse_hour_key(key)
+    return data_dir / "bronze" / f"date={day}" / f"hour={hour}.parquet"
+
+
+def write_parquet(rows: Iterable[dict], out_path: Path, batch_size: int = 50_000) -> int:
+    # Write to a .part file first so a crash never leaves a half file that looks finished.
+    part = out_path.with_name(out_path.name + ".part")
+    written = 0
+    try:
+        with pq.ParquetWriter(part, SCHEMA, compression="zstd") as writer:
+            batch = []
+            for row in rows:
+                batch.append(row)
+                if len(batch) >= batch_size:
+                    writer.write_table(pa.Table.from_pylist(batch, schema=SCHEMA))
+                    written += len(batch)
+                    batch = []
+            if batch:
+                writer.write_table(pa.Table.from_pylist(batch, schema=SCHEMA))
+                written += len(batch)
+        os.replace(part, out_path)
+    finally:
+        if part.exists():
+            part.unlink()
+    return written
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as f:
+        for block in iter(lambda: f.read(1 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def convert_hour(
+    key: str,
+    data_dir: Path,
+    manifest: Manifest,
+    chunks_factory: Callable[[str], Iterable[bytes]] = fetch_chunks,
+) -> HourResult:
+    key = normalize_hour_key(key)
+    if manifest.is_done(key):
+        return HourResult(key, "skipped")
+    out = hour_path(data_dir, key)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    stats = {"events_in": 0, "bad_lines": 0}
+    try:
+        events = iter_events(chunks_factory(hour_url(key)), stats)
+        kept = write_parquet((row for row in map(slim_event, events) if row is not None), out)
+    except Exception as exc:
+        manifest.mark_failed(key, f"{type(exc).__name__}: {exc}")
+        raise
+    size = out.stat().st_size
+    manifest.mark_done(key, events_in=stats["events_in"], events_kept=kept,
+                       bad_lines=stats["bad_lines"], bytes_out=size, sha256=_sha256(out))
+    return HourResult(key, "done", stats["events_in"], kept, stats["bad_lines"], size, out)

mergescout/archive/events.py

@@ -0,0 +1,55 @@
+import pyarrow as pa
+
+from mergescout.timeutil import parse_time
+
+KEPT_TYPES = frozenset({
+    "IssuesEvent", "IssueCommentEvent", "PullRequestEvent", "PullRequestReviewEvent", "PushEvent",
+})
+
+SCHEMA = pa.schema([
+    ("event_id", pa.string()),
+    ("type", pa.string()),
+    ("action", pa.string()),
+    ("created_at", pa.timestamp("us", tz="UTC")),
+    ("repo_id", pa.int64()),
+    ("repo_name", pa.string()),
+    ("actor_login", pa.string()),
+    ("number", pa.int64()),
+    ("is_pr", pa.bool_()),
+    ("issue_state", pa.string()),
+    ("assignee_count", pa.int32()),
+    ("label_names", pa.list_(pa.string())),
+    ("body_len", pa.int32()),
+    ("ref", pa.string()),
+])
+
+
+def slim_event(event: dict) -> dict | None:
+    etype = event.get("type")
+    if etype not in KEPT_TYPES:
+        return None
+    payload = event.get("payload") or {}
+    repo = event.get("repo") or {}
+    actor = event.get("actor") or {}
+    issue = payload.get("issue") or {}
+    pr = payload.get("pull_request") or {}
+    comment = payload.get("comment") or {}
+    labels = issue.get("labels")
+    assignees = issue.get("assignees")
+    body = comment.get("body") if etype == "IssueCommentEvent" else issue.get("body")
+    return {
+        "event_id": str(event.get("id")),
+        "type": etype,
+        "action": payload.get("action"),
+        "created_at": parse_time(event.get("created_at")),
+        "repo_id": repo.get("id"),
+        "repo_name": repo.get("name"),
+        "actor_login": actor.get("login"),
+        "number": issue.get("number") or pr.get("number") or payload.get("number"),
+        "is_pr": etype in ("PullRequestEvent", "PullRequestReviewEvent") or "pull_request" in issue,
+        "issue_state": issue.get("state"),
+        "assignee_count": len(assignees) if assignees is not None else None,
+        "label_names": [lb["name"] for lb in labels if lb.get("name")] if labels is not None else None,
+        "body_len": len(body) if body is not None else None,
+        "ref": payload.get("ref") if etype == "PushEvent" else None,
+    }

mergescout/archive/manifest.py

@@ -0,0 +1,45 @@
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+class Manifest:
+    def __init__(self, path: Path):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self.db = sqlite3.connect(path)
+        self.db.row_factory = sqlite3.Row
+        with self.db:
+            self.db.execute(
+                "CREATE TABLE IF NOT EXISTS archive_hours ("
+                "hour_key TEXT PRIMARY KEY, status TEXT NOT NULL, events_in INTEGER, events_kept INTEGER, "
+                "bad_lines INTEGER, bytes_out INTEGER, sha256 TEXT, error TEXT, updated_at TEXT NOT NULL)"
+            )
+
+    def get(self, key: str) -> dict | None:
+        row = self.db.execute("SELECT * FROM archive_hours WHERE hour_key = ?", (key,)).fetchone()
+        return dict(row) if row else None
+
+    def is_done(self, key: str) -> bool:
+        row = self.get(key)
+        return bool(row and row["status"] == "done")
+
+    def mark_done(self, key, *, events_in, events_kept, bad_lines, bytes_out, sha256) -> None:
+        with self.db:
+            self.db.execute(
+                "INSERT OR REPLACE INTO archive_hours VALUES (?, 'done', ?, ?, ?, ?, ?, NULL, ?)",
+                (key, events_in, events_kept, bad_lines, bytes_out, sha256, _now()),
+            )
+
+    def mark_failed(self, key: str, error: str) -> None:
+        with self.db:
+            self.db.execute(
+                "INSERT OR REPLACE INTO archive_hours VALUES (?, 'failed', NULL, NULL, NULL, NULL, NULL, ?, ?)",
+                (key, error[:500], _now()),
+            )
+
+    def close(self) -> None:
+        self.db.close()

mergescout/archive/stream.py

@@ -0,0 +1,83 @@
+import zlib
+from collections.abc import Iterable, Iterator
+from datetime import date
+
+import httpx
+import orjson
+
+ARCHIVE_BASE = "https://data.gharchive.org"
+
+
+class ArchiveStreamError(Exception):
+    pass
+
+
+def normalize_hour_key(key: str) -> str:
+    parts = key.split("-")
+    if len(parts) != 4:
+        raise ValueError(f"Hour key must look like 2026-10-01-15, got {key!r}")
+    try:
+        day = date(int(parts[0]), int(parts[1]), int(parts[2]))
+        hour = int(parts[3])
+    except ValueError as exc:
+        raise ValueError(f"Hour key must look like 2026-10-01-15, got {key!r}") from exc
+    if not 0 <= hour <= 23:
+        raise ValueError(f"Hour must be 0-23, got {hour}")
+    return f"{day.isoformat()}-{hour}"
+
+
+def parse_hour_key(key: str) -> tuple[str, int]:
+    norm = normalize_hour_key(key)
+    day, hour = norm.rsplit("-", 1)
+    return day, int(hour)
+
+
+def hour_url(key: str) -> str:
+    return f"{ARCHIVE_BASE}/{normalize_hour_key(key)}.json.gz"
+
+
+def fetch_chunks(url: str, client: httpx.Client | None = None, chunk_size: int = 1 << 20) -> Iterator[bytes]:
+    own = client is None
+    client = client or httpx.Client(timeout=httpx.Timeout(60.0, connect=15.0), follow_redirects=True)
+    try:
+        with client.stream("GET", url) as resp:
+            resp.raise_for_status()
+            # Raw bytes, so httpx never tries to unzip for us.
+            yield from resp.iter_raw(chunk_size)
+    finally:
+        if own:
+            client.close()
+
+
+def iter_lines(chunks: Iterable[bytes]) -> Iterator[bytes]:
+    decomp = zlib.decompressobj(zlib.MAX_WBITS | 16)
+    buffer = b""
+    try:
+        for chunk in chunks:
+            data = decomp.decompress(chunk)
+            # A file can hold several gzip parts back to back.
+            while decomp.unused_data:
+                leftover = decomp.unused_data
+                decomp = zlib.decompressobj(zlib.MAX_WBITS | 16)
+                data += decomp.decompress(leftover)
+            buffer += data
+            *lines, buffer = buffer.split(b"\n")
+            yield from (line for line in lines if line)
+        buffer += decomp.flush()
+    except zlib.error as exc:
+        raise ArchiveStreamError(f"Archive file is corrupt: {exc}") from exc
+    if not decomp.eof:
+        raise ArchiveStreamError("Archive file ended early (download cut off).")
+    if buffer.strip():
+        yield buffer
+
+
+def iter_events(chunks: Iterable[bytes], stats: dict) -> Iterator[dict]:
+    for line in iter_lines(chunks):
+        try:
+            event = orjson.loads(line)
+        except orjson.JSONDecodeError:
+            stats["bad_lines"] += 1
+            continue
+        stats["events_in"] += 1
+        yield event

mergescout/bench/__init__.py

@@ -0,0 +1 @@
+"""Daily samplers that turn the crawled repo list into open-source benchmarks."""

mergescout/bench/__main__.py

@@ -0,0 +1,135 @@
+import argparse
+import gzip
+import json
+import os
+import re
+import sys
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pyarrow.parquet as pq
+
+from mergescout.bench.ci import TokenPool, aggregate, fetch_all, todays_share
+from mergescout.bench.deploys import aggregate_deploys, repo_deploys
+from mergescout.bench.stars import aggregate_stars, read_stars
+from mergescout.config import load_config
+from mergescout.github.releases import Releases
+from mergescout.store.sync import STATE_TAG, safe_extract
+
+LATEST_TAG = "bench-latest"
+DAILY = re.compile(r"^(?P<kind>[a-z]+)-(\d{4}-\d{2}-\d{2})\.ndjson\.gz$")
+
+
+def week_tag(day: datetime) -> str:
+    # One release per ISO week keeps every release small.
+    return f"bench-{day:%G-W%V}"
+
+
+def load_repos(basics: Path, now: datetime, active_days: int = 30, min_stars: int = 100) -> list[dict]:
+    """Repos from the discovery sweep that pushed recently: the only ones with push runs to read."""
+    since, best = now - timedelta(days=active_days), {}
+    for path in sorted(basics.rglob("*.parquet")):
+        for r in pq.read_table(path, columns=["name", "stars", "language", "pushed_at"]).to_pylist():
+            if not r["name"] or (r["stars"] or 0) < min_stars or not r["pushed_at"] or r["pushed_at"] < since:
+                continue
+            key = r["name"].lower()
+            if key not in best or (r["stars"] or 0) > best[key]["stars"]:
+                best[key] = {"name": r["name"], "language": r["language"], "stars": r["stars"]}
+    return sorted(best.values(), key=lambda r: r["name"].lower())
+
+
+def _read(path: Path) -> list[dict]:
+    with gzip.open(path, "rt", encoding="utf-8") as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def publish(data_dir: Path, releases, records: list[dict], now: datetime, days: int = 35, log=print, kind: str = "ci",
+            build=aggregate) -> dict:
+    """Store today's sample privately, then rebuild the benchmarks from the last five weeks of samples of that kind."""
+    folder = data_dir / "bench"
+    folder.mkdir(parents=True, exist_ok=True)
+    name = f"{kind}-{now:%Y-%m-%d}.ndjson.gz"
+    today = folder / name
+    # A second run on the same day adds to that day's file instead of replacing it.
+    release = releases.get(week_tag(now))
+    earlier = next((a for a in releases.assets(release) if a["name"] == name), None) if release else None
+    merged = {}
+    if earlier:
+        releases.download_asset(earlier, today)
+        merged = {r["repo"].lower(): r for r in _read(today)}
+    merged.update({r["repo"].lower(): r for r in records})
+    records = list(merged.values())
+    with gzip.open(today, "wt", encoding="utf-8") as f:
+        f.writelines(json.dumps(r) + "\n" for r in records)
+    releases.upload(week_tag(now), today)
+    first = (now - timedelta(days=days)).strftime("%Y-%m-%d")
+    every = list(records)
+    for tag in sorted({week_tag(now - timedelta(days=d)) for d in range(days + 1)}):
+        release = releases.get(tag)
+        for asset in releases.assets(release) if release else []:
+            m = DAILY.match(asset["name"])
+            if not m or m["kind"] != kind or asset["name"] == name or m[2] < first:
+                continue
+            local = folder / asset["name"]
+            if not local.exists():
+                releases.download_asset(asset, local)
+            every += _read(local)
+    agg = build(every, now)
+    out = folder / f"{kind}-benchmarks.json"
+    out.write_text(json.dumps(agg, indent=1, sort_keys=True), encoding="utf-8")
+    releases.upload(LATEST_TAG, out)
+    log(f"benchmarks: {agg['repos']} repos, {len(agg['groups'])} groups with enough repos")
+    return agg
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m mergescout.bench",
+                                     description="Sample CI statistics of crawled repos into open benchmarks.")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    ci = sub.add_parser("ci", help="Read today's share of the active repos' push runs and rebuild the benchmarks")
+    stars = sub.add_parser("stars", help="Read today's share of the active repos' weekly star history and rebuild its benchmarks")
+    deploys = sub.add_parser("deploys", help="Read today's share of the active repos' deployments and releases (2 requests "
+                                             "each) and rebuild the shipping-frequency benchmarks")
+    for p in (ci, stars, deploys):
+        p.add_argument("--workers", type=int, default=10)
+        p.add_argument("--max-wait", type=int, default=3600, help="seconds to wait in total for hourly limits to reset")
+        p.add_argument("--limit", type=int, default=0, help="read at most this many repos (0: all of today's share)")
+    parser.add_argument("--repo", default=os.environ.get("MS_STORE_REPO") or os.environ.get("GITHUB_REPOSITORY"),
+                        help="owner/name of the repo whose releases hold the data")
+    args = parser.parse_args(argv)
+    store_token = os.environ.get("MS_RELEASE_TOKEN")
+    tokens = [os.environ.get(k) for k in ("GITHUB_TOKEN", "MS_APP_TOKEN", "MS_ACTIONS_TOKEN")]
+    if not store_token or not args.repo or not any(tokens):
+        print("Set MS_RELEASE_TOKEN, MS_STORE_REPO (or GITHUB_REPOSITORY) and GITHUB_TOKEN.", file=sys.stderr)
+        return 1
+    log = lambda msg: print(msg, flush=True)  # noqa: E731
+    now, data_dir = datetime.now(timezone.utc), load_config().data_dir
+    releases = Releases(store_token, args.repo)
+    basics = data_dir / "discover" / "basics"
+    if not basics.exists():
+        # Only the sweep results are needed, never the crawl state.
+        bundle = data_dir / "outbox" / "discover.tar"
+        if not releases.download(STATE_TAG, "discover.tar", bundle):
+            print("No sweep results in the releases yet.", file=sys.stderr)
+            return 1
+        safe_extract(bundle, data_dir)
+        bundle.unlink()
+    repos = load_repos(basics, now)
+    share = todays_share([r["name"] for r in repos], now)
+    picked = set(share[:args.limit] if args.limit else share)
+    todo = [r for r in repos if r["name"] in picked]
+    log(f"{len(repos)} active repos; today's share: {len(todo)}")
+    pool = TokenPool(tokens, max_wait=args.max_wait)
+    readers = {"ci": (None, aggregate, "push runs"), "stars": (read_stars, aggregate_stars, "star history"),
+               "deploys": (repo_deploys, aggregate_deploys, "deployment and release history")}
+    read, build, what = readers[args.cmd]
+    got = fetch_all(pool, todo, now, workers=args.workers, log=log, read=read)
+    # The requests are counted so each sampler is seen to stay inside the day's REST budget.
+    log(f"{len(got['records'])} repos with {what}, {got['requests']} requests "
+        f"({got['requests'] / max(len(todo), 1):.2f} a repo), {'complete' if got['complete'] else 'stopped at the hourly limits'}")
+    publish(data_dir, releases, got["records"], now, log=log, kind=args.cmd, build=build)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())