nfl-bigquery 0.1.0__tar.gz

nfl_bigquery-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,38 @@
+name: release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - run: uv python install 3.13
+      - name: Create venv
+        run: uv venv
+      - name: Install build tools
+        run: uv pip install build
+      - run: uv run python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: build
+    environment:
+      name: pypi
+      url: https://pypi.org/p/nfl-bigquery
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+      - uses: pypa/gh-action-pypi-publish@release/v1

nfl_bigquery-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,28 @@
+name: test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+      - name: Install
+        run: uv sync --extra dev
+      - name: Test
+        run: uv run pytest -m "not network"
+      - name: Lint
+        run: uv run ruff check nfl_bigquery tests
+      - name: Type check
+        run: uv run pyright nfl_bigquery tests
+      - name: Build
+        run: uv run python -m build

nfl_bigquery-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+# .gitignore
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.coverage
+htmlcov/

nfl_bigquery-0.1.0/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changelog
+
+## 0.1.0
+- `sync`: season-chunked idempotent load of `nfl_plays` (372-col nflfastR pbp), `games`
+  (schedules), `weekly_player_stats`; `--resume` via `_nfl_ingest_runs`.
+- `players`: `dim_players` MERGE upsert on `gsis_id`.
+- `docs`: llm / bq-apply / markdown / dictionary / dbt renderers from dictionary-seeded
+  ColumnSpecs + enrichment overlay.
+- `verify`: internal consistency + plays→weekly reconciliation for passing/rushing/receiving
+  yards and TDs (6 metrics, 8 checks total).
+- Schema-drift guard test against nflverse fixtures.

nfl_bigquery-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jason Blahovec
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nfl_bigquery-0.1.0/PKG-INFO

@@ -0,0 +1,73 @@
+Metadata-Version: 2.4
+Name: nfl-bigquery
+Version: 0.1.0
+Summary: NFL play-by-play → BigQuery: idempotent nflverse ingestion + LLM-friendly docs + verification
+Project-URL: Homepage, https://github.com/blahovec-labs/nfl-bigquery
+Project-URL: Issues, https://github.com/blahovec-labs/nfl-bigquery/issues
+Project-URL: Changelog, https://github.com/blahovec-labs/nfl-bigquery/blob/main/CHANGELOG.md
+Author: Jason Blahovec
+License: MIT
+License-File: LICENSE
+Keywords: bigquery,data-engineering,nfl,nflfastr,nflverse,play-by-play
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Requires-Python: >=3.11
+Requires-Dist: db-dtypes<2.0,>=1.0
+Requires-Dist: google-cloud-bigquery<4.0,>=3.20
+Requires-Dist: nflreadpy
+Requires-Dist: pandas<3.0,>=2.0
+Requires-Dist: pyarrow<19.0,>=15.0
+Provides-Extra: dev
+Requires-Dist: build>=1.2.0; extra == 'dev'
+Requires-Dist: pyright>=1.1.380; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# nfl-bigquery
+
+NFL play-by-play → BigQuery. Idempotent nflverse ingestion + LLM-friendly docs + verification.
+Fourth in the `*-bigquery` family (`statcast-bigquery`, `nhl-bigquery`, `yfinance-bigquery`).
+
+## Install
+```bash
+pip install nfl-bigquery
+gcloud auth application-default login
+```
+
+## Sync
+```bash
+# Backfill 1999–present into a shared dataset (idempotent, resumable)
+nfl-bigquery sync --seasons 1999-2025 --resume \
+    --pbp-table myproject.mydataset.nfl_plays
+# Player dimension
+nfl-bigquery players --players-table myproject.mydataset.dim_players
+```
+
+Writes `nfl_plays`, `games`, `weekly_player_stats` (season-chunked DELETE→INSERT) and
+`dim_players` (MERGE on `gsis_id`), plus a `_nfl_ingest_runs` log for `--resume`.
+
+## Docs & verify
+```bash
+nfl-bigquery docs --format llm
+nfl-bigquery docs --format bq-apply --table myproject.mydataset.nfl_plays
+nfl-bigquery verify --source internal --season 2024 \
+    --pbp-table myproject.mydataset.nfl_plays
+```
+
+The `verify` command runs 8 checks: duplicate-play detection, orphan-play detection, and
+plays→weekly reconciliation for **passing/rushing/receiving yards and TDs** (6 metrics).
+The `--tolerance` flag (default 0.0) sets the maximum allowed absolute aggregate diff;
+lateral plays cause small legitimate yard diffs, so consider `--tolerance 5` for yards.
+
+## Data source
+All data from [nflverse](https://nflreadpy.nflverse.com/) via `nflreadpy`. Play-by-play is
+nflfastR's 372-column feed (EPA/WPA/air-yards/CPOE modeled), back to 1999.
+
+MIT licensed.

nfl_bigquery-0.1.0/README.md

@@ -0,0 +1,41 @@
+# nfl-bigquery
+
+NFL play-by-play → BigQuery. Idempotent nflverse ingestion + LLM-friendly docs + verification.
+Fourth in the `*-bigquery` family (`statcast-bigquery`, `nhl-bigquery`, `yfinance-bigquery`).
+
+## Install
+```bash
+pip install nfl-bigquery
+gcloud auth application-default login
+```
+
+## Sync
+```bash
+# Backfill 1999–present into a shared dataset (idempotent, resumable)
+nfl-bigquery sync --seasons 1999-2025 --resume \
+    --pbp-table myproject.mydataset.nfl_plays
+# Player dimension
+nfl-bigquery players --players-table myproject.mydataset.dim_players
+```
+
+Writes `nfl_plays`, `games`, `weekly_player_stats` (season-chunked DELETE→INSERT) and
+`dim_players` (MERGE on `gsis_id`), plus a `_nfl_ingest_runs` log for `--resume`.
+
+## Docs & verify
+```bash
+nfl-bigquery docs --format llm
+nfl-bigquery docs --format bq-apply --table myproject.mydataset.nfl_plays
+nfl-bigquery verify --source internal --season 2024 \
+    --pbp-table myproject.mydataset.nfl_plays
+```
+
+The `verify` command runs 8 checks: duplicate-play detection, orphan-play detection, and
+plays→weekly reconciliation for **passing/rushing/receiving yards and TDs** (6 metrics).
+The `--tolerance` flag (default 0.0) sets the maximum allowed absolute aggregate diff;
+lateral plays cause small legitimate yard diffs, so consider `--tolerance 5` for yards.
+
+## Data source
+All data from [nflverse](https://nflreadpy.nflverse.com/) via `nflreadpy`. Play-by-play is
+nflfastR's 372-column feed (EPA/WPA/air-yards/CPOE modeled), back to 1999.
+
+MIT licensed.

nfl_bigquery-0.1.0/nfl_bigquery/__init__.py

@@ -0,0 +1,3 @@
+from nfl_bigquery._version import __version__
+
+__all__ = ["__version__"]

nfl_bigquery-0.1.0/nfl_bigquery/_transform_util.py

@@ -0,0 +1,19 @@
+"""Shared transform helper: reindex to schema columns + stamp ingested_at."""
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import pandas as pd
+
+from nfl_bigquery.schema import ColumnSpec, spec_names
+
+
+def align_to_schema(
+    df: pd.DataFrame, schema: list[ColumnSpec], *, stamp: bool = True
+) -> pd.DataFrame:
+    df = df.copy()
+    if stamp:
+        df["ingested_at"] = datetime.now(UTC)
+    cols = spec_names(schema)
+    # Reindex adds missing NULLABLE columns as NA and drops columns not in schema.
+    return df.reindex(columns=cols)

nfl_bigquery-0.1.0/nfl_bigquery/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

nfl_bigquery-0.1.0/nfl_bigquery/cli.py

@@ -0,0 +1,145 @@
+"""nfl-bigquery CLI entrypoint."""
+from __future__ import annotations
+
+import argparse
+import logging
+
+from nfl_bigquery._version import __version__
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="nfl-bigquery")
+    parser.add_argument("--version", action="version", version=__version__)
+    sub = parser.add_subparsers(dest="command")
+
+    sy = sub.add_parser("sync", help="Fetch nflverse data and write to BigQuery")
+    sy.add_argument("--seasons", required=True,
+                    help="e.g. 1999-2025 | 2023,2025 | 2024 | latest")
+    sy.add_argument("--pbp-table", required=True, help="project.dataset.nfl_plays")
+    sy.add_argument("--games-table")
+    sy.add_argument("--weekly-stats-table")
+    sy.add_argument("--runs-table")
+    sy.add_argument("--skip-games", action="store_true")
+    sy.add_argument("--skip-weekly-stats", action="store_true")
+    sy.add_argument("--resume", action="store_true")
+    sy.add_argument("--dry-run", action="store_true")
+
+    pl = sub.add_parser("players", help="Upsert the dim_players dimension")
+    pl.add_argument("--players-table", required=True, help="project.dataset.dim_players")
+    pl.add_argument("--dry-run", action="store_true")
+
+    dc = sub.add_parser("docs", help="Render table documentation")
+    dc.add_argument("--format", required=True,
+                    choices=["bq-apply", "llm", "dictionary", "markdown", "dbt"])
+    dc.add_argument("--table", help="project.dataset.table (required for bq-apply)")
+    dc.add_argument("--output", default="-", help="output path or - for stdout")
+
+    vf = sub.add_parser("verify", help="Run data-quality checks")
+    vf.add_argument("--source", default="internal", choices=["internal"])
+    vf.add_argument("--season", type=int, required=True)
+    vf.add_argument("--pbp-table", required=True)
+    vf.add_argument("--games-table")
+    vf.add_argument("--weekly-stats-table")
+    vf.add_argument("--tolerance", type=float, default=1.0)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
+    parser = build_parser()
+    ns = parser.parse_args(argv)
+    if not getattr(ns, "command", None):
+        parser.print_help()
+        return 0
+    if ns.command == "sync":
+        from google.cloud import bigquery
+
+        from nfl_bigquery.client import NflverseClient
+        from nfl_bigquery.runs import RunsTable
+        from nfl_bigquery.sync import run_sync
+        from nfl_bigquery.writer import BigQueryWriter
+
+        bq = bigquery.Client()
+        return run_sync(ns, client=NflverseClient(), writer=BigQueryWriter(client=bq),
+                        runs=RunsTable(client=bq))
+    if ns.command == "players":
+        from google.cloud import bigquery
+
+        from nfl_bigquery.client import NflverseClient
+        from nfl_bigquery.players.sync import run_players
+
+        return run_players(ns, client=NflverseClient(), bq_client=bigquery.Client())
+    if ns.command == "docs":
+        from nfl_bigquery.docs.renderers import (
+            render_bq_apply,
+            render_dbt,
+            render_dictionary_rows,
+            render_llm,
+            render_markdown,
+        )
+        from nfl_bigquery.docs.taxonomy import TABLES
+
+        if ns.format == "llm":
+            text = render_llm(TABLES)
+        elif ns.format == "markdown":
+            text = render_markdown(TABLES)
+        elif ns.format == "dbt":
+            text = render_dbt(TABLES)
+        elif ns.format == "bq-apply":
+            if not ns.table:
+                parser.error("--table is required for --format bq-apply")
+            from google.cloud import bigquery
+
+            bq = bigquery.Client()
+            table_name = ns.table.split(".")[-1]
+            if table_name not in TABLES:
+                parser.error(f"unknown table '{table_name}'; choices: {sorted(TABLES)}")
+            desc = {n: d for n, d in render_bq_apply(table_name)}
+            fetched = bq.get_table(ns.table)
+            new_schema = [
+                bigquery.SchemaField(
+                    f.name, f.field_type, mode=f.mode,
+                    description=desc.get(f.name, f.description or ""),
+                )
+                for f in fetched.schema
+            ]
+            fetched.schema = new_schema
+            bq.update_table(fetched, ["schema"])
+            print(f"applied descriptions to {ns.table}")
+            return 0
+        else:  # dictionary
+            import json
+
+            text = json.dumps(render_dictionary_rows(TABLES), indent=2)
+        if ns.output == "-":
+            print(text)
+        else:
+            from pathlib import Path
+
+            Path(ns.output).write_text(text, encoding="utf-8")
+        return 0
+    if ns.command == "verify":
+        from google.cloud import bigquery
+
+        from nfl_bigquery.verify.internal import run_internal
+        from nfl_bigquery.writer import TableRef
+
+        pbp = TableRef.parse(ns.pbp_table)
+        games = ns.games_table or f"{pbp.project}.{pbp.dataset}.games"
+        stats = ns.weekly_stats_table or f"{pbp.project}.{pbp.dataset}.weekly_player_stats"
+        results = run_internal(
+            bigquery.Client(), plays_ref=str(pbp), games_ref=games, stats_ref=stats,
+            season=ns.season, tolerance=ns.tolerance)
+        failed = 0
+        for r in results:
+            mark = "PASS" if r.passed else "FAIL"
+            print(f"[{mark}] {r.name}: {r.detail}")
+            if not r.passed:
+                failed += 1
+        return 1 if failed else 0
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

nfl_bigquery-0.1.0/nfl_bigquery/client.py

@@ -0,0 +1,48 @@
+# nfl_bigquery/client.py
+"""Thin nflreadpy wrapper: fetch a season's parquet, convert to pandas, retry."""
+from __future__ import annotations
+
+import logging
+import time
+
+import nflreadpy as nfl
+import pandas as pd
+
+log = logging.getLogger(__name__)
+
+
+class NflverseClient:
+    def __init__(self, max_retries: int = 5, backoff_seconds: float = 2.0) -> None:
+        self.max_retries = max_retries
+        self.backoff_seconds = backoff_seconds
+
+    def _retry(self, fn, *args) -> pd.DataFrame:
+        last: Exception | None = None
+        for attempt in range(1, self.max_retries + 1):
+            try:
+                return fn(*args).to_pandas()
+            except Exception as e:  # noqa: BLE001 - retry any transient download error
+                last = e
+                wait = self.backoff_seconds * (2 ** (attempt - 1))
+                log.warning(
+                    "nflverse load failed (attempt %d/%d): %s; retrying in %.1fs",
+                    attempt,
+                    self.max_retries,
+                    e,
+                    wait,
+                )
+                if attempt < self.max_retries:
+                    time.sleep(wait)
+        raise RuntimeError(f"nflverse load failed after {self.max_retries} attempts") from last
+
+    def load_pbp(self, season: int) -> pd.DataFrame:
+        return self._retry(nfl.load_pbp, [season])
+
+    def load_schedules(self, season: int) -> pd.DataFrame:
+        return self._retry(nfl.load_schedules, [season])
+
+    def load_player_stats(self, season: int) -> pd.DataFrame:
+        return self._retry(nfl.load_player_stats, [season])
+
+    def load_players(self) -> pd.DataFrame:
+        return self._retry(nfl.load_players)