ffl-bigquery 0.1.0__py3-none-any.whl

ffl_bigquery/__init__.py

@@ -0,0 +1,3 @@
+from ffl_bigquery._version import __version__
+
+__all__ = ["__version__"]

ffl_bigquery/_schema_samples/__init__.py

@@ -0,0 +1,54 @@
+"""Parquet schema samples shipped inside the wheel.
+
+Six of the season-chunked nflverse tables (ff_opportunity, snap_counts,
+injuries, ftn_charting, participation) plus ff_rankings build their BigQuery
+schema at import time by sampling a real upstream frame's dtypes
+(`schema_gen.specs_from_frame`). That sample has to be readable from an
+*installed* package, not just a repo checkout: the previous approach --
+`Path(__file__).resolve().parents[3] / "tests/fixtures/nflverse/<x>.parquet"`
+-- resolves to the repo root only when `ffl_bigquery/` sits inside a checkout
+three directories below that fixture. Once pip unpacks the wheel into
+`site-packages/`, `parents[3]` lands outside site-packages entirely (`tests/`
+isn't even shipped), so every one of those six modules raised
+`FileNotFoundError` at import time -- and because `load_all_specs()` imports
+all of them eagerly, that took down `sync-nflverse` for every `--tables`
+value, including `--dry-run`.
+
+These parquet files (~230 KB total) are the fix: they live inside the
+installed package itself and are located via `importlib.resources`, which
+resolves correctly in a repo checkout, an editable install, and a built
+wheel alike (`pyproject.toml` force-includes them into the wheel since
+hatchling does not pick up non-Python files under `packages=` by default).
+
+The same files also back the fidelity tests in
+`tests/test_play_level_tables.py` and `tests/test_nflverse_tables_simple.py`,
+via this same `read_sample`/`sample_path` helpers, so there is exactly one
+copy of each parquet on disk -- not a package copy plus a test copy.
+"""
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+from importlib import resources
+from pathlib import Path
+
+import pandas as pd
+
+
+@contextmanager
+def sample_path(name: str) -> Iterator[Path]:
+    """Yield a real on-disk Path to `<name>.parquet`.
+
+    `importlib.resources.as_file` (not a bare `Path(__file__)`) is required
+    because a zipped/compressed install has no on-disk file to hand back
+    directly -- `as_file` materializes one (a temp copy, if needed) for the
+    duration of the `with` block.
+    """
+    with resources.as_file(resources.files(__name__) / f"{name}.parquet") as p:
+        yield p
+
+
+def read_sample(name: str) -> pd.DataFrame:
+    """Read `<name>.parquet` from this package's bundled schema samples."""
+    with sample_path(name) as p:
+        return pd.read_parquet(p)

ffl_bigquery/_transform_util.py

@@ -0,0 +1,18 @@
+"""Shared transform helper: reindex to schema columns + stamp ingested_at."""
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import pandas as pd
+
+from ffl_bigquery.schema import ColumnSpec, spec_names
+
+
+def align_to_schema(
+    df: pd.DataFrame, schema: list[ColumnSpec], *, stamp: bool = True
+) -> pd.DataFrame:
+    df = df.copy()
+    if stamp:
+        df["ingested_at"] = datetime.now(UTC)
+    # Reindex adds missing NULLABLE columns as NA and drops columns not in schema.
+    return df.reindex(columns=spec_names(schema))

ffl_bigquery/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

ffl_bigquery/adp/ffc.py

@@ -0,0 +1,66 @@
+"""Fantasy Football Calculator ADP fetch.
+
+Terms (their published API docs): free for personal and commercial use, attribution
+requested, data updates once daily — do not poll frequently.
+
+FFC signals "no data for this year/format" with HTTP 200 in two different shapes,
+both observed 2026-07-29:
+
+    standard/2007 -> {"status": "Success", "meta": {...}, "players": []}
+    ppr/2007      -> {"status": "Error", "errors": "No ADP data found."}
+
+Emptiness is therefore decided by the absence of players, never by `status`.
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+from ffl_bigquery.adp.schema import FFC_FORMATS
+from ffl_bigquery.http import ThrottledSession
+
+log = logging.getLogger(__name__)
+
+FFC_BASE_URL = "https://fantasyfootballcalculator.com/api/v1/adp"
+
+
+@dataclass(frozen=True)
+class FfcResponse:
+    players: list[dict]
+    total_drafts: int | None
+    window_start: str | None
+    window_end: str | None
+
+    @property
+    def is_empty(self) -> bool:
+        return not self.players
+
+
+def fetch_ffc(
+    session: ThrottledSession,
+    *,
+    season: int,
+    scoring_format: str,
+    teams: int,
+) -> FfcResponse:
+    if scoring_format not in FFC_FORMATS:
+        raise ValueError(
+            f"unknown scoring_format {scoring_format!r}; expected one of {FFC_FORMATS}"
+        )
+    payload = session.get_json(
+        f"{FFC_BASE_URL}/{scoring_format}", {"teams": teams, "year": season}
+    )
+    players = payload.get("players") or []
+    meta = payload.get("meta") or {}
+    if not players:
+        log.info(
+            "FFC empty: season=%s format=%s teams=%s status=%s errors=%s",
+            season, scoring_format, teams,
+            payload.get("status"), payload.get("errors"),
+        )
+    return FfcResponse(
+        players=list(players),
+        total_drafts=meta.get("total_drafts"),
+        window_start=meta.get("start_date"),
+        window_end=meta.get("end_date"),
+    )

ffl_bigquery/adp/mfl.py

@@ -0,0 +1,81 @@
+"""MyFantasyLeague ADP fetch (export?TYPE=adp).
+
+Free, no key, but the endpoint refuses requests without a real User-Agent.
+
+Three shapes matter, all observed 2026-07-29:
+  * every numeric field is a STRING ("averagePick": "3.28") — casting is the
+    transform's job, not this module's
+  * pre-2011 returns {"adp": {"totalPicks": "0", "totalDrafts": "0"}} with no
+    "player" key at all
+  * MFL's JSON derives from XML, so a single-element list collapses to a bare
+    object; "player" may be a dict
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+from ffl_bigquery.http import ThrottledSession
+
+log = logging.getLogger(__name__)
+
+MFL_BASE_URL = "https://api.myfantasyleague.com"
+
+
+def _as_int(value: Any) -> int | None:
+    if value is None or value == "":
+        return None
+    try:
+        return int(str(value).strip())
+    except (TypeError, ValueError):
+        return None
+
+
+@dataclass(frozen=True)
+class MflResponse:
+    players: list[dict]
+    total_drafts: int | None
+    total_picks: int | None
+
+    @property
+    def is_empty(self) -> bool:
+        return not self.players
+
+
+def fetch_mfl(
+    session: ThrottledSession,
+    *,
+    season: int,
+    teams: int | None = None,
+    is_ppr: bool | None = None,
+    is_keeper: bool | None = None,
+    is_mock: bool | None = None,
+) -> MflResponse:
+    params: dict[str, Any] = {"TYPE": "adp", "JSON": 1}
+    if teams is not None:
+        params["FCOUNT"] = teams
+    if is_ppr is not None:
+        params["IS_PPR"] = int(is_ppr)
+    if is_keeper is not None:
+        params["IS_KEEPER"] = int(is_keeper)
+    if is_mock is not None:
+        params["IS_MOCK"] = int(is_mock)
+
+    payload = session.get_json(f"{MFL_BASE_URL}/{season}/export", params)
+    adp = payload.get("adp") or {}
+    raw = adp.get("player")
+    if raw is None:
+        players: list[dict] = []
+    elif isinstance(raw, dict):
+        # XML-derived JSON collapses a single-element list into an object.
+        players = [raw]
+    else:
+        players = list(raw)
+    if not players:
+        log.info("MFL empty: season=%s params=%s", season, params)
+    return MflResponse(
+        players=players,
+        total_drafts=_as_int(adp.get("totalDrafts")),
+        total_picks=_as_int(adp.get("totalPicks")),
+    )

ffl_bigquery/adp/resolve.py

@@ -0,0 +1,132 @@
+"""Resolve ff_adp rows to nflverse gsis_id.
+
+The two sources take different paths, and this asymmetry is not incidental:
+
+  * MFL publishes its own player id, which IS one of the 20 id systems in
+    ff_playerids -> exact join on mfl_id.
+  * FFC publishes an internal player_id that appears in NO nflverse id system
+    (verified against all 20 columns) -> name-based join only.
+
+Ambiguity is refused rather than resolved. A wrong gsis_id silently attributes one
+player's draft market to another; a NULL is visible and countable.
+"""
+from __future__ import annotations
+
+import logging
+from typing import cast
+
+import pandas as pd
+
+log = logging.getLogger(__name__)
+
+# ffverse merge_name convention, verified against load_ff_playerids():
+# lowercase, drop name suffixes, drop periods and apostrophes, keep hyphens.
+_SUFFIXES = {"jr", "sr", "ii", "iii", "iv", "v"}
+
+
+def normalize_merge_name(name: str) -> str:
+    if not isinstance(name, str):
+        return ""
+    s = name.lower().replace(".", "").replace("'", "").replace("’", "")
+    tokens = [t for t in s.split() if t]
+    while len(tokens) > 1 and tokens[-1] in _SUFFIXES:
+        tokens.pop()
+    return " ".join(tokens)
+
+
+def _notna_rows(df: pd.DataFrame, col: str) -> pd.DataFrame:
+    """Boolean-mask row filtering (``df[df[col].notna()]``).
+
+    Without pandas-stubs, pyright resolves ``DataFrame.__getitem__(Series[bool])``
+    to an ambiguous overload, and the filtered frame's type then infects every
+    downstream attribute access on it (``.map``, ``.fillna``, ``.values``, ...)
+    with its own error. Asserting the known-correct return type once, at the
+    filter itself, is cheaper and more precise than re-suppressing each of
+    those cascaded call sites individually.
+    """
+    mask = cast(pd.Series, df[col].notna())
+    return cast(pd.DataFrame, df[mask])
+
+
+def _eq_mask(df: pd.DataFrame, col: str, value: str) -> pd.Series:
+    """Same ambiguity as `_notna_rows`, for an equality mask instead of notna."""
+    return cast(pd.Series, df[col].eq(value))
+
+
+def _mfl_map(xref: pd.DataFrame) -> pd.Series:
+    x = _notna_rows(xref, "gsis_id")
+    # pd.to_numeric(...).astype("Int64") is a genuine pandas-stub gap (also
+    # suppressed at writer.py:45 for the same reason); cast the result so the
+    # gap doesn't cascade into every downstream use of `keys` below.
+    keys = cast(pd.Series, pd.to_numeric(x["mfl_id"], errors="coerce").astype("Int64"))  # type: ignore[union-attr]
+    # Filter on the parsed key's own null-ness, NOT Series.dropna() on the
+    # constructed map: dropna() only drops NaN *values*, not a NaN *index*. A
+    # xref row with a populated gsis_id but a missing/non-numeric mfl_id would
+    # otherwise survive into the map at index pd.NA — and Series.map() treats
+    # that as a real match against any ADP row whose own malformed
+    # source_player_id also parsed to pd.NA, silently attributing one player's
+    # draft market to a different, unrelated player.
+    valid = keys.notna()
+    return pd.Series(x["gsis_id"].values[valid.to_numpy()], index=keys[valid])
+
+
+def _name_map(xref: pd.DataFrame) -> dict[tuple[str, str], str]:
+    x = _notna_rows(xref, "gsis_id").copy()
+    x["_key"] = list(
+        zip(
+            x["merge_name"].map(normalize_merge_name),
+            x["position"].fillna("").astype(str),
+            strict=False,
+        )
+    )
+    counts = x["_key"].value_counts()
+    # Refuse any (merge_name, position) that maps to more than one player.
+    # Series.map() genuinely accepts a Series as a value-lookup mapping (not
+    # just a function) at runtime; the bundled pandas stub's `.map()` overload
+    # only types the function form, so this one has no cast-able boundary.
+    unique = x[x["_key"].map(counts) == 1]  # type: ignore[arg-type]
+    dropped = len(x) - len(unique)
+    if dropped:
+        log.info("refused %d ambiguous name matches in xref", dropped)
+    return dict(zip(unique["_key"], unique["gsis_id"], strict=False))
+
+
+def resolve_gsis_ids(adp: pd.DataFrame, xref: pd.DataFrame) -> pd.DataFrame:
+    if adp.empty:
+        return adp
+    out = adp.copy()
+    resolved = pd.Series([pd.NA] * len(out), index=out.index, dtype="object")
+
+    is_mfl = _eq_mask(out, "source", "mfl")
+    if is_mfl.any():
+        # pd.to_numeric(...).astype("Int64") is the same genuine stub gap as
+        # in `_mfl_map` above.
+        ids = cast(
+            pd.Series,
+            pd.to_numeric(
+                out.loc[is_mfl, "source_player_id"], errors="coerce"
+            ).astype("Int64"),  # type: ignore[union-attr]
+        )
+        resolved.loc[is_mfl] = ids.map(_mfl_map(xref)).values
+
+    is_ffc = _eq_mask(out, "source", "ffc")
+    if is_ffc.any():
+        name_map = _name_map(xref)
+        keys = list(
+            zip(
+                out.loc[is_ffc, "player_name"].map(normalize_merge_name),
+                out.loc[is_ffc, "position"].fillna("").astype(str),
+                strict=False,
+            )
+        )
+        resolved.loc[is_ffc] = [name_map.get(k, pd.NA) for k in keys]
+
+    out["gsis_id"] = resolved
+    log.info("resolved %d/%d ADP rows to gsis_id", out["gsis_id"].notna().sum(), len(out))
+    return out
+
+
+def resolution_rate(adp: pd.DataFrame) -> float:
+    if adp.empty:
+        return 0.0
+    return float(adp["gsis_id"].notna().mean())

ffl_bigquery/adp/schema.py

@@ -0,0 +1,193 @@
+"""ff_adp: one row per player per ADP snapshot per league configuration.
+
+Snapshot grain is not an aesthetic choice. FFC's start_date/end_date query
+parameters are silently ignored (verified 2026-07-29: requests for 2026-06-01..15
+and 2026-07-01..15 both returned the identical current window), so intra-preseason
+ADP drift cannot be backfilled — it exists only if captured forward, daily.
+"""
+from __future__ import annotations
+
+from ffl_bigquery.partition import TimePartition
+from ffl_bigquery.schema import INGESTED_AT_SPEC, ColumnSpec
+
+ADP_SOURCES: tuple[str, ...] = ("ffc", "mfl")
+FFC_FORMATS: tuple[str, ...] = (
+    "standard", "ppr", "half-ppr", "2qb", "dynasty", "rookie",
+)
+
+FF_ADP_KEYS: list[str] = [
+    "source", "season", "scoring_format", "teams", "snapshot_date", "source_player_id",
+]
+
+FF_ADP_PARTITION = TimePartition(
+    field="snapshot_date",
+    clustering=["season", "source", "scoring_format", "teams"],
+)
+
+
+def _spec(
+    name: str,
+    type: str,
+    mode: str,
+    short: str,
+    definition: str,
+    *,
+    tags: list[str] | None = None,
+    valid_range: tuple[float, float] | None = None,
+    valid_values: list[str] | None = None,
+    example: object | None = None,
+    gotchas: list[str] | None = None,
+    source_field: str = "",
+) -> ColumnSpec:
+    return ColumnSpec(
+        name=name, type=type, mode=mode, short_description=short,  # type: ignore[arg-type]
+        business_definition=definition, semantic_tags=tags or [],
+        valid_range=valid_range, valid_values=valid_values, example_value=example,
+        gotchas=gotchas or [], source_field=source_field or name,
+        deprecated_in_year=None,
+    )
+
+
+FF_ADP_SCHEMA: list[ColumnSpec] = [
+    _spec("source", "STRING", "REQUIRED",
+          "Which ADP market this row came from.",
+          "Provider of the ADP observation. 'ffc' = Fantasy Football Calculator, "
+          "'mfl' = MyFantasyLeague. The two are independent markets and their ADP "
+          "values are not interchangeable.",
+          tags=["identifier", "primary_key", "dimension"],
+          valid_values=["ffc", "mfl"], example="ffc"),
+    _spec("season", "INT64", "REQUIRED",
+          "NFL season the draft was for.",
+          "Season year of the fantasy draft. FFC has data from 2010; MFL from 2011. "
+          "Earlier years return empty responses.",
+          tags=["identifier", "primary_key"], valid_range=(2010.0, 2100.0), example=2026),
+    _spec("scoring_format", "STRING", "REQUIRED",
+          "Scoring system of the drafts aggregated into this ADP.",
+          "League scoring format. FFC exposes standard/ppr/half-ppr/2qb/dynasty/rookie, "
+          "though half-ppr history is shallow. For MFL rows this is derived from the "
+          "IS_PPR request parameter.",
+          tags=["identifier", "primary_key", "dimension"], example="ppr",
+          gotchas=["half-ppr returned no data for 2010 or 2015; treat pre-recent "
+                   "half-ppr as unavailable rather than zero."]),
+    _spec("teams", "INT64", "REQUIRED",
+          "League size the ADP was aggregated over.",
+          "Number of teams in the drafts aggregated into this ADP value (FFC 'teams' "
+          "parameter, MFL 'FCOUNT'). ADP is only comparable within a league size.",
+          tags=["identifier", "primary_key", "dimension"],
+          valid_range=(4.0, 32.0), example=12),
+    _spec("snapshot_date", "DATE", "REQUIRED",
+          "Date this ADP observation was captured.",
+          "UTC date the sync ran. This is the drift axis: the same (source, season, "
+          "format, teams, player) appears once per capture day, so day-over-day "
+          "movement is queryable. Not the date the drafts occurred.",
+          tags=["identifier", "primary_key", "partition_key"], example="2026-07-29",
+          gotchas=["Rows only exist for days the cron actually ran. Gaps are real "
+                   "gaps, not zero-movement days."]),
+    _spec("source_player_id", "STRING", "REQUIRED",
+          "The provider's own player identifier.",
+          "Player id as issued by the source. For 'mfl' this is the MFL id and joins "
+          "ff_player_xref.mfl_id directly. For 'ffc' this is FFC's internal id, which "
+          "appears in NO nflverse id system — FFC rows resolve by name instead.",
+          tags=["identifier", "primary_key"], example="925",
+          gotchas=["FFC ids are not portable. Never attempt to join them to an "
+                   "nflverse id column."]),
+    _spec("gsis_id", "STRING", "NULLABLE",
+          "Resolved nflverse player id, if resolution succeeded.",
+          "Canonical nflverse gsis_id, resolved via ff_player_xref. NULL when "
+          "resolution failed. Unresolved rows are retained deliberately — dropping "
+          "them would silently thin the market.",
+          tags=["identifier", "join_key"], example="00-0033873",
+          gotchas=["Always NULL-guard before joining to nfl_plays or "
+                   "ff_points_weekly, or unresolved players vanish from results."]),
+    _spec("player_name", "STRING", "NULLABLE",
+          "Player name as published by the source.",
+          "Source-provided display name. Present for FFC; absent for MFL, whose ADP "
+          "export returns ids only.",
+          tags=["dimension"], example="Adrian Peterson"),
+    _spec("position", "STRING", "NULLABLE",
+          "Player position as published by the source.",
+          "Source-provided position. Present for FFC; absent for MFL.",
+          tags=["dimension"], example="RB"),
+    _spec("team", "STRING", "NULLABLE",
+          "NFL team as published by the source.",
+          "Source-provided team abbreviation at time of capture. Present for FFC; "
+          "absent for MFL. May disagree with nflverse abbreviations.",
+          tags=["dimension"], example="MIN"),
+    _spec("adp", "FLOAT64", "REQUIRED",
+          "Average draft position.",
+          "Mean overall pick number at which this player was selected across the "
+          "aggregated drafts. Lower is earlier.",
+          tags=["metric"], valid_range=(1.0, 500.0), example=1.8),
+    _spec("adp_formatted", "STRING", "NULLABLE",
+          "ADP rendered as round.pick.",
+          "FFC's human-readable round.pick form of adp, e.g. '1.02'. Provided by FFC "
+          "only; NULL for MFL.",
+          tags=["metric"], example="1.02"),
+    _spec("adp_stdev", "FLOAT64", "NULLABLE",
+          "Standard deviation of pick number.",
+          "Dispersion of the pick numbers behind adp — a consensus measure. High "
+          "stdev means drafters disagreed. FFC only; NULL for MFL.",
+          tags=["metric"], valid_range=(0.0, 200.0), example=1.0),
+    _spec("adp_earliest_pick", "INT64", "NULLABLE",
+          "Earliest (smallest) pick number observed.",
+          "The soonest this player was taken in any aggregated draft. Maps from FFC's "
+          "'high' field and MFL's 'minPick'.",
+          tags=["metric"], valid_range=(1.0, 500.0), example=1,
+          source_field="high|minPick",
+          gotchas=["FFC calls this 'high' because it is the highest draft position — "
+                   "which is the LOWEST number. Renamed here to prevent inverted "
+                   "comparisons."]),
+    _spec("adp_latest_pick", "INT64", "NULLABLE",
+          "Latest (largest) pick number observed.",
+          "The latest this player was taken in any aggregated draft. Maps from FFC's "
+          "'low' field and MFL's 'maxPick'.",
+          tags=["metric"], valid_range=(1.0, 500.0), example=5,
+          source_field="low|maxPick"),
+    _spec("times_drafted", "INT64", "NULLABLE",
+          "Drafts this player was selected in.",
+          "Count of aggregated drafts in which this player was taken. FFC "
+          "'times_drafted'; MFL 'draftsSelectedIn'. The denominator for "
+          "draft_selected_pct.",
+          tags=["metric"], example=329, source_field="times_drafted|draftsSelectedIn"),
+    _spec("draft_selected_pct", "FLOAT64", "NULLABLE",
+          "Share of drafts the player was selected in.",
+          "MFL 'draftSelPct' as a percentage (0-100). NULL for FFC.",
+          tags=["metric"], valid_range=(0.0, 100.0), example=13.0,
+          source_field="draftSelPct"),
+    _spec("source_rank", "INT64", "NULLABLE",
+          "The provider's own ADP rank ordering.",
+          "Rank by adp as published by the source. MFL 'rank'; NULL for FFC, where "
+          "row order conveys rank.",
+          tags=["metric"], example=1, source_field="rank"),
+    _spec("total_drafts", "INT64", "NULLABLE",
+          "Drafts in the aggregation pool.",
+          "Total drafts the provider aggregated for this request. FFC "
+          "meta.total_drafts; MFL adp.totalDrafts. A sample-size caveat: a 2026 "
+          "July snapshot may rest on far fewer drafts than a September one.",
+          tags=["metric", "quality"], example=3673),
+    _spec("bye", "INT64", "NULLABLE",
+          "Player's bye week that season.",
+          "Bye week as published by FFC. NULL for MFL.",
+          tags=["dimension"], valid_range=(1.0, 18.0), example=6),
+    _spec("window_start_date", "DATE", "NULLABLE",
+          "First draft date in the provider's aggregation window.",
+          "FFC meta.start_date — the earliest draft included. NULL for MFL, which "
+          "does not publish a window.",
+          tags=["quality"], example="2026-07-22",
+          gotchas=["This is the provider's window, NOT a request parameter. FFC "
+                   "ignores start_date/end_date on input."]),
+    _spec("window_end_date", "DATE", "NULLABLE",
+          "Last draft date in the provider's aggregation window.",
+          "FFC meta.end_date — the latest draft included. NULL for MFL.",
+          tags=["quality"], example="2026-07-29"),
+    _spec("is_keeper", "BOOL", "NULLABLE",
+          "Whether keeper leagues were included.",
+          "MFL IS_KEEPER request slicer. NULL for FFC, which does not expose it.",
+          tags=["dimension"], example=False),
+    _spec("is_mock", "BOOL", "NULLABLE",
+          "Whether mock drafts were included.",
+          "MFL IS_MOCK request slicer. NULL for FFC. Mock-only ADP rests on very "
+          "few drafts (measured: 12 for 2025) and should be read with care.",
+          tags=["dimension"], example=False),
+    INGESTED_AT_SPEC,
+]