ffl-bigquery 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/CHANGELOG.md +20 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/PKG-INFO +1 -1
- ffl_bigquery-0.3.3/ffl_bigquery/_atomic.py +83 -0
- ffl_bigquery-0.3.3/ffl_bigquery/_version.py +1 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/writer.py +8 -17
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/pyproject.toml +1 -1
- ffl_bigquery-0.3.3/tests/test_writer_atomic.py +103 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_writer_write_season.py +5 -5
- ffl_bigquery-0.3.2/ffl_bigquery/_version.py +0 -1
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/.github/workflows/ci.yml +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/.github/workflows/release.yml +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/.gitignore +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/LICENSE +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/README.md +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/bash.exe.stackdump +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/ff_opportunity.parquet +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/ff_rankings.parquet +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/ftn_charting.parquet +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/injuries.parquet +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/participation.parquet +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/snap_counts.parquet +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_transform_util.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/ffc.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/mfl.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/resolve.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/schema.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/sync.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/transform.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/cli.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coaches/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coaches/schema.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coaches/sync.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coaches/transform.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coordinators/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coordinators/schema.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coordinators/sync.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coordinators/wikipedia.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/dst_scoring.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/dst_weekly.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/kicker_scoring.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/kicker_weekly.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/personnel.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/points_weekly.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/scheme_week.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/http.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/driver.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/runs.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/spec.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/depth_charts.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/ftn_charting.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/injuries.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/opportunity.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/participation.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/rankings.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/snap_counts.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/partition.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/runs.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/schema.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/schema_gen.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/verify/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/verify/adp.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/verify/tables.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/xref/__init__.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/xref/schema.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/xref/sync.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/xref/transform.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/conftest.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/ff_playerids_sample.csv +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/ffc_empty_error.json +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/ffc_empty_success.json +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/ffc_ppr_2015.json +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/mfl_adp_2015.json +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/mfl_adp_empty_2005.json +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/mfl_adp_single_player.json +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/nflverse/depth_charts_legacy.parquet +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/nflverse/depth_charts_modern.parquet +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/wikipedia_seasons.json +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_ffc.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_mfl.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_resolve.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_schema.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_sync.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_transform.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_cli.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_clustering_types_are_valid.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_coaches.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_coordinators.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_depth_charts.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_dst_scoring.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_dst_weekly.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_http.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_kicker_scoring.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_kicker_weekly.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_nflverse_driver.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_nflverse_runs.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_nflverse_tables_registry.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_nflverse_tables_simple.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_partition.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_personnel.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_play_level_tables.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_points_weekly.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_runs.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_schema.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_schema_gen.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_scheme_week.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_verify_adp.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_verify_dst.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_verify_kicker.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_verify_tables.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_writer.py +0 -0
- {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_xref.py +0 -0
|
@@ -2,6 +2,26 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## 0.3.3 — 2026-11-21
|
|
6
|
+
|
|
7
|
+
Season writes are now atomic. `BigQueryWriter.write_season` ran
|
|
8
|
+
`DELETE ... WHERE season = @season` as its own query and then a separate load job. A
|
|
9
|
+
BigQuery load job can't join a transaction, so those were two commits: if the load
|
|
10
|
+
failed, the season's rows were already gone and stayed gone until the next successful
|
|
11
|
+
run. That is the failure that wiped 2026-06-02 from statcast-bigquery. It hit every
|
|
12
|
+
season-chunked nflverse table here, and each write replaces a whole season.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- `write_season` loads into a short-lived `_staging_*` table first (6-hour expiry as a
|
|
17
|
+
backstop), then runs `DELETE` + `INSERT ... SELECT` in one multi-statement
|
|
18
|
+
transaction, then drops the staging table (`ffl_bigquery/_atomic.py`). A failed load
|
|
19
|
+
no longer deletes anything; a failed swap rolls back. An empty season is still a
|
|
20
|
+
DELETE-only write, now inside a transaction. Idempotency is unchanged: re-running a
|
|
21
|
+
season still replaces it wholesale.
|
|
22
|
+
- `merge_rows` (ADP, xref, rankings, coordinators) was already safe: a single `MERGE`
|
|
23
|
+
statement.
|
|
24
|
+
|
|
5
25
|
## 0.3.2 — 2026-09-29
|
|
6
26
|
|
|
7
27
|
`ff_rankings` MERGE key now includes `page_type`: `(scrape_date, ecr_type, page_type, id)`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ffl-bigquery
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Historical fantasy football ADP, usage, and coaching/scheme data → BigQuery: the fantasy layer on top of nfl-bigquery
|
|
5
5
|
Project-URL: Homepage, https://github.com/blahovec-labs/ffl-bigquery
|
|
6
6
|
Project-URL: Issues, https://github.com/blahovec-labs/ffl-bigquery/issues
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Atomic replace-a-slice for the BigQuery writers.
|
|
2
|
+
|
|
3
|
+
A BigQuery load job cannot run inside a multi-statement transaction, so a plain
|
|
4
|
+
"DELETE, then load" is two independent commits: if the load fails, the DELETE has
|
|
5
|
+
already happened and the slice is simply gone. That is how statcast-bigquery lost
|
|
6
|
+
every 2026-06-02 pitch; this is the same helper as its 0.5.1 fix.
|
|
7
|
+
|
|
8
|
+
So: load the new rows into a short-lived staging table first (the step that
|
|
9
|
+
actually fails), then DELETE + INSERT ... SELECT in ONE transaction, then drop the
|
|
10
|
+
staging table. A failed load never touches the target; a failed swap rolls back.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import datetime as _dt
|
|
16
|
+
import logging
|
|
17
|
+
import uuid
|
|
18
|
+
|
|
19
|
+
import pandas as pd
|
|
20
|
+
from google.cloud import bigquery
|
|
21
|
+
|
|
22
|
+
log = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
# Safety net: if the process dies between load and drop, BigQuery expires it.
|
|
25
|
+
_STAGING_TTL = _dt.timedelta(hours=6)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def atomic_replace(
|
|
29
|
+
client: bigquery.Client,
|
|
30
|
+
target: str,
|
|
31
|
+
df: pd.DataFrame,
|
|
32
|
+
schema: list[bigquery.SchemaField],
|
|
33
|
+
where_sql: str,
|
|
34
|
+
query_parameters: list | None = None,
|
|
35
|
+
) -> int:
|
|
36
|
+
"""Replace the rows of `target` matching `where_sql` with `df`, atomically.
|
|
37
|
+
|
|
38
|
+
`target` is "project.dataset.table". An empty `df` is a DELETE-only swap
|
|
39
|
+
(still transactional, no staging table). Returns len(df).
|
|
40
|
+
"""
|
|
41
|
+
job_config = bigquery.QueryJobConfig(query_parameters=query_parameters or [])
|
|
42
|
+
delete_stmt = f"DELETE FROM `{target}` WHERE {where_sql};"
|
|
43
|
+
|
|
44
|
+
if len(df) == 0:
|
|
45
|
+
client.query_and_wait(
|
|
46
|
+
f"BEGIN TRANSACTION;\n{delete_stmt}\nCOMMIT TRANSACTION;",
|
|
47
|
+
job_config=job_config,
|
|
48
|
+
)
|
|
49
|
+
return 0
|
|
50
|
+
|
|
51
|
+
project, dataset, table = target.split(".")
|
|
52
|
+
staging = f"{project}.{dataset}._staging_{table}_{uuid.uuid4().hex[:12]}"
|
|
53
|
+
staging_table = bigquery.Table(staging, schema=schema)
|
|
54
|
+
staging_table.expires = _dt.datetime.now(_dt.UTC) + _STAGING_TTL
|
|
55
|
+
|
|
56
|
+
columns = ", ".join(f"`{f.name}`" for f in schema)
|
|
57
|
+
try:
|
|
58
|
+
client.create_table(staging_table)
|
|
59
|
+
log.info("load %d rows -> staging %s", len(df), staging)
|
|
60
|
+
client.load_table_from_dataframe(
|
|
61
|
+
df,
|
|
62
|
+
staging,
|
|
63
|
+
job_config=bigquery.LoadJobConfig(
|
|
64
|
+
schema=schema,
|
|
65
|
+
write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
|
|
66
|
+
source_format=bigquery.SourceFormat.PARQUET,
|
|
67
|
+
),
|
|
68
|
+
).result()
|
|
69
|
+
|
|
70
|
+
log.info("swap %s WHERE %s (atomic)", target, where_sql)
|
|
71
|
+
client.query_and_wait(
|
|
72
|
+
"BEGIN TRANSACTION;\n"
|
|
73
|
+
f"{delete_stmt}\n"
|
|
74
|
+
f"INSERT INTO `{target}` ({columns}) SELECT {columns} FROM `{staging}`;\n"
|
|
75
|
+
"COMMIT TRANSACTION;",
|
|
76
|
+
job_config=job_config,
|
|
77
|
+
)
|
|
78
|
+
finally:
|
|
79
|
+
try:
|
|
80
|
+
client.delete_table(staging, not_found_ok=True)
|
|
81
|
+
except Exception: # never mask the real error; TTL cleans up regardless
|
|
82
|
+
log.warning("could not drop staging table %s (expires in %s)", staging, _STAGING_TTL)
|
|
83
|
+
return len(df)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.3"
|
|
@@ -14,6 +14,7 @@ import pandas as pd
|
|
|
14
14
|
from google.cloud import bigquery
|
|
15
15
|
from google.cloud.exceptions import NotFound
|
|
16
16
|
|
|
17
|
+
from ffl_bigquery._atomic import atomic_replace
|
|
17
18
|
from ffl_bigquery.partition import ClusterOnly, SeasonRangePartition, TimePartition
|
|
18
19
|
from ffl_bigquery.schema import ColumnSpec, to_bq_schema
|
|
19
20
|
|
|
@@ -195,7 +196,7 @@ class BigQueryWriter:
|
|
|
195
196
|
season: int,
|
|
196
197
|
schema: list[ColumnSpec],
|
|
197
198
|
) -> WriteSeasonResult:
|
|
198
|
-
"""
|
|
199
|
+
"""Replace the rows for `season` with `df`, atomically. Returns a
|
|
199
200
|
`WriteSeasonResult` (an int of rows actually written, plus `.dropped`).
|
|
200
201
|
|
|
201
202
|
The DELETE runs even when df is empty: a season that legitimately goes
|
|
@@ -224,15 +225,9 @@ class BigQueryWriter:
|
|
|
224
225
|
passed through unchanged -- some callers legitimately have none.
|
|
225
226
|
"""
|
|
226
227
|
bq_schema = to_bq_schema(schema)
|
|
227
|
-
|
|
228
|
-
f"DELETE FROM `{ref}` WHERE season = @season",
|
|
229
|
-
job_config=bigquery.QueryJobConfig(
|
|
230
|
-
query_parameters=[
|
|
231
|
-
bigquery.ScalarQueryParameter("season", "INT64", season)
|
|
232
|
-
],
|
|
233
|
-
),
|
|
234
|
-
).result()
|
|
228
|
+
params = [bigquery.ScalarQueryParameter("season", "INT64", season)]
|
|
235
229
|
if df.empty:
|
|
230
|
+
atomic_replace(self.client, str(ref), df, bq_schema, "season = @season", params)
|
|
236
231
|
log.info("season %s of %s is empty; deleted only", season, ref)
|
|
237
232
|
return WriteSeasonResult(0, 0)
|
|
238
233
|
|
|
@@ -255,17 +250,13 @@ class BigQueryWriter:
|
|
|
255
250
|
df = df.loc[~(is_null | is_mismatched)]
|
|
256
251
|
|
|
257
252
|
if df.empty:
|
|
253
|
+
atomic_replace(self.client, str(ref), df, bq_schema, "season = @season", params)
|
|
258
254
|
log.info(
|
|
259
255
|
"season %s of %s: all rows dropped by season guard", season, ref,
|
|
260
256
|
)
|
|
261
257
|
return WriteSeasonResult(0, dropped)
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
job_config=bigquery.LoadJobConfig(
|
|
266
|
-
write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
|
|
267
|
-
schema=bq_schema,
|
|
268
|
-
),
|
|
269
|
-
).result()
|
|
258
|
+
# Stage, then DELETE + INSERT in one transaction: a failed load leaves the
|
|
259
|
+
# season's existing rows untouched instead of deleted (see _atomic.py).
|
|
260
|
+
atomic_replace(self.client, str(ref), df, bq_schema, "season = @season", params)
|
|
270
261
|
log.info("wrote %d rows to %s season=%s", len(df), ref, season)
|
|
271
262
|
return WriteSeasonResult(len(df), dropped)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""write_season must never leave a season deleted but not replaced.
|
|
2
|
+
|
|
3
|
+
It used to run `DELETE ... WHERE season = @season` as its own query and then a
|
|
4
|
+
separate load job. A BigQuery load job can't join a transaction, so those were two
|
|
5
|
+
commits: a failed load left the whole season gone until the next successful run.
|
|
6
|
+
That is the failure that wiped 2026-06-02 from statcast-bigquery (fixed there in
|
|
7
|
+
0.5.1).
|
|
8
|
+
|
|
9
|
+
Contract now: load into a short-lived staging table FIRST, then DELETE + INSERT ...
|
|
10
|
+
SELECT in ONE multi-statement transaction, then drop the staging table. A failed
|
|
11
|
+
load issues no DELETE at all; a failed swap rolls back.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import cast
|
|
16
|
+
from unittest.mock import MagicMock
|
|
17
|
+
|
|
18
|
+
import pandas as pd
|
|
19
|
+
import pytest
|
|
20
|
+
from google.cloud import bigquery
|
|
21
|
+
|
|
22
|
+
from ffl_bigquery.schema import BqMode, BqType, ColumnSpec
|
|
23
|
+
from ffl_bigquery.writer import BigQueryWriter, TableRef
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _spec(name: str, type: str = "STRING", mode: str = "NULLABLE") -> ColumnSpec:
|
|
27
|
+
return ColumnSpec(
|
|
28
|
+
name=name, type=cast(BqType, type), mode=cast(BqMode, mode),
|
|
29
|
+
short_description="x", business_definition="x", semantic_tags=[],
|
|
30
|
+
valid_range=None, valid_values=None, example_value=None, gotchas=[],
|
|
31
|
+
source_field=name, deprecated_in_year=None,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
SCHEMA = [_spec("season", "INT64", "REQUIRED"), _spec("val", "FLOAT64")]
|
|
36
|
+
REF = TableRef.parse("p.d.tbl")
|
|
37
|
+
TARGET = "p.d.tbl"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _df() -> pd.DataFrame:
|
|
41
|
+
return pd.DataFrame({"season": [2025, 2025], "val": [1.0, 2.0]})
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _sql(client: MagicMock) -> list[str]:
|
|
45
|
+
return [c.args[0] for c in client.query_and_wait.call_args_list] + [
|
|
46
|
+
c.args[0] for c in client.query.call_args_list
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_failed_load_deletes_nothing():
|
|
51
|
+
c = MagicMock(spec=bigquery.Client)
|
|
52
|
+
c.load_table_from_dataframe.return_value.result.side_effect = RuntimeError(
|
|
53
|
+
"400 load failed"
|
|
54
|
+
)
|
|
55
|
+
with pytest.raises(RuntimeError):
|
|
56
|
+
BigQueryWriter(client=c).write_season(REF, _df(), season=2025, schema=SCHEMA)
|
|
57
|
+
assert not any("DELETE" in sql for sql in _sql(c))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_load_targets_staging_not_the_live_table():
|
|
61
|
+
c = MagicMock(spec=bigquery.Client)
|
|
62
|
+
BigQueryWriter(client=c).write_season(REF, _df(), season=2025, schema=SCHEMA)
|
|
63
|
+
destination = str(c.load_table_from_dataframe.call_args.args[1])
|
|
64
|
+
assert destination != TARGET
|
|
65
|
+
assert "_staging_" in destination
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_delete_and_insert_commit_in_one_transaction():
|
|
69
|
+
c = MagicMock(spec=bigquery.Client)
|
|
70
|
+
BigQueryWriter(client=c).write_season(REF, _df(), season=2025, schema=SCHEMA)
|
|
71
|
+
swaps = [sql for sql in _sql(c) if "DELETE" in sql]
|
|
72
|
+
assert len(swaps) == 1
|
|
73
|
+
sql = swaps[0]
|
|
74
|
+
assert sql.lstrip().startswith("BEGIN TRANSACTION")
|
|
75
|
+
assert sql.rstrip().endswith("COMMIT TRANSACTION;")
|
|
76
|
+
assert "DELETE FROM `p.d.tbl` WHERE season = @season" in sql
|
|
77
|
+
assert sql.index("DELETE") < sql.index("INSERT INTO")
|
|
78
|
+
staging = str(c.load_table_from_dataframe.call_args.args[1])
|
|
79
|
+
assert f"FROM `{staging}`" in sql
|
|
80
|
+
params = c.query_and_wait.call_args.kwargs["job_config"].query_parameters
|
|
81
|
+
assert params[0].name == "season" and params[0].value == 2025
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_staging_table_dropped_even_when_swap_fails():
|
|
85
|
+
c = MagicMock(spec=bigquery.Client)
|
|
86
|
+
c.query_and_wait.side_effect = RuntimeError("transaction aborted")
|
|
87
|
+
with pytest.raises(RuntimeError):
|
|
88
|
+
BigQueryWriter(client=c).write_season(REF, _df(), season=2025, schema=SCHEMA)
|
|
89
|
+
staging = str(c.load_table_from_dataframe.call_args.args[1])
|
|
90
|
+
dropped = [str(call.args[0]) for call in c.delete_table.call_args_list]
|
|
91
|
+
assert staging in dropped
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_empty_season_is_a_transactional_delete_with_no_staging():
|
|
95
|
+
c = MagicMock(spec=bigquery.Client)
|
|
96
|
+
BigQueryWriter(client=c).write_season(
|
|
97
|
+
REF, pd.DataFrame(columns=["season", "val"]), season=2025, schema=SCHEMA,
|
|
98
|
+
)
|
|
99
|
+
assert not c.load_table_from_dataframe.called
|
|
100
|
+
assert not c.create_table.called
|
|
101
|
+
swaps = [sql for sql in _sql(c) if "DELETE" in sql]
|
|
102
|
+
assert len(swaps) == 1
|
|
103
|
+
assert swaps[0].lstrip().startswith("BEGIN TRANSACTION")
|
|
@@ -21,14 +21,14 @@ SCHEMA = [_spec("season", "INT64", "REQUIRED"), _spec("val", "FLOAT64")]
|
|
|
21
21
|
REF = TableRef.parse("p.d.tbl")
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
def
|
|
24
|
+
def test_write_season_replaces_that_season():
|
|
25
25
|
c = MagicMock(spec=bigquery.Client)
|
|
26
26
|
df = pd.DataFrame({"season": [2020, 2020], "val": [1.0, 2.0]})
|
|
27
27
|
n = BigQueryWriter(client=c).write_season(REF, df, season=2020, schema=SCHEMA)
|
|
28
28
|
assert n == 2
|
|
29
|
-
sql = c.
|
|
29
|
+
sql = c.query_and_wait.call_args[0][0]
|
|
30
30
|
assert "DELETE FROM `p.d.tbl` WHERE season = @season" in sql
|
|
31
|
-
params = c.
|
|
31
|
+
params = c.query_and_wait.call_args.kwargs["job_config"].query_parameters
|
|
32
32
|
assert params[0].name == "season" and params[0].value == 2020
|
|
33
33
|
assert c.load_table_from_dataframe.called
|
|
34
34
|
|
|
@@ -48,7 +48,7 @@ def test_write_season_still_deletes_when_df_is_empty():
|
|
|
48
48
|
REF, pd.DataFrame(columns=["season", "val"]), season=2020, schema=SCHEMA,
|
|
49
49
|
)
|
|
50
50
|
assert n == 0
|
|
51
|
-
assert "DELETE FROM" in c.
|
|
51
|
+
assert "DELETE FROM" in c.query_and_wait.call_args[0][0]
|
|
52
52
|
assert not c.load_table_from_dataframe.called
|
|
53
53
|
|
|
54
54
|
|
|
@@ -109,7 +109,7 @@ def test_write_season_deletes_even_when_filtering_empties_the_frame():
|
|
|
109
109
|
df = pd.DataFrame({"season": [2026, 2026], "val": [1.0, 2.0]})
|
|
110
110
|
n = BigQueryWriter(client=c).write_season(REF, df, season=2025, schema=SCHEMA)
|
|
111
111
|
assert n == 0
|
|
112
|
-
assert "DELETE FROM" in c.
|
|
112
|
+
assert "DELETE FROM" in c.query_and_wait.call_args[0][0]
|
|
113
113
|
assert not c.load_table_from_dataframe.called
|
|
114
114
|
|
|
115
115
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.3.2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/ff_opportunity.parquet
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/participation.parquet
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/nflverse/depth_charts_legacy.parquet
RENAMED
|
File without changes
|
{ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/nflverse/depth_charts_modern.parquet
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|