ffl-bigquery 0.3.2__tar.gz → 0.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/CHANGELOG.md +20 -0
  2. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/PKG-INFO +1 -1
  3. ffl_bigquery-0.3.3/ffl_bigquery/_atomic.py +83 -0
  4. ffl_bigquery-0.3.3/ffl_bigquery/_version.py +1 -0
  5. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/writer.py +8 -17
  6. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/pyproject.toml +1 -1
  7. ffl_bigquery-0.3.3/tests/test_writer_atomic.py +103 -0
  8. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_writer_write_season.py +5 -5
  9. ffl_bigquery-0.3.2/ffl_bigquery/_version.py +0 -1
  10. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/.github/workflows/ci.yml +0 -0
  11. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/.github/workflows/release.yml +0 -0
  12. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/.gitignore +0 -0
  13. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/LICENSE +0 -0
  14. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/README.md +0 -0
  15. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/bash.exe.stackdump +0 -0
  16. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/__init__.py +0 -0
  17. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/__init__.py +0 -0
  18. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/ff_opportunity.parquet +0 -0
  19. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/ff_rankings.parquet +0 -0
  20. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/ftn_charting.parquet +0 -0
  21. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/injuries.parquet +0 -0
  22. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/participation.parquet +0 -0
  23. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_schema_samples/snap_counts.parquet +0 -0
  24. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/_transform_util.py +0 -0
  25. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/__init__.py +0 -0
  26. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/ffc.py +0 -0
  27. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/mfl.py +0 -0
  28. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/resolve.py +0 -0
  29. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/schema.py +0 -0
  30. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/sync.py +0 -0
  31. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/adp/transform.py +0 -0
  32. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/cli.py +0 -0
  33. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coaches/__init__.py +0 -0
  34. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coaches/schema.py +0 -0
  35. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coaches/sync.py +0 -0
  36. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coaches/transform.py +0 -0
  37. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coordinators/__init__.py +0 -0
  38. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coordinators/schema.py +0 -0
  39. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coordinators/sync.py +0 -0
  40. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/coordinators/wikipedia.py +0 -0
  41. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/__init__.py +0 -0
  42. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/dst_scoring.py +0 -0
  43. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/dst_weekly.py +0 -0
  44. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/kicker_scoring.py +0 -0
  45. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/kicker_weekly.py +0 -0
  46. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/personnel.py +0 -0
  47. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/points_weekly.py +0 -0
  48. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/derive/scheme_week.py +0 -0
  49. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/http.py +0 -0
  50. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/__init__.py +0 -0
  51. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/driver.py +0 -0
  52. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/runs.py +0 -0
  53. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/spec.py +0 -0
  54. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/__init__.py +0 -0
  55. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/depth_charts.py +0 -0
  56. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/ftn_charting.py +0 -0
  57. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/injuries.py +0 -0
  58. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/opportunity.py +0 -0
  59. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/participation.py +0 -0
  60. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/rankings.py +0 -0
  61. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/nflverse/tables/snap_counts.py +0 -0
  62. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/partition.py +0 -0
  63. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/runs.py +0 -0
  64. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/schema.py +0 -0
  65. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/schema_gen.py +0 -0
  66. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/verify/__init__.py +0 -0
  67. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/verify/adp.py +0 -0
  68. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/verify/tables.py +0 -0
  69. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/xref/__init__.py +0 -0
  70. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/xref/schema.py +0 -0
  71. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/xref/sync.py +0 -0
  72. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/ffl_bigquery/xref/transform.py +0 -0
  73. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/conftest.py +0 -0
  74. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/ff_playerids_sample.csv +0 -0
  75. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/ffc_empty_error.json +0 -0
  76. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/ffc_empty_success.json +0 -0
  77. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/ffc_ppr_2015.json +0 -0
  78. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/mfl_adp_2015.json +0 -0
  79. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/mfl_adp_empty_2005.json +0 -0
  80. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/mfl_adp_single_player.json +0 -0
  81. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/nflverse/depth_charts_legacy.parquet +0 -0
  82. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/nflverse/depth_charts_modern.parquet +0 -0
  83. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/fixtures/wikipedia_seasons.json +0 -0
  84. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_ffc.py +0 -0
  85. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_mfl.py +0 -0
  86. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_resolve.py +0 -0
  87. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_schema.py +0 -0
  88. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_sync.py +0 -0
  89. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_adp_transform.py +0 -0
  90. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_cli.py +0 -0
  91. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_clustering_types_are_valid.py +0 -0
  92. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_coaches.py +0 -0
  93. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_coordinators.py +0 -0
  94. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_depth_charts.py +0 -0
  95. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_dst_scoring.py +0 -0
  96. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_dst_weekly.py +0 -0
  97. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_http.py +0 -0
  98. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_kicker_scoring.py +0 -0
  99. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_kicker_weekly.py +0 -0
  100. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_nflverse_driver.py +0 -0
  101. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_nflverse_runs.py +0 -0
  102. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_nflverse_tables_registry.py +0 -0
  103. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_nflverse_tables_simple.py +0 -0
  104. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_partition.py +0 -0
  105. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_personnel.py +0 -0
  106. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_play_level_tables.py +0 -0
  107. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_points_weekly.py +0 -0
  108. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_runs.py +0 -0
  109. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_schema.py +0 -0
  110. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_schema_gen.py +0 -0
  111. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_scheme_week.py +0 -0
  112. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_verify_adp.py +0 -0
  113. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_verify_dst.py +0 -0
  114. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_verify_kicker.py +0 -0
  115. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_verify_tables.py +0 -0
  116. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_writer.py +0 -0
  117. {ffl_bigquery-0.3.2 → ffl_bigquery-0.3.3}/tests/test_xref.py +0 -0
@@ -2,6 +2,26 @@
2
2
 
3
3
  All notable changes to this project will be documented in this file.
4
4
 
5
+ ## 0.3.3 — 2026-11-21
6
+
7
+ Season writes are now atomic. `BigQueryWriter.write_season` ran
8
+ `DELETE ... WHERE season = @season` as its own query and then a separate load job. A
9
+ BigQuery load job can't join a transaction, so those were two commits: if the load
10
+ failed, the season's rows were already gone and stayed gone until the next successful
11
+ run. That is the failure that wiped 2026-06-02 from statcast-bigquery. It hit every
12
+ season-chunked nflverse table here, and each write replaces a whole season.
13
+
14
+ ### Fixed
15
+
16
+ - `write_season` loads into a short-lived `_staging_*` table first (6-hour expiry as a
17
+ backstop), then runs `DELETE` + `INSERT ... SELECT` in one multi-statement
18
+ transaction, then drops the staging table (`ffl_bigquery/_atomic.py`). A failed load
19
+ no longer deletes anything; a failed swap rolls back. An empty season is still a
20
+ DELETE-only write, now inside a transaction. Idempotency is unchanged: re-running a
21
+ season still replaces it wholesale.
22
+ - `merge_rows` (ADP, xref, rankings, coordinators) was already safe: a single `MERGE`
23
+ statement.
24
+
5
25
  ## 0.3.2 — 2026-09-29
6
26
 
7
27
  `ff_rankings` MERGE key now includes `page_type`: `(scrape_date, ecr_type, page_type, id)`.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: ffl-bigquery
3
- Version: 0.3.2
3
+ Version: 0.3.3
4
4
  Summary: Historical fantasy football ADP, usage, and coaching/scheme data → BigQuery: the fantasy layer on top of nfl-bigquery
5
5
  Project-URL: Homepage, https://github.com/blahovec-labs/ffl-bigquery
6
6
  Project-URL: Issues, https://github.com/blahovec-labs/ffl-bigquery/issues
@@ -0,0 +1,83 @@
1
+ """Atomic replace-a-slice for the BigQuery writers.
2
+
3
+ A BigQuery load job cannot run inside a multi-statement transaction, so a plain
4
+ "DELETE, then load" is two independent commits: if the load fails, the DELETE has
5
+ already happened and the slice is simply gone. That is how statcast-bigquery lost
6
+ every 2026-06-02 pitch; this is the same helper as its 0.5.1 fix.
7
+
8
+ So: load the new rows into a short-lived staging table first (the step that
9
+ actually fails), then DELETE + INSERT ... SELECT in ONE transaction, then drop the
10
+ staging table. A failed load never touches the target; a failed swap rolls back.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import datetime as _dt
16
+ import logging
17
+ import uuid
18
+
19
+ import pandas as pd
20
+ from google.cloud import bigquery
21
+
22
+ log = logging.getLogger(__name__)
23
+
24
+ # Safety net: if the process dies between load and drop, BigQuery expires it.
25
+ _STAGING_TTL = _dt.timedelta(hours=6)
26
+
27
+
28
+ def atomic_replace(
29
+ client: bigquery.Client,
30
+ target: str,
31
+ df: pd.DataFrame,
32
+ schema: list[bigquery.SchemaField],
33
+ where_sql: str,
34
+ query_parameters: list | None = None,
35
+ ) -> int:
36
+ """Replace the rows of `target` matching `where_sql` with `df`, atomically.
37
+
38
+ `target` is "project.dataset.table". An empty `df` is a DELETE-only swap
39
+ (still transactional, no staging table). Returns len(df).
40
+ """
41
+ job_config = bigquery.QueryJobConfig(query_parameters=query_parameters or [])
42
+ delete_stmt = f"DELETE FROM `{target}` WHERE {where_sql};"
43
+
44
+ if len(df) == 0:
45
+ client.query_and_wait(
46
+ f"BEGIN TRANSACTION;\n{delete_stmt}\nCOMMIT TRANSACTION;",
47
+ job_config=job_config,
48
+ )
49
+ return 0
50
+
51
+ project, dataset, table = target.split(".")
52
+ staging = f"{project}.{dataset}._staging_{table}_{uuid.uuid4().hex[:12]}"
53
+ staging_table = bigquery.Table(staging, schema=schema)
54
+ staging_table.expires = _dt.datetime.now(_dt.UTC) + _STAGING_TTL
55
+
56
+ columns = ", ".join(f"`{f.name}`" for f in schema)
57
+ try:
58
+ client.create_table(staging_table)
59
+ log.info("load %d rows -> staging %s", len(df), staging)
60
+ client.load_table_from_dataframe(
61
+ df,
62
+ staging,
63
+ job_config=bigquery.LoadJobConfig(
64
+ schema=schema,
65
+ write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
66
+ source_format=bigquery.SourceFormat.PARQUET,
67
+ ),
68
+ ).result()
69
+
70
+ log.info("swap %s WHERE %s (atomic)", target, where_sql)
71
+ client.query_and_wait(
72
+ "BEGIN TRANSACTION;\n"
73
+ f"{delete_stmt}\n"
74
+ f"INSERT INTO `{target}` ({columns}) SELECT {columns} FROM `{staging}`;\n"
75
+ "COMMIT TRANSACTION;",
76
+ job_config=job_config,
77
+ )
78
+ finally:
79
+ try:
80
+ client.delete_table(staging, not_found_ok=True)
81
+ except Exception: # never mask the real error; TTL cleans up regardless
82
+ log.warning("could not drop staging table %s (expires in %s)", staging, _STAGING_TTL)
83
+ return len(df)
@@ -0,0 +1 @@
1
+ __version__ = "0.3.3"
@@ -14,6 +14,7 @@ import pandas as pd
14
14
  from google.cloud import bigquery
15
15
  from google.cloud.exceptions import NotFound
16
16
 
17
+ from ffl_bigquery._atomic import atomic_replace
17
18
  from ffl_bigquery.partition import ClusterOnly, SeasonRangePartition, TimePartition
18
19
  from ffl_bigquery.schema import ColumnSpec, to_bq_schema
19
20
 
@@ -195,7 +196,7 @@ class BigQueryWriter:
195
196
  season: int,
196
197
  schema: list[ColumnSpec],
197
198
  ) -> WriteSeasonResult:
198
- """DELETE rows for `season`, then append `df`. Returns a
199
+ """Replace the rows for `season` with `df`, atomically. Returns a
199
200
  `WriteSeasonResult` (an int of rows actually written, plus `.dropped`).
200
201
 
201
202
  The DELETE runs even when df is empty: a season that legitimately goes
@@ -224,15 +225,9 @@ class BigQueryWriter:
224
225
  passed through unchanged -- some callers legitimately have none.
225
226
  """
226
227
  bq_schema = to_bq_schema(schema)
227
- self.client.query(
228
- f"DELETE FROM `{ref}` WHERE season = @season",
229
- job_config=bigquery.QueryJobConfig(
230
- query_parameters=[
231
- bigquery.ScalarQueryParameter("season", "INT64", season)
232
- ],
233
- ),
234
- ).result()
228
+ params = [bigquery.ScalarQueryParameter("season", "INT64", season)]
235
229
  if df.empty:
230
+ atomic_replace(self.client, str(ref), df, bq_schema, "season = @season", params)
236
231
  log.info("season %s of %s is empty; deleted only", season, ref)
237
232
  return WriteSeasonResult(0, 0)
238
233
 
@@ -255,17 +250,13 @@ class BigQueryWriter:
255
250
  df = df.loc[~(is_null | is_mismatched)]
256
251
 
257
252
  if df.empty:
253
+ atomic_replace(self.client, str(ref), df, bq_schema, "season = @season", params)
258
254
  log.info(
259
255
  "season %s of %s: all rows dropped by season guard", season, ref,
260
256
  )
261
257
  return WriteSeasonResult(0, dropped)
262
- self.client.load_table_from_dataframe(
263
- df,
264
- str(ref),
265
- job_config=bigquery.LoadJobConfig(
266
- write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
267
- schema=bq_schema,
268
- ),
269
- ).result()
258
+ # Stage, then DELETE + INSERT in one transaction: a failed load leaves the
259
+ # season's existing rows untouched instead of deleted (see _atomic.py).
260
+ atomic_replace(self.client, str(ref), df, bq_schema, "season = @season", params)
270
261
  log.info("wrote %d rows to %s season=%s", len(df), ref, season)
271
262
  return WriteSeasonResult(len(df), dropped)
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "ffl-bigquery"
3
- version = "0.3.2"
3
+ version = "0.3.3"
4
4
  description = "Historical fantasy football ADP, usage, and coaching/scheme data → BigQuery: the fantasy layer on top of nfl-bigquery"
5
5
  readme = "README.md"
6
6
  license = { text = "MIT" }
@@ -0,0 +1,103 @@
1
+ """write_season must never leave a season deleted but not replaced.
2
+
3
+ It used to run `DELETE ... WHERE season = @season` as its own query and then a
4
+ separate load job. A BigQuery load job can't join a transaction, so those were two
5
+ commits: a failed load left the whole season gone until the next successful run.
6
+ That is the failure that wiped 2026-06-02 from statcast-bigquery (fixed there in
7
+ 0.5.1).
8
+
9
+ Contract now: load into a short-lived staging table FIRST, then DELETE + INSERT ...
10
+ SELECT in ONE multi-statement transaction, then drop the staging table. A failed
11
+ load issues no DELETE at all; a failed swap rolls back.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ from typing import cast
16
+ from unittest.mock import MagicMock
17
+
18
+ import pandas as pd
19
+ import pytest
20
+ from google.cloud import bigquery
21
+
22
+ from ffl_bigquery.schema import BqMode, BqType, ColumnSpec
23
+ from ffl_bigquery.writer import BigQueryWriter, TableRef
24
+
25
+
26
+ def _spec(name: str, type: str = "STRING", mode: str = "NULLABLE") -> ColumnSpec:
27
+ return ColumnSpec(
28
+ name=name, type=cast(BqType, type), mode=cast(BqMode, mode),
29
+ short_description="x", business_definition="x", semantic_tags=[],
30
+ valid_range=None, valid_values=None, example_value=None, gotchas=[],
31
+ source_field=name, deprecated_in_year=None,
32
+ )
33
+
34
+
35
+ SCHEMA = [_spec("season", "INT64", "REQUIRED"), _spec("val", "FLOAT64")]
36
+ REF = TableRef.parse("p.d.tbl")
37
+ TARGET = "p.d.tbl"
38
+
39
+
40
+ def _df() -> pd.DataFrame:
41
+ return pd.DataFrame({"season": [2025, 2025], "val": [1.0, 2.0]})
42
+
43
+
44
+ def _sql(client: MagicMock) -> list[str]:
45
+ return [c.args[0] for c in client.query_and_wait.call_args_list] + [
46
+ c.args[0] for c in client.query.call_args_list
47
+ ]
48
+
49
+
50
+ def test_failed_load_deletes_nothing():
51
+ c = MagicMock(spec=bigquery.Client)
52
+ c.load_table_from_dataframe.return_value.result.side_effect = RuntimeError(
53
+ "400 load failed"
54
+ )
55
+ with pytest.raises(RuntimeError):
56
+ BigQueryWriter(client=c).write_season(REF, _df(), season=2025, schema=SCHEMA)
57
+ assert not any("DELETE" in sql for sql in _sql(c))
58
+
59
+
60
+ def test_load_targets_staging_not_the_live_table():
61
+ c = MagicMock(spec=bigquery.Client)
62
+ BigQueryWriter(client=c).write_season(REF, _df(), season=2025, schema=SCHEMA)
63
+ destination = str(c.load_table_from_dataframe.call_args.args[1])
64
+ assert destination != TARGET
65
+ assert "_staging_" in destination
66
+
67
+
68
+ def test_delete_and_insert_commit_in_one_transaction():
69
+ c = MagicMock(spec=bigquery.Client)
70
+ BigQueryWriter(client=c).write_season(REF, _df(), season=2025, schema=SCHEMA)
71
+ swaps = [sql for sql in _sql(c) if "DELETE" in sql]
72
+ assert len(swaps) == 1
73
+ sql = swaps[0]
74
+ assert sql.lstrip().startswith("BEGIN TRANSACTION")
75
+ assert sql.rstrip().endswith("COMMIT TRANSACTION;")
76
+ assert "DELETE FROM `p.d.tbl` WHERE season = @season" in sql
77
+ assert sql.index("DELETE") < sql.index("INSERT INTO")
78
+ staging = str(c.load_table_from_dataframe.call_args.args[1])
79
+ assert f"FROM `{staging}`" in sql
80
+ params = c.query_and_wait.call_args.kwargs["job_config"].query_parameters
81
+ assert params[0].name == "season" and params[0].value == 2025
82
+
83
+
84
+ def test_staging_table_dropped_even_when_swap_fails():
85
+ c = MagicMock(spec=bigquery.Client)
86
+ c.query_and_wait.side_effect = RuntimeError("transaction aborted")
87
+ with pytest.raises(RuntimeError):
88
+ BigQueryWriter(client=c).write_season(REF, _df(), season=2025, schema=SCHEMA)
89
+ staging = str(c.load_table_from_dataframe.call_args.args[1])
90
+ dropped = [str(call.args[0]) for call in c.delete_table.call_args_list]
91
+ assert staging in dropped
92
+
93
+
94
+ def test_empty_season_is_a_transactional_delete_with_no_staging():
95
+ c = MagicMock(spec=bigquery.Client)
96
+ BigQueryWriter(client=c).write_season(
97
+ REF, pd.DataFrame(columns=["season", "val"]), season=2025, schema=SCHEMA,
98
+ )
99
+ assert not c.load_table_from_dataframe.called
100
+ assert not c.create_table.called
101
+ swaps = [sql for sql in _sql(c) if "DELETE" in sql]
102
+ assert len(swaps) == 1
103
+ assert swaps[0].lstrip().startswith("BEGIN TRANSACTION")
@@ -21,14 +21,14 @@ SCHEMA = [_spec("season", "INT64", "REQUIRED"), _spec("val", "FLOAT64")]
21
21
  REF = TableRef.parse("p.d.tbl")
22
22
 
23
23
 
24
- def test_write_season_deletes_that_season_then_loads():
24
+ def test_write_season_replaces_that_season():
25
25
  c = MagicMock(spec=bigquery.Client)
26
26
  df = pd.DataFrame({"season": [2020, 2020], "val": [1.0, 2.0]})
27
27
  n = BigQueryWriter(client=c).write_season(REF, df, season=2020, schema=SCHEMA)
28
28
  assert n == 2
29
- sql = c.query.call_args[0][0]
29
+ sql = c.query_and_wait.call_args[0][0]
30
30
  assert "DELETE FROM `p.d.tbl` WHERE season = @season" in sql
31
- params = c.query.call_args.kwargs["job_config"].query_parameters
31
+ params = c.query_and_wait.call_args.kwargs["job_config"].query_parameters
32
32
  assert params[0].name == "season" and params[0].value == 2020
33
33
  assert c.load_table_from_dataframe.called
34
34
 
@@ -48,7 +48,7 @@ def test_write_season_still_deletes_when_df_is_empty():
48
48
  REF, pd.DataFrame(columns=["season", "val"]), season=2020, schema=SCHEMA,
49
49
  )
50
50
  assert n == 0
51
- assert "DELETE FROM" in c.query.call_args[0][0]
51
+ assert "DELETE FROM" in c.query_and_wait.call_args[0][0]
52
52
  assert not c.load_table_from_dataframe.called
53
53
 
54
54
 
@@ -109,7 +109,7 @@ def test_write_season_deletes_even_when_filtering_empties_the_frame():
109
109
  df = pd.DataFrame({"season": [2026, 2026], "val": [1.0, 2.0]})
110
110
  n = BigQueryWriter(client=c).write_season(REF, df, season=2025, schema=SCHEMA)
111
111
  assert n == 0
112
- assert "DELETE FROM" in c.query.call_args[0][0]
112
+ assert "DELETE FROM" in c.query_and_wait.call_args[0][0]
113
113
  assert not c.load_table_from_dataframe.called
114
114
 
115
115
 
@@ -1 +0,0 @@
1
- __version__ = "0.3.2"
File without changes
File without changes
File without changes