megaton 2.3.0__tar.gz → 2.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. {megaton-2.3.0 → megaton-2.4.0}/PKG-INFO +1 -1
  2. {megaton-2.3.0 → megaton-2.4.0}/megaton/ga4.py +10 -3
  3. {megaton-2.3.0 → megaton-2.4.0}/megaton/gsheet.py +42 -3
  4. {megaton-2.3.0 → megaton-2.4.0}/megaton/gsheet_lowlevel.py +22 -3
  5. {megaton-2.3.0 → megaton-2.4.0}/megaton/services/sheets_service.py +107 -16
  6. {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/PKG-INFO +1 -1
  7. {megaton-2.3.0 → megaton-2.4.0}/pyproject.toml +1 -1
  8. {megaton-2.3.0 → megaton-2.4.0}/tests/test_ga4_report_core.py +7 -0
  9. {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_core.py +33 -0
  10. {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_request_builders.py +18 -0
  11. {megaton-2.3.0 → megaton-2.4.0}/tests/test_sheets_service.py +297 -0
  12. {megaton-2.3.0 → megaton-2.4.0}/LICENSE +0 -0
  13. {megaton-2.3.0 → megaton-2.4.0}/README.md +0 -0
  14. {megaton-2.3.0 → megaton-2.4.0}/megaton/__init__.py +0 -0
  15. {megaton-2.3.0 → megaton-2.4.0}/megaton/__main__.py +0 -0
  16. {megaton-2.3.0 → megaton-2.4.0}/megaton/_result.py +0 -0
  17. {megaton-2.3.0 → megaton-2.4.0}/megaton/auth/__init__.py +0 -0
  18. {megaton-2.3.0 → megaton-2.4.0}/megaton/auth/google_auth.py +0 -0
  19. {megaton-2.3.0 → megaton-2.4.0}/megaton/auth/provider.py +0 -0
  20. {megaton-2.3.0 → megaton-2.4.0}/megaton/bq.py +0 -0
  21. {megaton-2.3.0 → megaton-2.4.0}/megaton/cli.py +0 -0
  22. {megaton-2.3.0 → megaton-2.4.0}/megaton/constants.py +0 -0
  23. {megaton-2.3.0 → megaton-2.4.0}/megaton/dates.py +0 -0
  24. {megaton-2.3.0 → megaton-2.4.0}/megaton/errors.py +0 -0
  25. {megaton-2.3.0 → megaton-2.4.0}/megaton/files.py +0 -0
  26. {megaton-2.3.0 → megaton-2.4.0}/megaton/gdrive.py +0 -0
  27. {megaton-2.3.0 → megaton-2.4.0}/megaton/google_api.py +0 -0
  28. {megaton-2.3.0 → megaton-2.4.0}/megaton/install/__init__.py +0 -0
  29. {megaton-2.3.0 → megaton-2.4.0}/megaton/install/install_bigquery.py +0 -0
  30. {megaton-2.3.0 → megaton-2.4.0}/megaton/install/install_ga4.py +0 -0
  31. {megaton-2.3.0 → megaton-2.4.0}/megaton/os.py +0 -0
  32. {megaton-2.3.0 → megaton-2.4.0}/megaton/py.typed +0 -0
  33. {megaton-2.3.0 → megaton-2.4.0}/megaton/query.py +0 -0
  34. {megaton-2.3.0 → megaton-2.4.0}/megaton/recipes/__init__.py +0 -0
  35. {megaton-2.3.0 → megaton-2.4.0}/megaton/recipes/config_loader.py +0 -0
  36. {megaton-2.3.0 → megaton-2.4.0}/megaton/retry_utils.py +0 -0
  37. {megaton-2.3.0 → megaton-2.4.0}/megaton/searchconsole.py +0 -0
  38. {megaton-2.3.0 → megaton-2.4.0}/megaton/services/__init__.py +0 -0
  39. {megaton-2.3.0 → megaton-2.4.0}/megaton/services/bq_service.py +0 -0
  40. {megaton-2.3.0 → megaton-2.4.0}/megaton/services/gsc_service.py +0 -0
  41. {megaton-2.3.0 → megaton-2.4.0}/megaton/services/sheet_namespace.py +0 -0
  42. {megaton-2.3.0 → megaton-2.4.0}/megaton/start.py +0 -0
  43. {megaton-2.3.0 → megaton-2.4.0}/megaton/state.py +0 -0
  44. {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/__init__.py +0 -0
  45. {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/classify.py +0 -0
  46. {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/ga4.py +0 -0
  47. {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/table.py +0 -0
  48. {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/text.py +0 -0
  49. {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/traffic.py +0 -0
  50. {megaton-2.3.0 → megaton-2.4.0}/megaton/ui/__init__.py +0 -0
  51. {megaton-2.3.0 → megaton-2.4.0}/megaton/ui/widgets.py +0 -0
  52. {megaton-2.3.0 → megaton-2.4.0}/megaton/utils.py +0 -0
  53. {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/SOURCES.txt +0 -0
  54. {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/dependency_links.txt +0 -0
  55. {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/entry_points.txt +0 -0
  56. {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/requires.txt +0 -0
  57. {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/top_level.txt +0 -0
  58. {megaton-2.3.0 → megaton-2.4.0}/setup.cfg +0 -0
  59. {megaton-2.3.0 → megaton-2.4.0}/tests/test_auth.py +0 -0
  60. {megaton-2.3.0 → megaton-2.4.0}/tests/test_auth_provider.py +0 -0
  61. {megaton-2.3.0 → megaton-2.4.0}/tests/test_auto_install.py +0 -0
  62. {megaton-2.3.0 → megaton-2.4.0}/tests/test_bq_core_extra.py +0 -0
  63. {megaton-2.3.0 → megaton-2.4.0}/tests/test_bq_state_api.py +0 -0
  64. {megaton-2.3.0 → megaton-2.4.0}/tests/test_clean_parameter.py +0 -0
  65. {megaton-2.3.0 → megaton-2.4.0}/tests/test_config_loader.py +0 -0
  66. {megaton-2.3.0 → megaton-2.4.0}/tests/test_dates.py +0 -0
  67. {megaton-2.3.0 → megaton-2.4.0}/tests/test_errors_and_init.py +0 -0
  68. {megaton-2.3.0 → megaton-2.4.0}/tests/test_failure_contracts.py +0 -0
  69. {megaton-2.3.0 → megaton-2.4.0}/tests/test_ga4_property_show.py +0 -0
  70. {megaton-2.3.0 → megaton-2.4.0}/tests/test_ga4_report_retry.py +0 -0
  71. {megaton-2.3.0 → megaton-2.4.0}/tests/test_ga4_state_and_run.py +0 -0
  72. {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsc_service.py +0 -0
  73. {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_a1_helpers.py +0 -0
  74. {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_batch_read.py +0 -0
  75. {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_facade.py +0 -0
  76. {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_retry_core.py +0 -0
  77. {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_retry_resolvers.py +0 -0
  78. {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_timeout.py +0 -0
  79. {megaton-2.3.0 → megaton-2.4.0}/tests/test_megaton_auth_bridge.py +0 -0
  80. {megaton-2.3.0 → megaton-2.4.0}/tests/test_normalize_queries.py +0 -0
  81. {megaton-2.3.0 → megaton-2.4.0}/tests/test_phase0_regressions.py +0 -0
  82. {megaton-2.3.0 → megaton-2.4.0}/tests/test_programmatic_api.py +0 -0
  83. {megaton-2.3.0 → megaton-2.4.0}/tests/test_query_contract.py +0 -0
  84. {megaton-2.3.0 → megaton-2.4.0}/tests/test_query_regressions.py +0 -0
  85. {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_dates_to_sheet.py +0 -0
  86. {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_month_window.py +0 -0
  87. {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_prep.py +0 -0
  88. {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_result.py +0 -0
  89. {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_run_multi.py +0 -0
  90. {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_run_ranges.py +0 -0
  91. {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_show_download.py +0 -0
  92. {megaton-2.3.0 → megaton-2.4.0}/tests/test_retry_utils.py +0 -0
  93. {megaton-2.3.0 → megaton-2.4.0}/tests/test_run_all.py +0 -0
  94. {megaton-2.3.0 → megaton-2.4.0}/tests/test_sc_wrapper.py +0 -0
  95. {megaton-2.3.0 → megaton-2.4.0}/tests/test_search_apply_if.py +0 -0
  96. {megaton-2.3.0 → megaton-2.4.0}/tests/test_search_dimension_filter.py +0 -0
  97. {megaton-2.3.0 → megaton-2.4.0}/tests/test_search_filter.py +0 -0
  98. {megaton-2.3.0 → megaton-2.4.0}/tests/test_search_result.py +0 -0
  99. {megaton-2.3.0 → megaton-2.4.0}/tests/test_session_retry.py +0 -0
  100. {megaton-2.3.0 → megaton-2.4.0}/tests/test_sheet_namespace.py +0 -0
  101. {megaton-2.3.0 → megaton-2.4.0}/tests/test_sheet_save_options.py +0 -0
  102. {megaton-2.3.0 → megaton-2.4.0}/tests/test_sheets_read.py +0 -0
  103. {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_classify.py +0 -0
  104. {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_ga4.py +0 -0
  105. {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_pipeline_gsc_like.py +0 -0
  106. {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_table.py +0 -0
  107. {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_table_normalize_thresholds_df.py +0 -0
  108. {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_text.py +0 -0
  109. {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_traffic.py +0 -0
  110. {megaton-2.3.0 → megaton-2.4.0}/tests/test_ui_lazy_import.py +0 -0
  111. {megaton-2.3.0 → megaton-2.4.0}/tests/test_upsert_to_csv.py +0 -0
  112. {megaton-2.3.0 → megaton-2.4.0}/tests/test_utils.py +0 -0
  113. {megaton-2.3.0 → megaton-2.4.0}/tests/test_wrap_and_chain.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: megaton
3
- Version: 2.3.0
3
+ Version: 2.4.0
4
4
  Summary: Google analytics toolkit for Python, CLI and Notebooks: GA4, Search Console, Sheets and BigQuery.
5
5
  Author-email: Makoto Shimizu <aa.analyst.ga@gmail.com>
6
6
  License-Expression: MIT
@@ -715,13 +715,20 @@ class MegatonGA4(object):
715
715
  TIME (in HH:MM:SS format)
716
716
  """
717
717
  type = type.replace('TYPE_', '')
718
- if type in ['INTEGER', 'HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS']:
718
+ if type == 'INTEGER':
719
719
  try:
720
720
  return int(value)
721
721
  except:
722
722
  return value
723
- elif type in ['FLOAT']:
724
- return float(value)
723
+ elif type in ['FLOAT', 'HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS',
724
+ 'CURRENCY', 'FEET', 'METERS', 'KILOMETERS', 'MILES', 'STANDARD']:
725
+ try:
726
+ number = float(value)
727
+ if type in ['HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS'] and number.is_integer():
728
+ return int(number)
729
+ return number
730
+ except (TypeError, ValueError):
731
+ return value
725
732
  else:
726
733
  return value
727
734
 
@@ -459,18 +459,57 @@ class MegatonGS(object):
459
459
  """
460
460
  return self.last_row + 1
461
461
 
462
- def get_records(self, *, max_retries: Optional[int] = None, backoff_factor: Optional[float] = None):
462
+ def get_records(self, *, max_retries: Optional[int] = None, backoff_factor: Optional[float] = None,
463
+ value_render_option=None, numericise_ignore=None, preserve_dates=False):
463
464
  """Read all records with header row as keys."""
464
465
  if not self._driver:
465
466
  LOGGER.error("Please select a sheet first.")
466
467
  return
467
- return self._maybe_retry(
468
+ options = {}
469
+ if value_render_option is not None:
470
+ options["value_render_option"] = value_render_option
471
+ if numericise_ignore is not None:
472
+ options["numericise_ignore"] = numericise_ignore
473
+ records = self._maybe_retry(
468
474
  "Google Sheets read records",
469
- lambda: self._driver.get_all_records(),
475
+ lambda: self._driver.get_all_records(**options),
470
476
  max_retries=max_retries,
471
477
  backoff_factor=backoff_factor,
472
478
  retry_on_requests=True,
473
479
  )
480
+ if not preserve_dates or not records:
481
+ return records
482
+ if value_render_option != "UNFORMATTED_VALUE" or numericise_ignore != ["all"]:
483
+ raise ValueError("preserve_dates requires unformatted, non-numericised records")
484
+ title = self._driver.title.replace("'", "''")
485
+ last_cell = gspread.utils.rowcol_to_a1(len(records) + 1, len(records[0]))
486
+ metadata = self._maybe_retry(
487
+ "Google Sheets read date formats",
488
+ lambda: self.parent._driver.fetch_sheet_metadata(params={
489
+ "ranges": "'" + title + "'!A1:" + last_cell,
490
+ "fields": "sheets(data(startRow,startColumn,rowData(values(effectiveValue,effectiveFormat.numberFormat))))",
491
+ "includeGridData": True,
492
+ }),
493
+ max_retries=max_retries, backoff_factor=backoff_factor,
494
+ retry_on_requests=True,
495
+ )
496
+ headers = list(records[0])
497
+ for sheet in metadata.get("sheets", []):
498
+ for grid in sheet.get("data", []):
499
+ for offset, row in enumerate(grid.get("rowData", [])):
500
+ index = grid.get("startRow", 0) + offset - 1
501
+ if not 0 <= index < len(records):
502
+ continue
503
+ for ci, cell in enumerate(row.get("values", []), grid.get("startColumn", 0)):
504
+ fmt = cell.get("effectiveFormat", {}).get("numberFormat", {})
505
+ serial = cell.get("effectiveValue", {}).get("numberValue")
506
+ if ci >= len(headers) or serial is None or fmt.get("type") not in ("DATE", "DATE_TIME", "TIME"):
507
+ continue
508
+ key = headers[ci]
509
+ if records[index][key] != serial:
510
+ raise ValueError("Sheet changed while reading date formats; no write performed")
511
+ records[index][key] = gsheet_lowlevel._NativeDate(serial, fmt)
512
+ return records
474
513
 
475
514
  @property
476
515
  def data(self):
@@ -18,6 +18,8 @@ from __future__ import annotations
18
18
  import datetime as _dt
19
19
  import logging
20
20
  import math
21
+ from dataclasses import dataclass
22
+ from numbers import Integral, Real
21
23
  import re
22
24
  import time
23
25
  from collections.abc import Iterable
@@ -282,6 +284,14 @@ _DATETIME_RE = re.compile(
282
284
  _SHEETS_EPOCH_DATE = _dt.date(1899, 12, 30)
283
285
 
284
286
 
287
+ @dataclass(frozen=True)
288
+ class _NativeDate:
289
+ """A persisted date cell, without a lossy serial-to-datetime round trip."""
290
+
291
+ serial: float
292
+ number_format: dict
293
+
294
+
285
295
  def _is_missing_scalar(value: Any) -> bool:
286
296
  if value is None:
287
297
  return True
@@ -301,12 +311,12 @@ def cell_value(value: Any, *, parse_formulas: bool = True) -> dict:
301
311
  return {"stringValue": ""}
302
312
  if isinstance(value, bool):
303
313
  return {"boolValue": value}
304
- if isinstance(value, int):
314
+ if isinstance(value, Integral):
305
315
  return {"numberValue": float(value)}
306
- if isinstance(value, float):
316
+ if isinstance(value, Real):
307
317
  if math.isnan(value) or math.isinf(value):
308
318
  return {"stringValue": ""}
309
- return {"numberValue": value}
319
+ return {"numberValue": float(value)}
310
320
  if isinstance(value, str):
311
321
  s = value.strip()
312
322
  if not s:
@@ -349,6 +359,15 @@ def cell_data(
349
359
  ISO date and naive ISO datetime strings are written as real Sheets serials
350
360
  with a display format. Other strings stay strings.
351
361
  """
362
+ if isinstance(value, _NativeDate):
363
+ return {
364
+ "userEnteredValue": {"numberValue": value.serial},
365
+ "userEnteredFormat": {"numberFormat": dict(value.number_format)},
366
+ }
367
+ if _is_missing_scalar(value):
368
+ return {"userEnteredValue": cell_value(value, parse_formulas=parse_formulas)}
369
+ if parse_dates and isinstance(value, _dt.date):
370
+ value = value.isoformat()
352
371
  if isinstance(value, str):
353
372
  if not parse_formulas and not parse_dates:
354
373
  return {"userEnteredValue": cell_value(value, parse_formulas=False)}
@@ -1,15 +1,38 @@
1
1
  """Google Sheets service wrapper."""
2
2
 
3
3
  import logging
4
+ from datetime import datetime, timedelta
5
+ from numbers import Real
4
6
  from typing import Optional
5
7
 
6
8
  import pandas as pd
7
9
 
8
- from .. import errors, gsheet, mount_google_drive
10
+ from .. import errors, gsheet, gsheet_lowlevel, mount_google_drive
9
11
 
10
12
  logger = logging.getLogger(__name__)
11
13
 
12
14
 
15
+ def _public_sheet_frame(frame):
16
+ """Keep native serial/format carriers out of public results and sizing."""
17
+ def public_value(value):
18
+ if not isinstance(value, gsheet_lowlevel._NativeDate):
19
+ return value
20
+ instant = datetime(1899, 12, 30) + timedelta(days=value.serial)
21
+ kind = value.number_format.get("type")
22
+ if kind == "TIME":
23
+ return instant.time()
24
+ if kind == "DATE" and instant == instant.replace(hour=0, minute=0, second=0, microsecond=0):
25
+ return instant.date()
26
+ return instant
27
+ result = frame.copy()
28
+ for column in frame.columns:
29
+ if any(isinstance(value, gsheet_lowlevel._NativeDate) for value in frame[column]):
30
+ result[column] = pd.Series(
31
+ [public_value(value) for value in frame[column]], index=frame.index, dtype=object,
32
+ )
33
+ return result
34
+
35
+
13
36
  class SheetsService:
14
37
  def __init__(self, app):
15
38
  self.app = app
@@ -447,6 +470,9 @@ class SheetsService:
447
470
  self.app.gs.sheet.get_records(
448
471
  max_retries=max_retries,
449
472
  backoff_factor=backoff_factor,
473
+ value_render_option="UNFORMATTED_VALUE",
474
+ numericise_ignore=["all"],
475
+ preserve_dates=True,
450
476
  )
451
477
  )
452
478
  except Exception as exc:
@@ -457,9 +483,10 @@ class SheetsService:
457
483
 
458
484
  df_new = df_new.copy()
459
485
  if df_new.empty:
460
- return df_existing
486
+ return _public_sheet_frame(df_existing)
461
487
  if df_existing.empty:
462
488
  try:
489
+ public_frame = _public_sheet_frame(df_new)
463
490
  self.app.gs.sheet.overwrite_data(
464
491
  df_new,
465
492
  include_index=False,
@@ -467,7 +494,7 @@ class SheetsService:
467
494
  backoff_factor=backoff_factor,
468
495
  )
469
496
  self._apply_write_options(
470
- df_new,
497
+ public_frame,
471
498
  auto_width=auto_width,
472
499
  freeze_header=freeze_header,
473
500
  width_min=width_min,
@@ -478,27 +505,75 @@ class SheetsService:
478
505
  backoff_factor=backoff_factor,
479
506
  )
480
507
  print(f"'{sheet_name}' シートへ {len(df_new)} 行を書き込みました。")
481
- return df_new
508
+ return public_frame
482
509
  except Exception as exc:
483
510
  logger.exception("Failed to overwrite sheet '%s' (empty upsert)", sheet_name)
484
511
  print(f"'{sheet_name}' シートへの書き込みに失敗しました: {exc}")
485
512
  raise
486
513
 
487
514
  df_existing = df_existing.copy()
488
- for key in keys:
489
- if key in df_existing.columns:
490
- df_existing[key] = df_existing[key].astype(str).str.strip()
491
- if key in df_new.columns:
492
- df_new[key] = df_new[key].astype(str).str.strip()
493
-
515
+ # Legacy formatted dates are parsed only when the incoming column has
516
+ # an explicit datetime dtype. Native cells retain their serial/format.
517
+ date_keys = [key for key in keys if pd.api.types.is_datetime64_any_dtype(df_new[key])]
518
+ for key in date_keys:
519
+ if key in df_existing:
520
+ def date_value(value):
521
+ if isinstance(value, gsheet_lowlevel._NativeDate):
522
+ return value
523
+ if isinstance(value, Real) and not pd.isna(value):
524
+ return gsheet_lowlevel._NativeDate(float(value), {
525
+ "type": "DATE_TIME", "pattern": "yyyy-mm-dd hh:mm:ss.000",
526
+ })
527
+ return pd.Timestamp(value) if not pd.isna(value) else value
528
+ df_existing[key] = df_existing[key].map(date_value)
494
529
  try:
495
- keys_to_remove = set(tuple(row) for row in df_new[keys].drop_duplicates().values)
496
- mask = df_existing[keys].apply(tuple, axis=1).isin(keys_to_remove)
530
+ def normalize(value):
531
+ cell = gsheet_lowlevel.cell_data(value)["userEnteredValue"]
532
+ if "numberValue" in cell:
533
+ number = cell["numberValue"]
534
+ return str(int(number)) if float(number).is_integer() else repr(float(number))
535
+ return str(next(iter(cell.values()), "")).strip()
536
+ def key_value(value):
537
+ cell = gsheet_lowlevel.cell_data(value)["userEnteredValue"]
538
+ kind, content = next(iter(cell.items()), ("stringValue", ""))
539
+ if kind == "stringValue" and content == "":
540
+ return ""
541
+ return (kind, content)
542
+ new_keys = df_new[keys].apply(lambda col: col.map(key_value))
543
+ existing_keys = df_existing[keys].apply(lambda col: col.map(key_value))
544
+ new_tuples = new_keys.apply(tuple, axis=1)
545
+ existing_tuples = existing_keys.apply(tuple, axis=1)
546
+ incomplete = new_keys.eq("").any(axis=1)
547
+ keys_to_remove = set(new_tuples[~incomplete])
548
+ mask = existing_tuples.isin(keys_to_remove)
497
549
  except KeyError as exc:
498
550
  raise ValueError(f"Existing sheet is missing upsert keys: {keys}") from exc
499
551
 
552
+ # Incomplete keys can have historical duplicates. Keep those rows when
553
+ # the incoming row is already present; never collapse ambiguous history.
554
+ skip_new = pd.Series(False, index=df_new.index)
555
+ history_indexes = {}
556
+ for index, key_tuple in existing_tuples.items():
557
+ history_indexes.setdefault(key_tuple, []).append(index)
558
+ for index in df_new.index[incomplete]:
559
+ matches = history_indexes.get(new_tuples.loc[index], [])
560
+ history = df_existing.loc[matches].reindex(columns=df_new.columns)
561
+ row = df_new.loc[index].astype(object)
562
+ row = row.where(pd.notna(row), "")
563
+ history = history.astype(object).where(pd.notna(history), "")
564
+ row = row.map(lambda value: gsheet_lowlevel.cell_data(value)["userEnteredValue"])
565
+ history = history.apply(lambda col: col.map(
566
+ lambda value: gsheet_lowlevel.cell_data(value)["userEnteredValue"]
567
+ ))
568
+ if not history.empty and history.eq(row, axis="columns").all(axis=1).any():
569
+ skip_new.loc[index] = True
570
+ elif len(history) > 1:
571
+ raise ValueError("Cannot update an incomplete upsert key with ambiguous historical rows; no write performed")
572
+ else:
573
+ mask.loc[matches] = True
574
+
500
575
  df_cleaned = df_existing[~mask]
501
- df_combined = pd.concat([df_cleaned, df_new], ignore_index=True)
576
+ df_combined = pd.concat([df_cleaned, df_new[~skip_new]], ignore_index=True)
502
577
 
503
578
  sort_cols = sort_by or list(keys)
504
579
  if sort_cols:
@@ -506,7 +581,22 @@ class SheetsService:
506
581
  if missing_sort:
507
582
  print(f"ソート対象の列が見つかりません: {missing_sort}")
508
583
  else:
509
- df_combined.sort_values(by=sort_cols, inplace=True)
584
+ def sort_value(value):
585
+ cell = gsheet_lowlevel.cell_data(value)
586
+ fmt = cell.get("userEnteredFormat", {}).get("numberFormat", {})
587
+ if fmt.get("type") in ("DATE", "DATE_TIME", "TIME"):
588
+ return cell["userEnteredValue"]["numberValue"]
589
+ return value
590
+ sort_frame = df_combined[sort_cols].apply(
591
+ lambda col: col.map(sort_value)
592
+ )
593
+ for col in sort_cols:
594
+ try:
595
+ sort_frame[col].sort_values()
596
+ except TypeError:
597
+ sort_frame[col] = sort_frame[col].map(normalize)
598
+ order = sort_frame.sort_values(by=sort_cols, kind="stable").index
599
+ df_combined = df_combined.loc[order]
510
600
 
511
601
  if columns:
512
602
  for col in columns:
@@ -515,6 +605,7 @@ class SheetsService:
515
605
  df_combined = df_combined[columns]
516
606
 
517
607
  try:
608
+ public_frame = _public_sheet_frame(df_combined)
518
609
  self.app.gs.sheet.overwrite_data(
519
610
  df_combined,
520
611
  include_index=False,
@@ -522,7 +613,7 @@ class SheetsService:
522
613
  backoff_factor=backoff_factor,
523
614
  )
524
615
  self._apply_write_options(
525
- df_combined,
616
+ public_frame,
526
617
  auto_width=auto_width,
527
618
  freeze_header=freeze_header,
528
619
  width_min=width_min,
@@ -533,7 +624,7 @@ class SheetsService:
533
624
  backoff_factor=backoff_factor,
534
625
  )
535
626
  print(f"'{sheet_name}' シートを更新しました(新規 {len(df_new)} 行、削除 {mask.sum()} 行)。")
536
- return df_combined
627
+ return public_frame
537
628
  except Exception as exc:
538
629
  logger.exception("Failed to overwrite sheet '%s' (upsert)", sheet_name)
539
630
  print(f"'{sheet_name}' シートへの書き込みに失敗しました: {exc}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: megaton
3
- Version: 2.3.0
3
+ Version: 2.4.0
4
4
  Summary: Google analytics toolkit for Python, CLI and Notebooks: GA4, Search Console, Sheets and BigQuery.
5
5
  Author-email: Makoto Shimizu <aa.analyst.ga@gmail.com>
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "megaton"
7
- version = "2.3.0"
7
+ version = "2.4.0"
8
8
  description = "Google analytics toolkit for Python, CLI and Notebooks: GA4, Search Console, Sheets and BigQuery."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -52,6 +52,13 @@ def test_convert_metric_numeric_and_fallback():
52
52
  assert report._convert_metric("10", "TYPE_INTEGER") == 10
53
53
  assert report._convert_metric("1.5", "TYPE_FLOAT") == 1.5
54
54
  assert report._convert_metric("not-number", "TYPE_INTEGER") == "not-number"
55
+ for kind in ("SECONDS", "HOURS", "MINUTES", "MILLISECONDS", "CURRENCY", "METERS"):
56
+ assert report._convert_metric("673.8335", "TYPE_" + kind) == 673.8335
57
+ assert report._convert_metric("not-number", "TYPE_" + kind) == "not-number"
58
+ for kind in ("SECONDS", "HOURS", "MINUTES", "MILLISECONDS"):
59
+ assert type(report._convert_metric("1234", "TYPE_" + kind)) is int
60
+ assert type(report._convert_metric("1234.5", "TYPE_" + kind)) is float
61
+ assert pd.DataFrame({"duration": [report._convert_metric("1234", "TYPE_" + kind)]}).to_csv(index=False) == "duration\n1234\n"
55
62
 
56
63
 
57
64
  def test_parse_response_converts_metric_values():
@@ -15,6 +15,39 @@ def _new_gs():
15
15
  return gs
16
16
 
17
17
 
18
+ def test_records_native_read_disables_numericising_text_identifiers():
19
+ calls = []
20
+ def get_all_records(**kwargs):
21
+ calls.append(kwargs)
22
+ return [{"source": "nan", "code": "0012", "users": 3}]
23
+ sheet = MegatonGS.Sheet(SimpleNamespace())
24
+ sheet._driver = SimpleNamespace(get_all_records=get_all_records)
25
+ assert sheet.get_records()[0]["source"] == "nan"
26
+ assert calls[-1] == {}
27
+ assert sheet.get_records(value_render_option="UNFORMATTED_VALUE", numericise_ignore=["all"])[0]["code"] == "0012"
28
+ assert calls[-1] == {"value_render_option": "UNFORMATTED_VALUE", "numericise_ignore": ["all"]}
29
+
30
+
31
+ def test_preserve_dates_requires_consistent_native_read():
32
+ sheet = MegatonGS.Sheet(SimpleNamespace())
33
+ sheet._driver = SimpleNamespace(get_all_records=lambda **kw: [{"date": "2024-01-01"}])
34
+ with pytest.raises(ValueError, match="unformatted"):
35
+ sheet.get_records(preserve_dates=True)
36
+
37
+
38
+ def test_preserve_dates_stops_when_metadata_and_values_disagree():
39
+ metadata = {"sheets": [{"data": [{"rowData": [{}, {"values": [{
40
+ "effectiveValue": {"numberValue": 45293},
41
+ "effectiveFormat": {"numberFormat": {"type": "DATE", "pattern": "yyyy-mm-dd"}},
42
+ }]}]}]}]}
43
+ sheet = MegatonGS.Sheet(SimpleNamespace(_driver=SimpleNamespace(
44
+ fetch_sheet_metadata=lambda **kw: metadata,
45
+ )))
46
+ sheet._driver = SimpleNamespace(title="report", get_all_records=lambda **kw: [{"date": 45292}])
47
+ with pytest.raises(ValueError, match="Sheet changed"):
48
+ sheet.get_records(value_render_option="UNFORMATTED_VALUE", numericise_ignore=["all"], preserve_dates=True)
49
+
50
+
18
51
  def _api_error(message: str, code: int = 403) -> gspread.exceptions.APIError:
19
52
  resp = requests.Response()
20
53
  resp.status_code = code
@@ -1,5 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
+ from datetime import date
3
4
  import pandas as pd
4
5
 
5
6
  from megaton.gsheet_lowlevel import (
@@ -16,6 +17,17 @@ def test_cell_value_preserves_string_codes() -> None:
16
17
  assert cell_value("03311187") == {"stringValue": "03311187"}
17
18
 
18
19
 
20
+ def test_nullable_numeric_columns_remain_sheet_numbers():
21
+ from megaton.gsheet_lowlevel import dataframe_update_cells_rows
22
+
23
+ frame = pd.DataFrame({"count": pd.Series([12, pd.NA], dtype="Int64"),
24
+ "code": ["0012", "0007"]})
25
+ rows = dataframe_update_cells_rows(frame)
26
+ assert rows[1]["values"][0]["userEnteredValue"] == {"numberValue": 12.0}
27
+ assert rows[1]["values"][1]["userEnteredValue"] == {"stringValue": "0012"}
28
+ assert rows[2]["values"][0]["userEnteredValue"] == {"stringValue": ""}
29
+
30
+
19
31
  def test_cell_value_builds_scalar_user_entered_values() -> None:
20
32
  assert cell_value(True) == {"boolValue": True}
21
33
  assert cell_value(1000) == {"numberValue": 1000.0}
@@ -34,6 +46,12 @@ def test_cell_data_writes_iso_dates_as_sheet_serials() -> None:
34
46
  }
35
47
 
36
48
 
49
+ def test_python_date_scalars_write_real_sheet_dates():
50
+ assert cell_data(date(2026, 6, 23)) == cell_data("2026-06-23")
51
+ assert cell_data(date(2026, 6, 23), parse_dates=False) == {
52
+ "userEnteredValue": {"stringValue": "2026-06-23"}}
53
+
54
+
37
55
  def test_cell_data_writes_naive_iso_datetimes_as_sheet_serials() -> None:
38
56
  result = cell_data("2026-06-23 19:05")
39
57
 
@@ -4,6 +4,8 @@ import pandas as pd
4
4
  import pytest
5
5
 
6
6
  from megaton import errors
7
+ from megaton import gsheet_lowlevel
8
+ from megaton.gsheet import MegatonGS
7
9
  from megaton.services.sheets_service import SheetsService
8
10
 
9
11
 
@@ -28,6 +30,7 @@ class _FakeSheet:
28
30
  return self.parent._sheets.get(self._name, [])
29
31
 
30
32
  def get_records(self, **_kwargs):
33
+ self.parent.last_read_options = _kwargs
31
34
  return self.data
32
35
 
33
36
  def select(self, name: str):
@@ -112,6 +115,300 @@ def _make_app(gs):
112
115
  )
113
116
 
114
117
 
118
+ class _RoundTripSheet(_FakeSheet):
119
+ """Execute real record reading and atomic cell building against a fake grid."""
120
+
121
+ def __init__(self, parent):
122
+ super().__init__(parent)
123
+ self.cells = []
124
+ self.ws = SimpleNamespace(
125
+ title="report", id=1, row_count=100, col_count=10,
126
+ get_all_records=self._records,
127
+ )
128
+ api = SimpleNamespace(batch_update=self._batch, fetch_sheet_metadata=self._metadata)
129
+ self.real = MegatonGS.Sheet(SimpleNamespace(_driver=api))
130
+ self.real._driver = self.ws
131
+
132
+ def _records(self, **kwargs):
133
+ headers = [cell["userEnteredValue"]["stringValue"] for cell in self.cells[0]]
134
+ return [dict(zip(headers, [next(iter(cell.get("userEnteredValue", {}).values()), "")
135
+ for cell in row])) for row in self.cells[1:]] if self.cells else []
136
+
137
+ def _metadata(self, **kwargs):
138
+ self.last_metadata_params = kwargs["params"]
139
+ return {"sheets": [{"data": [{"rowData": [{"values": [
140
+ {"effectiveValue": cell.get("userEnteredValue", {}),
141
+ "effectiveFormat": cell.get("userEnteredFormat", {})} for cell in row
142
+ ]} for row in self.cells]}]}]}
143
+
144
+ def _batch(self, payload):
145
+ for request in payload["requests"]:
146
+ update = request.get("updateCells", {})
147
+ if "rows" in update:
148
+ old = self.cells
149
+ self.cells = [record["values"] for record in update["rows"]]
150
+ for ri, row in enumerate(self.cells):
151
+ for ci, cell in enumerate(row):
152
+ cell.pop("userEnteredFormat", None)
153
+ if ri < len(old) and ci < len(old[ri]):
154
+ cell["userEnteredFormat"] = old[ri][ci].get("userEnteredFormat", {})
155
+ repeat = request.get("repeatCell", {})
156
+ if repeat:
157
+ region = repeat["range"]
158
+ for ri in range(region["startRowIndex"], region["endRowIndex"]):
159
+ self.cells[ri][region["startColumnIndex"]]["userEnteredFormat"] = repeat["cell"]["userEnteredFormat"]
160
+
161
+ def get_records(self, **kwargs):
162
+ if not self.cells:
163
+ return []
164
+ return self.real.get_records(**kwargs)
165
+
166
+ def overwrite_data(self, df, include_index=False, **kwargs):
167
+ self.parent.last_written = df.copy()
168
+ return self.real._atomic_overwrite(
169
+ df, row=1, include_index=include_index, max_retries=None, backoff_factor=None,
170
+ )
171
+
172
+
173
+ @pytest.mark.parametrize("date", [
174
+ "2024-01-01", "2024-01-01 10:30:15", pd.Timestamp("2024-01-01 10:30:15"),
175
+ pd.Timestamp("2024-01-01 10:30:15.123456"),
176
+ ])
177
+ def test_upsert_date_keys_survive_real_cell_roundtrip(date):
178
+ gs = _FakeGS("https://example.com/sheet", {"report": []})
179
+ gs.sheet = _RoundTripSheet(gs)
180
+ service = SheetsService(_make_app(gs))
181
+ for count in (1, 2, 3):
182
+ result = service.upsert_df(gs.url, "report", pd.DataFrame([{"date": date, "count": count}]), keys=["date"])
183
+ assert len(result) == 1
184
+ assert result["count"].tolist() == [count]
185
+ assert len(gs.sheet.cells) == 2
186
+
187
+
188
+ def test_upsert_preserves_distinct_subsecond_keys_and_nonkey_date_formats():
189
+ gs = _FakeGS("https://example.com/sheet", {"report": []})
190
+ gs.sheet = _RoundTripSheet(gs)
191
+ service = SheetsService(_make_app(gs))
192
+ first = pd.DataFrame([
193
+ {"key": "2024-01-01 10:30:15.100000", "n": 9, "date": "2024-01-02"},
194
+ {"key": "2024-01-01 10:30:15.200000", "n": 100, "date": "2024-01-03"},
195
+ ])
196
+ service.upsert_df(gs.url, "report", first, keys=["key"], sort_by=["n"])
197
+ second = pd.DataFrame([{"key": "2024-01-01 10:30:15.100000", "n": 101, "date": "2024-01-02"}])
198
+ result = service.upsert_df(gs.url, "report", second, keys=["key"], sort_by=["n"])
199
+ assert len(result) == 2
200
+ assert result["n"].tolist() == [100, 101]
201
+ for row, expected in zip(gs.sheet.cells[1:], ("2024-01-03", "2024-01-02")):
202
+ assert row[2]["userEnteredValue"] == gsheet_lowlevel.cell_data(expected)["userEnteredValue"]
203
+ assert row[2]["userEnteredFormat"]["numberFormat"]["type"] == "DATE"
204
+
205
+
206
+ def test_retained_nonkey_date_moves_to_unformatted_row_with_original_format():
207
+ gs = _FakeGS("https://example.com/sheet", {"report": []})
208
+ gs.sheet = _RoundTripSheet(gs)
209
+ service = SheetsService(_make_app(gs))
210
+ service.upsert_df(gs.url, "report", pd.DataFrame([
211
+ {"key": "a", "n": 100, "date": "2024-01-02"},
212
+ {"key": "b", "n": 9, "date": ""},
213
+ ]), keys=["key"])
214
+ original = {"type": "DATE", "pattern": "dd/mm/yyyy"}
215
+ gs.sheet.cells[1][2]["userEnteredFormat"]["numberFormat"] = original
216
+ service.upsert_df(gs.url, "report", pd.DataFrame([
217
+ {"key": "c", "n": 10, "date": ""},
218
+ ]), keys=["key"], sort_by=["n"])
219
+ assert gs.sheet.cells[3][0]["userEnteredValue"]["stringValue"] == "a"
220
+ assert gs.sheet.cells[3][2]["userEnteredValue"] == {"numberValue": 45293.0}
221
+ assert gs.sheet.cells[3][2]["userEnteredFormat"]["numberFormat"] == original
222
+
223
+
224
+ def test_upsert_numeric_sort_and_no_dataframe_map(monkeypatch):
225
+ monkeypatch.delattr(pd.DataFrame, "map", raising=False)
226
+ gs = _FakeGS("https://example.com/sheet", {"report": [{"key": "a", "n": 9}, {"key": "b", "n": 100}]})
227
+ result = SheetsService(_make_app(gs)).upsert_df(
228
+ gs.url, "report", pd.DataFrame([{"key": "c", "n": 10}]), keys=["key"], sort_by=["n"],
229
+ )
230
+ assert result["n"].tolist() == [9, 10, 100]
231
+
232
+
233
+ @pytest.mark.parametrize("date", ["2024-01-01", "2024-01-01 10:30:15.123456"])
234
+ def test_upsert_returns_public_dates_and_sizes_without_internal_objects(date):
235
+ gs = _FakeGS("https://example.com/sheet", {"report": []})
236
+ gs.sheet = _RoundTripSheet(gs)
237
+ gs.sheet.id = 1
238
+ widths = []
239
+ gs._driver.batch_update = widths.append
240
+ service = SheetsService(_make_app(gs))
241
+ service.upsert_df(gs.url, "report", pd.DataFrame([{"key": "a", "date": date}]), keys=["key"])
242
+ original_cell = gs.sheet.cells[1][1]["userEnteredValue"].copy()
243
+ result = service.upsert_df(gs.url, "report", pd.DataFrame([
244
+ {"key": "b", "date": "2024-01-02"},
245
+ ]), keys=["key"], auto_width=True)
246
+ assert not any(isinstance(value, gsheet_lowlevel._NativeDate) for value in result["date"])
247
+ assert pd.Timestamp(result["date"].iloc[0]) == pd.Timestamp(date)
248
+ assert "_NativeDate" not in result.to_csv(index=False)
249
+ assert (pd.to_datetime(result["date"].map(pd.Timestamp)) - pd.Timestamp("2024-01-01")).notna().all()
250
+ date_width = widths[-1]["requests"][1]["updateDimensionProperties"]["properties"]["pixelSize"]
251
+ assert 50 <= date_width < 300
252
+ assert gs.sheet.cells[1][1]["userEnteredValue"] == original_cell
253
+ assert gs.sheet.last_metadata_params["ranges"] == "'report'!A1:B2"
254
+ empty = service.upsert_df(gs.url, "report", pd.DataFrame(columns=["key", "date"]), keys=["key"])
255
+ assert not any(isinstance(value, gsheet_lowlevel._NativeDate) for value in empty["date"])
256
+ assert len(empty) == 2
257
+
258
+
259
+ def test_date_serial_and_textual_serial_are_distinct_upsert_keys():
260
+ gs = _FakeGS("https://example.com/sheet", {"report": []})
261
+ gs.sheet = _RoundTripSheet(gs)
262
+ service = SheetsService(_make_app(gs))
263
+ incoming = pd.DataFrame([{"key": "2024-01-01", "n": 1}, {"key": "45292", "n": 2}])
264
+ for _ in range(3):
265
+ result = service.upsert_df(gs.url, "report", incoming, keys=["key"])
266
+ assert len(result) == 2
267
+ assert result["n"].sum() == 3
268
+
269
+
270
+ def test_upsert_numeric_keys_are_not_inferred_as_dates():
271
+ gs = _FakeGS("https://example.com/sheet", {"report": []})
272
+ gs.sheet = _RoundTripSheet(gs)
273
+ service = SheetsService(_make_app(gs))
274
+ for _ in range(2):
275
+ result = service.upsert_df(gs.url, "report", pd.DataFrame([
276
+ {"key": 45292, "n": 1}, {"key": "0012", "n": 2},
277
+ ]), keys=["key"])
278
+ assert len(result) == 2
279
+ assert any(isinstance(value, (int, float)) and value == 45292 for value in result["key"])
280
+ assert any(value == "0012" for value in result["key"])
281
+ assert all(cell.get("userEnteredFormat", {}).get("numberFormat", {}).get("type") != "DATE"
282
+ for row in gs.sheet.cells[1:] for cell in row)
283
+
284
+
285
+ def test_incomplete_date_keys_are_idempotent_after_cell_roundtrip():
286
+ gs = _FakeGS("https://example.com/sheet", {"report": []})
287
+ gs.sheet = _RoundTripSheet(gs)
288
+ service = SheetsService(_make_app(gs))
289
+ incoming = pd.DataFrame([{"date": "2024-01-01", "medium": None, "n": 1}])
290
+ for _ in range(3):
291
+ result = service.upsert_df(gs.url, "report", incoming, keys=["date", "medium"])
292
+ assert len(result) == 1
293
+
294
+
295
+ def test_upsert_sorts_native_and_new_dates_on_the_same_timeline():
296
+ gs = _FakeGS("https://example.com/sheet", {"report": []})
297
+ gs.sheet = _RoundTripSheet(gs)
298
+ service = SheetsService(_make_app(gs))
299
+ service.upsert_df(gs.url, "report", pd.DataFrame([
300
+ {"date": "1899-12-29", "n": 1}, {"date": "1900-01-09", "n": 3},
301
+ ]), keys=["date"])
302
+ result = service.upsert_df(gs.url, "report", pd.DataFrame([
303
+ {"date": pd.Timestamp("1899-12-31"), "n": 2},
304
+ ]), keys=["date"])
305
+ assert result["n"].tolist() == [1, 2, 3]
306
+
307
+
308
+ def test_upsert_mixed_sort_fallback_preserves_numeric_secondary_sort():
309
+ gs = _FakeGS("https://example.com/sheet", {"report": [{"key": "a", "group": 1, "n": 100}]})
310
+ result = SheetsService(_make_app(gs)).upsert_df(gs.url, "report", pd.DataFrame([
311
+ {"key": "b", "group": "1", "n": 9}, {"key": "c", "group": "1", "n": 10},
312
+ ]), keys=["key"], sort_by=["group", "n"])
313
+ assert result["n"].tolist() == [9, 10, 100]
314
+
315
+
316
+ def test_upsert_preserves_numeric_and_text_key_types():
317
+ gs = _FakeGS("https://example.com/sheet", {"report": [
318
+ {"month": 202608, "code": "0012", "users": 1},
319
+ {"month": 202609, "code": "0012", "users": 2},
320
+ ]})
321
+ gs.sheet.select("report")
322
+ service = SheetsService(_make_app(gs))
323
+ result = service.upsert_df("https://example.com/sheet", "report", pd.DataFrame([
324
+ {"month": 202609, "code": "0012", "users": 3},
325
+ ]), keys=["month", "code"])
326
+ assert result["month"].tolist() == [202608, 202609]
327
+ assert result["code"].tolist() == ["0012", "0012"]
328
+ assert result["users"].tolist() == [1, 3]
329
+
330
+
331
+ def test_upsert_date_keys_match_formatted_history_without_duplicates():
332
+ gs = _FakeGS("https://example.com/sheet", {"report": [
333
+ {"ym": "2026/8/1", "count": 1}, {"ym": "2026/9/1", "count": 2}]})
334
+ gs.sheet.select("report")
335
+ service = SheetsService(_make_app(gs))
336
+ result = service.upsert_df("https://example.com/sheet", "report", pd.DataFrame([
337
+ {"ym": pd.Timestamp("2026-09-01"), "count": 3}]), keys=["ym"])
338
+ assert result["ym"].tolist() == [pd.Timestamp("2026-08-01"), pd.Timestamp("2026-09-01")]
339
+ assert result["count"].tolist() == [1, 3]
340
+
341
+
342
+ @pytest.mark.parametrize("missing", [None, float("nan"), pd.NA, ""])
343
+ def test_upsert_blank_keys_remain_idempotent_and_preserve_history(missing):
344
+ gs = _FakeGS("https://example.com/sheet", {"report": [
345
+ {"month": 202608, "medium": "", "users": 0},
346
+ {"month": 202609, "medium": "", "users": 0},
347
+ {"month": 202609, "medium": "", "users": 0},
348
+ {"month": 202609, "medium": "organic", "users": 2}]})
349
+ gs.sheet.select("report")
350
+ service = SheetsService(_make_app(gs))
351
+ incoming = pd.DataFrame([
352
+ {"month": 202609, "medium": missing, "users": 0},
353
+ {"month": 202609, "medium": "organic", "users": 3}])
354
+ for _ in range(2):
355
+ service.upsert_df("https://example.com/sheet", "report", incoming, keys=["month", "medium"])
356
+ assert len(gs.last_written) == 4
357
+ assert gs.last_written["users"].sum() == 3
358
+ assert len(gs.last_written[gs.last_written["medium"].eq("")]) == 3
359
+
360
+
361
+ def test_upsert_blank_key_ambiguous_change_stops_before_write():
362
+ gs = _FakeGS("https://example.com/sheet", {"report": [
363
+ {"medium": "", "users": 1}, {"medium": "", "users": 2}]})
364
+ service = SheetsService(_make_app(gs))
365
+ with pytest.raises(ValueError, match="ambiguous historical"):
366
+ service.upsert_df("https://example.com/sheet", "report",
367
+ pd.DataFrame([{"medium": None, "users": 3}]), keys=["medium"])
368
+ assert gs.last_written is None
369
+
370
+
371
+ def test_upsert_literal_nan_is_not_a_blank_key():
372
+ gs = _FakeGS("https://example.com/sheet", {"report": [{"medium": "nan", "users": 2}]})
373
+ service = SheetsService(_make_app(gs))
374
+ result = service.upsert_df("https://example.com/sheet", "report",
375
+ pd.DataFrame([{"medium": None, "users": 0}]), keys=["medium"])
376
+ assert len(result) == 2
377
+ assert result["users"].sum() == 2
378
+ assert gs.last_read_options["value_render_option"] == "UNFORMATTED_VALUE"
379
+ assert gs.last_read_options["numericise_ignore"] == ["all"]
380
+
381
+
382
+ def test_upsert_native_date_serial_keys_match_datetime_input():
383
+ serial = (pd.Timestamp("2026-09-01") - pd.Timestamp("1899-12-30")).days
384
+ gs = _FakeGS("https://example.com/sheet", {"report": [{"ym": serial, "count": 1}]})
385
+ service = SheetsService(_make_app(gs))
386
+ result = service.upsert_df("https://example.com/sheet", "report",
387
+ pd.DataFrame([{"ym": pd.Timestamp("2026-09-01"), "count": 3}]), keys=["ym"])
388
+ assert result["ym"].tolist() == [pd.Timestamp("2026-09-01")]
389
+ assert result["count"].tolist() == [3]
390
+
391
+
392
+ def test_upsert_new_null_key_is_idempotent_after_sheets_blank_roundtrip():
393
+ gs = _FakeGS("https://example.com/sheet", {"report": []})
394
+ service = SheetsService(_make_app(gs))
395
+ incoming = pd.DataFrame([{"medium": pd.NA, "source": None, "users": 0}])
396
+ for _ in range(3):
397
+ result = service.upsert_df("https://example.com/sheet", "report", incoming,
398
+ keys=["medium", "source"])
399
+ assert len(result) == 1
400
+ gs._sheets["report"] = result.astype(object).where(pd.notna(result), "").to_dict("records")
401
+
402
+
403
+ def test_upsert_single_blank_history_row_can_be_updated():
404
+ gs = _FakeGS("https://example.com/sheet", {"report": [{"medium": "", "users": 1}]})
405
+ service = SheetsService(_make_app(gs))
406
+ result = service.upsert_df("https://example.com/sheet", "report",
407
+ pd.DataFrame([{"medium": None, "users": 3}]), keys=["medium"])
408
+ assert len(result) == 1
409
+ assert result["users"].sum() == 3
410
+
411
+
115
412
  def test_upsert_df_deduplicates_and_sorts():
116
413
  existing = [
117
414
  {"month": "2024-01", "clinic": "A", "users": 1},
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes