megaton 2.3.0__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {megaton-2.3.0 → megaton-2.4.0}/PKG-INFO +1 -1
- {megaton-2.3.0 → megaton-2.4.0}/megaton/ga4.py +10 -3
- {megaton-2.3.0 → megaton-2.4.0}/megaton/gsheet.py +42 -3
- {megaton-2.3.0 → megaton-2.4.0}/megaton/gsheet_lowlevel.py +22 -3
- {megaton-2.3.0 → megaton-2.4.0}/megaton/services/sheets_service.py +107 -16
- {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/PKG-INFO +1 -1
- {megaton-2.3.0 → megaton-2.4.0}/pyproject.toml +1 -1
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_ga4_report_core.py +7 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_core.py +33 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_request_builders.py +18 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_sheets_service.py +297 -0
- {megaton-2.3.0 → megaton-2.4.0}/LICENSE +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/README.md +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/__init__.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/__main__.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/_result.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/auth/__init__.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/auth/google_auth.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/auth/provider.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/bq.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/cli.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/constants.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/dates.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/errors.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/files.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/gdrive.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/google_api.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/install/__init__.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/install/install_bigquery.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/install/install_ga4.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/os.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/py.typed +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/query.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/recipes/__init__.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/recipes/config_loader.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/retry_utils.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/searchconsole.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/services/__init__.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/services/bq_service.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/services/gsc_service.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/services/sheet_namespace.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/start.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/state.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/__init__.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/classify.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/ga4.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/table.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/text.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/transform/traffic.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/ui/__init__.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/ui/widgets.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton/utils.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/SOURCES.txt +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/dependency_links.txt +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/entry_points.txt +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/requires.txt +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/megaton.egg-info/top_level.txt +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/setup.cfg +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_auth.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_auth_provider.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_auto_install.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_bq_core_extra.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_bq_state_api.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_clean_parameter.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_config_loader.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_dates.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_errors_and_init.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_failure_contracts.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_ga4_property_show.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_ga4_report_retry.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_ga4_state_and_run.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsc_service.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_a1_helpers.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_batch_read.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_facade.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_retry_core.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_retry_resolvers.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_gsheet_timeout.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_megaton_auth_bridge.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_normalize_queries.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_phase0_regressions.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_programmatic_api.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_query_contract.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_query_regressions.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_dates_to_sheet.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_month_window.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_prep.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_result.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_run_multi.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_run_ranges.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_report_show_download.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_retry_utils.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_run_all.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_sc_wrapper.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_search_apply_if.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_search_dimension_filter.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_search_filter.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_search_result.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_session_retry.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_sheet_namespace.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_sheet_save_options.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_sheets_read.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_classify.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_ga4.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_pipeline_gsc_like.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_table.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_table_normalize_thresholds_df.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_text.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_transform_traffic.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_ui_lazy_import.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_upsert_to_csv.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_utils.py +0 -0
- {megaton-2.3.0 → megaton-2.4.0}/tests/test_wrap_and_chain.py +0 -0
|
@@ -715,13 +715,20 @@ class MegatonGA4(object):
|
|
|
715
715
|
TIME (in HH:MM:SS format)
|
|
716
716
|
"""
|
|
717
717
|
type = type.replace('TYPE_', '')
|
|
718
|
-
if type
|
|
718
|
+
if type == 'INTEGER':
|
|
719
719
|
try:
|
|
720
720
|
return int(value)
|
|
721
721
|
except:
|
|
722
722
|
return value
|
|
723
|
-
elif type in ['FLOAT'
|
|
724
|
-
|
|
723
|
+
elif type in ['FLOAT', 'HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS',
|
|
724
|
+
'CURRENCY', 'FEET', 'METERS', 'KILOMETERS', 'MILES', 'STANDARD']:
|
|
725
|
+
try:
|
|
726
|
+
number = float(value)
|
|
727
|
+
if type in ['HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS'] and number.is_integer():
|
|
728
|
+
return int(number)
|
|
729
|
+
return number
|
|
730
|
+
except (TypeError, ValueError):
|
|
731
|
+
return value
|
|
725
732
|
else:
|
|
726
733
|
return value
|
|
727
734
|
|
|
@@ -459,18 +459,57 @@ class MegatonGS(object):
|
|
|
459
459
|
"""
|
|
460
460
|
return self.last_row + 1
|
|
461
461
|
|
|
462
|
-
def get_records(self, *, max_retries: Optional[int] = None, backoff_factor: Optional[float] = None
|
|
462
|
+
def get_records(self, *, max_retries: Optional[int] = None, backoff_factor: Optional[float] = None,
|
|
463
|
+
value_render_option=None, numericise_ignore=None, preserve_dates=False):
|
|
463
464
|
"""Read all records with header row as keys."""
|
|
464
465
|
if not self._driver:
|
|
465
466
|
LOGGER.error("Please select a sheet first.")
|
|
466
467
|
return
|
|
467
|
-
|
|
468
|
+
options = {}
|
|
469
|
+
if value_render_option is not None:
|
|
470
|
+
options["value_render_option"] = value_render_option
|
|
471
|
+
if numericise_ignore is not None:
|
|
472
|
+
options["numericise_ignore"] = numericise_ignore
|
|
473
|
+
records = self._maybe_retry(
|
|
468
474
|
"Google Sheets read records",
|
|
469
|
-
lambda: self._driver.get_all_records(),
|
|
475
|
+
lambda: self._driver.get_all_records(**options),
|
|
470
476
|
max_retries=max_retries,
|
|
471
477
|
backoff_factor=backoff_factor,
|
|
472
478
|
retry_on_requests=True,
|
|
473
479
|
)
|
|
480
|
+
if not preserve_dates or not records:
|
|
481
|
+
return records
|
|
482
|
+
if value_render_option != "UNFORMATTED_VALUE" or numericise_ignore != ["all"]:
|
|
483
|
+
raise ValueError("preserve_dates requires unformatted, non-numericised records")
|
|
484
|
+
title = self._driver.title.replace("'", "''")
|
|
485
|
+
last_cell = gspread.utils.rowcol_to_a1(len(records) + 1, len(records[0]))
|
|
486
|
+
metadata = self._maybe_retry(
|
|
487
|
+
"Google Sheets read date formats",
|
|
488
|
+
lambda: self.parent._driver.fetch_sheet_metadata(params={
|
|
489
|
+
"ranges": "'" + title + "'!A1:" + last_cell,
|
|
490
|
+
"fields": "sheets(data(startRow,startColumn,rowData(values(effectiveValue,effectiveFormat.numberFormat))))",
|
|
491
|
+
"includeGridData": True,
|
|
492
|
+
}),
|
|
493
|
+
max_retries=max_retries, backoff_factor=backoff_factor,
|
|
494
|
+
retry_on_requests=True,
|
|
495
|
+
)
|
|
496
|
+
headers = list(records[0])
|
|
497
|
+
for sheet in metadata.get("sheets", []):
|
|
498
|
+
for grid in sheet.get("data", []):
|
|
499
|
+
for offset, row in enumerate(grid.get("rowData", [])):
|
|
500
|
+
index = grid.get("startRow", 0) + offset - 1
|
|
501
|
+
if not 0 <= index < len(records):
|
|
502
|
+
continue
|
|
503
|
+
for ci, cell in enumerate(row.get("values", []), grid.get("startColumn", 0)):
|
|
504
|
+
fmt = cell.get("effectiveFormat", {}).get("numberFormat", {})
|
|
505
|
+
serial = cell.get("effectiveValue", {}).get("numberValue")
|
|
506
|
+
if ci >= len(headers) or serial is None or fmt.get("type") not in ("DATE", "DATE_TIME", "TIME"):
|
|
507
|
+
continue
|
|
508
|
+
key = headers[ci]
|
|
509
|
+
if records[index][key] != serial:
|
|
510
|
+
raise ValueError("Sheet changed while reading date formats; no write performed")
|
|
511
|
+
records[index][key] = gsheet_lowlevel._NativeDate(serial, fmt)
|
|
512
|
+
return records
|
|
474
513
|
|
|
475
514
|
@property
|
|
476
515
|
def data(self):
|
|
@@ -18,6 +18,8 @@ from __future__ import annotations
|
|
|
18
18
|
import datetime as _dt
|
|
19
19
|
import logging
|
|
20
20
|
import math
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from numbers import Integral, Real
|
|
21
23
|
import re
|
|
22
24
|
import time
|
|
23
25
|
from collections.abc import Iterable
|
|
@@ -282,6 +284,14 @@ _DATETIME_RE = re.compile(
|
|
|
282
284
|
_SHEETS_EPOCH_DATE = _dt.date(1899, 12, 30)
|
|
283
285
|
|
|
284
286
|
|
|
287
|
+
@dataclass(frozen=True)
|
|
288
|
+
class _NativeDate:
|
|
289
|
+
"""A persisted date cell, without a lossy serial-to-datetime round trip."""
|
|
290
|
+
|
|
291
|
+
serial: float
|
|
292
|
+
number_format: dict
|
|
293
|
+
|
|
294
|
+
|
|
285
295
|
def _is_missing_scalar(value: Any) -> bool:
|
|
286
296
|
if value is None:
|
|
287
297
|
return True
|
|
@@ -301,12 +311,12 @@ def cell_value(value: Any, *, parse_formulas: bool = True) -> dict:
|
|
|
301
311
|
return {"stringValue": ""}
|
|
302
312
|
if isinstance(value, bool):
|
|
303
313
|
return {"boolValue": value}
|
|
304
|
-
if isinstance(value,
|
|
314
|
+
if isinstance(value, Integral):
|
|
305
315
|
return {"numberValue": float(value)}
|
|
306
|
-
if isinstance(value,
|
|
316
|
+
if isinstance(value, Real):
|
|
307
317
|
if math.isnan(value) or math.isinf(value):
|
|
308
318
|
return {"stringValue": ""}
|
|
309
|
-
return {"numberValue": value}
|
|
319
|
+
return {"numberValue": float(value)}
|
|
310
320
|
if isinstance(value, str):
|
|
311
321
|
s = value.strip()
|
|
312
322
|
if not s:
|
|
@@ -349,6 +359,15 @@ def cell_data(
|
|
|
349
359
|
ISO date and naive ISO datetime strings are written as real Sheets serials
|
|
350
360
|
with a display format. Other strings stay strings.
|
|
351
361
|
"""
|
|
362
|
+
if isinstance(value, _NativeDate):
|
|
363
|
+
return {
|
|
364
|
+
"userEnteredValue": {"numberValue": value.serial},
|
|
365
|
+
"userEnteredFormat": {"numberFormat": dict(value.number_format)},
|
|
366
|
+
}
|
|
367
|
+
if _is_missing_scalar(value):
|
|
368
|
+
return {"userEnteredValue": cell_value(value, parse_formulas=parse_formulas)}
|
|
369
|
+
if parse_dates and isinstance(value, _dt.date):
|
|
370
|
+
value = value.isoformat()
|
|
352
371
|
if isinstance(value, str):
|
|
353
372
|
if not parse_formulas and not parse_dates:
|
|
354
373
|
return {"userEnteredValue": cell_value(value, parse_formulas=False)}
|
|
@@ -1,15 +1,38 @@
|
|
|
1
1
|
"""Google Sheets service wrapper."""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
from datetime import datetime, timedelta
|
|
5
|
+
from numbers import Real
|
|
4
6
|
from typing import Optional
|
|
5
7
|
|
|
6
8
|
import pandas as pd
|
|
7
9
|
|
|
8
|
-
from .. import errors, gsheet, mount_google_drive
|
|
10
|
+
from .. import errors, gsheet, gsheet_lowlevel, mount_google_drive
|
|
9
11
|
|
|
10
12
|
logger = logging.getLogger(__name__)
|
|
11
13
|
|
|
12
14
|
|
|
15
|
+
def _public_sheet_frame(frame):
|
|
16
|
+
"""Keep native serial/format carriers out of public results and sizing."""
|
|
17
|
+
def public_value(value):
|
|
18
|
+
if not isinstance(value, gsheet_lowlevel._NativeDate):
|
|
19
|
+
return value
|
|
20
|
+
instant = datetime(1899, 12, 30) + timedelta(days=value.serial)
|
|
21
|
+
kind = value.number_format.get("type")
|
|
22
|
+
if kind == "TIME":
|
|
23
|
+
return instant.time()
|
|
24
|
+
if kind == "DATE" and instant == instant.replace(hour=0, minute=0, second=0, microsecond=0):
|
|
25
|
+
return instant.date()
|
|
26
|
+
return instant
|
|
27
|
+
result = frame.copy()
|
|
28
|
+
for column in frame.columns:
|
|
29
|
+
if any(isinstance(value, gsheet_lowlevel._NativeDate) for value in frame[column]):
|
|
30
|
+
result[column] = pd.Series(
|
|
31
|
+
[public_value(value) for value in frame[column]], index=frame.index, dtype=object,
|
|
32
|
+
)
|
|
33
|
+
return result
|
|
34
|
+
|
|
35
|
+
|
|
13
36
|
class SheetsService:
|
|
14
37
|
def __init__(self, app):
|
|
15
38
|
self.app = app
|
|
@@ -447,6 +470,9 @@ class SheetsService:
|
|
|
447
470
|
self.app.gs.sheet.get_records(
|
|
448
471
|
max_retries=max_retries,
|
|
449
472
|
backoff_factor=backoff_factor,
|
|
473
|
+
value_render_option="UNFORMATTED_VALUE",
|
|
474
|
+
numericise_ignore=["all"],
|
|
475
|
+
preserve_dates=True,
|
|
450
476
|
)
|
|
451
477
|
)
|
|
452
478
|
except Exception as exc:
|
|
@@ -457,9 +483,10 @@ class SheetsService:
|
|
|
457
483
|
|
|
458
484
|
df_new = df_new.copy()
|
|
459
485
|
if df_new.empty:
|
|
460
|
-
return df_existing
|
|
486
|
+
return _public_sheet_frame(df_existing)
|
|
461
487
|
if df_existing.empty:
|
|
462
488
|
try:
|
|
489
|
+
public_frame = _public_sheet_frame(df_new)
|
|
463
490
|
self.app.gs.sheet.overwrite_data(
|
|
464
491
|
df_new,
|
|
465
492
|
include_index=False,
|
|
@@ -467,7 +494,7 @@ class SheetsService:
|
|
|
467
494
|
backoff_factor=backoff_factor,
|
|
468
495
|
)
|
|
469
496
|
self._apply_write_options(
|
|
470
|
-
|
|
497
|
+
public_frame,
|
|
471
498
|
auto_width=auto_width,
|
|
472
499
|
freeze_header=freeze_header,
|
|
473
500
|
width_min=width_min,
|
|
@@ -478,27 +505,75 @@ class SheetsService:
|
|
|
478
505
|
backoff_factor=backoff_factor,
|
|
479
506
|
)
|
|
480
507
|
print(f"'{sheet_name}' シートへ {len(df_new)} 行を書き込みました。")
|
|
481
|
-
return
|
|
508
|
+
return public_frame
|
|
482
509
|
except Exception as exc:
|
|
483
510
|
logger.exception("Failed to overwrite sheet '%s' (empty upsert)", sheet_name)
|
|
484
511
|
print(f"'{sheet_name}' シートへの書き込みに失敗しました: {exc}")
|
|
485
512
|
raise
|
|
486
513
|
|
|
487
514
|
df_existing = df_existing.copy()
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
515
|
+
# Legacy formatted dates are parsed only when the incoming column has
|
|
516
|
+
# an explicit datetime dtype. Native cells retain their serial/format.
|
|
517
|
+
date_keys = [key for key in keys if pd.api.types.is_datetime64_any_dtype(df_new[key])]
|
|
518
|
+
for key in date_keys:
|
|
519
|
+
if key in df_existing:
|
|
520
|
+
def date_value(value):
|
|
521
|
+
if isinstance(value, gsheet_lowlevel._NativeDate):
|
|
522
|
+
return value
|
|
523
|
+
if isinstance(value, Real) and not pd.isna(value):
|
|
524
|
+
return gsheet_lowlevel._NativeDate(float(value), {
|
|
525
|
+
"type": "DATE_TIME", "pattern": "yyyy-mm-dd hh:mm:ss.000",
|
|
526
|
+
})
|
|
527
|
+
return pd.Timestamp(value) if not pd.isna(value) else value
|
|
528
|
+
df_existing[key] = df_existing[key].map(date_value)
|
|
494
529
|
try:
|
|
495
|
-
|
|
496
|
-
|
|
530
|
+
def normalize(value):
|
|
531
|
+
cell = gsheet_lowlevel.cell_data(value)["userEnteredValue"]
|
|
532
|
+
if "numberValue" in cell:
|
|
533
|
+
number = cell["numberValue"]
|
|
534
|
+
return str(int(number)) if float(number).is_integer() else repr(float(number))
|
|
535
|
+
return str(next(iter(cell.values()), "")).strip()
|
|
536
|
+
def key_value(value):
|
|
537
|
+
cell = gsheet_lowlevel.cell_data(value)["userEnteredValue"]
|
|
538
|
+
kind, content = next(iter(cell.items()), ("stringValue", ""))
|
|
539
|
+
if kind == "stringValue" and content == "":
|
|
540
|
+
return ""
|
|
541
|
+
return (kind, content)
|
|
542
|
+
new_keys = df_new[keys].apply(lambda col: col.map(key_value))
|
|
543
|
+
existing_keys = df_existing[keys].apply(lambda col: col.map(key_value))
|
|
544
|
+
new_tuples = new_keys.apply(tuple, axis=1)
|
|
545
|
+
existing_tuples = existing_keys.apply(tuple, axis=1)
|
|
546
|
+
incomplete = new_keys.eq("").any(axis=1)
|
|
547
|
+
keys_to_remove = set(new_tuples[~incomplete])
|
|
548
|
+
mask = existing_tuples.isin(keys_to_remove)
|
|
497
549
|
except KeyError as exc:
|
|
498
550
|
raise ValueError(f"Existing sheet is missing upsert keys: {keys}") from exc
|
|
499
551
|
|
|
552
|
+
# Incomplete keys can have historical duplicates. Keep those rows when
|
|
553
|
+
# the incoming row is already present; never collapse ambiguous history.
|
|
554
|
+
skip_new = pd.Series(False, index=df_new.index)
|
|
555
|
+
history_indexes = {}
|
|
556
|
+
for index, key_tuple in existing_tuples.items():
|
|
557
|
+
history_indexes.setdefault(key_tuple, []).append(index)
|
|
558
|
+
for index in df_new.index[incomplete]:
|
|
559
|
+
matches = history_indexes.get(new_tuples.loc[index], [])
|
|
560
|
+
history = df_existing.loc[matches].reindex(columns=df_new.columns)
|
|
561
|
+
row = df_new.loc[index].astype(object)
|
|
562
|
+
row = row.where(pd.notna(row), "")
|
|
563
|
+
history = history.astype(object).where(pd.notna(history), "")
|
|
564
|
+
row = row.map(lambda value: gsheet_lowlevel.cell_data(value)["userEnteredValue"])
|
|
565
|
+
history = history.apply(lambda col: col.map(
|
|
566
|
+
lambda value: gsheet_lowlevel.cell_data(value)["userEnteredValue"]
|
|
567
|
+
))
|
|
568
|
+
if not history.empty and history.eq(row, axis="columns").all(axis=1).any():
|
|
569
|
+
skip_new.loc[index] = True
|
|
570
|
+
elif len(history) > 1:
|
|
571
|
+
raise ValueError("Cannot update an incomplete upsert key with ambiguous historical rows; no write performed")
|
|
572
|
+
else:
|
|
573
|
+
mask.loc[matches] = True
|
|
574
|
+
|
|
500
575
|
df_cleaned = df_existing[~mask]
|
|
501
|
-
df_combined = pd.concat([df_cleaned, df_new], ignore_index=True)
|
|
576
|
+
df_combined = pd.concat([df_cleaned, df_new[~skip_new]], ignore_index=True)
|
|
502
577
|
|
|
503
578
|
sort_cols = sort_by or list(keys)
|
|
504
579
|
if sort_cols:
|
|
@@ -506,7 +581,22 @@ class SheetsService:
|
|
|
506
581
|
if missing_sort:
|
|
507
582
|
print(f"ソート対象の列が見つかりません: {missing_sort}")
|
|
508
583
|
else:
|
|
509
|
-
|
|
584
|
+
def sort_value(value):
|
|
585
|
+
cell = gsheet_lowlevel.cell_data(value)
|
|
586
|
+
fmt = cell.get("userEnteredFormat", {}).get("numberFormat", {})
|
|
587
|
+
if fmt.get("type") in ("DATE", "DATE_TIME", "TIME"):
|
|
588
|
+
return cell["userEnteredValue"]["numberValue"]
|
|
589
|
+
return value
|
|
590
|
+
sort_frame = df_combined[sort_cols].apply(
|
|
591
|
+
lambda col: col.map(sort_value)
|
|
592
|
+
)
|
|
593
|
+
for col in sort_cols:
|
|
594
|
+
try:
|
|
595
|
+
sort_frame[col].sort_values()
|
|
596
|
+
except TypeError:
|
|
597
|
+
sort_frame[col] = sort_frame[col].map(normalize)
|
|
598
|
+
order = sort_frame.sort_values(by=sort_cols, kind="stable").index
|
|
599
|
+
df_combined = df_combined.loc[order]
|
|
510
600
|
|
|
511
601
|
if columns:
|
|
512
602
|
for col in columns:
|
|
@@ -515,6 +605,7 @@ class SheetsService:
|
|
|
515
605
|
df_combined = df_combined[columns]
|
|
516
606
|
|
|
517
607
|
try:
|
|
608
|
+
public_frame = _public_sheet_frame(df_combined)
|
|
518
609
|
self.app.gs.sheet.overwrite_data(
|
|
519
610
|
df_combined,
|
|
520
611
|
include_index=False,
|
|
@@ -522,7 +613,7 @@ class SheetsService:
|
|
|
522
613
|
backoff_factor=backoff_factor,
|
|
523
614
|
)
|
|
524
615
|
self._apply_write_options(
|
|
525
|
-
|
|
616
|
+
public_frame,
|
|
526
617
|
auto_width=auto_width,
|
|
527
618
|
freeze_header=freeze_header,
|
|
528
619
|
width_min=width_min,
|
|
@@ -533,7 +624,7 @@ class SheetsService:
|
|
|
533
624
|
backoff_factor=backoff_factor,
|
|
534
625
|
)
|
|
535
626
|
print(f"'{sheet_name}' シートを更新しました(新規 {len(df_new)} 行、削除 {mask.sum()} 行)。")
|
|
536
|
-
return
|
|
627
|
+
return public_frame
|
|
537
628
|
except Exception as exc:
|
|
538
629
|
logger.exception("Failed to overwrite sheet '%s' (upsert)", sheet_name)
|
|
539
630
|
print(f"'{sheet_name}' シートへの書き込みに失敗しました: {exc}")
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "megaton"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.4.0"
|
|
8
8
|
description = "Google analytics toolkit for Python, CLI and Notebooks: GA4, Search Console, Sheets and BigQuery."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -52,6 +52,13 @@ def test_convert_metric_numeric_and_fallback():
|
|
|
52
52
|
assert report._convert_metric("10", "TYPE_INTEGER") == 10
|
|
53
53
|
assert report._convert_metric("1.5", "TYPE_FLOAT") == 1.5
|
|
54
54
|
assert report._convert_metric("not-number", "TYPE_INTEGER") == "not-number"
|
|
55
|
+
for kind in ("SECONDS", "HOURS", "MINUTES", "MILLISECONDS", "CURRENCY", "METERS"):
|
|
56
|
+
assert report._convert_metric("673.8335", "TYPE_" + kind) == 673.8335
|
|
57
|
+
assert report._convert_metric("not-number", "TYPE_" + kind) == "not-number"
|
|
58
|
+
for kind in ("SECONDS", "HOURS", "MINUTES", "MILLISECONDS"):
|
|
59
|
+
assert type(report._convert_metric("1234", "TYPE_" + kind)) is int
|
|
60
|
+
assert type(report._convert_metric("1234.5", "TYPE_" + kind)) is float
|
|
61
|
+
assert pd.DataFrame({"duration": [report._convert_metric("1234", "TYPE_" + kind)]}).to_csv(index=False) == "duration\n1234\n"
|
|
55
62
|
|
|
56
63
|
|
|
57
64
|
def test_parse_response_converts_metric_values():
|
|
@@ -15,6 +15,39 @@ def _new_gs():
|
|
|
15
15
|
return gs
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
def test_records_native_read_disables_numericising_text_identifiers():
|
|
19
|
+
calls = []
|
|
20
|
+
def get_all_records(**kwargs):
|
|
21
|
+
calls.append(kwargs)
|
|
22
|
+
return [{"source": "nan", "code": "0012", "users": 3}]
|
|
23
|
+
sheet = MegatonGS.Sheet(SimpleNamespace())
|
|
24
|
+
sheet._driver = SimpleNamespace(get_all_records=get_all_records)
|
|
25
|
+
assert sheet.get_records()[0]["source"] == "nan"
|
|
26
|
+
assert calls[-1] == {}
|
|
27
|
+
assert sheet.get_records(value_render_option="UNFORMATTED_VALUE", numericise_ignore=["all"])[0]["code"] == "0012"
|
|
28
|
+
assert calls[-1] == {"value_render_option": "UNFORMATTED_VALUE", "numericise_ignore": ["all"]}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_preserve_dates_requires_consistent_native_read():
|
|
32
|
+
sheet = MegatonGS.Sheet(SimpleNamespace())
|
|
33
|
+
sheet._driver = SimpleNamespace(get_all_records=lambda **kw: [{"date": "2024-01-01"}])
|
|
34
|
+
with pytest.raises(ValueError, match="unformatted"):
|
|
35
|
+
sheet.get_records(preserve_dates=True)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_preserve_dates_stops_when_metadata_and_values_disagree():
|
|
39
|
+
metadata = {"sheets": [{"data": [{"rowData": [{}, {"values": [{
|
|
40
|
+
"effectiveValue": {"numberValue": 45293},
|
|
41
|
+
"effectiveFormat": {"numberFormat": {"type": "DATE", "pattern": "yyyy-mm-dd"}},
|
|
42
|
+
}]}]}]}]}
|
|
43
|
+
sheet = MegatonGS.Sheet(SimpleNamespace(_driver=SimpleNamespace(
|
|
44
|
+
fetch_sheet_metadata=lambda **kw: metadata,
|
|
45
|
+
)))
|
|
46
|
+
sheet._driver = SimpleNamespace(title="report", get_all_records=lambda **kw: [{"date": 45292}])
|
|
47
|
+
with pytest.raises(ValueError, match="Sheet changed"):
|
|
48
|
+
sheet.get_records(value_render_option="UNFORMATTED_VALUE", numericise_ignore=["all"], preserve_dates=True)
|
|
49
|
+
|
|
50
|
+
|
|
18
51
|
def _api_error(message: str, code: int = 403) -> gspread.exceptions.APIError:
|
|
19
52
|
resp = requests.Response()
|
|
20
53
|
resp.status_code = code
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
from datetime import date
|
|
3
4
|
import pandas as pd
|
|
4
5
|
|
|
5
6
|
from megaton.gsheet_lowlevel import (
|
|
@@ -16,6 +17,17 @@ def test_cell_value_preserves_string_codes() -> None:
|
|
|
16
17
|
assert cell_value("03311187") == {"stringValue": "03311187"}
|
|
17
18
|
|
|
18
19
|
|
|
20
|
+
def test_nullable_numeric_columns_remain_sheet_numbers():
|
|
21
|
+
from megaton.gsheet_lowlevel import dataframe_update_cells_rows
|
|
22
|
+
|
|
23
|
+
frame = pd.DataFrame({"count": pd.Series([12, pd.NA], dtype="Int64"),
|
|
24
|
+
"code": ["0012", "0007"]})
|
|
25
|
+
rows = dataframe_update_cells_rows(frame)
|
|
26
|
+
assert rows[1]["values"][0]["userEnteredValue"] == {"numberValue": 12.0}
|
|
27
|
+
assert rows[1]["values"][1]["userEnteredValue"] == {"stringValue": "0012"}
|
|
28
|
+
assert rows[2]["values"][0]["userEnteredValue"] == {"stringValue": ""}
|
|
29
|
+
|
|
30
|
+
|
|
19
31
|
def test_cell_value_builds_scalar_user_entered_values() -> None:
|
|
20
32
|
assert cell_value(True) == {"boolValue": True}
|
|
21
33
|
assert cell_value(1000) == {"numberValue": 1000.0}
|
|
@@ -34,6 +46,12 @@ def test_cell_data_writes_iso_dates_as_sheet_serials() -> None:
|
|
|
34
46
|
}
|
|
35
47
|
|
|
36
48
|
|
|
49
|
+
def test_python_date_scalars_write_real_sheet_dates():
|
|
50
|
+
assert cell_data(date(2026, 6, 23)) == cell_data("2026-06-23")
|
|
51
|
+
assert cell_data(date(2026, 6, 23), parse_dates=False) == {
|
|
52
|
+
"userEnteredValue": {"stringValue": "2026-06-23"}}
|
|
53
|
+
|
|
54
|
+
|
|
37
55
|
def test_cell_data_writes_naive_iso_datetimes_as_sheet_serials() -> None:
|
|
38
56
|
result = cell_data("2026-06-23 19:05")
|
|
39
57
|
|
|
@@ -4,6 +4,8 @@ import pandas as pd
|
|
|
4
4
|
import pytest
|
|
5
5
|
|
|
6
6
|
from megaton import errors
|
|
7
|
+
from megaton import gsheet_lowlevel
|
|
8
|
+
from megaton.gsheet import MegatonGS
|
|
7
9
|
from megaton.services.sheets_service import SheetsService
|
|
8
10
|
|
|
9
11
|
|
|
@@ -28,6 +30,7 @@ class _FakeSheet:
|
|
|
28
30
|
return self.parent._sheets.get(self._name, [])
|
|
29
31
|
|
|
30
32
|
def get_records(self, **_kwargs):
|
|
33
|
+
self.parent.last_read_options = _kwargs
|
|
31
34
|
return self.data
|
|
32
35
|
|
|
33
36
|
def select(self, name: str):
|
|
@@ -112,6 +115,300 @@ def _make_app(gs):
|
|
|
112
115
|
)
|
|
113
116
|
|
|
114
117
|
|
|
118
|
+
class _RoundTripSheet(_FakeSheet):
|
|
119
|
+
"""Execute real record reading and atomic cell building against a fake grid."""
|
|
120
|
+
|
|
121
|
+
def __init__(self, parent):
|
|
122
|
+
super().__init__(parent)
|
|
123
|
+
self.cells = []
|
|
124
|
+
self.ws = SimpleNamespace(
|
|
125
|
+
title="report", id=1, row_count=100, col_count=10,
|
|
126
|
+
get_all_records=self._records,
|
|
127
|
+
)
|
|
128
|
+
api = SimpleNamespace(batch_update=self._batch, fetch_sheet_metadata=self._metadata)
|
|
129
|
+
self.real = MegatonGS.Sheet(SimpleNamespace(_driver=api))
|
|
130
|
+
self.real._driver = self.ws
|
|
131
|
+
|
|
132
|
+
def _records(self, **kwargs):
|
|
133
|
+
headers = [cell["userEnteredValue"]["stringValue"] for cell in self.cells[0]]
|
|
134
|
+
return [dict(zip(headers, [next(iter(cell.get("userEnteredValue", {}).values()), "")
|
|
135
|
+
for cell in row])) for row in self.cells[1:]] if self.cells else []
|
|
136
|
+
|
|
137
|
+
def _metadata(self, **kwargs):
|
|
138
|
+
self.last_metadata_params = kwargs["params"]
|
|
139
|
+
return {"sheets": [{"data": [{"rowData": [{"values": [
|
|
140
|
+
{"effectiveValue": cell.get("userEnteredValue", {}),
|
|
141
|
+
"effectiveFormat": cell.get("userEnteredFormat", {})} for cell in row
|
|
142
|
+
]} for row in self.cells]}]}]}
|
|
143
|
+
|
|
144
|
+
def _batch(self, payload):
|
|
145
|
+
for request in payload["requests"]:
|
|
146
|
+
update = request.get("updateCells", {})
|
|
147
|
+
if "rows" in update:
|
|
148
|
+
old = self.cells
|
|
149
|
+
self.cells = [record["values"] for record in update["rows"]]
|
|
150
|
+
for ri, row in enumerate(self.cells):
|
|
151
|
+
for ci, cell in enumerate(row):
|
|
152
|
+
cell.pop("userEnteredFormat", None)
|
|
153
|
+
if ri < len(old) and ci < len(old[ri]):
|
|
154
|
+
cell["userEnteredFormat"] = old[ri][ci].get("userEnteredFormat", {})
|
|
155
|
+
repeat = request.get("repeatCell", {})
|
|
156
|
+
if repeat:
|
|
157
|
+
region = repeat["range"]
|
|
158
|
+
for ri in range(region["startRowIndex"], region["endRowIndex"]):
|
|
159
|
+
self.cells[ri][region["startColumnIndex"]]["userEnteredFormat"] = repeat["cell"]["userEnteredFormat"]
|
|
160
|
+
|
|
161
|
+
def get_records(self, **kwargs):
|
|
162
|
+
if not self.cells:
|
|
163
|
+
return []
|
|
164
|
+
return self.real.get_records(**kwargs)
|
|
165
|
+
|
|
166
|
+
def overwrite_data(self, df, include_index=False, **kwargs):
|
|
167
|
+
self.parent.last_written = df.copy()
|
|
168
|
+
return self.real._atomic_overwrite(
|
|
169
|
+
df, row=1, include_index=include_index, max_retries=None, backoff_factor=None,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@pytest.mark.parametrize("date", [
|
|
174
|
+
"2024-01-01", "2024-01-01 10:30:15", pd.Timestamp("2024-01-01 10:30:15"),
|
|
175
|
+
pd.Timestamp("2024-01-01 10:30:15.123456"),
|
|
176
|
+
])
|
|
177
|
+
def test_upsert_date_keys_survive_real_cell_roundtrip(date):
|
|
178
|
+
gs = _FakeGS("https://example.com/sheet", {"report": []})
|
|
179
|
+
gs.sheet = _RoundTripSheet(gs)
|
|
180
|
+
service = SheetsService(_make_app(gs))
|
|
181
|
+
for count in (1, 2, 3):
|
|
182
|
+
result = service.upsert_df(gs.url, "report", pd.DataFrame([{"date": date, "count": count}]), keys=["date"])
|
|
183
|
+
assert len(result) == 1
|
|
184
|
+
assert result["count"].tolist() == [count]
|
|
185
|
+
assert len(gs.sheet.cells) == 2
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def test_upsert_preserves_distinct_subsecond_keys_and_nonkey_date_formats():
|
|
189
|
+
gs = _FakeGS("https://example.com/sheet", {"report": []})
|
|
190
|
+
gs.sheet = _RoundTripSheet(gs)
|
|
191
|
+
service = SheetsService(_make_app(gs))
|
|
192
|
+
first = pd.DataFrame([
|
|
193
|
+
{"key": "2024-01-01 10:30:15.100000", "n": 9, "date": "2024-01-02"},
|
|
194
|
+
{"key": "2024-01-01 10:30:15.200000", "n": 100, "date": "2024-01-03"},
|
|
195
|
+
])
|
|
196
|
+
service.upsert_df(gs.url, "report", first, keys=["key"], sort_by=["n"])
|
|
197
|
+
second = pd.DataFrame([{"key": "2024-01-01 10:30:15.100000", "n": 101, "date": "2024-01-02"}])
|
|
198
|
+
result = service.upsert_df(gs.url, "report", second, keys=["key"], sort_by=["n"])
|
|
199
|
+
assert len(result) == 2
|
|
200
|
+
assert result["n"].tolist() == [100, 101]
|
|
201
|
+
for row, expected in zip(gs.sheet.cells[1:], ("2024-01-03", "2024-01-02")):
|
|
202
|
+
assert row[2]["userEnteredValue"] == gsheet_lowlevel.cell_data(expected)["userEnteredValue"]
|
|
203
|
+
assert row[2]["userEnteredFormat"]["numberFormat"]["type"] == "DATE"
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def test_retained_nonkey_date_moves_to_unformatted_row_with_original_format():
|
|
207
|
+
gs = _FakeGS("https://example.com/sheet", {"report": []})
|
|
208
|
+
gs.sheet = _RoundTripSheet(gs)
|
|
209
|
+
service = SheetsService(_make_app(gs))
|
|
210
|
+
service.upsert_df(gs.url, "report", pd.DataFrame([
|
|
211
|
+
{"key": "a", "n": 100, "date": "2024-01-02"},
|
|
212
|
+
{"key": "b", "n": 9, "date": ""},
|
|
213
|
+
]), keys=["key"])
|
|
214
|
+
original = {"type": "DATE", "pattern": "dd/mm/yyyy"}
|
|
215
|
+
gs.sheet.cells[1][2]["userEnteredFormat"]["numberFormat"] = original
|
|
216
|
+
service.upsert_df(gs.url, "report", pd.DataFrame([
|
|
217
|
+
{"key": "c", "n": 10, "date": ""},
|
|
218
|
+
]), keys=["key"], sort_by=["n"])
|
|
219
|
+
assert gs.sheet.cells[3][0]["userEnteredValue"]["stringValue"] == "a"
|
|
220
|
+
assert gs.sheet.cells[3][2]["userEnteredValue"] == {"numberValue": 45293.0}
|
|
221
|
+
assert gs.sheet.cells[3][2]["userEnteredFormat"]["numberFormat"] == original
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def test_upsert_numeric_sort_and_no_dataframe_map(monkeypatch):
|
|
225
|
+
monkeypatch.delattr(pd.DataFrame, "map", raising=False)
|
|
226
|
+
gs = _FakeGS("https://example.com/sheet", {"report": [{"key": "a", "n": 9}, {"key": "b", "n": 100}]})
|
|
227
|
+
result = SheetsService(_make_app(gs)).upsert_df(
|
|
228
|
+
gs.url, "report", pd.DataFrame([{"key": "c", "n": 10}]), keys=["key"], sort_by=["n"],
|
|
229
|
+
)
|
|
230
|
+
assert result["n"].tolist() == [9, 10, 100]
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@pytest.mark.parametrize("date", ["2024-01-01", "2024-01-01 10:30:15.123456"])
|
|
234
|
+
def test_upsert_returns_public_dates_and_sizes_without_internal_objects(date):
|
|
235
|
+
gs = _FakeGS("https://example.com/sheet", {"report": []})
|
|
236
|
+
gs.sheet = _RoundTripSheet(gs)
|
|
237
|
+
gs.sheet.id = 1
|
|
238
|
+
widths = []
|
|
239
|
+
gs._driver.batch_update = widths.append
|
|
240
|
+
service = SheetsService(_make_app(gs))
|
|
241
|
+
service.upsert_df(gs.url, "report", pd.DataFrame([{"key": "a", "date": date}]), keys=["key"])
|
|
242
|
+
original_cell = gs.sheet.cells[1][1]["userEnteredValue"].copy()
|
|
243
|
+
result = service.upsert_df(gs.url, "report", pd.DataFrame([
|
|
244
|
+
{"key": "b", "date": "2024-01-02"},
|
|
245
|
+
]), keys=["key"], auto_width=True)
|
|
246
|
+
assert not any(isinstance(value, gsheet_lowlevel._NativeDate) for value in result["date"])
|
|
247
|
+
assert pd.Timestamp(result["date"].iloc[0]) == pd.Timestamp(date)
|
|
248
|
+
assert "_NativeDate" not in result.to_csv(index=False)
|
|
249
|
+
assert (pd.to_datetime(result["date"].map(pd.Timestamp)) - pd.Timestamp("2024-01-01")).notna().all()
|
|
250
|
+
date_width = widths[-1]["requests"][1]["updateDimensionProperties"]["properties"]["pixelSize"]
|
|
251
|
+
assert 50 <= date_width < 300
|
|
252
|
+
assert gs.sheet.cells[1][1]["userEnteredValue"] == original_cell
|
|
253
|
+
assert gs.sheet.last_metadata_params["ranges"] == "'report'!A1:B2"
|
|
254
|
+
empty = service.upsert_df(gs.url, "report", pd.DataFrame(columns=["key", "date"]), keys=["key"])
|
|
255
|
+
assert not any(isinstance(value, gsheet_lowlevel._NativeDate) for value in empty["date"])
|
|
256
|
+
assert len(empty) == 2
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def test_date_serial_and_textual_serial_are_distinct_upsert_keys():
|
|
260
|
+
gs = _FakeGS("https://example.com/sheet", {"report": []})
|
|
261
|
+
gs.sheet = _RoundTripSheet(gs)
|
|
262
|
+
service = SheetsService(_make_app(gs))
|
|
263
|
+
incoming = pd.DataFrame([{"key": "2024-01-01", "n": 1}, {"key": "45292", "n": 2}])
|
|
264
|
+
for _ in range(3):
|
|
265
|
+
result = service.upsert_df(gs.url, "report", incoming, keys=["key"])
|
|
266
|
+
assert len(result) == 2
|
|
267
|
+
assert result["n"].sum() == 3
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def test_upsert_numeric_keys_are_not_inferred_as_dates():
|
|
271
|
+
gs = _FakeGS("https://example.com/sheet", {"report": []})
|
|
272
|
+
gs.sheet = _RoundTripSheet(gs)
|
|
273
|
+
service = SheetsService(_make_app(gs))
|
|
274
|
+
for _ in range(2):
|
|
275
|
+
result = service.upsert_df(gs.url, "report", pd.DataFrame([
|
|
276
|
+
{"key": 45292, "n": 1}, {"key": "0012", "n": 2},
|
|
277
|
+
]), keys=["key"])
|
|
278
|
+
assert len(result) == 2
|
|
279
|
+
assert any(isinstance(value, (int, float)) and value == 45292 for value in result["key"])
|
|
280
|
+
assert any(value == "0012" for value in result["key"])
|
|
281
|
+
assert all(cell.get("userEnteredFormat", {}).get("numberFormat", {}).get("type") != "DATE"
|
|
282
|
+
for row in gs.sheet.cells[1:] for cell in row)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def test_incomplete_date_keys_are_idempotent_after_cell_roundtrip():
|
|
286
|
+
gs = _FakeGS("https://example.com/sheet", {"report": []})
|
|
287
|
+
gs.sheet = _RoundTripSheet(gs)
|
|
288
|
+
service = SheetsService(_make_app(gs))
|
|
289
|
+
incoming = pd.DataFrame([{"date": "2024-01-01", "medium": None, "n": 1}])
|
|
290
|
+
for _ in range(3):
|
|
291
|
+
result = service.upsert_df(gs.url, "report", incoming, keys=["date", "medium"])
|
|
292
|
+
assert len(result) == 1
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def test_upsert_sorts_native_and_new_dates_on_the_same_timeline():
|
|
296
|
+
gs = _FakeGS("https://example.com/sheet", {"report": []})
|
|
297
|
+
gs.sheet = _RoundTripSheet(gs)
|
|
298
|
+
service = SheetsService(_make_app(gs))
|
|
299
|
+
service.upsert_df(gs.url, "report", pd.DataFrame([
|
|
300
|
+
{"date": "1899-12-29", "n": 1}, {"date": "1900-01-09", "n": 3},
|
|
301
|
+
]), keys=["date"])
|
|
302
|
+
result = service.upsert_df(gs.url, "report", pd.DataFrame([
|
|
303
|
+
{"date": pd.Timestamp("1899-12-31"), "n": 2},
|
|
304
|
+
]), keys=["date"])
|
|
305
|
+
assert result["n"].tolist() == [1, 2, 3]
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def test_upsert_mixed_sort_fallback_preserves_numeric_secondary_sort():
|
|
309
|
+
gs = _FakeGS("https://example.com/sheet", {"report": [{"key": "a", "group": 1, "n": 100}]})
|
|
310
|
+
result = SheetsService(_make_app(gs)).upsert_df(gs.url, "report", pd.DataFrame([
|
|
311
|
+
{"key": "b", "group": "1", "n": 9}, {"key": "c", "group": "1", "n": 10},
|
|
312
|
+
]), keys=["key"], sort_by=["group", "n"])
|
|
313
|
+
assert result["n"].tolist() == [9, 10, 100]
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def test_upsert_preserves_numeric_and_text_key_types():
|
|
317
|
+
gs = _FakeGS("https://example.com/sheet", {"report": [
|
|
318
|
+
{"month": 202608, "code": "0012", "users": 1},
|
|
319
|
+
{"month": 202609, "code": "0012", "users": 2},
|
|
320
|
+
]})
|
|
321
|
+
gs.sheet.select("report")
|
|
322
|
+
service = SheetsService(_make_app(gs))
|
|
323
|
+
result = service.upsert_df("https://example.com/sheet", "report", pd.DataFrame([
|
|
324
|
+
{"month": 202609, "code": "0012", "users": 3},
|
|
325
|
+
]), keys=["month", "code"])
|
|
326
|
+
assert result["month"].tolist() == [202608, 202609]
|
|
327
|
+
assert result["code"].tolist() == ["0012", "0012"]
|
|
328
|
+
assert result["users"].tolist() == [1, 3]
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def test_upsert_date_keys_match_formatted_history_without_duplicates():
|
|
332
|
+
gs = _FakeGS("https://example.com/sheet", {"report": [
|
|
333
|
+
{"ym": "2026/8/1", "count": 1}, {"ym": "2026/9/1", "count": 2}]})
|
|
334
|
+
gs.sheet.select("report")
|
|
335
|
+
service = SheetsService(_make_app(gs))
|
|
336
|
+
result = service.upsert_df("https://example.com/sheet", "report", pd.DataFrame([
|
|
337
|
+
{"ym": pd.Timestamp("2026-09-01"), "count": 3}]), keys=["ym"])
|
|
338
|
+
assert result["ym"].tolist() == [pd.Timestamp("2026-08-01"), pd.Timestamp("2026-09-01")]
|
|
339
|
+
assert result["count"].tolist() == [1, 3]
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
@pytest.mark.parametrize("missing", [None, float("nan"), pd.NA, ""])
|
|
343
|
+
def test_upsert_blank_keys_remain_idempotent_and_preserve_history(missing):
|
|
344
|
+
gs = _FakeGS("https://example.com/sheet", {"report": [
|
|
345
|
+
{"month": 202608, "medium": "", "users": 0},
|
|
346
|
+
{"month": 202609, "medium": "", "users": 0},
|
|
347
|
+
{"month": 202609, "medium": "", "users": 0},
|
|
348
|
+
{"month": 202609, "medium": "organic", "users": 2}]})
|
|
349
|
+
gs.sheet.select("report")
|
|
350
|
+
service = SheetsService(_make_app(gs))
|
|
351
|
+
incoming = pd.DataFrame([
|
|
352
|
+
{"month": 202609, "medium": missing, "users": 0},
|
|
353
|
+
{"month": 202609, "medium": "organic", "users": 3}])
|
|
354
|
+
for _ in range(2):
|
|
355
|
+
service.upsert_df("https://example.com/sheet", "report", incoming, keys=["month", "medium"])
|
|
356
|
+
assert len(gs.last_written) == 4
|
|
357
|
+
assert gs.last_written["users"].sum() == 3
|
|
358
|
+
assert len(gs.last_written[gs.last_written["medium"].eq("")]) == 3
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def test_upsert_blank_key_ambiguous_change_stops_before_write():
|
|
362
|
+
gs = _FakeGS("https://example.com/sheet", {"report": [
|
|
363
|
+
{"medium": "", "users": 1}, {"medium": "", "users": 2}]})
|
|
364
|
+
service = SheetsService(_make_app(gs))
|
|
365
|
+
with pytest.raises(ValueError, match="ambiguous historical"):
|
|
366
|
+
service.upsert_df("https://example.com/sheet", "report",
|
|
367
|
+
pd.DataFrame([{"medium": None, "users": 3}]), keys=["medium"])
|
|
368
|
+
assert gs.last_written is None
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def test_upsert_literal_nan_is_not_a_blank_key():
|
|
372
|
+
gs = _FakeGS("https://example.com/sheet", {"report": [{"medium": "nan", "users": 2}]})
|
|
373
|
+
service = SheetsService(_make_app(gs))
|
|
374
|
+
result = service.upsert_df("https://example.com/sheet", "report",
|
|
375
|
+
pd.DataFrame([{"medium": None, "users": 0}]), keys=["medium"])
|
|
376
|
+
assert len(result) == 2
|
|
377
|
+
assert result["users"].sum() == 2
|
|
378
|
+
assert gs.last_read_options["value_render_option"] == "UNFORMATTED_VALUE"
|
|
379
|
+
assert gs.last_read_options["numericise_ignore"] == ["all"]
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def test_upsert_native_date_serial_keys_match_datetime_input():
|
|
383
|
+
serial = (pd.Timestamp("2026-09-01") - pd.Timestamp("1899-12-30")).days
|
|
384
|
+
gs = _FakeGS("https://example.com/sheet", {"report": [{"ym": serial, "count": 1}]})
|
|
385
|
+
service = SheetsService(_make_app(gs))
|
|
386
|
+
result = service.upsert_df("https://example.com/sheet", "report",
|
|
387
|
+
pd.DataFrame([{"ym": pd.Timestamp("2026-09-01"), "count": 3}]), keys=["ym"])
|
|
388
|
+
assert result["ym"].tolist() == [pd.Timestamp("2026-09-01")]
|
|
389
|
+
assert result["count"].tolist() == [3]
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def test_upsert_new_null_key_is_idempotent_after_sheets_blank_roundtrip():
|
|
393
|
+
gs = _FakeGS("https://example.com/sheet", {"report": []})
|
|
394
|
+
service = SheetsService(_make_app(gs))
|
|
395
|
+
incoming = pd.DataFrame([{"medium": pd.NA, "source": None, "users": 0}])
|
|
396
|
+
for _ in range(3):
|
|
397
|
+
result = service.upsert_df("https://example.com/sheet", "report", incoming,
|
|
398
|
+
keys=["medium", "source"])
|
|
399
|
+
assert len(result) == 1
|
|
400
|
+
gs._sheets["report"] = result.astype(object).where(pd.notna(result), "").to_dict("records")
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def test_upsert_single_blank_history_row_can_be_updated():
|
|
404
|
+
gs = _FakeGS("https://example.com/sheet", {"report": [{"medium": "", "users": 1}]})
|
|
405
|
+
service = SheetsService(_make_app(gs))
|
|
406
|
+
result = service.upsert_df("https://example.com/sheet", "report",
|
|
407
|
+
pd.DataFrame([{"medium": None, "users": 3}]), keys=["medium"])
|
|
408
|
+
assert len(result) == 1
|
|
409
|
+
assert result["users"].sum() == 3
|
|
410
|
+
|
|
411
|
+
|
|
115
412
|
def test_upsert_df_deduplicates_and_sorts():
|
|
116
413
|
existing = [
|
|
117
414
|
{"month": "2024-01", "clinic": "A", "users": 1},
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|