megaton 2.2.0__tar.gz → 2.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. {megaton-2.2.0 → megaton-2.4.0}/PKG-INFO +43 -4
  2. {megaton-2.2.0 → megaton-2.4.0}/README.md +39 -2
  3. megaton-2.4.0/megaton/__main__.py +5 -0
  4. megaton-2.4.0/megaton/cli.py +69 -0
  5. {megaton-2.2.0 → megaton-2.4.0}/megaton/constants.py +3 -0
  6. {megaton-2.2.0 → megaton-2.4.0}/megaton/ga4.py +17 -5
  7. {megaton-2.2.0 → megaton-2.4.0}/megaton/gsheet.py +42 -3
  8. {megaton-2.2.0 → megaton-2.4.0}/megaton/gsheet_lowlevel.py +22 -3
  9. megaton-2.4.0/megaton/query.py +304 -0
  10. {megaton-2.2.0 → megaton-2.4.0}/megaton/services/gsc_service.py +12 -6
  11. {megaton-2.2.0 → megaton-2.4.0}/megaton/services/sheets_service.py +107 -16
  12. {megaton-2.2.0 → megaton-2.4.0}/megaton/start.py +26 -21
  13. {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/PKG-INFO +43 -4
  14. {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/SOURCES.txt +6 -0
  15. megaton-2.4.0/megaton.egg-info/entry_points.txt +2 -0
  16. {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/requires.txt +3 -0
  17. {megaton-2.2.0 → megaton-2.4.0}/pyproject.toml +8 -2
  18. {megaton-2.2.0 → megaton-2.4.0}/tests/test_ga4_report_core.py +7 -0
  19. {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_core.py +33 -0
  20. {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_request_builders.py +18 -0
  21. megaton-2.4.0/tests/test_query_contract.py +224 -0
  22. megaton-2.4.0/tests/test_query_regressions.py +241 -0
  23. {megaton-2.2.0 → megaton-2.4.0}/tests/test_sheets_service.py +297 -0
  24. {megaton-2.2.0 → megaton-2.4.0}/LICENSE +0 -0
  25. {megaton-2.2.0 → megaton-2.4.0}/megaton/__init__.py +0 -0
  26. {megaton-2.2.0 → megaton-2.4.0}/megaton/_result.py +0 -0
  27. {megaton-2.2.0 → megaton-2.4.0}/megaton/auth/__init__.py +0 -0
  28. {megaton-2.2.0 → megaton-2.4.0}/megaton/auth/google_auth.py +0 -0
  29. {megaton-2.2.0 → megaton-2.4.0}/megaton/auth/provider.py +0 -0
  30. {megaton-2.2.0 → megaton-2.4.0}/megaton/bq.py +0 -0
  31. {megaton-2.2.0 → megaton-2.4.0}/megaton/dates.py +0 -0
  32. {megaton-2.2.0 → megaton-2.4.0}/megaton/errors.py +0 -0
  33. {megaton-2.2.0 → megaton-2.4.0}/megaton/files.py +0 -0
  34. {megaton-2.2.0 → megaton-2.4.0}/megaton/gdrive.py +0 -0
  35. {megaton-2.2.0 → megaton-2.4.0}/megaton/google_api.py +0 -0
  36. {megaton-2.2.0 → megaton-2.4.0}/megaton/install/__init__.py +0 -0
  37. {megaton-2.2.0 → megaton-2.4.0}/megaton/install/install_bigquery.py +0 -0
  38. {megaton-2.2.0 → megaton-2.4.0}/megaton/install/install_ga4.py +0 -0
  39. {megaton-2.2.0 → megaton-2.4.0}/megaton/os.py +0 -0
  40. {megaton-2.2.0 → megaton-2.4.0}/megaton/py.typed +0 -0
  41. {megaton-2.2.0 → megaton-2.4.0}/megaton/recipes/__init__.py +0 -0
  42. {megaton-2.2.0 → megaton-2.4.0}/megaton/recipes/config_loader.py +0 -0
  43. {megaton-2.2.0 → megaton-2.4.0}/megaton/retry_utils.py +0 -0
  44. {megaton-2.2.0 → megaton-2.4.0}/megaton/searchconsole.py +0 -0
  45. {megaton-2.2.0 → megaton-2.4.0}/megaton/services/__init__.py +0 -0
  46. {megaton-2.2.0 → megaton-2.4.0}/megaton/services/bq_service.py +0 -0
  47. {megaton-2.2.0 → megaton-2.4.0}/megaton/services/sheet_namespace.py +0 -0
  48. {megaton-2.2.0 → megaton-2.4.0}/megaton/state.py +0 -0
  49. {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/__init__.py +0 -0
  50. {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/classify.py +0 -0
  51. {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/ga4.py +0 -0
  52. {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/table.py +0 -0
  53. {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/text.py +0 -0
  54. {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/traffic.py +0 -0
  55. {megaton-2.2.0 → megaton-2.4.0}/megaton/ui/__init__.py +0 -0
  56. {megaton-2.2.0 → megaton-2.4.0}/megaton/ui/widgets.py +0 -0
  57. {megaton-2.2.0 → megaton-2.4.0}/megaton/utils.py +0 -0
  58. {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/dependency_links.txt +0 -0
  59. {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/top_level.txt +0 -0
  60. {megaton-2.2.0 → megaton-2.4.0}/setup.cfg +0 -0
  61. {megaton-2.2.0 → megaton-2.4.0}/tests/test_auth.py +0 -0
  62. {megaton-2.2.0 → megaton-2.4.0}/tests/test_auth_provider.py +0 -0
  63. {megaton-2.2.0 → megaton-2.4.0}/tests/test_auto_install.py +0 -0
  64. {megaton-2.2.0 → megaton-2.4.0}/tests/test_bq_core_extra.py +0 -0
  65. {megaton-2.2.0 → megaton-2.4.0}/tests/test_bq_state_api.py +0 -0
  66. {megaton-2.2.0 → megaton-2.4.0}/tests/test_clean_parameter.py +0 -0
  67. {megaton-2.2.0 → megaton-2.4.0}/tests/test_config_loader.py +0 -0
  68. {megaton-2.2.0 → megaton-2.4.0}/tests/test_dates.py +0 -0
  69. {megaton-2.2.0 → megaton-2.4.0}/tests/test_errors_and_init.py +0 -0
  70. {megaton-2.2.0 → megaton-2.4.0}/tests/test_failure_contracts.py +0 -0
  71. {megaton-2.2.0 → megaton-2.4.0}/tests/test_ga4_property_show.py +0 -0
  72. {megaton-2.2.0 → megaton-2.4.0}/tests/test_ga4_report_retry.py +0 -0
  73. {megaton-2.2.0 → megaton-2.4.0}/tests/test_ga4_state_and_run.py +0 -0
  74. {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsc_service.py +0 -0
  75. {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_a1_helpers.py +0 -0
  76. {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_batch_read.py +0 -0
  77. {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_facade.py +0 -0
  78. {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_retry_core.py +0 -0
  79. {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_retry_resolvers.py +0 -0
  80. {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_timeout.py +0 -0
  81. {megaton-2.2.0 → megaton-2.4.0}/tests/test_megaton_auth_bridge.py +0 -0
  82. {megaton-2.2.0 → megaton-2.4.0}/tests/test_normalize_queries.py +0 -0
  83. {megaton-2.2.0 → megaton-2.4.0}/tests/test_phase0_regressions.py +0 -0
  84. {megaton-2.2.0 → megaton-2.4.0}/tests/test_programmatic_api.py +0 -0
  85. {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_dates_to_sheet.py +0 -0
  86. {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_month_window.py +0 -0
  87. {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_prep.py +0 -0
  88. {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_result.py +0 -0
  89. {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_run_multi.py +0 -0
  90. {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_run_ranges.py +0 -0
  91. {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_show_download.py +0 -0
  92. {megaton-2.2.0 → megaton-2.4.0}/tests/test_retry_utils.py +0 -0
  93. {megaton-2.2.0 → megaton-2.4.0}/tests/test_run_all.py +0 -0
  94. {megaton-2.2.0 → megaton-2.4.0}/tests/test_sc_wrapper.py +0 -0
  95. {megaton-2.2.0 → megaton-2.4.0}/tests/test_search_apply_if.py +0 -0
  96. {megaton-2.2.0 → megaton-2.4.0}/tests/test_search_dimension_filter.py +0 -0
  97. {megaton-2.2.0 → megaton-2.4.0}/tests/test_search_filter.py +0 -0
  98. {megaton-2.2.0 → megaton-2.4.0}/tests/test_search_result.py +0 -0
  99. {megaton-2.2.0 → megaton-2.4.0}/tests/test_session_retry.py +0 -0
  100. {megaton-2.2.0 → megaton-2.4.0}/tests/test_sheet_namespace.py +0 -0
  101. {megaton-2.2.0 → megaton-2.4.0}/tests/test_sheet_save_options.py +0 -0
  102. {megaton-2.2.0 → megaton-2.4.0}/tests/test_sheets_read.py +0 -0
  103. {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_classify.py +0 -0
  104. {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_ga4.py +0 -0
  105. {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_pipeline_gsc_like.py +0 -0
  106. {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_table.py +0 -0
  107. {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_table_normalize_thresholds_df.py +0 -0
  108. {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_text.py +0 -0
  109. {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_traffic.py +0 -0
  110. {megaton-2.2.0 → megaton-2.4.0}/tests/test_ui_lazy_import.py +0 -0
  111. {megaton-2.2.0 → megaton-2.4.0}/tests/test_upsert_to_csv.py +0 -0
  112. {megaton-2.2.0 → megaton-2.4.0}/tests/test_utils.py +0 -0
  113. {megaton-2.2.0 → megaton-2.4.0}/tests/test_wrap_and_chain.py +0 -0
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: megaton
3
- Version: 2.2.0
4
- Summary: Python toolkit for running GA4 and Search Console reports in Colab/Notebooks and publishing results to Google Sheets, CSV, or BigQuery, with built-in retries and easy append/upsert.
3
+ Version: 2.4.0
4
+ Summary: Google analytics toolkit for Python, CLI and Notebooks: GA4, Search Console, Sheets and BigQuery.
5
5
  Author-email: Makoto Shimizu <aa.analyst.ga@gmail.com>
6
6
  License-Expression: MIT
7
7
  Project-URL: Homepage, https://github.com/mak00s/megaton
@@ -34,6 +34,8 @@ Requires-Dist: gspread>=5.7
34
34
  Requires-Dist: oauthlib>=3.2
35
35
  Requires-Dist: gspread-dataframe>=3.0
36
36
  Requires-Dist: pytz>=2023.3
37
+ Provides-Extra: bigquery
38
+ Requires-Dist: db-dtypes>=1.0; extra == "bigquery"
37
39
  Provides-Extra: notebook
38
40
  Requires-Dist: ipywidgets>=8.0; extra == "notebook"
39
41
  Provides-Extra: dev
@@ -49,7 +51,44 @@ Dynamic: license-file
49
51
  [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
50
52
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
51
53
 
52
- Megaton is a Python toolkit for working with Google Analytics 4, Google Search Console, Google Sheets, and BigQuery from Notebooks with minimal code. It focuses on fast iteration during analysis and provides a UX tailored for Notebook workflows.
54
+ Megaton is a Python and CLI toolkit for Google Analytics 4, Search Console,
55
+ Google Sheets and BigQuery. Run a single query without an application checkout;
56
+ use the same operations from scripts, Notebooks and MCP adapters.
57
+
58
+ ## Python and CLI
59
+
60
+ ```bash
61
+ pip install megaton
62
+ megaton query --params query.json --credential /path/to/google.json --output result.csv
63
+ megaton resources gsc --credential /path/to/google.json
64
+ python -m megaton query --params query.json --validate-only
65
+ ```
66
+
67
+ `query.json` selects the resource and dates explicitly:
68
+
69
+ ```json
70
+ {
71
+ "schema_version": "1.0",
72
+ "source": "ga4",
73
+ "property_id": "123456789",
74
+ "date_range": {"start": "2026-01-01", "end": "2026-01-31"},
75
+ "dimensions": ["date"],
76
+ "metrics": ["sessions"]
77
+ }
78
+ ```
79
+
80
+ ```python
81
+ from megaton.query import run_query
82
+
83
+ result = run_query(params, credential="/path/to/google.json")
84
+ df = result.df
85
+ result.save("result.csv")
86
+ ```
87
+
88
+ The core package provides single Google queries; `megaton-app` adds multi-step
89
+ workflows, Adobe, jobs, audits, validation and delivery. See the
90
+ [machine interface contract](docs/cli.md) for GSC/BQ requests, authentication,
91
+ JSON output and MCP integration. These commands are available from 2.3.0.
53
92
 
54
93
  ## Core Concepts
55
94
 
@@ -61,7 +100,7 @@ scheduled jobs from 2.1.3.
61
100
 
62
101
  - **Result objects** — Method chaining via `SearchResult` / `ReportResult`
63
102
  - **Simple flow** — Open → Set dates → Run → Save
64
- - **Notebook-first** — Designed for inspecting intermediate results at every step
103
+ - **Shared operations** — Explicit machine requests and interactive Notebook sessions
65
104
 
66
105
  ## Quick Start
67
106
 
@@ -4,7 +4,44 @@
4
4
  [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
5
5
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
6
6
 
7
- Megaton is a Python toolkit for working with Google Analytics 4, Google Search Console, Google Sheets, and BigQuery from Notebooks with minimal code. It focuses on fast iteration during analysis and provides a UX tailored for Notebook workflows.
7
+ Megaton is a Python and CLI toolkit for Google Analytics 4, Search Console,
8
+ Google Sheets and BigQuery. Run a single query without an application checkout;
9
+ use the same operations from scripts, Notebooks and MCP adapters.
10
+
11
+ ## Python and CLI
12
+
13
+ ```bash
14
+ pip install megaton
15
+ megaton query --params query.json --credential /path/to/google.json --output result.csv
16
+ megaton resources gsc --credential /path/to/google.json
17
+ python -m megaton query --params query.json --validate-only
18
+ ```
19
+
20
+ `query.json` selects the resource and dates explicitly:
21
+
22
+ ```json
23
+ {
24
+ "schema_version": "1.0",
25
+ "source": "ga4",
26
+ "property_id": "123456789",
27
+ "date_range": {"start": "2026-01-01", "end": "2026-01-31"},
28
+ "dimensions": ["date"],
29
+ "metrics": ["sessions"]
30
+ }
31
+ ```
32
+
33
+ ```python
34
+ from megaton.query import run_query
35
+
36
+ result = run_query(params, credential="/path/to/google.json")
37
+ df = result.df
38
+ result.save("result.csv")
39
+ ```
40
+
41
+ The core package provides single Google queries; `megaton-app` adds multi-step
42
+ workflows, Adobe, jobs, audits, validation and delivery. See the
43
+ [machine interface contract](docs/cli.md) for GSC/BQ requests, authentication,
44
+ JSON output and MCP integration. These commands are available from 2.3.0.
8
45
 
9
46
  ## Core Concepts
10
47
 
@@ -16,7 +53,7 @@ scheduled jobs from 2.1.3.
16
53
 
17
54
  - **Result objects** — Method chaining via `SearchResult` / `ReportResult`
18
55
  - **Simple flow** — Open → Set dates → Run → Save
19
- - **Notebook-first** — Designed for inspecting intermediate results at every step
56
+ - **Shared operations** — Explicit machine requests and interactive Notebook sessions
20
57
 
21
58
  ## Quick Start
22
59
 
@@ -0,0 +1,5 @@
1
+ """Support python -m megaton."""
2
+
3
+ from .cli import main
4
+
5
+ raise SystemExit(main())
@@ -0,0 +1,69 @@
1
+ """Installed, checkout-independent single-query interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import contextlib
7
+ import json
8
+ import sys
9
+ from pathlib import Path
10
+
11
+ from .query import (
12
+ QueryValidationError,
13
+ _validate_auth,
14
+ list_resources,
15
+ run_query,
16
+ validate_query,
17
+ )
18
+
19
+
20
+ def main(argv: list[str] | None = None) -> int:
21
+ parser = argparse.ArgumentParser(prog="megaton")
22
+ commands = parser.add_subparsers(dest="command", required=True)
23
+ query = commands.add_parser("query", help="Execute an explicit GA4/GSC/BQ request")
24
+ query.add_argument("--params", required=True, help="JSON request file; - for stdin")
25
+ query.add_argument("--output", help="CSV or JSON file; default is JSON on stdout")
26
+ query.add_argument("--validate-only", action="store_true")
27
+ resources = commands.add_parser("resources", help="List accessible resources")
28
+ resources.add_argument("source", choices=["ga4", "gsc", "bigquery"])
29
+ resources.add_argument("--project-id")
30
+ for command in (query, resources):
31
+ command.add_argument("--credential", help="Explicit Google credential JSON file")
32
+ command.add_argument("--cache-key", help="Existing Megaton OAuth cache identifier")
33
+ command.add_argument("--adc", action="store_true", help="Explicitly use ADC for BigQuery")
34
+ args = parser.parse_args(argv)
35
+ try:
36
+ # Legacy Python APIs may print; stdout must remain one JSON document.
37
+ with contextlib.redirect_stdout(sys.stderr):
38
+ if args.command == "query":
39
+ raw = sys.stdin.read() if args.params == "-" else Path(args.params).read_text(encoding="utf-8")
40
+ params = validate_query(json.loads(raw))
41
+ _validate_auth(params["source"], args.credential, args.cache_key, args.adc)
42
+ if args.output and Path(args.output).suffix.lower() not in {".csv", ".json"}:
43
+ raise QueryValidationError("output must end in .csv or .json")
44
+ if args.validate_only:
45
+ payload = {"status": "ok", "validated": True, "params": params}
46
+ else:
47
+ result = run_query(params, credential=args.credential, cache_key=args.cache_key, adc=args.adc)
48
+ if args.output:
49
+ result.save(args.output)
50
+ payload = {"status": "ok", "output": args.output, "row_count": len(result.df)}
51
+ else:
52
+ payload = result.to_dict()
53
+ else:
54
+ data = list_resources(args.source, credential=args.credential, project_id=args.project_id,
55
+ cache_key=args.cache_key, adc=args.adc)
56
+ payload = {"status": "ok", "source": args.source, "data": data}
57
+ print(json.dumps(payload, ensure_ascii=False))
58
+ return 0
59
+ except (QueryValidationError, json.JSONDecodeError) as exc:
60
+ print(json.dumps({"status": "error", "error_code": "INVALID_REQUEST", "message": str(exc)}))
61
+ return 2
62
+ except Exception as exc: # noqa: BLE001 - CLI boundary reports provider failures.
63
+ # Do not copy raw provider errors/SQL/credentials into machine output.
64
+ print(json.dumps({"status": "error", "error_code": "EXECUTION_FAILED", "error_type": type(exc).__name__}))
65
+ return 1
66
+
67
+
68
+ if __name__ == "__main__":
69
+ raise SystemExit(main())
@@ -15,3 +15,6 @@ DEFAULT_SCOPES = [
15
15
  GOOGLE_SHEET_GA4_TEMPLATE_URL = 'https://docs.google.com/spreadsheets/d/1Jr4au61Xy7gfGBf5yvNKdT43awsSv98zOs4FTfM6Nms'
16
16
  GOOGLE_SHEET_GA4_TEMPLATE_SHEET1 = '推奨CustomDimension'
17
17
  GOOGLE_SHEET_GA4_TEMPLATE_SHEET2 = '推奨BQ'
18
+ GSC_DIMENSIONS = frozenset({
19
+ "date", "hour", "country", "device", "page", "query", "searchAppearance", "month",
20
+ })
@@ -715,13 +715,20 @@ class MegatonGA4(object):
715
715
  TIME (in HH:MM:SS format)
716
716
  """
717
717
  type = type.replace('TYPE_', '')
718
- if type in ['INTEGER', 'HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS']:
718
+ if type == 'INTEGER':
719
719
  try:
720
720
  return int(value)
721
721
  except:
722
722
  return value
723
- elif type in ['FLOAT']:
724
- return float(value)
723
+ elif type in ['FLOAT', 'HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS',
724
+ 'CURRENCY', 'FEET', 'METERS', 'KILOMETERS', 'MILES', 'STANDARD']:
725
+ try:
726
+ number = float(value)
727
+ if type in ['HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS'] and number.is_integer():
728
+ return int(number)
729
+ return number
730
+ except (TypeError, ValueError):
731
+ return value
725
732
  else:
726
733
  return value
727
734
 
@@ -902,6 +909,9 @@ class MegatonGA4(object):
902
909
  metrics = metrics[:10]
903
910
 
904
911
  limit = kwargs.get('limit', 10000)
912
+ max_rows = kwargs.get('max_rows')
913
+ if max_rows is not None and (isinstance(max_rows, bool) or not isinstance(max_rows, int) or max_rows < 1):
914
+ raise ValueError("max_rows must be a positive integer")
905
915
  start_date = kwargs.get('start_date', self.start_date)
906
916
  end_date = kwargs.get('end_date', self.end_date)
907
917
  # Calendar tokens (prev-month-start etc., v1.5+) are resolved
@@ -933,6 +943,8 @@ class MegatonGA4(object):
933
943
 
934
944
  all_rows, offset, page = [], 0, 1
935
945
  while True:
946
+ if max_rows is not None:
947
+ request.limit = min(limit, max_rows - len(all_rows))
936
948
  (data, total_rows, headers, types) = self._request_report_api(
937
949
  offset,
938
950
  request,
@@ -942,11 +954,11 @@ class MegatonGA4(object):
942
954
  timeout=timeout,
943
955
  )
944
956
  if len(data) > 0:
945
- all_rows.extend(data)
957
+ all_rows.extend(data if max_rows is None else data[:max_rows - len(all_rows)])
946
958
  if offset == 0:
947
959
  LOGGER.info(f"Total {total_rows} rows found.")
948
960
  LOGGER.info(f" p{page}: retrieved #{offset + 1} - {offset + len(data)}")
949
- if offset + len(data) == total_rows:
961
+ if offset + len(data) >= total_rows or (max_rows is not None and len(all_rows) >= max_rows):
950
962
  break
951
963
  else:
952
964
  page += 1
@@ -459,18 +459,57 @@ class MegatonGS(object):
459
459
  """
460
460
  return self.last_row + 1
461
461
 
462
- def get_records(self, *, max_retries: Optional[int] = None, backoff_factor: Optional[float] = None):
462
+ def get_records(self, *, max_retries: Optional[int] = None, backoff_factor: Optional[float] = None,
463
+ value_render_option=None, numericise_ignore=None, preserve_dates=False):
463
464
  """Read all records with header row as keys."""
464
465
  if not self._driver:
465
466
  LOGGER.error("Please select a sheet first.")
466
467
  return
467
- return self._maybe_retry(
468
+ options = {}
469
+ if value_render_option is not None:
470
+ options["value_render_option"] = value_render_option
471
+ if numericise_ignore is not None:
472
+ options["numericise_ignore"] = numericise_ignore
473
+ records = self._maybe_retry(
468
474
  "Google Sheets read records",
469
- lambda: self._driver.get_all_records(),
475
+ lambda: self._driver.get_all_records(**options),
470
476
  max_retries=max_retries,
471
477
  backoff_factor=backoff_factor,
472
478
  retry_on_requests=True,
473
479
  )
480
+ if not preserve_dates or not records:
481
+ return records
482
+ if value_render_option != "UNFORMATTED_VALUE" or numericise_ignore != ["all"]:
483
+ raise ValueError("preserve_dates requires unformatted, non-numericised records")
484
+ title = self._driver.title.replace("'", "''")
485
+ last_cell = gspread.utils.rowcol_to_a1(len(records) + 1, len(records[0]))
486
+ metadata = self._maybe_retry(
487
+ "Google Sheets read date formats",
488
+ lambda: self.parent._driver.fetch_sheet_metadata(params={
489
+ "ranges": "'" + title + "'!A1:" + last_cell,
490
+ "fields": "sheets(data(startRow,startColumn,rowData(values(effectiveValue,effectiveFormat.numberFormat))))",
491
+ "includeGridData": True,
492
+ }),
493
+ max_retries=max_retries, backoff_factor=backoff_factor,
494
+ retry_on_requests=True,
495
+ )
496
+ headers = list(records[0])
497
+ for sheet in metadata.get("sheets", []):
498
+ for grid in sheet.get("data", []):
499
+ for offset, row in enumerate(grid.get("rowData", [])):
500
+ index = grid.get("startRow", 0) + offset - 1
501
+ if not 0 <= index < len(records):
502
+ continue
503
+ for ci, cell in enumerate(row.get("values", []), grid.get("startColumn", 0)):
504
+ fmt = cell.get("effectiveFormat", {}).get("numberFormat", {})
505
+ serial = cell.get("effectiveValue", {}).get("numberValue")
506
+ if ci >= len(headers) or serial is None or fmt.get("type") not in ("DATE", "DATE_TIME", "TIME"):
507
+ continue
508
+ key = headers[ci]
509
+ if records[index][key] != serial:
510
+ raise ValueError("Sheet changed while reading date formats; no write performed")
511
+ records[index][key] = gsheet_lowlevel._NativeDate(serial, fmt)
512
+ return records
474
513
 
475
514
  @property
476
515
  def data(self):
@@ -18,6 +18,8 @@ from __future__ import annotations
18
18
  import datetime as _dt
19
19
  import logging
20
20
  import math
21
+ from dataclasses import dataclass
22
+ from numbers import Integral, Real
21
23
  import re
22
24
  import time
23
25
  from collections.abc import Iterable
@@ -282,6 +284,14 @@ _DATETIME_RE = re.compile(
282
284
  _SHEETS_EPOCH_DATE = _dt.date(1899, 12, 30)
283
285
 
284
286
 
287
+ @dataclass(frozen=True)
288
+ class _NativeDate:
289
+ """A persisted date cell, without a lossy serial-to-datetime round trip."""
290
+
291
+ serial: float
292
+ number_format: dict
293
+
294
+
285
295
  def _is_missing_scalar(value: Any) -> bool:
286
296
  if value is None:
287
297
  return True
@@ -301,12 +311,12 @@ def cell_value(value: Any, *, parse_formulas: bool = True) -> dict:
301
311
  return {"stringValue": ""}
302
312
  if isinstance(value, bool):
303
313
  return {"boolValue": value}
304
- if isinstance(value, int):
314
+ if isinstance(value, Integral):
305
315
  return {"numberValue": float(value)}
306
- if isinstance(value, float):
316
+ if isinstance(value, Real):
307
317
  if math.isnan(value) or math.isinf(value):
308
318
  return {"stringValue": ""}
309
- return {"numberValue": value}
319
+ return {"numberValue": float(value)}
310
320
  if isinstance(value, str):
311
321
  s = value.strip()
312
322
  if not s:
@@ -349,6 +359,15 @@ def cell_data(
349
359
  ISO date and naive ISO datetime strings are written as real Sheets serials
350
360
  with a display format. Other strings stay strings.
351
361
  """
362
+ if isinstance(value, _NativeDate):
363
+ return {
364
+ "userEnteredValue": {"numberValue": value.serial},
365
+ "userEnteredFormat": {"numberFormat": dict(value.number_format)},
366
+ }
367
+ if _is_missing_scalar(value):
368
+ return {"userEnteredValue": cell_value(value, parse_formulas=parse_formulas)}
369
+ if parse_dates and isinstance(value, _dt.date):
370
+ value = value.isoformat()
352
371
  if isinstance(value, str):
353
372
  if not parse_formulas and not parse_dates:
354
373
  return {"userEnteredValue": cell_value(value, parse_formulas=False)}