megaton 2.2.0__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {megaton-2.2.0 → megaton-2.4.0}/PKG-INFO +43 -4
- {megaton-2.2.0 → megaton-2.4.0}/README.md +39 -2
- megaton-2.4.0/megaton/__main__.py +5 -0
- megaton-2.4.0/megaton/cli.py +69 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/constants.py +3 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/ga4.py +17 -5
- {megaton-2.2.0 → megaton-2.4.0}/megaton/gsheet.py +42 -3
- {megaton-2.2.0 → megaton-2.4.0}/megaton/gsheet_lowlevel.py +22 -3
- megaton-2.4.0/megaton/query.py +304 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/services/gsc_service.py +12 -6
- {megaton-2.2.0 → megaton-2.4.0}/megaton/services/sheets_service.py +107 -16
- {megaton-2.2.0 → megaton-2.4.0}/megaton/start.py +26 -21
- {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/PKG-INFO +43 -4
- {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/SOURCES.txt +6 -0
- megaton-2.4.0/megaton.egg-info/entry_points.txt +2 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/requires.txt +3 -0
- {megaton-2.2.0 → megaton-2.4.0}/pyproject.toml +8 -2
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_ga4_report_core.py +7 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_core.py +33 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_request_builders.py +18 -0
- megaton-2.4.0/tests/test_query_contract.py +224 -0
- megaton-2.4.0/tests/test_query_regressions.py +241 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_sheets_service.py +297 -0
- {megaton-2.2.0 → megaton-2.4.0}/LICENSE +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/__init__.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/_result.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/auth/__init__.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/auth/google_auth.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/auth/provider.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/bq.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/dates.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/errors.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/files.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/gdrive.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/google_api.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/install/__init__.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/install/install_bigquery.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/install/install_ga4.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/os.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/py.typed +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/recipes/__init__.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/recipes/config_loader.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/retry_utils.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/searchconsole.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/services/__init__.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/services/bq_service.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/services/sheet_namespace.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/state.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/__init__.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/classify.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/ga4.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/table.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/text.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/transform/traffic.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/ui/__init__.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/ui/widgets.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton/utils.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/dependency_links.txt +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/megaton.egg-info/top_level.txt +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/setup.cfg +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_auth.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_auth_provider.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_auto_install.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_bq_core_extra.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_bq_state_api.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_clean_parameter.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_config_loader.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_dates.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_errors_and_init.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_failure_contracts.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_ga4_property_show.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_ga4_report_retry.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_ga4_state_and_run.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsc_service.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_a1_helpers.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_batch_read.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_facade.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_lowlevel_retry_core.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_retry_resolvers.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_gsheet_timeout.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_megaton_auth_bridge.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_normalize_queries.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_phase0_regressions.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_programmatic_api.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_dates_to_sheet.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_month_window.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_prep.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_result.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_run_multi.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_run_ranges.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_report_show_download.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_retry_utils.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_run_all.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_sc_wrapper.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_search_apply_if.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_search_dimension_filter.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_search_filter.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_search_result.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_session_retry.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_sheet_namespace.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_sheet_save_options.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_sheets_read.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_classify.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_ga4.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_pipeline_gsc_like.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_table.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_table_normalize_thresholds_df.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_text.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_transform_traffic.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_ui_lazy_import.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_upsert_to_csv.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_utils.py +0 -0
- {megaton-2.2.0 → megaton-2.4.0}/tests/test_wrap_and_chain.py +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: megaton
|
|
3
|
-
Version: 2.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 2.4.0
|
|
4
|
+
Summary: Google analytics toolkit for Python, CLI and Notebooks: GA4, Search Console, Sheets and BigQuery.
|
|
5
5
|
Author-email: Makoto Shimizu <aa.analyst.ga@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/mak00s/megaton
|
|
@@ -34,6 +34,8 @@ Requires-Dist: gspread>=5.7
|
|
|
34
34
|
Requires-Dist: oauthlib>=3.2
|
|
35
35
|
Requires-Dist: gspread-dataframe>=3.0
|
|
36
36
|
Requires-Dist: pytz>=2023.3
|
|
37
|
+
Provides-Extra: bigquery
|
|
38
|
+
Requires-Dist: db-dtypes>=1.0; extra == "bigquery"
|
|
37
39
|
Provides-Extra: notebook
|
|
38
40
|
Requires-Dist: ipywidgets>=8.0; extra == "notebook"
|
|
39
41
|
Provides-Extra: dev
|
|
@@ -49,7 +51,44 @@ Dynamic: license-file
|
|
|
49
51
|
[](https://www.python.org/downloads/)
|
|
50
52
|
[](LICENSE)
|
|
51
53
|
|
|
52
|
-
Megaton is a Python toolkit for
|
|
54
|
+
Megaton is a Python and CLI toolkit for Google Analytics 4, Search Console,
|
|
55
|
+
Google Sheets and BigQuery. Run a single query without an application checkout;
|
|
56
|
+
use the same operations from scripts, Notebooks and MCP adapters.
|
|
57
|
+
|
|
58
|
+
## Python and CLI
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install megaton
|
|
62
|
+
megaton query --params query.json --credential /path/to/google.json --output result.csv
|
|
63
|
+
megaton resources gsc --credential /path/to/google.json
|
|
64
|
+
python -m megaton query --params query.json --validate-only
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`query.json` selects the resource and dates explicitly:
|
|
68
|
+
|
|
69
|
+
```json
|
|
70
|
+
{
|
|
71
|
+
"schema_version": "1.0",
|
|
72
|
+
"source": "ga4",
|
|
73
|
+
"property_id": "123456789",
|
|
74
|
+
"date_range": {"start": "2026-01-01", "end": "2026-01-31"},
|
|
75
|
+
"dimensions": ["date"],
|
|
76
|
+
"metrics": ["sessions"]
|
|
77
|
+
}
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from megaton.query import run_query
|
|
82
|
+
|
|
83
|
+
result = run_query(params, credential="/path/to/google.json")
|
|
84
|
+
df = result.df
|
|
85
|
+
result.save("result.csv")
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
The core package provides single Google queries; `megaton-app` adds multi-step
|
|
89
|
+
workflows, Adobe, jobs, audits, validation and delivery. See the
|
|
90
|
+
[machine interface contract](docs/cli.md) for GSC/BQ requests, authentication,
|
|
91
|
+
JSON output and MCP integration. These commands are available from 2.3.0.
|
|
53
92
|
|
|
54
93
|
## Core Concepts
|
|
55
94
|
|
|
@@ -61,7 +100,7 @@ scheduled jobs from 2.1.3.
|
|
|
61
100
|
|
|
62
101
|
- **Result objects** — Method chaining via `SearchResult` / `ReportResult`
|
|
63
102
|
- **Simple flow** — Open → Set dates → Run → Save
|
|
64
|
-
- **
|
|
103
|
+
- **Shared operations** — Explicit machine requests and interactive Notebook sessions
|
|
65
104
|
|
|
66
105
|
## Quick Start
|
|
67
106
|
|
|
@@ -4,7 +4,44 @@
|
|
|
4
4
|
[](https://www.python.org/downloads/)
|
|
5
5
|
[](LICENSE)
|
|
6
6
|
|
|
7
|
-
Megaton is a Python toolkit for
|
|
7
|
+
Megaton is a Python and CLI toolkit for Google Analytics 4, Search Console,
|
|
8
|
+
Google Sheets and BigQuery. Run a single query without an application checkout;
|
|
9
|
+
use the same operations from scripts, Notebooks and MCP adapters.
|
|
10
|
+
|
|
11
|
+
## Python and CLI
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install megaton
|
|
15
|
+
megaton query --params query.json --credential /path/to/google.json --output result.csv
|
|
16
|
+
megaton resources gsc --credential /path/to/google.json
|
|
17
|
+
python -m megaton query --params query.json --validate-only
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
`query.json` selects the resource and dates explicitly:
|
|
21
|
+
|
|
22
|
+
```json
|
|
23
|
+
{
|
|
24
|
+
"schema_version": "1.0",
|
|
25
|
+
"source": "ga4",
|
|
26
|
+
"property_id": "123456789",
|
|
27
|
+
"date_range": {"start": "2026-01-01", "end": "2026-01-31"},
|
|
28
|
+
"dimensions": ["date"],
|
|
29
|
+
"metrics": ["sessions"]
|
|
30
|
+
}
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from megaton.query import run_query
|
|
35
|
+
|
|
36
|
+
result = run_query(params, credential="/path/to/google.json")
|
|
37
|
+
df = result.df
|
|
38
|
+
result.save("result.csv")
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The core package provides single Google queries; `megaton-app` adds multi-step
|
|
42
|
+
workflows, Adobe, jobs, audits, validation and delivery. See the
|
|
43
|
+
[machine interface contract](docs/cli.md) for GSC/BQ requests, authentication,
|
|
44
|
+
JSON output and MCP integration. These commands are available from 2.3.0.
|
|
8
45
|
|
|
9
46
|
## Core Concepts
|
|
10
47
|
|
|
@@ -16,7 +53,7 @@ scheduled jobs from 2.1.3.
|
|
|
16
53
|
|
|
17
54
|
- **Result objects** — Method chaining via `SearchResult` / `ReportResult`
|
|
18
55
|
- **Simple flow** — Open → Set dates → Run → Save
|
|
19
|
-
- **
|
|
56
|
+
- **Shared operations** — Explicit machine requests and interactive Notebook sessions
|
|
20
57
|
|
|
21
58
|
## Quick Start
|
|
22
59
|
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Installed, checkout-independent single-query interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import contextlib
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from .query import (
|
|
12
|
+
QueryValidationError,
|
|
13
|
+
_validate_auth,
|
|
14
|
+
list_resources,
|
|
15
|
+
run_query,
|
|
16
|
+
validate_query,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def main(argv: list[str] | None = None) -> int:
|
|
21
|
+
parser = argparse.ArgumentParser(prog="megaton")
|
|
22
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
23
|
+
query = commands.add_parser("query", help="Execute an explicit GA4/GSC/BQ request")
|
|
24
|
+
query.add_argument("--params", required=True, help="JSON request file; - for stdin")
|
|
25
|
+
query.add_argument("--output", help="CSV or JSON file; default is JSON on stdout")
|
|
26
|
+
query.add_argument("--validate-only", action="store_true")
|
|
27
|
+
resources = commands.add_parser("resources", help="List accessible resources")
|
|
28
|
+
resources.add_argument("source", choices=["ga4", "gsc", "bigquery"])
|
|
29
|
+
resources.add_argument("--project-id")
|
|
30
|
+
for command in (query, resources):
|
|
31
|
+
command.add_argument("--credential", help="Explicit Google credential JSON file")
|
|
32
|
+
command.add_argument("--cache-key", help="Existing Megaton OAuth cache identifier")
|
|
33
|
+
command.add_argument("--adc", action="store_true", help="Explicitly use ADC for BigQuery")
|
|
34
|
+
args = parser.parse_args(argv)
|
|
35
|
+
try:
|
|
36
|
+
# Legacy Python APIs may print; stdout must remain one JSON document.
|
|
37
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
38
|
+
if args.command == "query":
|
|
39
|
+
raw = sys.stdin.read() if args.params == "-" else Path(args.params).read_text(encoding="utf-8")
|
|
40
|
+
params = validate_query(json.loads(raw))
|
|
41
|
+
_validate_auth(params["source"], args.credential, args.cache_key, args.adc)
|
|
42
|
+
if args.output and Path(args.output).suffix.lower() not in {".csv", ".json"}:
|
|
43
|
+
raise QueryValidationError("output must end in .csv or .json")
|
|
44
|
+
if args.validate_only:
|
|
45
|
+
payload = {"status": "ok", "validated": True, "params": params}
|
|
46
|
+
else:
|
|
47
|
+
result = run_query(params, credential=args.credential, cache_key=args.cache_key, adc=args.adc)
|
|
48
|
+
if args.output:
|
|
49
|
+
result.save(args.output)
|
|
50
|
+
payload = {"status": "ok", "output": args.output, "row_count": len(result.df)}
|
|
51
|
+
else:
|
|
52
|
+
payload = result.to_dict()
|
|
53
|
+
else:
|
|
54
|
+
data = list_resources(args.source, credential=args.credential, project_id=args.project_id,
|
|
55
|
+
cache_key=args.cache_key, adc=args.adc)
|
|
56
|
+
payload = {"status": "ok", "source": args.source, "data": data}
|
|
57
|
+
print(json.dumps(payload, ensure_ascii=False))
|
|
58
|
+
return 0
|
|
59
|
+
except (QueryValidationError, json.JSONDecodeError) as exc:
|
|
60
|
+
print(json.dumps({"status": "error", "error_code": "INVALID_REQUEST", "message": str(exc)}))
|
|
61
|
+
return 2
|
|
62
|
+
except Exception as exc: # noqa: BLE001 - CLI boundary reports provider failures.
|
|
63
|
+
# Do not copy raw provider errors/SQL/credentials into machine output.
|
|
64
|
+
print(json.dumps({"status": "error", "error_code": "EXECUTION_FAILED", "error_type": type(exc).__name__}))
|
|
65
|
+
return 1
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
if __name__ == "__main__":
|
|
69
|
+
raise SystemExit(main())
|
|
@@ -15,3 +15,6 @@ DEFAULT_SCOPES = [
|
|
|
15
15
|
GOOGLE_SHEET_GA4_TEMPLATE_URL = 'https://docs.google.com/spreadsheets/d/1Jr4au61Xy7gfGBf5yvNKdT43awsSv98zOs4FTfM6Nms'
|
|
16
16
|
GOOGLE_SHEET_GA4_TEMPLATE_SHEET1 = '推奨CustomDimension'
|
|
17
17
|
GOOGLE_SHEET_GA4_TEMPLATE_SHEET2 = '推奨BQ'
|
|
18
|
+
GSC_DIMENSIONS = frozenset({
|
|
19
|
+
"date", "hour", "country", "device", "page", "query", "searchAppearance", "month",
|
|
20
|
+
})
|
|
@@ -715,13 +715,20 @@ class MegatonGA4(object):
|
|
|
715
715
|
TIME (in HH:MM:SS format)
|
|
716
716
|
"""
|
|
717
717
|
type = type.replace('TYPE_', '')
|
|
718
|
-
if type
|
|
718
|
+
if type == 'INTEGER':
|
|
719
719
|
try:
|
|
720
720
|
return int(value)
|
|
721
721
|
except:
|
|
722
722
|
return value
|
|
723
|
-
elif type in ['FLOAT'
|
|
724
|
-
|
|
723
|
+
elif type in ['FLOAT', 'HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS',
|
|
724
|
+
'CURRENCY', 'FEET', 'METERS', 'KILOMETERS', 'MILES', 'STANDARD']:
|
|
725
|
+
try:
|
|
726
|
+
number = float(value)
|
|
727
|
+
if type in ['HOURS', 'MINUTES', 'SECONDS', 'MILLISECONDS'] and number.is_integer():
|
|
728
|
+
return int(number)
|
|
729
|
+
return number
|
|
730
|
+
except (TypeError, ValueError):
|
|
731
|
+
return value
|
|
725
732
|
else:
|
|
726
733
|
return value
|
|
727
734
|
|
|
@@ -902,6 +909,9 @@ class MegatonGA4(object):
|
|
|
902
909
|
metrics = metrics[:10]
|
|
903
910
|
|
|
904
911
|
limit = kwargs.get('limit', 10000)
|
|
912
|
+
max_rows = kwargs.get('max_rows')
|
|
913
|
+
if max_rows is not None and (isinstance(max_rows, bool) or not isinstance(max_rows, int) or max_rows < 1):
|
|
914
|
+
raise ValueError("max_rows must be a positive integer")
|
|
905
915
|
start_date = kwargs.get('start_date', self.start_date)
|
|
906
916
|
end_date = kwargs.get('end_date', self.end_date)
|
|
907
917
|
# Calendar tokens (prev-month-start etc., v1.5+) are resolved
|
|
@@ -933,6 +943,8 @@ class MegatonGA4(object):
|
|
|
933
943
|
|
|
934
944
|
all_rows, offset, page = [], 0, 1
|
|
935
945
|
while True:
|
|
946
|
+
if max_rows is not None:
|
|
947
|
+
request.limit = min(limit, max_rows - len(all_rows))
|
|
936
948
|
(data, total_rows, headers, types) = self._request_report_api(
|
|
937
949
|
offset,
|
|
938
950
|
request,
|
|
@@ -942,11 +954,11 @@ class MegatonGA4(object):
|
|
|
942
954
|
timeout=timeout,
|
|
943
955
|
)
|
|
944
956
|
if len(data) > 0:
|
|
945
|
-
all_rows.extend(data)
|
|
957
|
+
all_rows.extend(data if max_rows is None else data[:max_rows - len(all_rows)])
|
|
946
958
|
if offset == 0:
|
|
947
959
|
LOGGER.info(f"Total {total_rows} rows found.")
|
|
948
960
|
LOGGER.info(f" p{page}: retrieved #{offset + 1} - {offset + len(data)}")
|
|
949
|
-
if offset + len(data)
|
|
961
|
+
if offset + len(data) >= total_rows or (max_rows is not None and len(all_rows) >= max_rows):
|
|
950
962
|
break
|
|
951
963
|
else:
|
|
952
964
|
page += 1
|
|
@@ -459,18 +459,57 @@ class MegatonGS(object):
|
|
|
459
459
|
"""
|
|
460
460
|
return self.last_row + 1
|
|
461
461
|
|
|
462
|
-
def get_records(self, *, max_retries: Optional[int] = None, backoff_factor: Optional[float] = None
|
|
462
|
+
def get_records(self, *, max_retries: Optional[int] = None, backoff_factor: Optional[float] = None,
|
|
463
|
+
value_render_option=None, numericise_ignore=None, preserve_dates=False):
|
|
463
464
|
"""Read all records with header row as keys."""
|
|
464
465
|
if not self._driver:
|
|
465
466
|
LOGGER.error("Please select a sheet first.")
|
|
466
467
|
return
|
|
467
|
-
|
|
468
|
+
options = {}
|
|
469
|
+
if value_render_option is not None:
|
|
470
|
+
options["value_render_option"] = value_render_option
|
|
471
|
+
if numericise_ignore is not None:
|
|
472
|
+
options["numericise_ignore"] = numericise_ignore
|
|
473
|
+
records = self._maybe_retry(
|
|
468
474
|
"Google Sheets read records",
|
|
469
|
-
lambda: self._driver.get_all_records(),
|
|
475
|
+
lambda: self._driver.get_all_records(**options),
|
|
470
476
|
max_retries=max_retries,
|
|
471
477
|
backoff_factor=backoff_factor,
|
|
472
478
|
retry_on_requests=True,
|
|
473
479
|
)
|
|
480
|
+
if not preserve_dates or not records:
|
|
481
|
+
return records
|
|
482
|
+
if value_render_option != "UNFORMATTED_VALUE" or numericise_ignore != ["all"]:
|
|
483
|
+
raise ValueError("preserve_dates requires unformatted, non-numericised records")
|
|
484
|
+
title = self._driver.title.replace("'", "''")
|
|
485
|
+
last_cell = gspread.utils.rowcol_to_a1(len(records) + 1, len(records[0]))
|
|
486
|
+
metadata = self._maybe_retry(
|
|
487
|
+
"Google Sheets read date formats",
|
|
488
|
+
lambda: self.parent._driver.fetch_sheet_metadata(params={
|
|
489
|
+
"ranges": "'" + title + "'!A1:" + last_cell,
|
|
490
|
+
"fields": "sheets(data(startRow,startColumn,rowData(values(effectiveValue,effectiveFormat.numberFormat))))",
|
|
491
|
+
"includeGridData": True,
|
|
492
|
+
}),
|
|
493
|
+
max_retries=max_retries, backoff_factor=backoff_factor,
|
|
494
|
+
retry_on_requests=True,
|
|
495
|
+
)
|
|
496
|
+
headers = list(records[0])
|
|
497
|
+
for sheet in metadata.get("sheets", []):
|
|
498
|
+
for grid in sheet.get("data", []):
|
|
499
|
+
for offset, row in enumerate(grid.get("rowData", [])):
|
|
500
|
+
index = grid.get("startRow", 0) + offset - 1
|
|
501
|
+
if not 0 <= index < len(records):
|
|
502
|
+
continue
|
|
503
|
+
for ci, cell in enumerate(row.get("values", []), grid.get("startColumn", 0)):
|
|
504
|
+
fmt = cell.get("effectiveFormat", {}).get("numberFormat", {})
|
|
505
|
+
serial = cell.get("effectiveValue", {}).get("numberValue")
|
|
506
|
+
if ci >= len(headers) or serial is None or fmt.get("type") not in ("DATE", "DATE_TIME", "TIME"):
|
|
507
|
+
continue
|
|
508
|
+
key = headers[ci]
|
|
509
|
+
if records[index][key] != serial:
|
|
510
|
+
raise ValueError("Sheet changed while reading date formats; no write performed")
|
|
511
|
+
records[index][key] = gsheet_lowlevel._NativeDate(serial, fmt)
|
|
512
|
+
return records
|
|
474
513
|
|
|
475
514
|
@property
|
|
476
515
|
def data(self):
|
|
@@ -18,6 +18,8 @@ from __future__ import annotations
|
|
|
18
18
|
import datetime as _dt
|
|
19
19
|
import logging
|
|
20
20
|
import math
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from numbers import Integral, Real
|
|
21
23
|
import re
|
|
22
24
|
import time
|
|
23
25
|
from collections.abc import Iterable
|
|
@@ -282,6 +284,14 @@ _DATETIME_RE = re.compile(
|
|
|
282
284
|
_SHEETS_EPOCH_DATE = _dt.date(1899, 12, 30)
|
|
283
285
|
|
|
284
286
|
|
|
287
|
+
@dataclass(frozen=True)
|
|
288
|
+
class _NativeDate:
|
|
289
|
+
"""A persisted date cell, without a lossy serial-to-datetime round trip."""
|
|
290
|
+
|
|
291
|
+
serial: float
|
|
292
|
+
number_format: dict
|
|
293
|
+
|
|
294
|
+
|
|
285
295
|
def _is_missing_scalar(value: Any) -> bool:
|
|
286
296
|
if value is None:
|
|
287
297
|
return True
|
|
@@ -301,12 +311,12 @@ def cell_value(value: Any, *, parse_formulas: bool = True) -> dict:
|
|
|
301
311
|
return {"stringValue": ""}
|
|
302
312
|
if isinstance(value, bool):
|
|
303
313
|
return {"boolValue": value}
|
|
304
|
-
if isinstance(value,
|
|
314
|
+
if isinstance(value, Integral):
|
|
305
315
|
return {"numberValue": float(value)}
|
|
306
|
-
if isinstance(value,
|
|
316
|
+
if isinstance(value, Real):
|
|
307
317
|
if math.isnan(value) or math.isinf(value):
|
|
308
318
|
return {"stringValue": ""}
|
|
309
|
-
return {"numberValue": value}
|
|
319
|
+
return {"numberValue": float(value)}
|
|
310
320
|
if isinstance(value, str):
|
|
311
321
|
s = value.strip()
|
|
312
322
|
if not s:
|
|
@@ -349,6 +359,15 @@ def cell_data(
|
|
|
349
359
|
ISO date and naive ISO datetime strings are written as real Sheets serials
|
|
350
360
|
with a display format. Other strings stay strings.
|
|
351
361
|
"""
|
|
362
|
+
if isinstance(value, _NativeDate):
|
|
363
|
+
return {
|
|
364
|
+
"userEnteredValue": {"numberValue": value.serial},
|
|
365
|
+
"userEnteredFormat": {"numberFormat": dict(value.number_format)},
|
|
366
|
+
}
|
|
367
|
+
if _is_missing_scalar(value):
|
|
368
|
+
return {"userEnteredValue": cell_value(value, parse_formulas=parse_formulas)}
|
|
369
|
+
if parse_dates and isinstance(value, _dt.date):
|
|
370
|
+
value = value.isoformat()
|
|
352
371
|
if isinstance(value, str):
|
|
353
372
|
if not parse_formulas and not parse_dates:
|
|
354
373
|
return {"userEnteredValue": cell_value(value, parse_formulas=False)}
|