megaton 1.0.0__tar.gz

megaton-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Mak Shimizu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

megaton-1.0.0/MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include LICENSE
+include requirements.txt

megaton-1.0.0/PKG-INFO

@@ -0,0 +1,101 @@
+Metadata-Version: 2.4
+Name: megaton
+Version: 1.0.0
+Summary: Utilities for Google Analytics, Google Analytics 4, Google Sheets, Search Console and Google Cloud Platform.
+Author: Makoto Shimizu
+Author-email: aa.analyst.ga@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Internet
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=1.3
+Requires-Dist: google-api-python-client>=2.89
+Requires-Dist: google-auth>=2.40.0
+Requires-Dist: google-api-core>=2.25.0
+Requires-Dist: google-auth-oauthlib>=1.0
+Requires-Dist: google-analytics-data>=0.18
+Requires-Dist: google-analytics-admin>=0.25.0
+Requires-Dist: google-cloud-bigquery>=3.11
+Requires-Dist: google-cloud-bigquery-datatransfer>=1.11
+Requires-Dist: gspread>=5.7
+Requires-Dist: oauthlib>=3.2
+Requires-Dist: ipywidgets>=8.0
+Requires-Dist: gspread-dataframe>=3.0
+Requires-Dist: pytz>=2023.3
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# megaton
+
+Megaton は Google Analytics 4、Google Search Console、Google Sheets、BigQuery を **Notebook から短いコードで扱う** ためのツールです。分析の試行錯誤を速く回すことを重視し、Notebook 向けの UX に特化しています。
+
+## コア概念
+
+- **結果オブジェクト**: `SearchResult` / `ReportResult` によるメソッドチェーン
+- **シンプルな流れ**: 開く → 期間設定 → 取得 → 保存
+- **Notebook 前提**: 途中結果の確認を前提にした設計
+
+## クイックスタート
+
+```python
+from megaton.start import Megaton
+
+mg = Megaton("/path/to/service_account.json")
+mg.report.set.dates("2024-01-01", "2024-01-31")
+mg.report.run(d=["date", "eventName"], m=["eventCount"])
+
+mg.open.sheet("https://docs.google.com/spreadsheets/d/...")
+mg.save.to.sheet("_ga_data", mg.report.data)
+```
+
+## もう少し実用的な例
+
+```python
+# 複数サイトの Search Console データを一括取得して整形
+result = (mg.search.run.all(
+    sites,
+    dimensions=['query', 'page'],
+    item_key='clinic',
+)
+    .categorize('query', by=query_map)
+    .categorize('page', by=page_map))
+
+mg.save.to.sheet('_query', result.df, sort_by='impressions')
+mg.upsert.to.csv(result.df, filename='query_master', keys=['clinic', 'query', 'page'], include_dates=False)
+```
+
+## インストール
+
+```bash
+# PyPI 公開版
+pip install megaton
+
+# 最新版（GitHub）
+pip install git+https://github.com/mak00s/megaton.git
+```
+
+## ドキュメント
+
+- 仕様の正は [api-reference.md](docs/api-reference.md) です。
+- [api-reference.md](docs/api-reference.md) - API 仕様の単一ソース
+- [design.md](docs/design.md) - 設計思想とトレードオフ
+- [cookbook.md](docs/cookbook.md) - 実用例集
+- [cheatsheet.md](docs/cheatsheet.md) - 1 行リファレンス
+
+## 変更履歴
+
+- [CHANGELOG.md](CHANGELOG.md)
+- [docs/changelog-archive.md](docs/changelog-archive.md) - 0.x 系の履歴
+
+## ライセンス
+
+MIT License

megaton-1.0.0/README.md

@@ -0,0 +1,65 @@
+# megaton
+
+Megaton は Google Analytics 4、Google Search Console、Google Sheets、BigQuery を **Notebook から短いコードで扱う** ためのツールです。分析の試行錯誤を速く回すことを重視し、Notebook 向けの UX に特化しています。
+
+## コア概念
+
+- **結果オブジェクト**: `SearchResult` / `ReportResult` によるメソッドチェーン
+- **シンプルな流れ**: 開く → 期間設定 → 取得 → 保存
+- **Notebook 前提**: 途中結果の確認を前提にした設計
+
+## クイックスタート
+
+```python
+from megaton.start import Megaton
+
+mg = Megaton("/path/to/service_account.json")
+mg.report.set.dates("2024-01-01", "2024-01-31")
+mg.report.run(d=["date", "eventName"], m=["eventCount"])
+
+mg.open.sheet("https://docs.google.com/spreadsheets/d/...")
+mg.save.to.sheet("_ga_data", mg.report.data)
+```
+
+## もう少し実用的な例
+
+```python
+# 複数サイトの Search Console データを一括取得して整形
+result = (mg.search.run.all(
+    sites,
+    dimensions=['query', 'page'],
+    item_key='clinic',
+)
+    .categorize('query', by=query_map)
+    .categorize('page', by=page_map))
+
+mg.save.to.sheet('_query', result.df, sort_by='impressions')
+mg.upsert.to.csv(result.df, filename='query_master', keys=['clinic', 'query', 'page'], include_dates=False)
+```
+
+## インストール
+
+```bash
+# PyPI 公開版
+pip install megaton
+
+# 最新版（GitHub）
+pip install git+https://github.com/mak00s/megaton.git
+```
+
+## ドキュメント
+
+- 仕様の正は [api-reference.md](docs/api-reference.md) です。
+- [api-reference.md](docs/api-reference.md) - API 仕様の単一ソース
+- [design.md](docs/design.md) - 設計思想とトレードオフ
+- [cookbook.md](docs/cookbook.md) - 実用例集
+- [cheatsheet.md](docs/cheatsheet.md) - 1 行リファレンス
+
+## 変更履歴
+
+- [CHANGELOG.md](CHANGELOG.md)
+- [docs/changelog-archive.md](docs/changelog-archive.md) - 0.x 系の履歴
+
+## ライセンス
+
+MIT License

megaton-1.0.0/megaton/__init__.py

@@ -0,0 +1,62 @@
+import os
+import sys
+from IPython.display import clear_output
+
+
+__all__ = ['start', 'mount_google_drive']
+
+
+def _is_colab() -> bool:
+    return 'google.colab' in sys.modules
+
+
+def _auto_install_enabled() -> bool:
+    env_value = os.environ.get("MEGATON_AUTO_INSTALL")
+    if env_value == "1":
+        return True
+    if env_value == "0":
+        return False
+    return _is_colab()
+
+
+def _print_install_help():
+    print(
+        "Megaton requires GA4 packages. Install with:\n"
+        "  pip install -U -q google-analytics-admin google-analytics-data\n"
+        "  pip install -U -q google-cloud-bigquery-datatransfer\n"
+        "Or set MEGATON_AUTO_INSTALL=1 in Colab."
+    )
+
+
+try:
+    # check if packages for GA4 are installed
+    from google.analytics.data import BetaAnalyticsDataClient
+    from google.analytics.admin import AnalyticsAdminServiceClient
+except ModuleNotFoundError:
+    if _auto_install_enabled():
+        clear_output()
+        print("Installing packages for GA4...")
+        from .install import install_ga4, install_bigquery
+
+        install_ga4.install()
+        install_bigquery.install()
+        clear_output()
+    else:
+        _print_install_help()
+        raise
+
+IS_COLAB = _is_colab()
+
+if IS_COLAB:
+    from google.colab import data_table
+    data_table.enable_dataframe_formatter()
+    data_table._DEFAULT_FORMATTERS[float] = lambda x: f"{x:.3f}"
+
+
+def mount_google_drive():
+    '''Mount Google Drive when running in Google Colab.'''
+    if not IS_COLAB:
+        print("Google Drive mounting is only available in Google Colab.")
+        return None
+    from . import gdrive
+    return gdrive.link_nbs()

megaton-1.0.0/megaton/auth/__init__.py

@@ -0,0 +1,6 @@
+"""Auth helpers."""
+
+from . import google_auth, provider
+from .google_auth import *  # noqa: F403
+
+__all__ = list(google_auth.__all__) + ["google_auth", "provider"]

megaton-1.0.0/megaton/auth/google_auth.py

@@ -0,0 +1,231 @@
+"""
+Functions for handling Authentications
+"""
+
+import json
+import logging
+import os
+from collections import defaultdict
+
+import google.oauth2.credentials
+from google.oauth2 import service_account
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+
+_REQUIRED_CONFIG_KEYS = frozenset(("auth_uri", "token_uri", "client_id"))
+LOGGER = logging.getLogger(__name__)
+
+
+def _is_service_account(json_text: str):
+    """Return true if the provided text is a JSON service credentials file."""
+    try:
+        key_obj = json.loads(json_text)
+    except json.JSONDecodeError:
+        return False
+    if not key_obj or key_obj.get('type', '') != 'service_account':
+        return False
+    return True
+
+
+def _is_service_account_json(json_path: str):
+    """Return true if the provided JSON file is for a service account."""
+    with open(json_path, 'r') as f:
+        return _is_service_account(f.read())
+
+
+def get_credential_type(client_config: dict):
+    """Gets a client type from client configuration loaded from a Google-format client secrets file.
+
+    Args:
+        client_config (Mapping[str, Any]): The client
+            configuration in the Google `client secrets`_ format.
+
+    Returns:
+        client_type [str]: The client type, either ``'service_account'`` or ``'web'`` or ``'installed'``
+    """
+    if client_config.get('type', '') == "service_account":
+        return "service_account"
+    elif "web" in client_config:
+        client_type = "web"
+    elif "installed" in client_config:
+        client_type = "installed"
+    else:
+        return None
+    config = client_config[client_type]
+    if _REQUIRED_CONFIG_KEYS.issubset(config.keys()):
+        return client_type
+
+
+def get_credential_type_from_file(json_path: str):
+    """Gets a client type from a Google client secrets file.
+
+        Args:
+            json_path (str): The path to the client secrets .json file.
+
+        Returns:
+            client_type [str]: The client type, either ``'service_account'`` or ``'web'`` or ``'installed'``
+        """
+    try:
+        with open(json_path, "r") as fp:
+            client_config = json.load(fp)
+    except (OSError, json.JSONDecodeError):
+        LOGGER.debug("Skipping non JSON credential file: %s", json_path)
+        return None
+
+    if not isinstance(client_config, dict):
+        LOGGER.debug('Credential file %s does not contain a JSON object; skipping.', json_path)
+        return None
+
+    return get_credential_type(client_config)
+
+
+def get_credential_type_from_info(info: dict) -> str:
+    if isinstance(info, dict):
+        if info.get("type") == "service_account":
+            return "service_account"
+        if "installed" in info:
+            return "installed"
+        if "web" in info:
+            return "web"
+    return "unknown"
+
+
+def get_json_files_from_dir(json_dir: str):
+    """Gets a list of valid credentials json files from a directory recursively"""
+    json_files = defaultdict(lambda: {})
+    for root, dirs, files in os.walk(json_dir):
+        for file in files:
+            if file.endswith('.json'):
+                client_type = get_credential_type_from_file(os.path.join(root, file))
+                if client_type == 'service_account':
+                    json_files['Service Account'][file] = os.path.join(root, file)
+                elif client_type in ['installed', 'web']:
+                    json_files['OAuth'][file] = os.path.join(root, file)
+    return json_files
+
+
+def get_cache_path(json_path: str):
+    """Gets the path to the Google user credentials based on the provided source file
+    """
+    dir_path = os.path.join(os.path.expanduser("~"), ".config")
+    os.makedirs(dir_path, exist_ok=True)
+    base_name = os.path.splitext(os.path.basename(json_path))[0]
+    return os.path.join(dir_path, f"cache_{base_name}.json")
+
+
+def save_credentials(file_path: str, credentials: Credentials):
+    """Save Credentials to cache file
+    """
+    cache_path = get_cache_path(file_path)
+    with open(cache_path, 'w') as w:
+        LOGGER.debug(f"saving credentials to {cache_path}")
+        w.write(credentials.to_json())
+    return credentials
+
+
+def load_credentials(file_path: str, scopes: list):
+    """Load Credentials from cache file
+    """
+    cache_path = get_cache_path(file_path)
+    if os.path.isfile(cache_path):
+        LOGGER.debug(f"loading credentials from {cache_path}")
+        return Credentials.from_authorized_user_file(cache_path, scopes=scopes)
+
+
+def load_service_account_credentials_from_info(info: dict, scopes: list):
+    if not isinstance(info, dict) or info.get("type") != "service_account":
+        raise ValueError("service_account info required")
+    credentials = service_account.Credentials.from_service_account_info(info, scopes=scopes)
+    if not credentials.valid:
+        request = google.auth.transport.requests.Request()
+        try:
+            credentials.refresh(request)
+        except google.auth.exceptions.RefreshError as exc:
+            email = info.get("client_email") or getattr(credentials, "service_account_email", None)
+            if email:
+                message = f"指定の {email} のサービスアカウントは存在しない、または無効です。"
+            else:
+                message = "指定したサービスアカウントは存在しない、または無効です。"
+            LOGGER.error(message)
+            LOGGER.debug(f"Service account refresh error detail: {exc}")
+            return None
+    return credentials
+
+
+def delete_credentials(cache_file: str = "creden-cache.json"):
+    """Delete Credentials cache file
+    """
+    if os.path.isfile(cache_file):
+        LOGGER.debug(f"deleting cache file {cache_file}")
+        os.remove(cache_file)
+
+
+def get_oauth_redirect(client_secret_file: str, scopes: list):
+    """Run OAuth2 Flow"""
+    flow = InstalledAppFlow.from_client_secrets_file(
+        client_secret_file,
+        scopes=scopes,
+        redirect_uri="urn:ietf:wg:oauth:2.0:oob"
+    )
+    auth_url, _ = flow.authorization_url(prompt="consent")
+    return flow, auth_url
+
+
+def get_oauth_redirect_from_info(info: dict, scopes: list):
+    # mirrors get_oauth_redirect(file, scopes) but uses in-memory client config
+    flow = InstalledAppFlow.from_client_config(info, scopes=scopes)
+    flow.redirect_uri = "urn:ietf:wg:oauth:2.0:oob"
+    auth_url, _ = flow.authorization_url(
+        prompt='consent',
+        access_type='offline',
+        include_granted_scopes='true'
+    )
+    return flow, auth_url
+
+
+def get_token(flow, code: str):
+    flow.fetch_token(code=code)
+    return flow.credentials
+
+
+def load_service_account_credentials_from_file(path: str, scopes: list):
+    """Gets service account credentials from JSON file at ``path``.
+
+    :param path: Path to credentials JSON file.
+    :param scopes: A list of scopes to use when authenticating to Google APIs.
+    :return: google.oauth2.service_account.Credentials
+    """
+    credentials = service_account.Credentials.from_service_account_file(path, scopes=scopes)
+    if not credentials.valid:
+        request = google.auth.transport.requests.Request()
+        try:
+            credentials.refresh(request)
+        except google.auth.exceptions.RefreshError as exc:
+            email = getattr(credentials, "service_account_email", None)
+            if email:
+                message = f"指定の {email} のサービスアカウントは存在しない、または無効です。"
+            else:
+                message = "指定したサービスアカウントは存在しない、または無効です。"
+            LOGGER.error(message)
+            LOGGER.debug("Service account refresh error detail: %s", exc)
+            return None
+    return credentials
+
+
+__all__ = [
+    "_is_service_account",
+    "_is_service_account_json",
+    "get_credential_type",
+    "get_credential_type_from_file",
+    "get_credential_type_from_info",
+    "get_json_files_from_dir",
+    "get_cache_path",
+    "save_credentials",
+    "load_credentials",
+    "load_service_account_credentials_from_info",
+    "delete_credentials",
+    "get_oauth_redirect",
+    "get_oauth_redirect_from_info",
+    "get_token",
+    "load_service_account_credentials_from_file",
+]

megaton-1.0.0/megaton/auth/provider.py

@@ -0,0 +1,117 @@
+"""Credential source resolution helpers."""
+
+from dataclasses import dataclass
+import base64
+import json
+import os
+from typing import Any, Optional, Tuple
+
+
+@dataclass(frozen=True)
+class CredentialSource:
+    raw: Any
+    origin: str
+    kind: str
+    value: Optional[str] = None
+    info: Optional[dict] = None
+    credential_type: Optional[str] = None
+    error: Optional[Exception] = None
+
+
+def get_credential_type_from_info(info: Optional[dict]) -> str:
+    from . import google_auth
+
+    return google_auth.get_credential_type_from_info(info)
+
+
+def parse_json_input(value: Any) -> Optional[dict]:
+    """Return dict if value looks like JSON (or base64 JSON); else None."""
+    if isinstance(value, dict):
+        return value
+    if not isinstance(value, str):
+        return None
+    s = value.strip()
+    if s.startswith("{") and s.endswith("}"):
+        try:
+            return json.loads(s)
+        except Exception:
+            return None
+    try:
+        decoded = base64.b64decode(s).decode("utf-8", errors="ignore")
+        ds = decoded.strip()
+        if ds.startswith("{") and ds.endswith("}"):
+            return json.loads(ds)
+    except Exception:
+        pass
+    return None
+
+
+def load_json_file(path: str) -> Tuple[Optional[dict], Optional[Exception]]:
+    try:
+        with open(path) as fp:
+            data = json.load(fp)
+    except Exception as exc:
+        return None, exc
+    if not isinstance(data, dict):
+        return None, ValueError("JSON object required")
+    return data, None
+
+
+def extract_email_from_file(path: str) -> Optional[str]:
+    info, _ = load_json_file(path)
+    if not isinstance(info, dict):
+        return None
+    return info.get("client_email")
+
+
+def resolve_credential_source(
+    credential: Any,
+    *,
+    env_var: str = "MEGATON_CREDS_JSON",
+    in_colab: bool = False,
+    colab_default: str = "/nbs",
+) -> CredentialSource:
+    origin = "explicit"
+    raw = credential
+    value = credential
+    if credential is None:
+        env_val = os.environ.get(env_var)
+        if env_val:
+            origin = "env"
+            value = env_val
+        elif in_colab:
+            origin = "colab_default"
+            value = colab_default
+        else:
+            return CredentialSource(raw=None, origin="none", kind="none")
+
+    if isinstance(value, dict):
+        info = value
+        ctype = get_credential_type_from_info(info)
+        return CredentialSource(raw=raw, origin=origin, kind="inline", info=info, credential_type=ctype)
+
+    if not isinstance(value, str):
+        return CredentialSource(raw=raw, origin=origin, kind="unknown")
+
+    info = parse_json_input(value)
+    if info is not None:
+        ctype = get_credential_type_from_info(info)
+        return CredentialSource(raw=raw, origin=origin, kind="inline", value=value, info=info, credential_type=ctype)
+
+    if os.path.isdir(value):
+        return CredentialSource(raw=raw, origin=origin, kind="directory", value=value)
+
+    if os.path.isfile(value):
+        info, error = load_json_file(value)
+        ctype = get_credential_type_from_info(info) if info else None
+        return CredentialSource(
+            raw=raw,
+            origin=origin,
+            kind="file",
+            value=value,
+            info=info,
+            credential_type=ctype,
+            error=error,
+        )
+
+    return CredentialSource(raw=raw, origin=origin, kind="unknown", value=value)