interloper-google-cloud 0.93.0__tar.gz → 0.94.2__tar.gz

interloper_google_cloud-0.94.2/PKG-INFO

@@ -0,0 +1,65 @@
+Metadata-Version: 2.3
+Name: interloper-google-cloud
+Version: 0.94.2
+Summary: Interloper Google Cloud integration: BigQuery, Cloud Storage and Google Sheets destinations
+Author: Guillaume Onfroy
+Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
+Requires-Dist: google-cloud-bigquery>=3.0
+Requires-Dist: google-cloud-storage>=2.0
+Requires-Dist: interloper-core
+Requires-Dist: interloper-pandas
+Requires-Dist: pyarrow>=14
+Requires-Dist: db-dtypes>=1.0
+Requires-Dist: pandas-gbq>=0.26.1
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# interloper-google-cloud
+
+Google Cloud integration for the Interloper framework. Provides BigQuery, Cloud Storage and Google Sheets destinations and the Google Cloud connection resource.
+
+## Google Sheets
+
+`GoogleSheetsDestination` writes assets into one spreadsheet, one tab per asset. It authenticates with the
+`GoogleCloudConnection`'s service-account key, or with ambient credentials when the key is empty.
+
+### Setup
+
+1. Create the spreadsheet and copy its id from the URL
+   (`https://docs.google.com/spreadsheets/d/<id>/edit`).
+2. Share it with the service account's email address (`client_email` in the key) as an **editor**. The
+   service account sees no spreadsheet it was not shared on.
+
+### Usage
+
+```python
+from interloper_google_cloud import GoogleCloudConnection, GoogleSheetsDestination
+
+destination = GoogleSheetsDestination(
+    connection=GoogleCloudConnection(service_account_key="..."),
+    spreadsheet_id="1AbC...",
+)
+```
+
+### Layout
+
+- Each asset is a tab titled after its table, or `{dataset}.{table}` when the asset has a dataset (at most 100
+  characters, Sheets' limit).
+- The first row is the header, written when the tab is created from the asset's schema. Later writes are
+  aligned to it: columns missing from the data are left empty, columns missing from the header are dropped with
+  a warning. The header is never altered.
+- Rows carry the partition column. Replacing a partition deletes its rows, then appends the new ones; Sheets
+  has no transactions, so the pair is not atomic.
+
+### Types
+
+Values are written raw: numbers and booleans as is, `None` as an empty cell, decimals as strings, dates and
+datetimes as ISO 8601 strings, dicts and lists as JSON. Time partitions are matched by comparing those ISO
+strings, which orders chronologically, so a time-partitioned asset's partition column must be a date or a
+datetime (or an ISO string). On read, empty cells come back as `None` and rows are reconciled against the
+asset's schema, restoring the declared types.
+
+### Limits
+
+Sheets caps a spreadsheet at 10 million cells, and replacing a partition reads the whole tab and rewrites it. The
+destination suits small reporting tables that people open in Sheets, not a warehouse.

interloper_google_cloud-0.94.2/README.md

@@ -0,0 +1,49 @@
+# interloper-google-cloud
+
+Google Cloud integration for the Interloper framework. Provides BigQuery, Cloud Storage and Google Sheets destinations and the Google Cloud connection resource.
+
+## Google Sheets
+
+`GoogleSheetsDestination` writes assets into one spreadsheet, one tab per asset. It authenticates with the
+`GoogleCloudConnection`'s service-account key, or with ambient credentials when the key is empty.
+
+### Setup
+
+1. Create the spreadsheet and copy its id from the URL
+   (`https://docs.google.com/spreadsheets/d/<id>/edit`).
+2. Share it with the service account's email address (`client_email` in the key) as an **editor**. The
+   service account sees no spreadsheet it was not shared on.
+
+### Usage
+
+```python
+from interloper_google_cloud import GoogleCloudConnection, GoogleSheetsDestination
+
+destination = GoogleSheetsDestination(
+    connection=GoogleCloudConnection(service_account_key="..."),
+    spreadsheet_id="1AbC...",
+)
+```
+
+### Layout
+
+- Each asset is a tab titled after its table, or `{dataset}.{table}` when the asset has a dataset (at most 100
+  characters, Sheets' limit).
+- The first row is the header, written when the tab is created from the asset's schema. Later writes are
+  aligned to it: columns missing from the data are left empty, columns missing from the header are dropped with
+  a warning. The header is never altered.
+- Rows carry the partition column. Replacing a partition deletes its rows, then appends the new ones; Sheets
+  has no transactions, so the pair is not atomic.
+
+### Types
+
+Values are written raw: numbers and booleans as is, `None` as an empty cell, decimals as strings, dates and
+datetimes as ISO 8601 strings, dicts and lists as JSON. Time partitions are matched by comparing those ISO
+strings, which orders chronologically, so a time-partitioned asset's partition column must be a date or a
+datetime (or an ISO string). On read, empty cells come back as `None` and rows are reconciled against the
+asset's schema, restoring the declared types.
+
+### Limits
+
+Sheets caps a spreadsheet at 10 million cells, and replacing a partition reads the whole tab and rewrites it. The
+destination suits small reporting tables that people open in Sheets, not a warehouse.

{interloper_google_cloud-0.93.0 → interloper_google_cloud-0.94.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "interloper-google-cloud"
-version = "0.93.0"
-description = "Interloper Google Cloud integration: BigQuery and Cloud Storage destinations"
+version = "0.94.2"
+description = "Interloper Google Cloud integration: BigQuery, Cloud Storage and Google Sheets destinations"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [

{interloper_google_cloud-0.93.0 → interloper_google_cloud-0.94.2}/pyproject.toml.orig

@@ -3,8 +3,8 @@
 # ###############
 [project]
 name = "interloper-google-cloud"
-version = "0.93.0"
-description = "Interloper Google Cloud integration: BigQuery and Cloud Storage destinations"
+version = "0.94.2"
+description = "Interloper Google Cloud integration: BigQuery, Cloud Storage and Google Sheets destinations"
 readme = "README.md"
 authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
 requires-python = ">=3.10"

{interloper_google_cloud-0.93.0 → interloper_google_cloud-0.94.2}/src/interloper_google_cloud/__init__.py

@@ -1,11 +1,13 @@
-"""Interloper Google Cloud integration: BigQuery and Cloud Storage destinations and connection."""
+"""Interloper Google Cloud integration: BigQuery, Cloud Storage and Google Sheets destinations and connection."""
 
 from interloper_google_cloud.bigquery import BigQueryDestination
 from interloper_google_cloud.connection import GoogleCloudConnection
 from interloper_google_cloud.gcs import GCSDestination
+from interloper_google_cloud.google_sheets import GoogleSheetsDestination
 
 __all__ = [
     "BigQueryDestination",
     "GCSDestination",
     "GoogleCloudConnection",
+    "GoogleSheetsDestination",
 ]

interloper_google_cloud-0.94.2/src/interloper_google_cloud/gcs/destination.py

@@ -0,0 +1,113 @@
+"""Google Cloud Storage destination implementation."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from functools import cached_property
+
+import google.auth
+from google.cloud import storage
+from google.cloud.exceptions import NotFound
+from google.oauth2 import service_account
+from interloper.destination import ObjectStoreDestination, StoredObject, destination
+from interloper.resource.fields import FetchField
+
+from interloper_google_cloud.connection import GoogleCloudConnection
+
+
+@destination(
+    key="gcs_destination",
+    name="Google Cloud Storage",
+    icon="icon:gcs",
+    tags=["Cloud"],
+)
+class GCSDestination(ObjectStoreDestination):
+    """Google Cloud Storage destination.
+
+    An :class:`~interloper.destination.ObjectStoreDestination` over Cloud
+    Storage: the base owns the hive layout, the formats and the row-count
+    metadata, so objects land at::
+
+        gs://{bucket}/{prefix}/{dataset}/{table}/data.{ext}
+        gs://{bucket}/{prefix}/{dataset}/{table}/{column}={partition}/data.{ext}
+    """
+
+    connection: GoogleCloudConnection
+
+    bucket: str = FetchField(
+        provider="connection.buckets",
+        label_key="name",
+        value_key="name",
+        description="Cloud Storage bucket",
+        discriminator=True,
+    )
+
+    @cached_property
+    def client(self) -> storage.Client:
+        """The Cloud Storage client every read and write goes through.
+
+        Built from the connection's service-account key when there is one,
+        else from ambient credentials (workload identity in-cluster).
+
+        Returns:
+            The client, cached per destination instance.
+        """
+        if self.connection and self.connection.service_account_key:
+            key_info = json.loads(self.connection.service_account_key)
+            credentials = service_account.Credentials.from_service_account_info(key_info)
+            return storage.Client(project=key_info.get("project_id"), credentials=credentials)
+        credentials, project = google.auth.default()
+        return storage.Client(project=project, credentials=credentials)
+
+    # -- Object hooks ----------------------------------------------------------
+
+    def put_object(self, name: str, payload: bytes, content_type: str, metadata: dict[str, str]) -> None:
+        """Upload a blob, overwriting any blob of the same name.
+
+        Args:
+            name: The blob name, relative to the bucket root.
+            payload: The blob's bytes.
+            content_type: The payload's media type.
+            metadata: Custom metadata to store on the blob.
+        """
+        blob = self.client.bucket(self.bucket).blob(name)
+        blob.metadata = metadata
+        blob.upload_from_string(payload, content_type=content_type)
+
+    def get_object(self, name: str) -> bytes | None:
+        """Download a blob.
+
+        Args:
+            name: The blob name, relative to the bucket root.
+
+        Returns:
+            The blob's bytes, or ``None`` when no blob has that name.
+        """
+        try:
+            return self.client.bucket(self.bucket).blob(name).download_as_bytes()
+        except NotFound:
+            return None
+
+    def list_objects(self, prefix: str) -> Iterator[StoredObject]:
+        """List the blobs under a name prefix; the listing carries their metadata.
+
+        Args:
+            prefix: The name prefix, ending with ``/``.
+
+        Yields:
+            One stored object per blob.
+        """
+        for blob in self.client.list_blobs(self.bucket, prefix=prefix):
+            yield StoredObject(name=blob.name, metadata=dict(blob.metadata or {}))
+
+    def object_uri(self, name: str) -> str:
+        """Return the ``gs://`` URI of a blob.
+
+        Args:
+            name: The blob name, relative to the bucket root.
+
+        Returns:
+            ``gs://{bucket}/{name}``.
+        """
+        return f"gs://{self.bucket}/{name}"

interloper_google_cloud-0.94.2/src/interloper_google_cloud/google_sheets/__init__.py

@@ -0,0 +1,7 @@
+"""Google Sheets destination implementation."""
+
+from interloper_google_cloud.google_sheets.destination import GoogleSheetsDestination
+
+__all__ = [
+    "GoogleSheetsDestination",
+]

interloper_google_cloud-0.94.2/src/interloper_google_cloud/google_sheets/destination.py

@@ -0,0 +1,466 @@
+"""Google Sheets destination implementation."""
+
+from __future__ import annotations
+
+import datetime
+import json
+import math
+import numbers
+import warnings
+from decimal import Decimal
+from functools import cached_property
+from typing import Any, get_origin
+from urllib.parse import quote
+
+import google.auth
+from google.auth.credentials import Credentials
+from google.auth.transport.requests import Request
+from google.oauth2 import service_account
+from interloper.destination import IOContext, destination
+from interloper.destination.database import DatabaseDestination, PartitionFilter
+from interloper.errors import ConfigError, DataNotFoundError
+from interloper.partitioning import Partition
+from interloper.representation import Representation
+from interloper.resource.fields import InputField
+from interloper.rest import RESTClient
+from interloper.schema import FieldSpec
+from interloper.utils.json import json_default, replace_non_finite
+
+from interloper_google_cloud.connection import GoogleCloudConnection
+
+SHEETS_SCOPE = "https://www.googleapis.com/auth/spreadsheets"
+
+_API_BASE = "https://sheets.googleapis.com/v4/spreadsheets"
+_MAX_TITLE_LENGTH = 100
+_APPEND_CHUNK_ROWS = 10_000
+_TIMEOUT = 60.0
+
+
+def _credentials(connection: GoogleCloudConnection | None) -> Credentials:
+    """Build Sheets-scoped credentials for a connection.
+
+    Args:
+        connection: The connection whose service-account key signs requests;
+            ``None`` or an empty key falls back to ambient credentials
+            (workload identity in-cluster).
+
+    Returns:
+        Credentials carrying the spreadsheets scope.
+    """
+    if connection and connection.service_account_key:
+        key_info = json.loads(connection.service_account_key)
+        return service_account.Credentials.from_service_account_info(key_info).with_scopes([SHEETS_SCOPE])
+    credentials, _ = google.auth.default(scopes=[SHEETS_SCOPE])
+    return credentials
+
+
+class _SheetsAPI:
+    """The Sheets REST API v4 calls one spreadsheet needs, over httpx2."""
+
+    def __init__(self, spreadsheet_id: str, credentials: Credentials, **client_kwargs: Any) -> None:
+        """Bind the API to a spreadsheet.
+
+        Args:
+            spreadsheet_id: The spreadsheet every call addresses.
+            credentials: Credentials whose bearer token authorizes each call.
+            **client_kwargs: Forwarded to the :class:`RESTClient` (a ``transport`` in tests).
+        """
+        self.spreadsheet_id = spreadsheet_id
+        self._credentials = credentials
+        self._path = "/" + quote(spreadsheet_id, safe="")
+        self._client = RESTClient(_API_BASE, timeout=_TIMEOUT, **client_kwargs)
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
+        """Send one authorized request, refreshing the token when it has expired.
+
+        Args:
+            method: The HTTP method.
+            path: The suffix to the spreadsheet's path (``""`` for the spreadsheet itself).
+            **kwargs: Forwarded to the client (``params``, ``json``).
+
+        Returns:
+            The decoded JSON response body.
+        """
+        if not self._credentials.valid:
+            self._credentials.refresh(Request())
+        response = self._client.request(
+            method, self._path + path, headers={"Authorization": f"Bearer {self._credentials.token}"}, **kwargs
+        )
+        response.raise_for_status()
+        return response.json()
+
+    @staticmethod
+    def _range(title: str) -> str:
+        """Render a tab title as a URL-safe A1 range covering the whole tab.
+
+        Args:
+            title: The tab title.
+
+        Returns:
+            The quoted, percent-encoded range.
+        """
+        return quote("'" + title.replace("'", "''") + "'", safe="")
+
+    def sheet_titles(self) -> set[str]:
+        """List the spreadsheet's tab titles.
+
+        Returns:
+            Every tab title.
+        """
+        body = self._request("GET", "", params={"fields": "sheets.properties.title"})
+        return {sheet["properties"]["title"] for sheet in body.get("sheets", [])}
+
+    def values(self, title: str) -> list[list[Any]]:
+        """Read every row of a tab, header first.
+
+        Values come back unformatted so numbers keep their precision; rows
+        are ragged, the API omitting trailing empty cells.
+
+        Args:
+            title: The tab title.
+
+        Returns:
+            The tab's rows.
+        """
+        body = self._request("GET", f"/values/{self._range(title)}", params={"valueRenderOption": "UNFORMATTED_VALUE"})
+        return body.get("values", [])
+
+    def append(self, title: str, rows: list[list[Any]]) -> None:
+        """Append rows after a tab's last row, in chunks.
+
+        Args:
+            title: The tab title.
+            rows: The rows to append.
+        """
+        for start in range(0, len(rows), _APPEND_CHUNK_ROWS):
+            self._request(
+                "POST",
+                f"/values/{self._range(title)}:append",
+                params={"valueInputOption": "RAW", "insertDataOption": "INSERT_ROWS"},
+                json={"values": rows[start : start + _APPEND_CHUNK_ROWS]},
+            )
+
+    def rewrite(self, title: str, rows: list[list[Any]]) -> None:
+        """Replace a tab's contents with rows.
+
+        Args:
+            title: The tab title.
+            rows: The rows the tab holds afterwards, header first.
+        """
+        self._request("POST", f"/values/{self._range(title)}:clear", json={})
+        self.append(title, rows)
+
+    def add_sheet(self, title: str) -> None:
+        """Add an empty tab to the spreadsheet.
+
+        Args:
+            title: The new tab's title.
+        """
+        self._request("POST", ":batchUpdate", json={"requests": [{"addSheet": {"properties": {"title": title}}}]})
+
+
+@destination(
+    key="google_sheets_destination",
+    name="Google Sheets",
+    icon="icon:google_sheets",
+    tags=["Cloud"],
+)
+class GoogleSheetsDestination(DatabaseDestination):
+    """Google Sheets destination.
+
+    The spreadsheet is the database and each asset is a tab, titled after its
+    table (``{dataset}.{table}`` when the asset has a dataset). The first row
+    is the header, fixed when the tab is created; every later row is a record
+    carrying the partition column like any other.
+
+    Sheets has no transactions: replacing a partition deletes its rows (a
+    read, then a rewrite of the rows kept) and then appends the new ones, so a
+    failure in between leaves the partition empty until the next write, and
+    concurrent writes to one tab are not safe.
+    """
+
+    connection: GoogleCloudConnection
+
+    spreadsheet_id: str = InputField(
+        label="Spreadsheet ID",
+        description="The id in the spreadsheet's URL",
+        info="Share the spreadsheet with the service account's email address (editor).",
+        discriminator=True,
+    )
+
+    @cached_property
+    def api(self) -> _SheetsAPI:
+        """The Sheets API every read and write goes through.
+
+        Returns:
+            The API bound to the spreadsheet, cached per destination instance.
+        """
+        return _SheetsAPI(self.spreadsheet_id, _credentials(self.connection))
+
+    # -- Helpers ---------------------------------------------------------------
+
+    @staticmethod
+    def _title(table: str, dataset: str | None) -> str:
+        """Return the tab title an asset's table maps to.
+
+        Args:
+            table: Target table name.
+            dataset: The asset's dataset, or ``None``.
+
+        Returns:
+            ``table``, or ``{dataset}.{table}`` when there is a dataset.
+
+        Raises:
+            ConfigError: If the title exceeds Sheets' 100-character limit.
+        """
+        title = f"{dataset}.{table}" if dataset else table
+        if len(title) > _MAX_TITLE_LENGTH:
+            raise ConfigError(
+                f"Sheet title '{title}' is {len(title)} characters long; Google Sheets allows {_MAX_TITLE_LENGTH}."
+            )
+        return title
+
+    # -- DatabaseDestination hooks ---------------------------------------------
+
+    def insert(self, table: str, dataset: str | None, data: Any, context: IOContext) -> None:
+        """Append data to its tab, creating the tab and its header on first write.
+
+        A new tab's header is the effective schema's columns (inferred from
+        the data when the context carries none); an existing tab keeps its
+        own. Rows are aligned to the header: header columns the data lacks
+        are left empty, and columns the header lacks are dropped with a
+        warning.
+
+        Args:
+            table: Target table name.
+            dataset: The asset's dataset, or ``None``.
+            data: The data in its native representation.
+            context: IO context carrying the effective schema.
+        """
+        title = self._title(table, dataset)
+        view = Representation.of(data)
+        schema = context.schema if context.schema is not None else view.infer()
+        columns = [spec.name for spec in schema.field_specs()]
+
+        rows: list[list[Any]] = []
+        if title not in self.api.sheet_titles():
+            self.api.add_sheet(title)
+            existing = []
+        else:
+            existing = self.api.values(title)
+        header = [str(cell) for cell in existing[0]] if existing else columns
+        if not existing:
+            rows.append(header)
+
+        extras = [column for column in dict.fromkeys([*columns, *view.columns]) if column not in header]
+        if extras:
+            warnings.warn(
+                f"Columns {extras} are not in the schema for '{title}' and will not be written.",
+                UserWarning,
+                stacklevel=2,
+            )
+        rows.extend([_cell(record.get(column)) for column in header] for record in view.records)
+        self.api.append(title, rows)
+
+    def delete(self, table: str, dataset: str | None, where: PartitionFilter | None) -> None:
+        """Delete the rows a filter selects, or every row but the header.
+
+        A tab that does not exist has nothing to delete, and a tab left
+        unchanged by the filter is not rewritten.
+
+        Args:
+            table: Target table name.
+            dataset: The asset's dataset, or ``None``.
+            where: The rows to delete; ``None`` for the whole table.
+        """
+        title = self._title(table, dataset)
+        if title not in self.api.sheet_titles():
+            return
+        existing = self.api.values(title)
+        if not existing:
+            return
+        header, body = existing[0], existing[1:]
+        kept = [] if where is None else [row for row in body if not _matches(header, row, where)]
+        if len(kept) < len(body):
+            self.api.rewrite(title, [header, *kept])
+
+    def select(self, table: str, dataset: str | None, where: PartitionFilter | None) -> list[dict[str, Any]]:
+        """Select the rows a filter selects, or every row, as records.
+
+        Empty cells read back as ``None``; every other value comes back as
+        Sheets stores it (numbers and booleans native, the rest strings).
+
+        Args:
+            table: Target table name.
+            dataset: The asset's dataset, or ``None``.
+            where: The rows to select; ``None`` for the whole table.
+
+        Returns:
+            The selected rows.
+
+        Raises:
+            DataNotFoundError: If the tab does not exist.
+        """
+        title = self._title(table, dataset)
+        if title not in self.api.sheet_titles():
+            raise DataNotFoundError(
+                f"Sheet '{title}' does not exist in spreadsheet '{self.spreadsheet_id}'. "
+                "Has the asset been materialized?"
+            )
+        existing = self.api.values(title)
+        if not existing:
+            return []
+        header, body = [str(cell) for cell in existing[0]], existing[1:]
+        return [
+            {column: _read(row[i] if i < len(row) else "") for i, column in enumerate(header)}
+            for row in body
+            if where is None or _matches(header, row, where)
+        ]
+
+    def read_partition(self, context: IOContext, partition: Partition | None) -> list[dict[str, Any]]:
+        """Load one partition, restoring the schema's types when the context carries one.
+
+        Dates and decimals are stored as strings and nested values as JSON,
+        so rows are reconciled against ``context.schema`` (JSON cells decoded
+        first) rather than handed back as Sheets holds them.
+
+        Args:
+            context: IO context carrying the asset and the effective schema.
+            partition: The partition to load, or ``None`` for the whole table.
+
+        Returns:
+            The partition's rows.
+        """
+        rows = super().read_partition(context, partition)
+        if context.schema is None:
+            return rows
+        specs = context.schema.field_specs()
+        json_columns = [spec.name for spec in specs if _is_json(spec)]
+        for row in rows:
+            for column in json_columns:
+                if isinstance(row.get(column), str):
+                    row[column] = json.loads(row[column])
+        return context.schema.reconcile(rows)
+
+    def count(self, table: str, dataset: str | None, column: str) -> dict[str, int]:
+        """Return row counts grouped by a column's stored value.
+
+        Args:
+            table: Target table name.
+            dataset: The asset's dataset, or ``None``.
+            column: The column to group by.
+
+        Returns:
+            Each distinct value, as a string, to its row count.
+
+        Raises:
+            DataNotFoundError: If the tab does not exist.
+        """
+        title = self._title(table, dataset)
+        if title not in self.api.sheet_titles():
+            raise DataNotFoundError(
+                f"Sheet '{title}' does not exist in spreadsheet '{self.spreadsheet_id}'. "
+                "Has the asset been materialized?"
+            )
+        existing = self.api.values(title)
+        if not existing or column not in existing[0]:
+            return {}
+        index = existing[0].index(column)
+        counts: dict[str, int] = {}
+        for row in existing[1:]:
+            value = _text(row[index] if index < len(row) else "")
+            counts[value] = counts.get(value, 0) + 1
+        return counts
+
+
+# -- Cell encoding -------------------------------------------------------------
+
+
+def _cell(value: Any) -> Any:
+    """Encode a value as a Sheets cell.
+
+    Args:
+        value: The value to encode.
+
+    Returns:
+        ``""`` for ``None`` and non-finite floats, booleans and numbers as
+        is, decimals as strings, dates and datetimes as ISO 8601, dicts and
+        lists as JSON, anything else as its string.
+    """
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, numbers.Integral):
+        return int(value)
+    if isinstance(value, numbers.Real):
+        number = float(value)
+        return number if math.isfinite(number) else ""
+    if isinstance(value, Decimal):
+        return str(value)
+    if isinstance(value, (datetime.date, datetime.datetime)):
+        return value.isoformat()
+    if isinstance(value, (dict, list)):
+        return json.dumps(replace_non_finite(value), default=json_default)
+    return str(value)
+
+
+def _read(cell: Any) -> Any:
+    """Decode a stored cell, an empty one reading as ``None``.
+
+    Args:
+        cell: The cell as the API returns it.
+
+    Returns:
+        The cell's value.
+    """
+    return None if cell == "" else cell
+
+
+def _text(cell: Any) -> str:
+    """Render a cell as the string it compares and groups by.
+
+    Args:
+        cell: A cell as stored or as :func:`_cell` encodes it.
+
+    Returns:
+        Strings as is, anything else as JSON (``5``, ``1.5``, ``true``).
+    """
+    return cell if isinstance(cell, str) else json.dumps(cell)
+
+
+def _matches(header: list[Any], row: list[Any], where: PartitionFilter) -> bool:
+    """Return whether a stored row falls in a partition filter.
+
+    Bounds compare as strings, which is chronological because the partition
+    column is stored as ISO 8601 (a date is a prefix of the datetimes it
+    contains).
+
+    Args:
+        header: The tab's header row.
+        row: A stored row, possibly ragged.
+        where: The filter to apply.
+
+    Returns:
+        ``True`` if the row's partition cell matches.
+    """
+    if where.column not in header:
+        return False
+    index = header.index(where.column)
+    cell = _text(row[index] if index < len(row) else "")
+    if where.bounds is None:
+        return cell == _text(_cell(where.value))
+    start, end = where.bounds
+    return _text(_cell(start)) <= cell < _text(_cell(end))
+
+
+def _is_json(spec: FieldSpec) -> bool:
+    """Return whether a field is stored as JSON text.
+
+    Args:
+        spec: The field spec.
+
+    Returns:
+        ``True`` for nested and repeated fields, and ``dict`` or ``list`` typed ones.
+    """
+    return spec.repeated or spec.fields is not None or (get_origin(spec.type) or spec.type) in (dict, list)

interloper_google_cloud-0.93.0/PKG-INFO

@@ -1,19 +0,0 @@
-Metadata-Version: 2.3
-Name: interloper-google-cloud
-Version: 0.93.0
-Summary: Interloper Google Cloud integration: BigQuery and Cloud Storage destinations
-Author: Guillaume Onfroy
-Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
-Requires-Dist: google-cloud-bigquery>=3.0
-Requires-Dist: google-cloud-storage>=2.0
-Requires-Dist: interloper-core
-Requires-Dist: interloper-pandas
-Requires-Dist: pyarrow>=14
-Requires-Dist: db-dtypes>=1.0
-Requires-Dist: pandas-gbq>=0.26.1
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-
-# interloper-google-cloud
-
-Google Cloud integration for the Interloper framework. Provides a BigQuery destination and Google Cloud connection resource.

interloper_google_cloud-0.93.0/README.md

@@ -1,3 +0,0 @@
-# interloper-google-cloud
-
-Google Cloud integration for the Interloper framework. Provides a BigQuery destination and Google Cloud connection resource.

interloper_google_cloud-0.93.0/src/interloper_google_cloud/gcs/destination.py

@@ -1,239 +0,0 @@
-"""Google Cloud Storage destination implementation."""
-
-from __future__ import annotations
-
-import json
-from functools import cached_property
-from typing import Any
-
-import google.auth
-from google.cloud import storage
-from google.cloud.exceptions import NotFound
-from google.oauth2 import service_account
-from interloper.destination import IOContext, destination
-from interloper.destination.base import Destination
-from interloper.errors import DataNotFoundError
-from interloper.partitioning import Partition
-from interloper.representation import Representation
-from interloper.resource.fields import FetchField, InputField, SelectField
-from interloper.schema import FieldSpec
-
-from interloper_google_cloud.connection import GoogleCloudConnection
-from interloper_google_cloud.gcs.formats import FORMATS, FileFormat
-
-# Custom blob metadata key carrying the partition's row count, so
-# partition_row_counts introspects from a single list call without downloads.
-_ROW_COUNT_METADATA_KEY = "row_count"
-
-
-@destination(
-    key="gcs_destination",
-    name="Google Cloud Storage",
-    icon="icon:gcs",
-    tags=["Cloud"],
-)
-class GCSDestination(Destination):
-    """Google Cloud Storage destination.
-
-    Writes one object per partition in a hive-partitioned layout::
-
-        gs://{bucket}/{prefix}/{dataset}/{table}/data.{ext}
-        gs://{bucket}/{prefix}/{dataset}/{table}/{column}={partition}/data.{ext}
-
-    Following the hive convention, the partition column lives in the *path
-    only*: it is dropped from partitioned file contents on write (external
-    readers like BigQuery external tables and DuckDB reject a duplicate
-    partition column) and re-injected from the partition on read, so
-    interloper round-trips stay lossless.
-    """
-
-    connection: GoogleCloudConnection
-
-    bucket: str = FetchField(
-        provider="connection.buckets",
-        label_key="name",
-        value_key="name",
-        description="Cloud Storage bucket",
-        discriminator=True,
-    )
-    format: str = SelectField(
-        default="parquet",
-        description="Output file format",
-        options=[
-            {"label": "Parquet", "value": "parquet"},
-            {"label": "JSONL", "value": "jsonl"},
-            {"label": "CSV", "value": "csv"},
-        ],
-    )
-    prefix: str | None = InputField(default=None, description="Path prefix inside the bucket")
-
-    @cached_property
-    def client(self) -> storage.Client:
-        """The Cloud Storage client every read and write goes through.
-
-        Built from the connection's service-account key when there is one,
-        else from ambient credentials (workload identity in-cluster).
-
-        Returns:
-            The client, cached per destination instance.
-        """
-        if self.connection and self.connection.service_account_key:
-            key_info = json.loads(self.connection.service_account_key)
-            credentials = service_account.Credentials.from_service_account_info(key_info)
-            return storage.Client(project=key_info.get("project_id"), credentials=credentials)
-        credentials, project = google.auth.default()
-        return storage.Client(project=project, credentials=credentials)
-
-    # -- Helpers ---------------------------------------------------------------
-
-    @property
-    def _format(self) -> FileFormat:
-        """The configured file format strategy.
-
-        Returns:
-            The strategy for the configured format.
-
-        """
-        return FORMATS[self.format]
-
-    def _asset_prefix(self, context: IOContext) -> str:
-        """Return the object-name prefix for an asset (no trailing slash).
-
-        Args:
-            context: The IO context naming the asset.
-
-        Returns:
-            The prefix every object for the asset sits under.
-
-        """
-        parts = [self.prefix or "", context.asset.dataset or "", context.asset.table]
-        return "/".join(part.strip("/") for part in parts if part and part.strip("/"))
-
-    def _blob_name(self, context: IOContext, partition: Partition | None) -> str:
-        """Build the object name for a partition.
-
-        Args:
-            context: The IO context naming the asset.
-            partition: The partition being addressed, or ``None`` for the whole.
-
-        Returns:
-            ``.../data.{ext}``, inside a ``{column}={id}`` segment for
-            partitions.
-        """
-        parts = [self._asset_prefix(context)]
-        if partition is not None:
-            assert context.asset.partitioning
-            parts.append(f"{context.asset.partitioning.column}={partition.id}")
-        parts.append(f"data.{self._format.extension}")
-        return "/".join(parts)
-
-    def _effective_specs(self, context: IOContext, partition: Partition | None) -> list[FieldSpec] | None:
-        """Return the field specs for a partition's file contents.
-
-        Partitions exclude the partition column (its value lives in the
-        object path).
-
-        Args:
-            context: The IO context carrying the schema.
-            partition: The partition being addressed, or ``None`` for the whole.
-
-        Returns:
-            The specs, or ``None`` when the context carries no schema.
-        """
-        if context.schema is None:
-            return None
-        specs = context.schema.field_specs()
-        if partition is not None:
-            assert context.asset.partitioning
-            specs = [spec for spec in specs if spec.name != context.asset.partitioning.column]
-        return specs
-
-    # -- Partition hooks -------------------------------------------------------
-
-    def write_partition(self, context: IOContext, partition: Partition | None, data: Any) -> None:
-        """Serialize one partition's data and upload it, overwriting its object.
-
-        The row count is stamped as blob metadata so introspection never has
-        to download data.
-
-        Args:
-            context: The IO context naming the asset.
-            partition: The partition being written, or ``None`` for the whole.
-            data: The rows to write.
-
-        """
-        rows = Representation.of(data).records
-        if partition is not None:
-            assert context.asset.partitioning
-            column = context.asset.partitioning.column
-            rows = [{k: v for k, v in row.items() if k != column} for row in rows]
-
-        payload = self._format.serialize(rows, self._effective_specs(context, partition))
-        blob = self.client.bucket(self.bucket).blob(self._blob_name(context, partition))
-        blob.metadata = {_ROW_COUNT_METADATA_KEY: str(len(rows))}
-        blob.upload_from_string(payload, content_type=self._format.content_type)
-
-    def read_partition(self, context: IOContext, partition: Partition | None) -> list[dict[str, Any]]:
-        """Download and parse one partition's object.
-
-        The partition column is re-injected from the partition, and rows are
-        reconciled against the context schema when one is set (restoring the
-        declared types — text formats read everything back as strings).
-
-        Args:
-            context: The IO context carrying the schema.
-            partition: The partition being read, or ``None`` for the whole.
-
-        Returns:
-            Rows as a list of dicts.
-
-        Raises:
-            DataNotFoundError: If the partition's object does not exist.
-        """
-        name = self._blob_name(context, partition)
-        try:
-            payload = self.client.bucket(self.bucket).blob(name).download_as_bytes()
-        except NotFound:
-            raise DataNotFoundError(
-                f"Object 'gs://{self.bucket}/{name}' does not exist. Has the asset been materialized?"
-            ) from None
-
-        rows = self._format.deserialize(payload)
-        if partition is not None:
-            assert context.asset.partitioning
-            column = context.asset.partitioning.column
-            rows = [{**row, column: partition.id} for row in rows]
-        if context.schema is not None:
-            rows = context.schema.reconcile(rows)
-        return rows
-
-    # -- Introspection ---------------------------------------------------------
-
-    def partition_row_counts(self, context: IOContext) -> dict[str, int]:
-        """Return row counts grouped by partition from a single list call.
-
-        Counts come from the ``row_count`` blob metadata stamped at write
-        time; objects missing it (written by other tools) are downloaded and
-        counted.
-
-        Args:
-            context: The IO context naming the asset.
-
-        Returns:
-            Mapping from partition value (as string) to row count.
-        """
-        assert context.asset.partitioning is not None
-        column = context.asset.partitioning.column
-        prefix = self._asset_prefix(context) + "/"
-
-        counts: dict[str, int] = {}
-        for blob in self.client.list_blobs(self.bucket, prefix=prefix):
-            segment = blob.name[len(prefix) :].split("/", 1)[0]
-            if not segment.startswith(f"{column}="):
-                continue
-            value = segment.split("=", 1)[1]
-            row_count = (blob.metadata or {}).get(_ROW_COUNT_METADATA_KEY)
-            if row_count is None:
-                row_count = len(self._format.deserialize(blob.download_as_bytes()))
-            counts[value] = counts.get(value, 0) + int(row_count)
-        return counts

interloper_google_cloud-0.93.0/src/interloper_google_cloud/gcs/formats.py

@@ -1,265 +0,0 @@
-"""File formats for the GCS destination: records ⇄ bytes, one class per format."""
-
-from __future__ import annotations
-
-import csv
-import datetime
-import io
-import json
-from abc import ABC, abstractmethod
-from decimal import Decimal
-from typing import Any, ClassVar
-
-import pyarrow as pa
-import pyarrow.parquet as pq
-from interloper.schema import FieldSpec
-
-from interloper_google_cloud.serialization import json_default, replace_non_finite
-
-
-class FileFormat(ABC):
-    """Serialization strategy for one file format.
-
-    ``serialize`` receives records (and the effective field specs, when a
-    schema is known) and returns the object payload; ``deserialize`` parses a
-    payload back into records.  Type restoration from text formats is the
-    caller's concern (``Schema.reconcile``) — formats only deal in bytes.
-    """
-
-    key: ClassVar[str]
-    extension: ClassVar[str]
-    content_type: ClassVar[str]
-
-    @abstractmethod
-    def serialize(self, rows: list[dict[str, Any]], specs: list[FieldSpec] | None) -> bytes:
-        """Serialize records into the format's byte payload.
-
-        Args:
-            rows: The records to serialize.
-            specs: The field specs, when the format uses a declared schema.
-
-        Returns:
-            The encoded payload.
-
-        """
-
-    @abstractmethod
-    def deserialize(self, payload: bytes) -> list[dict[str, Any]]:
-        """Parse a byte payload back into records.
-
-        Args:
-            payload: The encoded payload.
-
-        Returns:
-            One dict per record.
-
-        """
-
-
-class JSONLFormat(FileFormat):
-    """Newline-delimited JSON — the load format BigQuery and most lakes expect."""
-
-    key = "jsonl"
-    extension = "jsonl"
-    content_type = "application/jsonl"
-
-    def serialize(self, rows: list[dict[str, Any]], specs: list[FieldSpec] | None) -> bytes:
-        """Serialize records as one JSON object per line.
-
-        Non-finite floats become ``null`` (invalid JSON otherwise); dates and
-        decimals serialize via :func:`json_default`.
-
-        Args:
-            rows: The records to serialize.
-            specs: Accepted for the interface and unused by NDJSON.
-
-        Returns:
-            The NDJSON payload.
-        """
-        lines = (json.dumps(replace_non_finite(row), default=json_default) for row in rows)
-        return "\n".join(lines).encode()
-
-    def deserialize(self, payload: bytes) -> list[dict[str, Any]]:
-        """Parse NDJSON lines into records.
-
-        Args:
-            payload: The NDJSON payload.
-
-        Returns:
-            One dict per non-empty line.
-        """
-        return [json.loads(line) for line in payload.decode().splitlines() if line.strip()]
-
-
-class CSVFormat(FileFormat):
-    """CSV with a header row; every value is stored as a string."""
-
-    key = "csv"
-    extension = "csv"
-    content_type = "text/csv"
-
-    def serialize(self, rows: list[dict[str, Any]], specs: list[FieldSpec] | None) -> bytes:
-        """Serialize records as CSV (the first row's keys are the headers).
-
-        Args:
-            rows: The records to serialize.
-            specs: Accepted for the interface and unused by CSV.
-
-        Returns:
-            The CSV payload; empty for no rows.
-        """
-        if not rows:
-            return b""
-        buffer = io.StringIO()
-        writer = csv.DictWriter(buffer, fieldnames=list(rows[0].keys()))
-        writer.writeheader()
-        writer.writerows(rows)
-        return buffer.getvalue().encode()
-
-    def deserialize(self, payload: bytes) -> list[dict[str, Any]]:
-        """Parse CSV into records, reading empty cells as ``None``.
-
-        Args:
-            payload: The CSV payload.
-
-        Returns:
-            One dict per data row.
-        """
-        reader = csv.DictReader(io.StringIO(payload.decode()))
-        return [{k: (None if v == "" else v) for k, v in row.items()} for row in reader]
-
-
-class ParquetFormat(FileFormat):
-    """Parquet via pyarrow — full type fidelity, ideal for external tables."""
-
-    key = "parquet"
-    extension = "parquet"
-    content_type = "application/vnd.apache.parquet"
-
-    def serialize(self, rows: list[dict[str, Any]], specs: list[FieldSpec] | None) -> bytes:
-        """Serialize records as a Parquet file.
-
-        With field specs, the Arrow schema is built from them so the file
-        shape is deterministic and stable across partitions; without, pyarrow
-        infers it from the data.
-
-        Args:
-            rows: The records to serialize.
-            specs: The field specs the Arrow schema is built from, when given.
-
-        Returns:
-            The Parquet payload.
-        """
-        if specs is not None:
-            rows = _stringify_untyped(rows, specs)
-            table = pa.Table.from_pylist(rows, schema=pa.schema([_spec_to_arrow_field(spec) for spec in specs]))
-        else:
-            table = pa.Table.from_pylist(rows)
-        buffer = pa.BufferOutputStream()
-        pq.write_table(table, buffer)
-        return buffer.getvalue().to_pybytes()
-
-    def deserialize(self, payload: bytes) -> list[dict[str, Any]]:
-        """Read a Parquet payload into records (types preserved).
-
-        Args:
-            payload: The Parquet payload.
-
-        Returns:
-            One dict per row.
-        """
-        return pq.read_table(pa.BufferReader(payload)).to_pylist()
-
-
-FORMATS: dict[str, FileFormat] = {fmt.key: fmt for fmt in (ParquetFormat(), JSONLFormat(), CSVFormat())}
-
-
-def _spec_to_arrow_field(spec: FieldSpec) -> pa.Field:
-    """Map a field spec to an Arrow field (nested records as structs).
-
-    Args:
-        spec: The field spec to map.
-
-    Returns:
-        The Arrow field, wrapped in ``list_`` when the spec is repeated.
-    """
-    if spec.fields is not None:
-        arrow_type: pa.DataType = pa.struct([_spec_to_arrow_field(sub) for sub in spec.fields])
-    else:
-        arrow_type = _py_type_to_arrow_type(spec.type)
-    if spec.repeated:
-        arrow_type = pa.list_(arrow_type)
-    return pa.field(spec.name, arrow_type, nullable=spec.nullable)
-
-
-def _is_concrete_type(py_type: Any) -> bool:
-    """Return whether *py_type* is a real class the Arrow mapping can use.
-
-    ``typing.Any`` is excluded explicitly — on Python 3.11+ it *is* an
-    ``isinstance(..., type)``, but it types nothing.
-
-    Args:
-        py_type: The declared type to test.
-
-    Returns:
-        True when the type is a concrete class.
-
-    """
-    return py_type is not Any and isinstance(py_type, type)
-
-
-def _py_type_to_arrow_type(py_type: Any) -> pa.DataType:
-    """Map a Python *type* (from a FieldSpec) to an Arrow type.
-
-    Mirrors the BigQuery mapping (``Decimal`` → NUMERIC precision/scale) so
-    parquet files load into BigQuery without casts.
-
-    Args:
-        py_type: The declared Python type.
-
-    Returns:
-        The Arrow type.
-
-    """
-    if not _is_concrete_type(py_type):
-        return pa.string()  # typing.Any or unresolvable annotations
-    if issubclass(py_type, bool):
-        return pa.bool_()
-    if issubclass(py_type, int):
-        return pa.int64()
-    if issubclass(py_type, float):
-        return pa.float64()
-    if issubclass(py_type, Decimal):
-        return pa.decimal128(38, 9)
-    if issubclass(py_type, datetime.datetime):
-        return pa.timestamp("us")
-    if issubclass(py_type, datetime.date):
-        return pa.date32()
-    if issubclass(py_type, bytes):
-        return pa.binary()
-    return pa.string()
-
-
-def _stringify_untyped(rows: list[dict[str, Any]], specs: list[FieldSpec]) -> list[dict[str, Any]]:
-    """Stringify values of untyped (``Any``) scalar fields.
-
-    Those fields map to Arrow strings, and ``from_pylist`` does not coerce
-    e.g. ints into a string column — pre-convert so the write cannot fail on
-    a column the schema could not type.
-
-    Args:
-        rows: The records to rewrite.
-        specs: The field specs deciding which fields are untyped.
-
-    Returns:
-        The rows, copied only when a conversion applies.
-    """
-    untyped = {
-        spec.name for spec in specs if spec.fields is None and not spec.repeated and not _is_concrete_type(spec.type)
-    }
-    if not untyped:
-        return rows
-    return [
-        {k: (str(v) if k in untyped and v is not None and not isinstance(v, str) else v) for k, v in row.items()}
-        for row in rows
-    ]

interloper_google_cloud-0.93.0/src/interloper_google_cloud/serialization.py

@@ -1,53 +0,0 @@
-"""JSON serialization helpers shared by the Google Cloud destinations."""
-
-from __future__ import annotations
-
-import datetime
-import math
-from decimal import Decimal
-from typing import Any
-
-
-def replace_non_finite(obj: Any) -> Any:
-    """Recursively replace non-finite floats (``NaN``, ``Infinity``) with ``None``.
-
-    pandas represents missing numeric values as ``float('nan')``. Python's
-    ``json.dumps`` serialises these to the bare tokens ``NaN`` / ``Infinity``,
-    which are invalid JSON — BigQuery's load parser rejects them with "Parser
-    terminated before end of string". Neither BigQuery nor JSONL files have a
-    NaN concept, so map non-finite floats to ``None`` (JSON ``null``).
-
-    Args:
-        obj: The value or container to rewrite.
-
-    Returns:
-        The value with non-finite floats replaced.
-
-    """
-    if isinstance(obj, float):
-        return obj if math.isfinite(obj) else None
-    if isinstance(obj, dict):
-        return {k: replace_non_finite(v) for k, v in obj.items()}
-    if isinstance(obj, list):
-        return [replace_non_finite(v) for v in obj]
-    return obj
-
-
-def json_default(o: Any) -> Any:
-    """JSON serializer for types not handled by the default encoder.
-
-    Args:
-        o: The object the encoder could not serialize.
-
-    Returns:
-        A JSON-encodable stand-in.
-
-    Raises:
-        TypeError: If the object has no JSON representation.
-
-    """
-    if isinstance(o, (datetime.date, datetime.datetime)):
-        return o.isoformat()
-    if isinstance(o, Decimal):
-        return str(o)
-    raise TypeError(f"Object of type {type(o).__name__} is not JSON serializable")