interloper-google-cloud 0.72.0__tar.gz → 0.74.0__tar.gz

{interloper_google_cloud-0.72.0 → interloper_google_cloud-0.74.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: interloper-google-cloud
-Version: 0.72.0
+Version: 0.74.0
 Summary: Interloper Google Cloud integration: BigQuery and Cloud Storage destinations
 Author: Guillaume Onfroy
 Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>

{interloper_google_cloud-0.72.0 → interloper_google_cloud-0.74.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "interloper-google-cloud"
-version = "0.72.0"
+version = "0.74.0"
 description = "Interloper Google Cloud integration: BigQuery and Cloud Storage destinations"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -34,6 +34,7 @@ workspace = true
 line-length = 120
 
 [tool.ruff.lint]
+preview = true
 extend-select = [
     "E",
     "I",
@@ -41,8 +42,13 @@ extend-select = [
     "ANN001",
     "ANN201",
     "ANN202",
+    "DOC",
+    "D",
 ]
 
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = [
     "F401",
@@ -51,4 +57,8 @@ extend-select = [
 "tests/**" = [
     "ANN",
     "F811",
+    "D101",
+    "D102",
+    "D103",
+    "D104",
 ]

{interloper_google_cloud-0.72.0 → interloper_google_cloud-0.74.0}/pyproject.toml.orig

@@ -3,7 +3,7 @@
 # ###############
 [project]
 name = "interloper-google-cloud"
-version = "0.72.0"
+version = "0.74.0"
 description = "Interloper Google Cloud integration: BigQuery and Cloud Storage destinations"
 readme = "README.md"
 authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
@@ -35,8 +35,12 @@ interloper-pandas = { workspace = true }
 line-length = 120
 
 [tool.ruff.lint]
-extend-select = ["E", "I", "UP", "ANN001", "ANN201", "ANN202"]
+preview = true
+extend-select = ["E", "I", "UP", "ANN001", "ANN201", "ANN202", "DOC", "D"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
 
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["F401", "F403"]
-"tests/**" = ["ANN", "F811"]
+"tests/**" = ["ANN", "F811", "D101", "D102", "D103", "D104"]

{interloper_google_cloud-0.72.0 → interloper_google_cloud-0.74.0}/src/interloper_google_cloud/bigquery/destination.py

@@ -63,6 +63,14 @@ class BigQueryDestination(DatabaseDestination):
 
     @cached_property
     def client(self) -> bigquery.Client:
+        """The BigQuery client every write goes through.
+
+        Built from the connection's service-account key when there is one,
+        else from ambient credentials (workload identity in-cluster).
+
+        Returns:
+            The client, cached per destination instance.
+        """
         if self.connection and self.connection.service_account_key:
             key_info = json.loads(self.connection.service_account_key)
             credentials = service_account.Credentials.from_service_account_info(key_info)
@@ -522,6 +530,7 @@ class BigQueryDestination(DatabaseDestination):
     # -- Lifecycle -------------------------------------------------------------
 
     def dispose(self) -> None:
+        """Close the BigQuery client, if one was ever built."""
         if self.client:
             self.client.close()
 
@@ -544,6 +553,9 @@ def _schema_to_bq_fields(schema: type[Schema]) -> list[bigquery.SchemaField]:
 def _specs_to_bq_fields(specs: list[FieldSpec] | tuple[FieldSpec, ...]) -> list[bigquery.SchemaField]:
     """Map field specs to BigQuery field definitions.
 
+    Args:
+        specs: The field specs to map.
+
     Returns:
         One ``SchemaField`` per spec, carrying the spec's description.
     """
@@ -568,6 +580,9 @@ def _asset_description(asset: Any) -> str | None:
 
     This mirrors how ``Component.definition()`` derives descriptions.
 
+    Args:
+        asset: The asset whose docstring is read.
+
     Returns:
         The cleaned docstring, or ``None`` when the asset has none.
     """
@@ -751,6 +766,9 @@ def _partition_param(
 def _iso_string(value: Any) -> str:
     """Render a bound as an ISO-8601 string for STRING-typed columns.
 
+    Args:
+        value: The bound to render.
+
     Returns:
         The ISO rendering of *value*.
     """
@@ -758,7 +776,15 @@ def _iso_string(value: Any) -> str:
 
 
 def _py_type_to_bq_type(py_type: Any) -> str:
-    """Map a Python *type* (from a FieldSpec) to a BigQuery column type."""
+    """Map a Python *type* (from a FieldSpec) to a BigQuery column type.
+
+    Args:
+        py_type: The declared Python type.
+
+    Returns:
+        The BigQuery column type name.
+
+    """
     if not isinstance(py_type, type):
         return "STRING"  # typing.Any or unresolvable annotations
     if issubclass(py_type, bool):
@@ -813,7 +839,15 @@ def _infer_bq_schema(rows: list[dict[str, Any]]) -> list[bigquery.SchemaField]:
 
 
 def _py_to_bq_type(value: Any) -> str:
-    """Infer a BigQuery field type from a Python value."""
+    """Infer a BigQuery field type from a Python value.
+
+    Args:
+        value: The value to inspect.
+
+    Returns:
+        The inferred BigQuery field type name.
+
+    """
     if isinstance(value, bool):
         return "BOOLEAN"
     if isinstance(value, int):
@@ -832,7 +866,15 @@ def _py_to_bq_type(value: Any) -> str:
 
 
 def _bq_to_py_type(value: Any) -> str:
-    """Map a Python value to a BigQuery query parameter type."""
+    """Map a Python value to a BigQuery query parameter type.
+
+    Args:
+        value: The parameter value.
+
+    Returns:
+        The BigQuery query-parameter type name.
+
+    """
     if isinstance(value, bool):
         return "BOOL"
     if isinstance(value, int):

{interloper_google_cloud-0.72.0 → interloper_google_cloud-0.74.0}/src/interloper_google_cloud/connection.py

@@ -46,7 +46,17 @@ def _make_assertion(key_info: dict[str, Any], scope: str) -> str:
 
 
 async def _get_access_token(client: httpx.AsyncClient, key_info: dict[str, Any], scope: str) -> str:
-    """Exchange a service account JWT assertion for an access token."""
+    """Exchange a service account JWT assertion for an access token.
+
+    Args:
+        client: The HTTP client the exchange is made through.
+        key_info: The parsed service-account key.
+        scope: The OAuth scope requested.
+
+    Returns:
+        The bearer token.
+
+    """
     resp = await client.post(
         _TOKEN_URL,
         data={
@@ -61,6 +71,10 @@ async def _get_access_token(client: httpx.AsyncClient, key_info: dict[str, Any],
 async def _list_projects(client: httpx.AsyncClient, access_token: str) -> list[dict[str, str]]:
     """List the projects the credential has BigQuery access to, following pagination.
 
+    Args:
+        client: The HTTP client the listing is made through.
+        access_token: Bearer token for the listing.
+
     Returns:
         Project options with ``project_id`` and a display ``name``.
     """
@@ -90,6 +104,11 @@ async def _list_projects(client: httpx.AsyncClient, access_token: str) -> list[d
 async def _list_buckets(client: httpx.AsyncClient, access_token: str, project: str) -> list[dict[str, str]]:
     """List the Cloud Storage buckets in *project*, following pagination.
 
+    Args:
+        client: The HTTP client the listing is made through.
+        access_token: Bearer token for the listing.
+        project: The project whose buckets are listed.
+
     Returns:
         Bucket options with ``name``.
     """
@@ -141,6 +160,10 @@ class GoogleCloudConnection(Connection):
         JWT-bearer assertion from the service account key and exchanges it for
         an access token over httpx (only the signing uses google-auth), then
         pages through BigQuery's ``projects.list``.
+
+        Returns:
+            Project options for the field's dropdown.
+
         """
         key_info: dict[str, Any] = json.loads(self.service_account_key)
         async with httpx.AsyncClient(timeout=30) as client:
@@ -155,6 +178,10 @@ class GoogleCloudConnection(Connection):
         exchange as ``projects``, with the read-only storage scope; the project
         comes from the key itself (bucket names are global, but listing is
         per-project).
+
+        Returns:
+            Bucket options for the field's dropdown.
+
         """
         key_info: dict[str, Any] = json.loads(self.service_account_key)
         async with httpx.AsyncClient(timeout=30) as client:

{interloper_google_cloud-0.72.0 → interloper_google_cloud-0.74.0}/src/interloper_google_cloud/gcs/destination.py

@@ -69,6 +69,14 @@ class GCSDestination(PartitionedDestination):
 
     @cached_property
     def client(self) -> storage.Client:
+        """The Cloud Storage client every read and write goes through.
+
+        Built from the connection's service-account key when there is one,
+        else from ambient credentials (workload identity in-cluster).
+
+        Returns:
+            The client, cached per destination instance.
+        """
         if self.connection and self.connection.service_account_key:
             key_info = json.loads(self.connection.service_account_key)
             credentials = service_account.Credentials.from_service_account_info(key_info)
@@ -80,17 +88,34 @@ class GCSDestination(PartitionedDestination):
 
     @property
     def _format(self) -> FileFormat:
-        """The configured file format strategy."""
+        """The configured file format strategy.
+
+        Returns:
+            The strategy for the configured format.
+
+        """
         return FORMATS[self.format]
 
     def _asset_prefix(self, context: IOContext) -> str:
-        """Return the object-name prefix for an asset (no trailing slash)."""
+        """Return the object-name prefix for an asset (no trailing slash).
+
+        Args:
+            context: The IO context naming the asset.
+
+        Returns:
+            The prefix every object for the asset sits under.
+
+        """
         parts = [self.prefix or "", context.asset.dataset or "", context.asset.table]
         return "/".join(part.strip("/") for part in parts if part and part.strip("/"))
 
     def _blob_name(self, context: IOContext, partition: Partition | None) -> str:
         """Build the object name for a scope.
 
+        Args:
+            context: The IO context naming the asset.
+            partition: The partition being addressed, when scoped.
+
         Returns:
             ``.../data.{ext}``, inside a ``{column}={id}`` segment for
             partition scopes.
@@ -108,6 +133,10 @@ class GCSDestination(PartitionedDestination):
         Partition scopes exclude the partition column (its value lives in the
         object path).
 
+        Args:
+            context: The IO context carrying the schema.
+            partition: The partition being addressed, when scoped.
+
         Returns:
             The specs, or ``None`` when the context carries no schema.
         """
@@ -126,6 +155,12 @@ class GCSDestination(PartitionedDestination):
 
         The row count is stamped as blob metadata so introspection never has
         to download data.
+
+        Args:
+            context: The IO context naming the asset.
+            partition: The partition being written, when scoped.
+            data: The rows to write.
+
         """
         rows = Representation.of(data).to_records(data)
         if partition is not None:
@@ -145,6 +180,10 @@ class GCSDestination(PartitionedDestination):
         reconciled against the context schema when one is set (restoring the
         declared types — text formats read everything back as strings).
 
+        Args:
+            context: The IO context carrying the schema.
+            partition: The partition being read, when scoped.
+
         Returns:
             Rows as a list of dicts.
 
@@ -177,6 +216,9 @@ class GCSDestination(PartitionedDestination):
         time; objects missing it (written by other tools) are downloaded and
         counted.
 
+        Args:
+            context: The IO context naming the asset.
+
         Returns:
             Mapping from partition value (as string) to row count.
         """
@@ -199,5 +241,6 @@ class GCSDestination(PartitionedDestination):
     # -- Lifecycle -------------------------------------------------------------
 
     def dispose(self) -> None:
+        """Close the storage client, if one was ever built."""
         if self.client:
             self.client.close()

{interloper_google_cloud-0.72.0 → interloper_google_cloud-0.74.0}/src/interloper_google_cloud/gcs/formats.py

@@ -32,11 +32,28 @@ class FileFormat(ABC):
 
     @abstractmethod
     def serialize(self, rows: list[dict[str, Any]], specs: list[FieldSpec] | None) -> bytes:
-        """Serialize records into the format's byte payload."""
+        """Serialize records into the format's byte payload.
+
+        Args:
+            rows: The records to serialize.
+            specs: The field specs, when the format uses a declared schema.
+
+        Returns:
+            The encoded payload.
+
+        """
 
     @abstractmethod
     def deserialize(self, payload: bytes) -> list[dict[str, Any]]:
-        """Parse a byte payload back into records."""
+        """Parse a byte payload back into records.
+
+        Args:
+            payload: The encoded payload.
+
+        Returns:
+            One dict per record.
+
+        """
 
 
 class JSONLFormat(FileFormat):
@@ -52,6 +69,10 @@ class JSONLFormat(FileFormat):
         Non-finite floats become ``null`` (invalid JSON otherwise); dates and
         decimals serialize via :func:`json_default`.
 
+        Args:
+            rows: The records to serialize.
+            specs: Accepted for the interface and unused by NDJSON.
+
         Returns:
             The NDJSON payload.
         """
@@ -61,6 +82,9 @@ class JSONLFormat(FileFormat):
     def deserialize(self, payload: bytes) -> list[dict[str, Any]]:
         """Parse NDJSON lines into records.
 
+        Args:
+            payload: The NDJSON payload.
+
         Returns:
             One dict per non-empty line.
         """
@@ -77,6 +101,10 @@ class CSVFormat(FileFormat):
     def serialize(self, rows: list[dict[str, Any]], specs: list[FieldSpec] | None) -> bytes:
         """Serialize records as CSV (the first row's keys are the headers).
 
+        Args:
+            rows: The records to serialize.
+            specs: Accepted for the interface and unused by CSV.
+
         Returns:
             The CSV payload; empty for no rows.
         """
@@ -91,6 +119,9 @@ class CSVFormat(FileFormat):
     def deserialize(self, payload: bytes) -> list[dict[str, Any]]:
         """Parse CSV into records, reading empty cells as ``None``.
 
+        Args:
+            payload: The CSV payload.
+
         Returns:
             One dict per data row.
         """
@@ -112,6 +143,10 @@ class ParquetFormat(FileFormat):
         shape is deterministic and stable across partitions; without, pyarrow
         infers it from the data.
 
+        Args:
+            rows: The records to serialize.
+            specs: The field specs the Arrow schema is built from, when given.
+
         Returns:
             The Parquet payload.
         """
@@ -127,6 +162,9 @@ class ParquetFormat(FileFormat):
     def deserialize(self, payload: bytes) -> list[dict[str, Any]]:
         """Read a Parquet payload into records (types preserved).
 
+        Args:
+            payload: The Parquet payload.
+
         Returns:
             One dict per row.
         """
@@ -139,6 +177,9 @@ FORMATS: dict[str, FileFormat] = {fmt.key: fmt for fmt in (ParquetFormat(), JSON
 def _spec_to_arrow_field(spec: FieldSpec) -> pa.Field:
     """Map a field spec to an Arrow field (nested records as structs).
 
+    Args:
+        spec: The field spec to map.
+
     Returns:
         The Arrow field, wrapped in ``list_`` when the spec is repeated.
     """
@@ -156,6 +197,13 @@ def _is_concrete_type(py_type: Any) -> bool:
 
     ``typing.Any`` is excluded explicitly — on Python 3.11+ it *is* an
     ``isinstance(..., type)``, but it types nothing.
+
+    Args:
+        py_type: The declared type to test.
+
+    Returns:
+        True when the type is a concrete class.
+
     """
     return py_type is not Any and isinstance(py_type, type)
 
@@ -165,6 +213,13 @@ def _py_type_to_arrow_type(py_type: Any) -> pa.DataType:
 
     Mirrors the BigQuery mapping (``Decimal`` → NUMERIC precision/scale) so
     parquet files load into BigQuery without casts.
+
+    Args:
+        py_type: The declared Python type.
+
+    Returns:
+        The Arrow type.
+
     """
     if not _is_concrete_type(py_type):
         return pa.string()  # typing.Any or unresolvable annotations
@@ -192,6 +247,10 @@ def _stringify_untyped(rows: list[dict[str, Any]], specs: list[FieldSpec]) -> li
     e.g. ints into a string column — pre-convert so the write cannot fail on
     a column the schema could not type.
 
+    Args:
+        rows: The records to rewrite.
+        specs: The field specs deciding which fields are untyped.
+
     Returns:
         The rows, copied only when a conversion applies.
     """

{interloper_google_cloud-0.72.0 → interloper_google_cloud-0.74.0}/src/interloper_google_cloud/serialization.py

@@ -16,6 +16,13 @@ def replace_non_finite(obj: Any) -> Any:
     which are invalid JSON — BigQuery's load parser rejects them with "Parser
     terminated before end of string". Neither BigQuery nor JSONL files have a
     NaN concept, so map non-finite floats to ``None`` (JSON ``null``).
+
+    Args:
+        obj: The value or container to rewrite.
+
+    Returns:
+        The value with non-finite floats replaced.
+
     """
     if isinstance(obj, float):
         return obj if math.isfinite(obj) else None
@@ -27,7 +34,18 @@ def replace_non_finite(obj: Any) -> Any:
 
 
 def json_default(o: Any) -> Any:
-    """JSON serializer for types not handled by the default encoder."""
+    """JSON serializer for types not handled by the default encoder.
+
+    Args:
+        o: The object the encoder could not serialize.
+
+    Returns:
+        A JSON-encodable stand-in.
+
+    Raises:
+        TypeError: If the object has no JSON representation.
+
+    """
     if isinstance(o, (datetime.date, datetime.datetime)):
         return o.isoformat()
     if isinstance(o, Decimal):