boti-data 1.3.7__tar.gz → 1.3.8__tar.gz

{boti_data-1.3.7 → boti_data-1.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boti-data
-Version: 1.3.7
+Version: 1.3.8
 Summary: Data infrastructure for the Boti ecosystem
 Author: Luis Valverde
 Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>

{boti_data-1.3.7 → boti_data-1.3.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "boti-data"
-version = "1.3.7"
+version = "1.3.8"
 description = "Data infrastructure for the Boti ecosystem"
 readme = "README.md"
 requires-python = ">=3.13"

{boti_data-1.3.7 → boti_data-1.3.8}/src/boti_data/enrichment/protocol.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 from typing import Any, Protocol, runtime_checkable
 
+import dask.dataframe as dd
 import pandas as pd
 
 
@@ -17,13 +18,15 @@ import pandas as pd
 class DataFrameTransformer(Protocol):
     """Generic DataFrame transform contract.
 
-    Implementations receive a ``pd.DataFrame`` and arbitrary keyword
-    arguments (used by higher layers to pass context) and must return
-    a ``pd.DataFrame``.
+    Implementations receive a ``pd.DataFrame`` or ``dd.DataFrame`` and
+    arbitrary keyword arguments (used by higher layers to pass context)
+    and must return a frame of the same kind they were given.
 
     This protocol is intentionally minimal — ``**kwargs`` keeps it
     flexible so that etl-core can pass a ``TransformContext`` without
     boti-data knowing about it.
     """
 
-    async def transform(self, df: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: ...
+    async def transform(
+        self, df: pd.DataFrame | dd.DataFrame, **kwargs: Any
+    ) -> pd.DataFrame | dd.DataFrame: ...

{boti_data-1.3.7 → boti_data-1.3.8}/src/boti_data/enrichment/transformers.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 from typing import Any
 
+import dask.dataframe as dd
 import pandas as pd
 from boti.core.security import has_dunder_identifier
 
@@ -46,16 +47,30 @@ class TypeCaster:
         ``"int64"``, ``"float64"``, ``"bool"``, ``"string"``, etc.
 
     Missing columns are silently skipped.
+
+    Dask frames
+    -----------
+    ``dd.DataFrame.astype()`` has no ``errors=`` kwarg, and a dask
+    column's dtype must be uniform across every partition — so there is
+    no dask equivalent of pandas' per-column "cast what you can, leave
+    the rest" behaviour. Against a ``dd.DataFrame``, a cast that would
+    have been silently skipped under ``errors="ignore"`` instead raises
+    immediately. Only list columns you're confident are castable when
+    the input is a dask frame.
     """
 
     def __init__(self, casts: dict[str, str]) -> None:
         self._casts = dict(casts)
 
-    async def transform(self, df: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
+    async def transform(
+        self, df: pd.DataFrame | dd.DataFrame, **kwargs: Any
+    ) -> pd.DataFrame | dd.DataFrame:
         applicable = {col: dt for col, dt in self._casts.items() if col in df.columns}
-        if applicable:
-            df = df.astype(applicable, errors="ignore")
-        return df
+        if not applicable:
+            return df
+        if isinstance(df, dd.DataFrame):
+            return df.astype(applicable)
+        return df.astype(applicable, errors="ignore")
 
 
 class RowFilter:
@@ -79,7 +94,9 @@ class RowFilter:
     def __init__(self, predicates: list[str]) -> None:
         self._predicates = list(predicates)
 
-    async def transform(self, df: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
+    async def transform(
+        self, df: pd.DataFrame | dd.DataFrame, **kwargs: Any
+    ) -> pd.DataFrame | dd.DataFrame:
         for expr in self._predicates:
             _reject_dunder_expr(expr)
             mask = df.eval(expr)  # nosec CWE-94 -- dunder-chain rejected above; see class docstring's Security section
@@ -113,7 +130,9 @@ class DerivedColumn:
     def __init__(self, columns: dict[str, str]) -> None:
         self._columns = dict(columns)
 
-    async def transform(self, df: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
+    async def transform(
+        self, df: pd.DataFrame | dd.DataFrame, **kwargs: Any
+    ) -> pd.DataFrame | dd.DataFrame:
         for name, expr in self._columns.items():
             _reject_dunder_expr(expr)
             df[name] = df.eval(expr)  # nosec CWE-94 -- dunder-chain rejected above; see class docstring's Security section
@@ -141,7 +160,9 @@ class Deduplicator:
         self._subset = subset
         self._keep = keep
 
-    async def transform(self, df: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
+    async def transform(
+        self, df: pd.DataFrame | dd.DataFrame, **kwargs: Any
+    ) -> pd.DataFrame | dd.DataFrame:
         before = len(df)
         df = df.drop_duplicates(subset=self._subset, keep=self._keep).reset_index(
             drop=True,

{boti_data-1.3.7 → boti_data-1.3.8}/src/boti_data/parquet/schema_filters.py

@@ -73,47 +73,88 @@ def _string_field_names(resource: ParquetDataResource) -> set[str] | None:
     return string_fields or None
 
 
+def _temporal_field_tzs(resource: ParquetDataResource) -> dict[str, str | None]:
+    """Returns {field_name: tz} for schema fields with a genuine PyArrow
+    timestamp type (``tz`` is ``None`` for a naive/no-tz column)."""
+    schema = dataset_schema(resource)
+    if schema is None:
+        return {}
+    return {field.name: field.type.tz for field in schema if pa.types.is_timestamp(field.type)}
+
+
 def coerce_temporal_filters(
     resource: ParquetDataResource, filters: dict[str, Any]
 ) -> dict[str, Any]:
-    """Coerce date/datetime filter values to ISO strings for string-typed columns.
-
-    A ``date``/``datetime`` value compared against a string column (e.g. an
-    ISO-8601 date stored as text) has no PyArrow comparison kernel and would
-    raise ``ArrowNotImplementedError`` on both the pushdown and residual
-    paths. Rewriting the value to its ISO string makes the comparison a
-    (correct) lexicographic string comparison. Columns with a genuine
-    temporal type, and explicitly cast filters, are left untouched.
+    """Coerce filter values so their type matches the target column's PyArrow type.
+
+    Two mismatches share the same underlying failure mode — PyArrow's
+    filter-pushdown has no comparison kernel between mismatched temporal
+    representations, raising ``ArrowNotImplementedError``/``ArrowInvalid``
+    on both the pushdown and residual paths:
+
+    - A ``date``/``datetime`` value compared against a **string** column
+      (e.g. an ISO-8601 date stored as text) is rewritten to its ISO string,
+      making the comparison a correct lexicographic string comparison.
+    - A bare ``datetime.date`` value compared against a genuine **timestamp**
+      column (tz-aware or not) is promoted to a ``pd.Timestamp`` carrying
+      the column's own tz, since callers commonly build period filters via
+      ``pd.to_datetime(x).date()`` regardless of the target column's actual
+      precision (see ``prepare_period_filters()`` in ``gateway/normalization.py``).
+
+    Explicitly cast filters (``field__date__gte=...``) and values that
+    already carry full datetime precision are left untouched either way.
     """
     if not filters:
         return filters
-    string_fields = _string_field_names(resource)
-    if not string_fields:
+    string_fields = _string_field_names(resource) or set()
+    temporal_field_tzs = _temporal_field_tzs(resource)
+    if not string_fields and not temporal_field_tzs:
         return filters
-    return coerce_mapping(filters, string_fields)
+    return coerce_mapping(filters, string_fields, temporal_field_tzs)
 
 
-def coerce_mapping(filters: dict[str, Any], string_fields: set[str]) -> dict[str, Any]:
+def coerce_mapping(
+    filters: dict[str, Any],
+    string_fields: set[str],
+    temporal_field_tzs: dict[str, str | None] | None = None,
+) -> dict[str, Any]:
+    temporal_field_tzs = temporal_field_tzs or {}
     coerced: dict[str, Any] = {}
     for key, value in filters.items():
         if key in {"$and", "$or"} and isinstance(value, (list, tuple)):
             coerced[key] = [
-                coerce_mapping(sub, string_fields) if isinstance(sub, dict) else sub
+                coerce_mapping(sub, string_fields, temporal_field_tzs)
+                if isinstance(sub, dict)
+                else sub
                 for sub in value
             ]
         elif key == "$not" and isinstance(value, dict):
-            coerced[key] = coerce_mapping(value, string_fields)
+            coerced[key] = coerce_mapping(value, string_fields, temporal_field_tzs)
         elif str(key).startswith("$"):
             coerced[key] = value
         else:
-            field, casting, _op = parse_filter_key(key)
-            if casting is None and field in string_fields:
-                coerced[key] = stringify_temporal(value)
-            else:
-                coerced[key] = value
+            coerced[key] = _coerce_single_filter_value(
+                key, value, string_fields, temporal_field_tzs
+            )
     return coerced
 
 
+def _coerce_single_filter_value(
+    key: str,
+    value: Any,
+    string_fields: set[str],
+    temporal_field_tzs: dict[str, str | None],
+) -> Any:
+    field, casting, _op = parse_filter_key(key)
+    if casting is not None:
+        return value
+    if field in string_fields:
+        return stringify_temporal(value)
+    if field in temporal_field_tzs:
+        return promote_bare_date_to_timestamp(value, temporal_field_tzs[field])
+    return value
+
+
 def stringify_temporal(value: Any) -> Any:
     def convert(item: Any) -> Any:
         # pandas.Timestamp subclasses datetime, so it is handled here too.
@@ -128,6 +169,27 @@ def stringify_temporal(value: Any) -> Any:
     return convert(value)
 
 
+def promote_bare_date_to_timestamp(value: Any, tz: str | None) -> Any:
+    """Promote a bare ``datetime.date`` to a ``pd.Timestamp`` in ``tz``.
+
+    Values already carrying datetime precision (``pd.Timestamp``/``datetime.datetime``,
+    which subclasses ``date``) are left untouched — only a truly bare
+    ``date`` needs promotion to satisfy PyArrow's timestamp comparison kernel.
+    """
+
+    def convert(item: Any) -> Any:
+        if isinstance(item, dt.datetime):
+            return item
+        if isinstance(item, dt.date):
+            ts = pd.Timestamp(item)
+            return ts.tz_localize(tz) if tz else ts
+        return item
+
+    if isinstance(value, (list, tuple)):
+        return type(value)(convert(item) for item in value)
+    return convert(value)
+
+
 def raw_filters_to_expression(filters: list[Any] | None) -> ds.Expression | None:
     if not filters:
         return None

{boti_data-1.3.7 → boti_data-1.3.8}/src/boti_data/pipelines/sinks_common.py

@@ -108,13 +108,73 @@ def _validate_meta_matches_real_dtypes(frame: dd.DataFrame, *, sink_name: str) -
     )
 
 
+def _arrow_convertible(sample: pd.DataFrame) -> bool:
+    try:
+        pa.Table.from_pandas(sample, preserve_index=False)
+    except (pa.ArrowInvalid, pa.ArrowTypeError):
+        return False
+    return True
+
+
+def _validate_meta_object_columns_are_arrow_convertible(
+    frame: dd.DataFrame, *, sink_name: str
+) -> None:
+    """Guard against dask's meta_nonempty() sentinel-object corruption (wishlist #10).
+
+    Distinct from `_validate_meta_matches_real_dtypes()` above: that check
+    catches meta/real *dtype* divergence, but a genuine ``object``-dtype
+    column (e.g. a nested/struct parquet column, or one column of two
+    otherwise-identical branches merged via `HybridDataset` where the other
+    is entirely null in this slice) can have meta and real dtype agree
+    (both ``object``) while still breaking PyArrow at write time. On
+    pandas>=3.0, dask's `_nonempty_series()` has no per-partition data to
+    sample from `frame._meta` (always 0 rows) and falls back to a bare
+    `object()` placeholder for any generic object dtype -- a value with no
+    representable Arrow type. Left unchecked, that surfaces as an opaque
+    `pyarrow.lib.ArrowInvalid: Could not convert <object object at 0x...>`
+    at write time with nothing pointing back to the real column. Catch it
+    against the synthesized sample before any sink commits to a write.
+    """
+    object_columns = [
+        column for column, dtype in frame.dtypes.items() if pd.api.types.is_object_dtype(dtype)
+    ]
+    if not object_columns:
+        return
+    sample = frame._meta_nonempty
+    bad_columns = [column for column in object_columns if not _arrow_convertible(sample[[column]])]
+    if not bad_columns:
+        return
+    raise ValueError(
+        f"{sink_name} detected object-dtype column(s) with no PyArrow-representable "
+        f"value in dask's synthesized meta sample: {bad_columns!r}. This typically "
+        "happens with a nested/struct-typed column, or one entirely null in one branch "
+        "of a concat/merge (e.g. HybridDataset's historical vs live branches) while a "
+        "sibling branch has real values -- dask can't derive a placeholder PyArrow can "
+        "place. Project the load down to only the columns actually needed, or cast the "
+        "column to an explicit scalar dtype before writing."
+    )
+
+
 def prepare_partitioned_frame(
     frame: dd.DataFrame,
     *,
     partition_on: Sequence[str] | None,
     date_field: str | None,
     sink_name: str,
+    validate_arrow_convertible: bool = False,
 ) -> dd.DataFrame:
+    """Derive partition columns and validate the frame's dask meta before a write.
+
+    ``validate_arrow_convertible`` gates
+    `_validate_meta_object_columns_are_arrow_convertible()`: that guard
+    exists for PyArrow's schema-inference step, which only ``ParquetSink``
+    goes through -- ``CsvSink``/``JsonlSink`` write each partition's real
+    data straight through pandas' own ``to_csv``/``to_json`` and would
+    reject a perfectly writable object column for a failure mode they never
+    hit. `_validate_meta_matches_real_dtypes()` above stays unconditional:
+    a meta/real dtype mismatch is a signal of upstream corruption regardless
+    of sink type, not a PyArrow-specific concern.
+    """
     if partition_on:
         partition_columns = list(partition_on)
         missing = [name for name in partition_columns if name not in frame.columns]
@@ -129,6 +189,8 @@ def prepare_partitioned_frame(
             )
 
     _validate_meta_matches_real_dtypes(frame, sink_name=sink_name)
+    if validate_arrow_convertible:
+        _validate_meta_object_columns_are_arrow_convertible(frame, sink_name=sink_name)
     return frame
 
 

{boti_data-1.3.7 → boti_data-1.3.8}/src/boti_data/pipelines/sinks_parquet.py

@@ -96,6 +96,7 @@ class ParquetSink(_AsyncWriteViaThreadMixin, PicklableLifecycleCoreMixin, Lifecy
             partition_on=self.partition_on,
             date_field=date_field,
             sink_name="ParquetSink",
+            validate_arrow_convertible=True,
         )
         if persist:
             ddf = safe_persist(ddf)

{boti_data-1.3.7 → boti_data-1.3.8}/src/boti_data.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boti-data
-Version: 1.3.7
+Version: 1.3.8
 Summary: Data infrastructure for the Boti ecosystem
 Author: Luis Valverde
 Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>