datazone-sdk 6.0.1.dev9__tar.gz → 6.0.1.dev10__tar.gz

{datazone_sdk-6.0.1.dev9 → datazone_sdk-6.0.1.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datazone-sdk
-Version: 6.0.1.dev9
+Version: 6.0.1.dev10
 Summary: Database and Delta storage client library for working with Delta Lake tables
 Author: Team Enigma
 Author-email: enigma@energinet.dk

{datazone_sdk-6.0.1.dev9 → datazone_sdk-6.0.1.dev10}/datazone/deltastorage/table.py

@@ -61,13 +61,21 @@ def _dnf_to_sql(dnf: list[tuple]) -> str:
     return " AND ".join(sql_parts)
 
 
+def _empty_to_none(filters: list[tuple]) -> list[tuple] | None:
+    """Return ``None`` for an empty filter list.
+
+    ``DeltaTable.to_pyarrow_table`` treats ``None`` as "no filter", so callers
+    must pass ``None`` rather than an empty list when there is nothing to filter
+    on.
+    """
+    return filters if len(filters) > 0 else None
+
+
 def _filter_to_polars_expr(filter_: tuple) -> pl.Expr:
     """Convert a single tuple filter to a Polars expression.
 
-    Needed so the SDK's ``(column, op, value)`` filters can be pushed down as
-    predicates to the native Polars Delta reader in ``Table.read``, instead of
-    PyArrow filters, whose compute kernels fail on delta-rs ``string_view``
-    columns (``ArrowNotImplementedError``).
+    Used to apply residual partition predicates as a Polars post-filter in
+    ``Table.read`` after delta-rs partition pruning has narrowed the files read.
     """
     col, op, val = filter_
     if op == "=":
@@ -138,19 +146,16 @@ class Table:
     ) -> pl.DataFrame:
         """Read from Delta table.
 
-        All filters are pushed down to the native Delta reader. Partition
-        filters (including filters derived from generated partition columns)
-        result in partition pruning, while non-partition filters are pushed
-        down to the Parquet reader as predicates.
+        Filters are split into three groups so each is handled by the engine
+        best suited for it:
 
-        We use the native Polars Delta reader (`pl.scan_delta`) rather than
-        `DeltaTable.to_pyarrow_table(filters=...)` on purpose. The PyArrow
-        filter path evaluates predicates with PyArrow compute kernels, which
-        do not implement comparisons for the `string_view` type returned by
-        delta-rs. That mismatch raises errors such as
-        `ArrowNotImplementedError: Function 'greater_equal' has no kernel
-        matching input types (string_view, string_view)`. The native reader
-        evaluates predicates in its own engine and avoids this entirely.
+        * Partition filters (including filters derived from generated partition
+          columns) go to ``partitions=`` so delta-rs prunes whole partitions
+          before any data is read.
+        * Non-partition filters go to ``filters=`` and are pushed down to the
+          Parquet reader.
+        * Residual partition predicates that cannot be expressed as a partition
+          filter are applied afterwards as a Polars post-filter.
 
         Args:
             hyper_slice (HyperSlice): Hyper slice used to filter data.
@@ -159,29 +164,41 @@ class Table:
         if hyper_slice is None:
             hyper_slice = []
 
-        # Generated filters add predicates on generated partition columns
-        # (e.g. `date_utc` derived from `time_utc`) so the native reader can
-        # prune partitions even when the caller only filters the base column.
-        pushdown_slice = self.schema().add_generated_filters(hyper_slice)
-
-        # `credential_provider=None` makes Polars pass `storage_options` straight
-        # to the native object store, exactly like `dl.DeltaTable(...)` does. We
-        # avoid the default `credential_provider="auto"`, which would build a
-        # separate Polars-managed Azure credential and could diverge from the
-        # managed-identity / Azure CLI auth used everywhere else in this class.
-        lazy_frame = pl.scan_delta(
-            self.url,
-            storage_options=self.storage_options,
-            credential_provider=None,
-        )
+        # Generated filters are an optimization for partition pruning. Keep all
+        # partition filters out of row filters because partition values may be
+        # represented differently by PyArrow than by Delta's partition pruning.
+        partition_hyper_slice = self.schema().add_generated_filters(hyper_slice)
 
-        for filter_ in pushdown_slice:
-            lazy_frame = lazy_frame.filter(_filter_to_polars_expr(filter_))
+        delta_table = self.delta_table
+        partition_cols = delta_table.metadata().partition_columns
 
-        if columns is not None:
-            lazy_frame = lazy_frame.select(columns)
+        if len(hyper_slice) == 0:
+            file_filters = None
+            partition_filters = None
+            post_filters = []
+        else:
+            file_filters = _empty_to_none(
+                [f for f in hyper_slice if f[0] not in partition_cols]
+            )
+            partition_filters = _empty_to_none(
+                [f for f in partition_hyper_slice if f[0] in partition_cols]
+            )
+            post_filters = [
+                f
+                for f in hyper_slice
+                if f[0] in partition_cols and f not in partition_hyper_slice
+            ]
+
+        pyarrow_table_existing_data = delta_table.to_pyarrow_table(
+            columns=columns,
+            partitions=partition_filters,
+            filters=file_filters,
+        )
 
-        return lazy_frame.collect()
+        df = pl.from_arrow(pyarrow_table_existing_data)
+        for filter_ in post_filters:
+            df = df.filter(_filter_to_polars_expr(filter_))
+        return df
 
     def _to_writable_pyarrow_table(self, df: pl.DataFrame, schema: Schema) -> pa.Table:
         """Convert Polars dataframe to pyarrow table with casted schema.

{datazone_sdk-6.0.1.dev9 → datazone_sdk-6.0.1.dev10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datazone-sdk"
-version = "6.0.1.dev9"
+version = "6.0.1.dev10"
 description = "Database and Delta storage client library for working with Delta Lake tables"
 authors = [{ name = "Team Enigma", email = "enigma@energinet.dk" }]
 requires-python = ">=3.10"