datazone-sdk 6.0.1.dev6__tar.gz → 6.0.1.dev8__tar.gz

{datazone_sdk-6.0.1.dev6 → datazone_sdk-6.0.1.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datazone-sdk
-Version: 6.0.1.dev6
+Version: 6.0.1.dev8
 Summary: Database and Delta storage client library for working with Delta Lake tables
 Author: Team Enigma
 Author-email: enigma@energinet.dk

{datazone_sdk-6.0.1.dev6 → datazone_sdk-6.0.1.dev8}/datazone/deltastorage/table.py

@@ -46,14 +46,6 @@ def _dnf_to_sql(dnf: list[tuple]) -> str:
     return " AND ".join(sql_parts)
 
 
-def _is_safe_partition_filter(filter_: tuple) -> bool:
-    return filter_[1] in ["=", "in"]
-
-
-def _empty_to_none(filters: list[tuple]) -> list[tuple] | None:
-    return filters if len(filters) > 0 else None
-
-
 def _filter_to_polars_expr(filter_: tuple) -> pl.Expr:
     col, op, val = filter_
     if op == "=":
@@ -122,53 +114,55 @@ class Table:
     def read(
         self, hyper_slice: Optional[HyperSlice] = None, columns=None
     ) -> pl.DataFrame:
-        """Read from Delta table
+        """Read from Delta table.
+
+        All filters are pushed down to the native Delta reader. Partition
+        filters (including filters derived from generated partition columns)
+        result in partition pruning, while non-partition filters are pushed
+        down to the Parquet reader as predicates.
+
+        We use the native Polars Delta reader (`pl.scan_delta`) rather than
+        `DeltaTable.to_pyarrow_table(filters=...)` on purpose. The PyArrow
+        filter path evaluates predicates with PyArrow compute kernels, which
+        do not implement comparisons for the `string_view` type returned by
+        delta-rs. That mismatch raises errors such as
+        `ArrowNotImplementedError: Function 'greater_equal' has no kernel
+        matching input types (string_view, string_view)`. The native reader
+        evaluates predicates in its own engine and avoids this entirely.
+
+        Any further (post-)filtering that cannot be expressed as a pushdown
+        predicate is the responsibility of the caller.
 
         Args:
-            hyper_slice (HyperSlice): Hyper sliced used to filter data
+            hyper_slice (HyperSlice): Hyper slice used to filter data.
+            columns: Optional list of columns to project.
         """
         if hyper_slice is None:
             hyper_slice = []
 
-        # Generated filters are an optimization for partition pruning. Keep them out
-        # of row filters because partition values may be represented as strings by
-        # PyArrow even when the Delta schema has a richer logical type.
-        partition_hyper_slice = self.schema().add_generated_filters(hyper_slice)
+        # Generated filters add predicates on generated partition columns
+        # (e.g. `date_utc` derived from `time_utc`) so the native reader can
+        # prune partitions even when the caller only filters the base column.
+        pushdown_slice = self.schema().add_generated_filters(hyper_slice)
+
+        # `credential_provider=None` makes Polars pass `storage_options` straight
+        # to the native object store, exactly like `dl.DeltaTable(...)` does. We
+        # avoid the default `credential_provider="auto"`, which would build a
+        # separate Polars-managed Azure credential and could diverge from the
+        # managed-identity / Azure CLI auth used everywhere else in this class.
+        lazy_frame = pl.scan_delta(
+            self.url,
+            storage_options=self.storage_options,
+            credential_provider=None,
+        )
 
-        delta_table = self.delta_table
-        partition_cols = delta_table.metadata().partition_columns
+        for filter_ in pushdown_slice:
+            lazy_frame = lazy_frame.filter(_filter_to_polars_expr(filter_))
 
-        if len(hyper_slice) == 0:
-            file_filters = None
-            partition_filters = None
-            post_filters = []
-        else:
-            file_filters = _empty_to_none(
-                [f for f in hyper_slice if f[0] not in partition_cols]
-            )
-            partition_filters = _empty_to_none(
-                [
-                    f
-                    for f in partition_hyper_slice
-                    if f[0] in partition_cols and _is_safe_partition_filter(f)
-                ]
-            )
-            post_filters = [
-                f
-                for f in hyper_slice
-                if f[0] in partition_cols and not _is_safe_partition_filter(f)
-            ]
-
-        pyarrow_table_existing_data = delta_table.to_pyarrow_table(
-            columns=columns,
-            partitions=partition_filters,
-            filters=file_filters,
-        )
+        if columns is not None:
+            lazy_frame = lazy_frame.select(columns)
 
-        df = pl.from_arrow(pyarrow_table_existing_data)
-        for filter_ in post_filters:
-            df = df.filter(_filter_to_polars_expr(filter_))
-        return df
+        return lazy_frame.collect()
 
     def _to_writable_pyarrow_table(self, df: pl.DataFrame, schema: Schema) -> pa.Table:
         """Convert Polars dataframe to pyarrow table with casted schema.

{datazone_sdk-6.0.1.dev6 → datazone_sdk-6.0.1.dev8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datazone-sdk"
-version = "6.0.1.dev6"
+version = "6.0.1.dev8"
 description = "Database and Delta storage client library for working with Delta Lake tables"
 authors = [{ name = "Team Enigma", email = "enigma@energinet.dk" }]
 requires-python = ">=3.10"