datazone-sdk 6.0.1.dev10__tar.gz → 6.0.1.dev11__tar.gz

{datazone_sdk-6.0.1.dev10 → datazone_sdk-6.0.1.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datazone-sdk
-Version: 6.0.1.dev10
+Version: 6.0.1.dev11
 Summary: Database and Delta storage client library for working with Delta Lake tables
 Author: Team Enigma
 Author-email: enigma@energinet.dk

{datazone_sdk-6.0.1.dev10 → datazone_sdk-6.0.1.dev11}/datazone/deltastorage/table.py

@@ -12,12 +12,12 @@ from .slicing import HyperSlice
 def _sql_literal(value: Any) -> str:
     """Render a Python value as a *type-correct* SQL literal for Delta predicates.
 
-    Earlier versions quoted every value as a string (``f"{col} {op} '{val}'"``),
-    which worked because the old delta-rs/datafusion implicitly cast the string
-    literal to the column type. The upgraded datafusion type-checks predicates
-    strictly and rejects comparing a non-string column to a string literal
-    (e.g. ``Timestamp(us, "UTC") <= Utf8View``). So numbers are left unquoted,
-    booleans become ``TRUE``/``FALSE`` and ``None`` becomes ``NULL``; only real
+    datafusion type-checks predicates and does not always coerce a quoted
+    string literal to the column type (in particular not inside ``IN (...)``,
+    where a string compared to a timestamp column raises
+    ``Timestamp(us, "UTC") <= Utf8View``). Rendering each value with its real
+    type avoids relying on that coercion: numbers are left unquoted, booleans
+    become ``TRUE``/``FALSE`` and ``None`` becomes ``NULL``; only real
     strings/dates are quoted (and escaped).
     """
     if isinstance(value, dt.datetime):
@@ -38,8 +38,16 @@ def _dnf_to_sql(dnf: list[tuple]) -> str:
     """Convert a hyper slice (DNF expression) to a Delta predicate SQL string.
 
     Needed because delta-rs overwrite/delete operations accept a SQL predicate
-    string, not the tuple filter format the SDK uses everywhere else. ``in`` is
-    expanded to ``OR`` of equalities since the predicate dialect has no ``IN``.
+    string, not the tuple filter format the SDK uses everywhere else.
+
+    ``in`` is rendered as an OR-of-equalities rather than SQL ``IN (...)`` on
+    purpose: datafusion coerces a quoted string literal to the column type for
+    the binary ``=`` operator, but NOT inside ``IN (...)``. So a timestamp
+    filter written as ``time_utc IN ('2025-01-01T...')`` raises
+    ``Invalid comparison operation: Timestamp(us, "UTC") <= Utf8View``, while
+    ``(time_utc = '2025-01-01T...')`` is coerced and works. (For columns whose
+    type already matches the literal, e.g. ints, ``IN`` would be fine -- but the
+    OR form is correct for every column type.)
     """
     if len(dnf) == 0:
         return "1=1"
@@ -71,28 +79,6 @@ def _empty_to_none(filters: list[tuple]) -> list[tuple] | None:
     return filters if len(filters) > 0 else None
 
 
-def _filter_to_polars_expr(filter_: tuple) -> pl.Expr:
-    """Convert a single tuple filter to a Polars expression.
-
-    Used to apply residual partition predicates as a Polars post-filter in
-    ``Table.read`` after delta-rs partition pruning has narrowed the files read.
-    """
-    col, op, val = filter_
-    if op == "=":
-        return pl.col(col) == val
-    if op == "in":
-        return pl.col(col).is_in(val)
-    if op == ">=":
-        return pl.col(col) >= val
-    if op == "<=":
-        return pl.col(col) <= val
-    if op == ">":
-        return pl.col(col) > val
-    if op == "<":
-        return pl.col(col) < val
-    raise ValueError(f"Unsupported operation: {op}")
-
-
 class Table:
     def __init__(
         self,
@@ -146,7 +132,7 @@ class Table:
     ) -> pl.DataFrame:
         """Read from Delta table.
 
-        Filters are split into three groups so each is handled by the engine
+        Filters are split into two groups so each is handled by the engine
         best suited for it:
 
         * Partition filters (including filters derived from generated partition
@@ -154,8 +140,6 @@ class Table:
           before any data is read.
         * Non-partition filters go to ``filters=`` and are pushed down to the
           Parquet reader.
-        * Residual partition predicates that cannot be expressed as a partition
-          filter are applied afterwards as a Polars post-filter.
 
         Args:
             hyper_slice (HyperSlice): Hyper slice used to filter data.
@@ -164,10 +148,8 @@ class Table:
         if hyper_slice is None:
             hyper_slice = []
 
-        # Generated filters are an optimization for partition pruning. Keep all
-        # partition filters out of row filters because partition values may be
-        # represented differently by PyArrow than by Delta's partition pruning.
-        partition_hyper_slice = self.schema().add_generated_filters(hyper_slice)
+        # add generated filters to hyperslice
+        hyper_slice = self.schema().add_generated_filters(hyper_slice)
 
         delta_table = self.delta_table
         partition_cols = delta_table.metadata().partition_columns
@@ -175,19 +157,9 @@ class Table:
         if len(hyper_slice) == 0:
             file_filters = None
             partition_filters = None
-            post_filters = []
         else:
-            file_filters = _empty_to_none(
-                [f for f in hyper_slice if f[0] not in partition_cols]
-            )
-            partition_filters = _empty_to_none(
-                [f for f in partition_hyper_slice if f[0] in partition_cols]
-            )
-            post_filters = [
-                f
-                for f in hyper_slice
-                if f[0] in partition_cols and f not in partition_hyper_slice
-            ]
+            file_filters = hyper_slice
+            partition_filters = [f for f in hyper_slice if f[0] in partition_cols]
 
         pyarrow_table_existing_data = delta_table.to_pyarrow_table(
             columns=columns,
@@ -195,10 +167,7 @@ class Table:
             filters=file_filters,
         )
 
-        df = pl.from_arrow(pyarrow_table_existing_data)
-        for filter_ in post_filters:
-            df = df.filter(_filter_to_polars_expr(filter_))
-        return df
+        return pl.from_arrow(pyarrow_table_existing_data)
 
     def _to_writable_pyarrow_table(self, df: pl.DataFrame, schema: Schema) -> pa.Table:
         """Convert Polars dataframe to pyarrow table with casted schema.

{datazone_sdk-6.0.1.dev10 → datazone_sdk-6.0.1.dev11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datazone-sdk"
-version = "6.0.1.dev10"
+version = "6.0.1.dev11"
 description = "Database and Delta storage client library for working with Delta Lake tables"
 authors = [{ name = "Team Enigma", email = "enigma@energinet.dk" }]
 requires-python = ">=3.10"