datazone-sdk 6.0.1.dev8__tar.gz → 6.0.1.dev9__tar.gz

{datazone_sdk-6.0.1.dev8 → datazone_sdk-6.0.1.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datazone-sdk
-Version: 6.0.1.dev8
+Version: 6.0.1.dev9
 Summary: Database and Delta storage client library for working with Delta Lake tables
 Author: Team Enigma
 Author-email: enigma@energinet.dk

{datazone_sdk-6.0.1.dev8 → datazone_sdk-6.0.1.dev9}/datazone/deltastorage/generated_columns.py

@@ -112,8 +112,9 @@ class DateBucket(GeneratedColumn):
             case (">" | ">="):
                 return [(">=", date_from_timestamp(value))]
             case "in":
-                dates = [date_from_timestamp(timestamp) for timestamp in value]
-                return [("in", list(dict.fromkeys(dates)))]
+                # de-duplicate: multiple timestamps can bucket to the same date
+                dates = {date_from_timestamp(timestamp) for timestamp in value}
+                return [("in", sorted(dates))]
             case _:
                 # for other operations, we cannot make any
                 # useful filters on the generated column

{datazone_sdk-6.0.1.dev8 → datazone_sdk-6.0.1.dev9}/datazone/deltastorage/table.py

@@ -10,6 +10,16 @@ from .slicing import HyperSlice
 
 
 def _sql_literal(value: Any) -> str:
+    """Render a Python value as a *type-correct* SQL literal for Delta predicates.
+
+    Earlier versions quoted every value as a string (``f"{col} {op} '{val}'"``),
+    which worked because the old delta-rs/datafusion implicitly cast the string
+    literal to the column type. The upgraded datafusion type-checks predicates
+    strictly and rejects comparing a non-string column to a string literal
+    (e.g. ``Timestamp(us, "UTC") <= Utf8View``). So numbers are left unquoted,
+    booleans become ``TRUE``/``FALSE`` and ``None`` becomes ``NULL``; only real
+    strings/dates are quoted (and escaped).
+    """
     if isinstance(value, dt.datetime):
         return f"'{value.isoformat()}'"
     if isinstance(value, dt.date):
@@ -25,7 +35,12 @@ def _sql_literal(value: Any) -> str:
 
 
 def _dnf_to_sql(dnf: list[tuple]) -> str:
-    """Convert DNF expression to SQL expression."""
+    """Convert a hyper slice (DNF expression) to a Delta predicate SQL string.
+
+    Needed because delta-rs overwrite/delete operations accept a SQL predicate
+    string, not the tuple filter format the SDK uses everywhere else. ``in`` is
+    expanded to ``OR`` of equalities since the predicate dialect has no ``IN``.
+    """
     if len(dnf) == 0:
         return "1=1"
 
@@ -47,6 +62,13 @@ def _dnf_to_sql(dnf: list[tuple]) -> str:
 
 
 def _filter_to_polars_expr(filter_: tuple) -> pl.Expr:
+    """Convert a single tuple filter to a Polars expression.
+
+    Needed so the SDK's ``(column, op, value)`` filters can be pushed down as
+    predicates to the native Polars Delta reader in ``Table.read``, instead of
+    PyArrow filters, whose compute kernels fail on delta-rs ``string_view``
+    columns (``ArrowNotImplementedError``).
+    """
     col, op, val = filter_
     if op == "=":
         return pl.col(col) == val
@@ -130,9 +152,6 @@ class Table:
         matching input types (string_view, string_view)`. The native reader
         evaluates predicates in its own engine and avoids this entirely.
 
-        Any further (post-)filtering that cannot be expressed as a pushdown
-        predicate is the responsibility of the caller.
-
         Args:
             hyper_slice (HyperSlice): Hyper slice used to filter data.
             columns: Optional list of columns to project.

{datazone_sdk-6.0.1.dev8 → datazone_sdk-6.0.1.dev9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datazone-sdk"
-version = "6.0.1.dev8"
+version = "6.0.1.dev9"
 description = "Database and Delta storage client library for working with Delta Lake tables"
 authors = [{ name = "Team Enigma", email = "enigma@energinet.dk" }]
 requires-python = ">=3.10"