datazone-sdk 6.0.1.dev6__tar.gz → 6.0.1.dev7__tar.gz

{datazone_sdk-6.0.1.dev6 → datazone_sdk-6.0.1.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datazone-sdk
-Version: 6.0.1.dev6
+Version: 6.0.1.dev7
 Summary: Database and Delta storage client library for working with Delta Lake tables
 Author: Team Enigma
 Author-email: enigma@energinet.dk

{datazone_sdk-6.0.1.dev6 → datazone_sdk-6.0.1.dev7}/datazone/deltastorage/table.py

@@ -46,10 +46,6 @@ def _dnf_to_sql(dnf: list[tuple]) -> str:
     return " AND ".join(sql_parts)
 
 
-def _is_safe_partition_filter(filter_: tuple) -> bool:
-    return filter_[1] in ["=", "in"]
-
-
 def _empty_to_none(filters: list[tuple]) -> list[tuple] | None:
     return filters if len(filters) > 0 else None
 
@@ -130,9 +126,9 @@ class Table:
         if hyper_slice is None:
             hyper_slice = []
 
-        # Generated filters are an optimization for partition pruning. Keep them out
-        # of row filters because partition values may be represented as strings by
-        # PyArrow even when the Delta schema has a richer logical type.
+        # Generated filters are an optimization for partition pruning. Keep all
+        # partition filters out of row filters because partition values may be
+        # represented differently by PyArrow than by Delta's partition pruning.
         partition_hyper_slice = self.schema().add_generated_filters(hyper_slice)
 
         delta_table = self.delta_table
@@ -147,16 +143,12 @@ class Table:
                 [f for f in hyper_slice if f[0] not in partition_cols]
             )
             partition_filters = _empty_to_none(
-                [
-                    f
-                    for f in partition_hyper_slice
-                    if f[0] in partition_cols and _is_safe_partition_filter(f)
-                ]
+                [f for f in partition_hyper_slice if f[0] in partition_cols]
             )
             post_filters = [
                 f
                 for f in hyper_slice
-                if f[0] in partition_cols and not _is_safe_partition_filter(f)
+                if f[0] in partition_cols and f not in partition_hyper_slice
             ]
 
         pyarrow_table_existing_data = delta_table.to_pyarrow_table(

{datazone_sdk-6.0.1.dev6 → datazone_sdk-6.0.1.dev7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datazone-sdk"
-version = "6.0.1.dev6"
+version = "6.0.1.dev7"
 description = "Database and Delta storage client library for working with Delta Lake tables"
 authors = [{ name = "Team Enigma", email = "enigma@energinet.dk" }]
 requires-python = ">=3.10"