warpzone-sdk 15.0.1.dev1__tar.gz → 15.1.0__tar.gz

{warpzone_sdk-15.0.1.dev1 → warpzone_sdk-15.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warpzone-sdk
-Version: 15.0.1.dev1
+Version: 15.1.0
 Summary: The main objective of this package is to centralize logic used to interact with Azure Functions, Azure Service Bus and Azure Table Storage
 Author: Team Enigma
 Author-email: enigma@energinet.dk

{warpzone_sdk-15.0.1.dev1 → warpzone_sdk-15.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "warpzone-sdk"
-version = "15.0.1.dev1"
+version = "15.1.0"
 description = "The main objective of this package is to centralize logic used to interact with Azure Functions, Azure Service Bus and Azure Table Storage"
 authors = [{ name = "Team Enigma", email = "enigma@energinet.dk" }]
 requires-python = ">=3.10"

{warpzone_sdk-15.0.1.dev1 → warpzone_sdk-15.1.0}/warpzone/db/client.py

@@ -170,6 +170,7 @@ class WarpzoneDatabaseClient:
         time_interval: Optional[pdz.TimeInterval] = None,
         time_travel: Optional[pdz.TimeTravel] = None,
         filters: Optional[dict[str, object]] = None,
+        columns: Optional[list[str]] = None,
         include_validity_period_columns: bool = False,
         include_generated_columns: bool = False,
     ) -> pd.DataFrame:
@@ -188,13 +189,18 @@ class WarpzoneDatabaseClient:
             filters (Optional[dict[str, object]], optional): Filters to apply to the
             query.
             Defaults to None.
+            columns (Optional[list[str]], optional): Columns to return.
+            Selecting columns can significantly improve query performance.
+            Defaults to None, meaning all columns will be returned.
             include_validity_period_columns (bool, optional): Whether to include
             validity period columns in the result;
-            (e.g. `valid_from_time_utc`, `valid_to_time_utc`).
-            Defaults to False.
+            (`valid_from_time_utc`, `valid_to_time_utc`).
+            Defaults to False. If set to True while using `columns`-argument, make sure
+            to include these columns in the `columns`-list.
             include_generated_columns (bool, optional): Whether to include generated
             columns in the result; (e.g. `valid_from_time_utc`, `valid_to_time_utc`).
-            Defaults to False.
+            Defaults to False. If set to True while using `columns`-argument, make sure
+            to include these columns in the `columns`-list.
 
         Returns:
             pd.DataFrame: The result of the query.
@@ -214,7 +220,6 @@ class WarpzoneDatabaseClient:
                     hyper_slice.append((key, "in", value))
                 else:
                     hyper_slice.append((key, "=", value))
-
         if time_interval:
             hyper_slice.append(("time_utc", ">=", time_interval.left))
             hyper_slice.append(("time_utc", "<=", time_interval.right))
@@ -230,9 +235,9 @@ class WarpzoneDatabaseClient:
             valid_from_column="valid_from_time_utc",
             valid_to_column="valid_to_time_utc",
         )
-        hyper_slice.extend(tt_filter)
 
-        pl_df = table.read(hyper_slice=HyperSlice(hyper_slice))
+        hyper_slice.extend(tt_filter)
+        pl_df = table.read(hyper_slice=HyperSlice(hyper_slice), columns=columns)
 
         pd_df = pl_df.to_pandas()
 

{warpzone_sdk-15.0.1.dev1 → warpzone_sdk-15.1.0}/warpzone/deltastorage/store.py

@@ -2,6 +2,7 @@ import os
 
 import deltalake as dl
 import obstore as obs
+from deltalake.exceptions import TableNotFoundError as DeltaTableNotFoundError
 
 from .schema import Schema
 from .table import Table
@@ -76,10 +77,19 @@ class Store:
         Args:
             table_name (str): Table name
         """
-        return dl.DeltaTable.is_deltatable(
-            table_uri=self._get_table_uri(table_name),
-            storage_options=self.storage_options,
-        )
+        # For some reason `deltalake.DeltaTable.is_deltatable()` can be very slow.
+        # deltalake has an issue open about this:
+        # https://github.com/delta-io/delta-rs/issues/3942
+        # For now we catch the exception when trying to load the table
+        try:
+            _ = dl.DeltaTable(
+                table_uri=self._get_table_uri(table_name),
+                storage_options=self.storage_options,
+                without_files=True,
+            )
+        except DeltaTableNotFoundError:
+            return False
+        return True
 
     def create_table(
         self,

{warpzone_sdk-15.0.1.dev1 → warpzone_sdk-15.1.0}/warpzone/deltastorage/table.py

@@ -42,6 +42,7 @@ class Table:
         self.storage_options = storage_options
 
         self.table_name = self.table_uri.split("/")[-1]
+        self._delta_table = None
 
     def __repr__(self):
         return f"Table('{self.table_name}')"
@@ -49,12 +50,21 @@ class Table:
     @property
     def delta_table(self) -> dl.DeltaTable:
         """Get the Delta table object.
-        This is lazily computed to avoid using the same instance.
-        This is *important* because using the same instance can lead to transaction
+        As the `Table`-class is lazily initialized,
+        the `delta_table`-property is initialized on the first access
+        and saved for future use to minimize overhead.
+        It is *important* that this property is only initialized within
+        a lock when doing concurrent reads/writes
+        and not initialized when creating the `Table`-object.
+        This is important because using the same instance can lead to transaction
         issues in delta as DeltaTable uses metadata (transaction id) from
         the first time the object is instantiated.
         """
-        return dl.DeltaTable(self.table_uri, storage_options=self.storage_options)
+        if self._delta_table is None:
+            self._delta_table = dl.DeltaTable(
+                self.table_uri, storage_options=self.storage_options
+            )
+        return self._delta_table
 
     def partition_cols(self) -> list[str]:
         """Get the partition columns of the table"""
@@ -65,7 +75,9 @@ class Table:
         pa_schema = pa.schema(self.delta_table.schema())
         return Schema.from_arrow(pa_schema)
 
-    def read(self, hyper_slice: Optional[HyperSlice] = None) -> pl.DataFrame:
+    def read(
+        self, hyper_slice: Optional[HyperSlice] = None, columns=None
+    ) -> pl.DataFrame:
         """Read from Delta table
 
         Args:
@@ -88,6 +100,7 @@ class Table:
             partition_filters = [f for f in hyper_slice if f[0] in partition_cols]
 
         pyarrow_table_existing_data = delta_table.to_pyarrow_table(
+            columns=columns,
             partitions=partition_filters,
             filters=file_filters,
         )

{warpzone_sdk-15.0.1.dev1 → warpzone_sdk-15.1.0}/warpzone/monitor/traces.py

@@ -23,13 +23,6 @@ TRACING_IS_CONFIGURED = False
 
 def configure_tracing():
     global TRACING_IS_CONFIGURED
-    if TRACING_IS_CONFIGURED:
-        # tracing should only be set up once
-        # to avoid duplicated trace handling.
-        # Global variables is the pattern used
-        # by opentelemetry, so we use the same
-        return
-
     # Add thread locking to avoid race conditions during setup
     with _TRACING_LOCK:
         if TRACING_IS_CONFIGURED: