warpzone-sdk 15.0.0.dev13__tar.gz → 15.0.1.dev1__tar.gz

{warpzone_sdk-15.0.0.dev13 → warpzone_sdk-15.0.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warpzone-sdk
-Version: 15.0.0.dev13
+Version: 15.0.1.dev1
 Summary: The main objective of this package is to centralize logic used to interact with Azure Functions, Azure Service Bus and Azure Table Storage
 Author: Team Enigma
 Author-email: enigma@energinet.dk

{warpzone_sdk-15.0.0.dev13 → warpzone_sdk-15.0.1.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "warpzone-sdk"
-version = "15.0.0.dev13"
+version = "15.0.1.dev1"
 description = "The main objective of this package is to centralize logic used to interact with Azure Functions, Azure Service Bus and Azure Table Storage"
 authors = [{ name = "Team Enigma", email = "enigma@energinet.dk" }]
 requires-python = ">=3.10"

{warpzone_sdk-15.0.0.dev13 → warpzone_sdk-15.0.1.dev1}/warpzone/db/client.py

@@ -32,29 +32,51 @@ class WarpzoneDatabaseClient:
         self,
         path: str,
         storage_options: dict[str, str] | None = None,
+        table_prefix: str = "",
     ):
         self.store = Store(
             path=path,
             storage_options=storage_options,
         )
+        self.table_prefix = table_prefix
 
     @classmethod
-    def from_storage_account(
+    def from_resource_name(
         cls,
         storage_account: str,
         container_name: str = "datasets",
+        sub_path: str = "",
+        table_prefix: str = "",
         credential: (
             AzureNamedKeyCredential | AzureSasCredential | TokenCredential
         ) = DefaultAzureCredential(),
     ):
+        """Create a WarpzoneDatabaseClient from resource name (storage account).
+        This assumes the path of the delta lake is of the form:
+        abfss://{container_name}@{storage_account}.dfs.core.windows.net/{sub_path}
+
+        Args:
+            storage_account (str): Storage account name.
+            container_name (str, optional): Container name. Defaults to "datasets".
+            sub_path (str, optional): Sub-path within the container. Defaults to "".
+            table_prefix (str, optional): Table prefix to use (e.g. `mz_` for archive).
+            Defaults to "".
+            credential (optional): Azure credential to use.
+            Defaults to DefaultAzureCredential().
+        """
         path = f"abfss://{container_name}@{storage_account}.dfs.core.windows.net"
+        if sub_path:
+            path += f"/{sub_path}"
+
         token = credential.get_token("https://storage.azure.com/.default")
         storage_options = {
             "account_name": storage_account,
             "token": token.token,
         }
 
-        return cls(path=path, storage_options=storage_options)
+        return cls(
+            path=path, storage_options=storage_options, table_prefix=table_prefix
+        )
 
     def get_unit_and_multiple(self, timedelta: pd.Timedelta) -> tuple[str | None, int]:
         """
@@ -148,7 +170,35 @@ class WarpzoneDatabaseClient:
         time_interval: Optional[pdz.TimeInterval] = None,
         time_travel: Optional[pdz.TimeTravel] = None,
         filters: Optional[dict[str, object]] = None,
+        include_validity_period_columns: bool = False,
+        include_generated_columns: bool = False,
     ) -> pd.DataFrame:
+        """Query table.
+        Query defaults are set to match old Table Storage client behavior.
+        Time travel defaults to "as of now"
+        Validity period columns are dropped by default.
+        Generated columns are dropped by default.
+
+        Args:
+            table_name (str): Name of the table
+            time_interval (Optional[pdz.TimeInterval], optional): Time interval for the
+            query. Defaults to None.
+            time_travel (Optional[pdz.TimeTravel], optional): Time travel information.
+            Defaults to None.
+            filters (Optional[dict[str, object]], optional): Filters to apply to the
+            query.
+            Defaults to None.
+            include_validity_period_columns (bool, optional): Whether to include
+            validity period columns in the result;
+            (e.g. `valid_from_time_utc`, `valid_to_time_utc`).
+            Defaults to False.
+            include_generated_columns (bool, optional): Whether to include generated
+            columns in the result; (e.g. `valid_from_time_utc`, `valid_to_time_utc`).
+            Defaults to False.
+
+        Returns:
+            pd.DataFrame: The result of the query.
+        """
         # We do 'camelCaseToSnake_case' conversion here because the old
         # naming convention used in WarpZone was CamelCase, while the new
         # naming convention is snake_case. The goal is to remove this
@@ -191,16 +241,16 @@ class WarpzoneDatabaseClient:
         for col in pd_df.select_dtypes(include=["datetime", "datetimetz"]).columns:
             pd_df[col] = pd_df[col].dt.floor("s").dt.as_unit("ns")
 
-        # We remove the valid-from and valid-to columns, as well
-        # as any generated columns, as this was not present
-        # in the old solution (Azure Table Stroage)
-        generated_cols = []
-        for field in table.schema().fields:
-            if field.generated_as is not None:
-                generated_cols.append(field.column_name)
-
-        pd_df = pd_df.drop(
-            columns=["valid_from_time_utc", "valid_to_time_utc"] + generated_cols
-        )
+        # Drop generated columns
+        if not include_generated_columns:
+            generated_cols = []
+            for field in table.schema().fields:
+                if field.generated_as is not None:
+                    generated_cols.append(field.column_name)
+            pd_df = pd_df.drop(columns=generated_cols)
+
+        # Drop valid-from/to columns
+        if not include_validity_period_columns:
+            pd_df = pd_df.drop(columns=["valid_from_time_utc", "valid_to_time_utc"])
 
         return pd_df

{warpzone_sdk-15.0.0.dev13 → warpzone_sdk-15.0.1.dev1}/warpzone/function/integrations.py

@@ -62,7 +62,7 @@ def get_db_client() -> WarpzoneDatabaseClient:
 
 
 def get_delta_db_client() -> WarpzoneDeltaDatabaseClient:
-    db_client = WarpzoneDeltaDatabaseClient.from_storage_account(
+    db_client = WarpzoneDeltaDatabaseClient.from_resource_name(
         os.environ["OPERATIONAL_DATA_STORAGE_ACCOUNT"],
         credential=_credential,
     )

{warpzone_sdk-15.0.0.dev13 → warpzone_sdk-15.0.1.dev1}/warpzone/function/monitor.py

@@ -1,5 +1,4 @@
 import asyncio
-import logging
 from contextlib import contextmanager
 from typing import Callable
 
@@ -18,16 +17,7 @@ def configure_monitoring():
     """
     Configure logging and tracing on Azure Function to
     - export telemetry to App Insights
-    - suppress spamming logs
     """
-    # disable logging for HTTP calls to avoid log spamming
-    logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
-        logging.WARNING
-    )
-
-    # disable logging for Service Bus underlying uAMQP library to avoid log spamming
-    logging.getLogger("uamqp").setLevel(logging.WARNING)
-
     # configure tracer provider
     traces.configure_tracing()
 

{warpzone_sdk-15.0.0.dev13 → warpzone_sdk-15.0.1.dev1}/warpzone/monitor/logs.py

@@ -1,6 +1,7 @@
 # NOTE: OpenTelemetry logging to Azure is still in EXPERIMENTAL mode!
 import logging
 import os
+import threading
 from logging import StreamHandler
 
 from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
@@ -27,42 +28,45 @@ _NOISY_LOGGERS = [
 for _logger_name in _NOISY_LOGGERS:
     logging.getLogger(_logger_name).setLevel(logging.WARNING)
 
+_LOGGING_LOCK = threading.Lock()
 LOGGING_IS_CONFIGURED = False
 
 
 def configure_logging():
     global LOGGING_IS_CONFIGURED
-    if LOGGING_IS_CONFIGURED:
-        # logging should only be set up once
-        # to avoid duplicated log handling.
-        # Global variables is the pattern used
-        # by opentelemetry, so we use the same
-        return
+    # Add thread locking to avoid race conditions during setup
+    with _LOGGING_LOCK:
+        if LOGGING_IS_CONFIGURED:
+            # logging should only be set up once
+            # to avoid duplicated log handling.
+            # Global variables is the pattern used
+            # by opentelemetry, so we use the same
+            return
 
-    # set up logger provider based on the Azure Function resource
-    # (this is make sure App Insights can track the log source correctly)
-    # (https://learn.microsoft.com/en-us/azure/azure-monitor/app/opentelemetry-enable?tabs=net#set-the-cloud-role-name-and-the-cloud-role-instance)
-    resource = Resource.create({SERVICE_NAME: os.getenv("WEBSITE_SITE_NAME")})
-    logs.set_logger_provider(
-        LoggerProvider(
-            resource=resource,
+        # set up logger provider based on the Azure Function resource
+        # (this is make sure App Insights can track the log source correctly)
+        # (https://learn.microsoft.com/en-us/azure/azure-monitor/app/opentelemetry-enable?tabs=net#set-the-cloud-role-name-and-the-cloud-role-instance)
+        service_name = os.getenv("WEBSITE_SITE_NAME") or "unknown-service"
+        resource = Resource.create({SERVICE_NAME: service_name})
+        logs.set_logger_provider(
+            LoggerProvider(
+                resource=resource,
+            )
         )
-    )
 
-    # setup azure monitor log exporter to send telemetry to App Insights
-    try:
-        log_exporter = AzureMonitorLogExporter()
-    except ValueError:
-        # if no App Insights instrumentation key is set (e.g. when running unit tests),
-        # the exporter creation will fail. In this case we skip it
-        logger.warning(
-            "Cant set up logging to App Insights, as no instrumentation key is set."
-        )
-    else:
-        log_record_processor = BatchLogRecordProcessor(log_exporter)
-        logs.get_logger_provider().add_log_record_processor(log_record_processor)
+        # setup azure monitor log exporter to send telemetry to App Insights
+        try:
+            log_exporter = AzureMonitorLogExporter()
+        except ValueError:
+            logger.warning(
+                "Cant set up logging to App Insights,"
+                " as no connection string is set."
+            )
+        else:
+            log_record_processor = BatchLogRecordProcessor(log_exporter)
+            logs.get_logger_provider().add_log_record_processor(log_record_processor)
 
-    LOGGING_IS_CONFIGURED = True
+        LOGGING_IS_CONFIGURED = True
 
 
 def get_logger(name: str):

{warpzone_sdk-15.0.0.dev13 → warpzone_sdk-15.0.1.dev1}/warpzone/monitor/traces.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import threading
 from contextlib import contextmanager
 from logging import StreamHandler
 
@@ -16,6 +17,7 @@ logger.addHandler(StreamHandler())
 
 tracer = trace.get_tracer(__name__)
 
+_TRACING_LOCK = threading.Lock()
 TRACING_IS_CONFIGURED = False
 
 
@@ -28,34 +30,43 @@ def configure_tracing():
         # by opentelemetry, so we use the same
         return
 
-    # set up tracer provider based on the Azure Function resource
-    # (this is make sure App Insights can track the trace source correctly)
-    # (https://learn.microsoft.com/en-us/azure/azure-monitor/app/opentelemetry-enable?tabs=net#set-the-cloud-role-name-and-the-cloud-role-instance).
-    # We use the ALWAYS ON sampler since otherwise spans will not be
-    # recording upon creation
-    # (https://anecdotes.dev/opentelemetry-on-google-cloud-unraveling-the-mystery-f61f044c18be)
-    resource = Resource.create({SERVICE_NAME: os.getenv("WEBSITE_SITE_NAME")})
-    trace.set_tracer_provider(
-        TracerProvider(
-            sampler=ALWAYS_ON,
-            resource=resource,
+    # Add thread locking to avoid race conditions during setup
+    with _TRACING_LOCK:
+        if TRACING_IS_CONFIGURED:
+            # tracing should only be set up once
+            # to avoid duplicated trace handling.
+            # Global variables is the pattern used
+            # by opentelemetry, so we use the same
+            return
+
+        # set up tracer provider based on the Azure Function resource
+        # (this is make sure App Insights can track the trace source correctly)
+        # (https://learn.microsoft.com/en-us/azure/azure-monitor/app/opentelemetry-enable?tabs=net#set-the-cloud-role-name-and-the-cloud-role-instance).
+        # We use the ALWAYS ON sampler since otherwise spans will not be
+        # recording upon creation
+        # (https://anecdotes.dev/opentelemetry-on-google-cloud-unraveling-the-mystery-f61f044c18be)
+        service_name = os.getenv("WEBSITE_SITE_NAME") or "unknown-service"
+        resource = Resource.create({SERVICE_NAME: service_name})
+        trace.set_tracer_provider(
+            TracerProvider(
+                sampler=ALWAYS_ON,
+                resource=resource,
+            )
         )
-    )
 
-    # setup azure monitor trace exporter to send telemetry to App Insights
-    try:
-        trace_exporter = AzureMonitorTraceExporter()
-    except ValueError:
-        # if no App Insights instrumentation key is set (e.g. when running unit tests),
-        # the exporter creation will fail. In this case we skip it
-        logger.warning(
-            "Cant set up tracing to App Insights, as no instrumentation key is set."
-        )
-    else:
-        span_processor = BatchSpanProcessor(trace_exporter)
-        trace.get_tracer_provider().add_span_processor(span_processor)
-
-    TRACING_IS_CONFIGURED = True
+        # setup azure monitor trace exporter to send telemetry to App Insights
+        try:
+            trace_exporter = AzureMonitorTraceExporter()
+        except ValueError:
+            logger.warning(
+                "Cant set up tracing to App Insights,"
+                " as no connection string is set."
+            )
+        else:
+            span_processor = BatchSpanProcessor(trace_exporter)
+            trace.get_tracer_provider().add_span_processor(span_processor)
+
+        TRACING_IS_CONFIGURED = True
 
 
 @contextmanager

{warpzone_sdk-15.0.0.dev13 → warpzone_sdk-15.0.1.dev1}/warpzone/tablestorage/db/client.py

@@ -12,13 +12,10 @@ from azure.identity import DefaultAzureCredential
 
 from warpzone.blobstorage.client import WarpzoneBlobClient
 from warpzone.healthchecks import HealthCheckResult, check_health_of
-from warpzone.monitor import traces
 from warpzone.tablestorage.db import base_client
 from warpzone.tablestorage.db.table_config import DataType, TableMetadata
 from warpzone.tablestorage.tables.client import WarpzoneTableClient
 
-tracer = traces.get_tracer(__name__)
-
 
 class WarpzoneDatabaseClient:
     """Class to interact with Azure Table Storage for database queries
@@ -160,31 +157,22 @@ class WarpzoneDatabaseClient:
         filters: Optional[dict[str, object]] = None,
         use_cache: Optional[bool] = True,
     ) -> pd.DataFrame:
-        with tracer.start_as_current_span(
-            "WarpzoneDatabaseClient.query",
-            attributes={
-                "table_name": table_name,
-                "use_cache": use_cache,
-                "has_filters": filters is not None,
-                "has_time_interval": time_interval is not None,
-            },
-        ):
-            table_metadata = self.get_table_metadata(table_name)
-
-            match table_metadata.data_type:
-                case DataType.TIME_SERIES:
-                    df = self._query_time_series(
-                        table_metadata, time_interval, filters, use_cache
+        table_metadata = self.get_table_metadata(table_name)
+
+        match table_metadata.data_type:
+            case DataType.TIME_SERIES:
+                df = self._query_time_series(
+                    table_metadata, time_interval, filters, use_cache
+                )
+            case _:
+                if time_interval:
+                    raise ValueError(
+                        f"Table {table_name} is not a time series table,"
+                        " and cannot be queried with a time interval."
                     )
-                case _:
-                    if time_interval:
-                        raise ValueError(
-                            f"Table {table_name} is not a time series table,"
-                            " and cannot be queried with a time interval."
-                        )
-                    df = self._query_generic(table_metadata, None, filters, use_cache)
-
-            return df
+                df = self._query_generic(table_metadata, None, filters, use_cache)
+
+        return df
 
     def list_tables(self):
         return self._table_client.list_tables()