warpzone-sdk 14.3.0__tar.gz → 15.0.0.dev1__tar.gz

{warpzone_sdk-14.3.0 → warpzone_sdk-15.0.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warpzone-sdk
-Version: 14.3.0
+Version: 15.0.0.dev1
 Summary: The main objective of this package is to centralize logic used to interact with Azure Functions, Azure Service Bus and Azure Table Storage
 Author: Team Enigma
 Author-email: enigma@energinet.dk
@@ -17,8 +17,9 @@ Requires-Dist: azure-core-tracing-opentelemetry (>=1.0.0b12)
 Requires-Dist: azure-data-tables (>=12.4.0)
 Requires-Dist: azure-functions (>=1.12.0)
 Requires-Dist: azure-identity (>=1.15.0)
+Requires-Dist: azure-monitor-opentelemetry (>=1.8.4)
 Requires-Dist: azure-monitor-opentelemetry-exporter (>=1.0.0b36)
-Requires-Dist: azure-servicebus (>=7.8.0,<7.9.0)
+Requires-Dist: azure-servicebus (>=7.8.0)
 Requires-Dist: azure-storage-blob (>=12.14.1)
 Requires-Dist: cryptography (==43.0.3)
 Requires-Dist: datamazing (>=5.1.6)

{warpzone_sdk-14.3.0 → warpzone_sdk-15.0.0.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "warpzone-sdk"
-version = "14.3.0"
+version = "15.0.0.dev1"
 description = "The main objective of this package is to centralize logic used to interact with Azure Functions, Azure Service Bus and Azure Table Storage"
 authors = [{ name = "Team Enigma", email = "enigma@energinet.dk" }]
 requires-python = ">=3.10"
@@ -20,10 +20,11 @@ datamazing = ">=5.1.6"
 azure-core = ">=1.26.3"
 azure-data-tables = ">=12.4.0"
 azure-identity = ">=1.15.0"
-azure-servicebus = "~7.8.0" # pin to avoid breaking changes with tracing in 7.9.0
+azure-servicebus = ">=7.8.0"
 azure-storage-blob = ">=12.14.1"
 aiohttp = ">=3.8.3"
 azure-core-tracing-opentelemetry = ">=1.0.0b12"
+azure-monitor-opentelemetry = ">=1.8.4"
 azure-monitor-opentelemetry-exporter = ">=1.0.0b36"
 opentelemetry-sdk = ">=1.32.0"
 azure-functions = ">=1.12.0"

{warpzone_sdk-14.3.0 → warpzone_sdk-15.0.0.dev1}/warpzone/db/client.py

@@ -32,51 +32,29 @@ class WarpzoneDatabaseClient:
         self,
         path: str,
         storage_options: dict[str, str] | None = None,
-        table_prefix: str = "",
     ):
         self.store = Store(
             path=path,
             storage_options=storage_options,
         )
-        self.table_prefix = table_prefix
 
     @classmethod
-    def from_resource_name(
+    def from_storage_account(
         cls,
         storage_account: str,
         container_name: str = "datasets",
-        sub_path: str = "",
-        table_prefix: str = "",
         credential: (
             AzureNamedKeyCredential | AzureSasCredential | TokenCredential
         ) = DefaultAzureCredential(),
     ):
-        """Create a WarpzoneDatabaseClient from resource name (storage account).
-        This assumes the path of the delta lake is of the form:
-        abfss://{container_name}@{storage_account}.dfs.core.windows.net/{sub_path}
-
-        Args:
-            storage_account (str): Storage account name.
-            container_name (str, optional): Container name. Defaults to "datasets".
-            sub_path (str, optional): Sub-path within the container. Defaults to "".
-            table_prefix (str, optional): Table prefix to use (e.g. `mz_` for archive).
-            Defaults to "".
-            credential (optional): Azure credential to use.
-            Defaults to DefaultAzureCredential().
-        """
         path = f"abfss://{container_name}@{storage_account}.dfs.core.windows.net"
-        if sub_path:
-            path += f"/{sub_path}"
-
         token = credential.get_token("https://storage.azure.com/.default")
         storage_options = {
             "account_name": storage_account,
             "token": token.token,
         }
 
-        return cls(
-            path=path, storage_options=storage_options, table_prefix=table_prefix
-        )
+        return cls(path=path, storage_options=storage_options)
 
     def get_unit_and_multiple(self, timedelta: pd.Timedelta) -> tuple[str | None, int]:
         """
@@ -170,35 +148,7 @@ class WarpzoneDatabaseClient:
         time_interval: Optional[pdz.TimeInterval] = None,
         time_travel: Optional[pdz.TimeTravel] = None,
         filters: Optional[dict[str, object]] = None,
-        include_validity_period_columns: bool = False,
-        include_generated_columns: bool = False,
     ) -> pd.DataFrame:
-        """Query table.
-        Query defaults are set to match old Table Storage client behavior.
-        Time travel defaults to "as of now"
-        Validity period columns are dropped by default.
-        Generated columns are dropped by default.
-
-        Args:
-            table_name (str): Name of the table
-            time_interval (Optional[pdz.TimeInterval], optional): Time interval for the
-            query. Defaults to None.
-            time_travel (Optional[pdz.TimeTravel], optional): Time travel information.
-            Defaults to None.
-            filters (Optional[dict[str, object]], optional): Filters to apply to the
-            query.
-            Defaults to None.
-            include_validity_period_columns (bool, optional): Whether to include
-            validity period columns in the result;
-            (e.g. `valid_from_time_utc`, `valid_to_time_utc`).
-            Defaults to False.
-            include_generated_columns (bool, optional): Whether to include generated
-            columns in the result; (e.g. `valid_from_time_utc`, `valid_to_time_utc`).
-            Defaults to False.
-
-        Returns:
-            pd.DataFrame: The result of the query.
-        """
         # We do 'camelCaseToSnake_case' conversion here because the old
         # naming convention used in WarpZone was CamelCase, while the new
         # naming convention is snake_case. The goal is to remove this
@@ -241,16 +191,16 @@ class WarpzoneDatabaseClient:
         for col in pd_df.select_dtypes(include=["datetime", "datetimetz"]).columns:
             pd_df[col] = pd_df[col].dt.floor("s").dt.as_unit("ns")
 
-        # Drop generated columns
-        if not include_generated_columns:
-            generated_cols = []
-            for field in table.schema().fields:
-                if field.generated_as is not None:
-                    generated_cols.append(field.column_name)
-            pd_df = pd_df.drop(columns=generated_cols)
-
-        # Drop valid-from/to columns
-        if not include_validity_period_columns:
-            pd_df = pd_df.drop(columns=["valid_from_time_utc", "valid_to_time_utc"])
+        # We remove the valid-from and valid-to columns, as well
+        # as any generated columns, as this was not present
+        # in the old solution (Azure Table Stroage)
+        generated_cols = []
+        for field in table.schema().fields:
+            if field.generated_as is not None:
+                generated_cols.append(field.column_name)
+
+        pd_df = pd_df.drop(
+            columns=["valid_from_time_utc", "valid_to_time_utc"] + generated_cols
+        )
 
         return pd_df

{warpzone_sdk-14.3.0 → warpzone_sdk-15.0.0.dev1}/warpzone/function/integrations.py

@@ -62,7 +62,7 @@ def get_db_client() -> WarpzoneDatabaseClient:
 
 
 def get_delta_db_client() -> WarpzoneDeltaDatabaseClient:
-    db_client = WarpzoneDeltaDatabaseClient.from_resource_name(
+    db_client = WarpzoneDeltaDatabaseClient.from_storage_account(
         os.environ["OPERATIONAL_DATA_STORAGE_ACCOUNT"],
         credential=_credential,
     )

{warpzone_sdk-14.3.0 → warpzone_sdk-15.0.0.dev1}/warpzone/function/monitor.py

@@ -1,44 +1,23 @@
 import asyncio
-import logging
 from contextlib import contextmanager
 from typing import Callable
 
 import azure.functions as func
+from azure.monitor.opentelemetry import configure_azure_monitor
 
 from warpzone.function.types import SingleArgumentCallable
 from warpzone.monitor import logs, traces
 
+configure_azure_monitor()
+
 SUBJECT_IDENTIFIER = "<Subject>"
 
 tracer = traces.get_tracer(__name__)
 logger = logs.get_logger(__name__)
 
 
-def configure_monitoring():
-    """
-    Configure logging and tracing on Azure Function to
-    - export telemetry to App Insights
-    - suppress spamming logs
-    """
-    # disable logging for HTTP calls to avoid log spamming
-    logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
-        logging.WARNING
-    )
-
-    # disable logging for Service Bus underlying uAMQP library to avoid log spamming
-    logging.getLogger("uamqp").setLevel(logging.WARNING)
-
-    # configure tracer provider
-    traces.configure_tracing()
-
-    # configure logger provider
-    logs.configure_logging()
-
-
 @contextmanager
 def run_in_trace_context(context: func.Context):
-    configure_monitoring()
-
     trace_context = context.trace_context
     with traces.set_trace_context(
         trace_context.trace_parent, trace_context.trace_state

warpzone_sdk-15.0.0.dev1/warpzone/monitor/logs.py

@@ -0,0 +1,24 @@
+import logging
+
+# Suppress verbose logging from Azure SDK
+logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
+    logging.WARNING
+)
+logging.getLogger("azure.data.tables").setLevel(logging.WARNING)
+logging.getLogger("azure.storage.blob").setLevel(logging.WARNING)
+logging.getLogger("azure.servicebus").setLevel(logging.WARNING)
+logging.getLogger("uamqp").setLevel(logging.WARNING)
+logging.getLogger("azure.identity").setLevel(logging.WARNING)
+# Suppress Azure Functions host/worker logging (e.g., trigger details)
+logging.getLogger("azure_functions_worker").setLevel(logging.WARNING)
+logging.getLogger("azure.functions").setLevel(logging.WARNING)
+# Suppress Azure Monitor exporter logging (e.g., transmission success messages)
+logging.getLogger("azure.monitor.opentelemetry.exporter").setLevel(logging.WARNING)
+
+
+def get_logger(name: str):
+    # set up standard logger
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+
+    return logger

warpzone_sdk-15.0.0.dev1/warpzone/monitor/traces.py

@@ -0,0 +1,51 @@
+from contextlib import contextmanager
+
+from azure.core.settings import settings
+from opentelemetry import context, trace
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+settings.tracing_implementation = "opentelemetry"
+
+
+@contextmanager
+def set_trace_context(trace_parent: str, trace_state: str = ""):
+    """Context manager for setting the trace context
+
+    Args:
+        trace_parent (str): Trace parent ID
+        trace_state (str, optional): Trace state. Defaults to "".
+    """
+    carrier = {"traceparent": trace_parent, "tracestate": trace_state}
+    ctx = TraceContextTextMapPropagator().extract(carrier=carrier)
+
+    token = context.attach(ctx)  # attach context before run
+    try:
+        yield
+    finally:
+        context.detach(token)  # detach context after run
+
+
+def get_tracer(name: str):
+    tracer = trace.get_tracer(name)
+    return tracer
+
+
+def get_current_diagnostic_id() -> str:
+    """Gets diagnostic id from current span
+
+    The diagnostic id is a concatenation of operation-id and parent-id
+
+    Returns:
+        str: diagnostic id
+    """
+    span = trace.get_current_span()
+
+    if not span.is_recording():
+        return ""
+
+    operation_id = "{:016x}".format(span.context.trace_id)
+    parent_id = "{:016x}".format(span.context.span_id)
+
+    diagnostic_id = f"00-{operation_id}-{parent_id}-01"
+
+    return diagnostic_id

{warpzone_sdk-14.3.0 → warpzone_sdk-15.0.0.dev1}/warpzone/servicebus/events/client.py

@@ -56,9 +56,9 @@ class WarpzoneEventClient:
     def from_resource_name(
         cls,
         service_bus_namespace: str,
-        credential: AzureNamedKeyCredential
-        | AzureSasCredential
-        | TokenCredential = DefaultAzureCredential(),
+        credential: (
+            AzureNamedKeyCredential | AzureSasCredential | TokenCredential
+        ) = DefaultAzureCredential(),
     ):
         service_bus_client = ServiceBusClient(
             fully_qualified_namespace=f"{service_bus_namespace}.servicebus.windows.net",
@@ -125,22 +125,20 @@ class WarpzoneEventClient:
     ):
         typeguard.check_type(value=topic, expected_type=Topic)
         topic_name = topic.value
-        with traces.servicebus_send_span(
-            self._service_bus_client.fully_qualified_namespace, topic_name
-        ):
-            diagnostic_id = traces.get_current_diagnostic_id()
-
-            az_sdk_msg = ServiceBusMessage(
-                body=json.dumps(event_msg.event),
-                subject=event_msg.subject,
-                content_type="application/json",
-                message_id=event_msg.message_id,
-                application_properties={"Diagnostic-Id": diagnostic_id},
-                time_to_live=event_msg.time_to_live,
-            )
 
-            with self._get_topic_sender(topic_name) as sender:
-                sender.send_messages(message=az_sdk_msg)
+        diagnostic_id = traces.get_current_diagnostic_id()
+
+        az_sdk_msg = ServiceBusMessage(
+            body=json.dumps(event_msg.event),
+            subject=event_msg.subject,
+            content_type="application/json",
+            message_id=event_msg.message_id,
+            application_properties={"Diagnostic-Id": diagnostic_id},
+            time_to_live=event_msg.time_to_live,
+        )
+
+        with self._get_topic_sender(topic_name) as sender:
+            sender.send_messages(message=az_sdk_msg)
 
     def list_subscriptions(
         self,

{warpzone_sdk-14.3.0 → warpzone_sdk-15.0.0.dev1}/warpzone/tablestorage/db/client.py

@@ -12,10 +12,13 @@ from azure.identity import DefaultAzureCredential
 
 from warpzone.blobstorage.client import WarpzoneBlobClient
 from warpzone.healthchecks import HealthCheckResult, check_health_of
+from warpzone.monitor import traces
 from warpzone.tablestorage.db import base_client
 from warpzone.tablestorage.db.table_config import DataType, TableMetadata
 from warpzone.tablestorage.tables.client import WarpzoneTableClient
 
+tracer = traces.get_tracer(__name__)
+
 
 class WarpzoneDatabaseClient:
     """Class to interact with Azure Table Storage for database queries
@@ -157,22 +160,31 @@ class WarpzoneDatabaseClient:
         filters: Optional[dict[str, object]] = None,
         use_cache: Optional[bool] = True,
     ) -> pd.DataFrame:
-        table_metadata = self.get_table_metadata(table_name)
-
-        match table_metadata.data_type:
-            case DataType.TIME_SERIES:
-                df = self._query_time_series(
-                    table_metadata, time_interval, filters, use_cache
-                )
-            case _:
-                if time_interval:
-                    raise ValueError(
-                        f"Table {table_name} is not a time series table,"
-                        " and cannot be queried with a time interval."
+        with tracer.start_as_current_span(
+            "WarpzoneDatabaseClient.query",
+            attributes={
+                "table_name": table_name,
+                "use_cache": use_cache,
+                "has_filters": filters is not None,
+                "has_time_interval": time_interval is not None,
+            },
+        ):
+            table_metadata = self.get_table_metadata(table_name)
+
+            match table_metadata.data_type:
+                case DataType.TIME_SERIES:
+                    df = self._query_time_series(
+                        table_metadata, time_interval, filters, use_cache
                     )
-                df = self._query_generic(table_metadata, None, filters, use_cache)
-
-        return df
+                case _:
+                    if time_interval:
+                        raise ValueError(
+                            f"Table {table_name} is not a time series table,"
+                            " and cannot be queried with a time interval."
+                        )
+                    df = self._query_generic(table_metadata, None, filters, use_cache)
+
+            return df
 
     def list_tables(self):
         return self._table_client.list_tables()

warpzone_sdk-14.3.0/warpzone/monitor/logs.py

@@ -1,63 +0,0 @@
-# NOTE: OpenTelemetry logging to Azure is still in EXPERIMENTAL mode!
-import logging
-import os
-from logging import StreamHandler
-
-from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
-from opentelemetry import _logs as logs
-from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
-from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
-
-logger = logging.getLogger(__name__)
-logger.addHandler(StreamHandler())
-
-LOGGING_IS_CONFIGURED = False
-
-
-def configure_logging():
-    global LOGGING_IS_CONFIGURED
-    if LOGGING_IS_CONFIGURED:
-        # logging should only be set up once
-        # to avoid duplicated log handling.
-        # Global variables is the pattern used
-        # by opentelemetry, so we use the same
-        return
-
-    # set up logger provider based on the Azure Function resource
-    # (this is make sure App Insights can track the log source correctly)
-    # (https://learn.microsoft.com/en-us/azure/azure-monitor/app/opentelemetry-enable?tabs=net#set-the-cloud-role-name-and-the-cloud-role-instance)
-    resource = Resource.create({SERVICE_NAME: os.getenv("WEBSITE_SITE_NAME")})
-    logs.set_logger_provider(
-        LoggerProvider(
-            resource=resource,
-        )
-    )
-
-    # setup azure monitor log exporter to send telemetry to App Insights
-    try:
-        log_exporter = AzureMonitorLogExporter()
-    except ValueError:
-        # if no App Insights instrumentation key is set (e.g. when running unit tests),
-        # the exporter creation will fail. In this case we skip it
-        logger.warning(
-            "Cant set up logging to App Insights, as no instrumentation key is set."
-        )
-    else:
-        log_record_processor = BatchLogRecordProcessor(log_exporter)
-        logs.get_logger_provider().add_log_record_processor(log_record_processor)
-
-    LOGGING_IS_CONFIGURED = True
-
-
-def get_logger(name: str):
-    # set up standard logger
-    logger = logging.getLogger(name)
-    logger.setLevel(logging.INFO)
-
-    if not logger.hasHandlers():
-        # add OTEL handler
-        handler = LoggingHandler()
-        logger.addHandler(handler)
-
-    return logger

warpzone_sdk-14.3.0/warpzone/monitor/traces.py

@@ -1,136 +0,0 @@
-import logging
-import os
-from contextlib import contextmanager
-from logging import StreamHandler
-
-from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter
-from azure.servicebus._common import constants
-from opentelemetry import context, trace
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.sdk.trace.sampling import ALWAYS_ON
-from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
-
-logger = logging.getLogger(__name__)
-logger.addHandler(StreamHandler())
-
-tracer = trace.get_tracer(__name__)
-
-TRACING_IS_CONFIGURED = False
-
-
-def configure_tracing():
-    global TRACING_IS_CONFIGURED
-    if TRACING_IS_CONFIGURED:
-        # tracing should only be set up once
-        # to avoid duplicated trace handling.
-        # Global variables is the pattern used
-        # by opentelemetry, so we use the same
-        return
-
-    # set up tracer provider based on the Azure Function resource
-    # (this is make sure App Insights can track the trace source correctly)
-    # (https://learn.microsoft.com/en-us/azure/azure-monitor/app/opentelemetry-enable?tabs=net#set-the-cloud-role-name-and-the-cloud-role-instance).
-    # We use the ALWAYS ON sampler since otherwise spans will not be
-    # recording upon creation
-    # (https://anecdotes.dev/opentelemetry-on-google-cloud-unraveling-the-mystery-f61f044c18be)
-    resource = Resource.create({SERVICE_NAME: os.getenv("WEBSITE_SITE_NAME")})
-    trace.set_tracer_provider(
-        TracerProvider(
-            sampler=ALWAYS_ON,
-            resource=resource,
-        )
-    )
-
-    # setup azure monitor trace exporter to send telemetry to App Insights
-    try:
-        trace_exporter = AzureMonitorTraceExporter()
-    except ValueError:
-        # if no App Insights instrumentation key is set (e.g. when running unit tests),
-        # the exporter creation will fail. In this case we skip it
-        logger.warning(
-            "Cant set up tracing to App Insights, as no instrumentation key is set."
-        )
-    else:
-        span_processor = BatchSpanProcessor(trace_exporter)
-        trace.get_tracer_provider().add_span_processor(span_processor)
-
-    TRACING_IS_CONFIGURED = True
-
-
-@contextmanager
-def set_trace_context(trace_parent: str, trace_state: str = ""):
-    """Context manager for setting the trace context
-
-    Args:
-        trace_parent (str): Trace parent ID
-        trace_state (str, optional): Trace state. Defaults to "".
-    """
-    carrier = {"traceparent": trace_parent, "tracestate": trace_state}
-    ctx = TraceContextTextMapPropagator().extract(carrier=carrier)
-
-    token = context.attach(ctx)  # attach context before run
-    try:
-        yield
-    finally:
-        context.detach(token)  # detach context after run
-
-
-def get_tracer(name: str):
-    tracer = trace.get_tracer(name)
-    return tracer
-
-
-def get_current_diagnostic_id() -> str:
-    """Gets diagnostic id from current span
-
-    The diagnostic id is a concatenation of operation-id and parent-id
-
-    Returns:
-        str: diagnostic id
-    """
-    span = trace.get_current_span()
-
-    if not span.is_recording():
-        return ""
-
-    operation_id = "{:016x}".format(span.context.trace_id)
-    parent_id = "{:016x}".format(span.context.span_id)
-
-    diagnostic_id = f"00-{operation_id}-{parent_id}-01"
-
-    return diagnostic_id
-
-
-@contextmanager
-def servicebus_send_span(servicebus_url: str, topic_name: str) -> trace.Span:
-    """Start spans necessary for correct tracing
-
-    Args:
-        servicebus_url (str): The full servicebus url
-        topic_name (str): Name of topic
-
-    Yields:
-        trace.Span: the inner span
-    """
-    with tracer.start_as_current_span(
-        constants.SPAN_NAME_SEND, kind=trace.SpanKind.CLIENT
-    ) as send_span:
-        send_span.set_attributes(
-            {
-                constants.TRACE_COMPONENT_PROPERTY: constants.TRACE_NAMESPACE,
-                constants.TRACE_NAMESPACE_PROPERTY: constants.TRACE_NAMESPACE_PROPERTY,
-                constants.TRACE_BUS_DESTINATION_PROPERTY: topic_name,
-                constants.TRACE_PEER_ADDRESS_PROPERTY: servicebus_url,
-            }
-        )
-
-        with tracer.start_as_current_span(
-            constants.SPAN_NAME_MESSAGE, kind=trace.SpanKind.PRODUCER
-        ) as msg_span:
-            msg_span.set_attributes(
-                {constants.TRACE_NAMESPACE_PROPERTY: constants.TRACE_NAMESPACE}
-            )
-
-            yield msg_span