esd-services-api-client 2.6.2a155.dev2__tar.gz → 2.6.2a158.dev2__tar.gz

{esd_services_api_client-2.6.2a155.dev2 → esd_services_api_client-2.6.2a158.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: esd-services-api-client
-Version: 2.6.2a155.dev2
+Version: 2.6.2a158.dev2
 Summary: Python clients for ESD services
 License: Apache 2.0
 Author: ECCO Sneaks & Data
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: azure
 Provides-Extra: nexus
-Requires-Dist: adapta[datadog,storage] (>=3.3,<4.0)
+Requires-Dist: adapta[datadog,storage] (>=3.4,<3.5)
 Requires-Dist: azure-identity (>=1.7,<1.8) ; extra == "azure"
 Requires-Dist: dataclasses-json (>=0.6.0,<0.7.0)
 Requires-Dist: httpx (>=0.27.0,<0.28.0) ; extra == "nexus"

esd_services_api_client-2.6.2a158.dev2/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = 'v2.6.2a158.dev2'

{esd_services_api_client-2.6.2a155.dev2 → esd_services_api_client-2.6.2a158.dev2}/esd_services_api_client/nexus/README.md

@@ -302,10 +302,6 @@ async def main():
             "(value of y:{y})": {"y": payload.y},
             "(request_id:{request_id})": {"request_id": run_args.request_id}
         }
-    def tag_metrics(payload: MyAlgorithmPayload2, run_args: CrystalEntrypointArguments) -> dict[str, str]:
-        return {
-            "country": payload.y,
-        }
     with ThreadingHTTPServer(("localhost", 9876), MockRequestHandler) as server:
         server_thread = threading.Thread(target=server.serve_forever)
         server_thread.daemon = True
@@ -321,7 +317,6 @@ async def main():
             .inject_configuration(MyAlgorithmConfiguration)
             .inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)
             .with_log_enricher(tags_from_payload, enrich_from_payload)
-            .with_metric_tagger(tag_metrics)
         )
 
         await nexus.activate()

{esd_services_api_client-2.6.2a155.dev2 → esd_services_api_client-2.6.2a158.dev2}/esd_services_api_client/nexus/core/app_core.py

@@ -45,9 +45,6 @@ from esd_services_api_client.nexus.abstractions.logger_factory import (
     LoggerFactory,
     BootstrapLoggerFactory,
 )
-from esd_services_api_client.nexus.abstractions.metrics_provider_factory import (
-    MetricsProviderFactory,
-)
 from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
 from esd_services_api_client.nexus.algorithms import (
     BaselineAlgorithm,
@@ -141,14 +138,6 @@ class Nexus:
         ] | None = None
         self._log_enrichment_delimiter: str = ", "
 
-        self._metric_tagger: Callable[
-            [
-                AlgorithmPayload,
-                CrystalEntrypointArguments,
-            ],
-            dict[str, str],
-        ] | None = None
-
         attach_signal_handlers()
 
     @property
@@ -236,23 +225,6 @@ class Nexus:
         self._log_enrichment_delimiter = delimiter
         return self
 
-    def with_metric_tagger(
-        self,
-        tagger: Callable[
-            [
-                AlgorithmPayload,
-                CrystalEntrypointArguments,
-            ],
-            dict[str, str],
-        ]
-        | None = None,
-    ) -> "Nexus":
-        """
-        Adds a metric `enricher` to be used with injected metrics provider to assign additional tags to emitted metrics.
-        """
-        self._metric_tagger = tagger
-        return self
-
     def with_module(self, module: Type[Module]) -> "Nexus":
         """
         Adds a (custom) DI module into the DI container.
@@ -345,59 +317,28 @@ class Nexus:
 
         bootstrap_logger.start()
 
-        try:
-            logger_fixed_template = {}
-            logger_tags = {}
-            metric_tags = {}
-
-            for payload_type in self._payload_types:
+        for payload_type in self._payload_types:
+            try:
                 payload = await self._get_payload(payload_type=payload_type)
                 self._injector.binder.bind(
                     payload.__class__, to=payload, scope=singleton
                 )
-                logger_fixed_template |= (
-                    self._log_enricher(payload, self._run_args)
-                    if self._log_enricher
-                    else {}
+                logger_factory = LoggerFactory(
+                    fixed_template=None
+                    if not self._log_enricher
+                    else self._log_enricher(payload, self._run_args),
+                    fixed_template_delimiter=self._log_enrichment_delimiter,
+                    global_tags=self._log_tagger(payload, self._run_args),
                 )
-                logger_tags |= (
-                    self._log_tagger(payload, self._run_args)
-                    if self._log_tagger
-                    else {}
-                )
-                metric_tags |= (
-                    self._metric_tagger(payload, self._run_args)
-                    if self._metric_tagger
-                    else {}
+                # bind app-level LoggerFactory now
+                self._injector.binder.bind(
+                    logger_factory.__class__,
+                    to=logger_factory,
+                    scope=singleton,
                 )
-
-            logger_factory = LoggerFactory(
-                fixed_template=logger_fixed_template,
-                fixed_template_delimiter=self._log_enrichment_delimiter,
-                global_tags=logger_tags,
-            )
-            # bind app-level LoggerFactory now
-            self._injector.binder.bind(
-                logger_factory.__class__,
-                to=logger_factory,
-                scope=singleton,
-            )
-
-            # bind app-level MetricsProvider now
-            metrics_provider = MetricsProviderFactory(
-                global_tags=metric_tags,
-            ).create_provider()
-            self._injector.binder.bind(
-                metrics_provider.__class__,
-                to=metrics_provider,
-                scope=singleton,
-            )
-        except BaseException as ex:  # pylint: disable=broad-except
-            bootstrap_logger.error("Error reading algorithm payload", ex)
-
-            # ensure we flush bootstrap logger before we exit
-            bootstrap_logger.stop()
-            sys.exit(1)
+            except BaseException as ex:  # pylint: disable=broad-except
+                bootstrap_logger.error("Error reading algorithm payload", ex)
+                sys.exit(1)
 
         bootstrap_logger.stop()
 
@@ -459,9 +400,7 @@ class Nexus:
                     for on_complete_task_class in self._on_complete_tasks
                 ]
                 if len(on_complete_tasks) > 0:
-                    done, pending = await asyncio.wait(
-                        on_complete_tasks, return_when=asyncio.FIRST_EXCEPTION
-                    )
+                    done, pending = await asyncio.wait(on_complete_tasks)
                     if len(pending) > 0:
                         root_logger.warning(
                             "Some post-processing operations did not complete or failed. Please review application logs for more information"

{esd_services_api_client-2.6.2a155.dev2 → esd_services_api_client-2.6.2a158.dev2}/esd_services_api_client/nexus/core/app_dependencies.py

@@ -17,11 +17,13 @@
 #  limitations under the License.
 #
 
+import json
 import os
 import re
 from pydoc import locate
 from typing import final, Type, Any
 
+from adapta.metrics import MetricsProvider
 from adapta.storage.blob.base import StorageClient
 from adapta.storage.query_enabled_store import QueryEnabledStore
 from injector import Module, singleton, provider
@@ -49,6 +51,28 @@ from esd_services_api_client.nexus.core.serializers import (
 )
 
 
+@final
+class MetricsModule(Module):
+    """
+    Metrics provider module.
+    """
+
+    @singleton
+    @provider
+    def provide(self) -> MetricsProvider:
+        """
+        DI factory method.
+        """
+        metrics_class: Type[MetricsProvider] = locate(
+            os.getenv(
+                "NEXUS__METRIC_PROVIDER_CLASS",
+                "adapta.metrics.providers.datadog_provider.DatadogMetricsProvider",
+            )
+        )
+        metrics_settings = json.loads(os.getenv("NEXUS__METRIC_PROVIDER_CONFIGURATION"))
+        return metrics_class(**metrics_settings)
+
+
 @final
 class BootstrapLoggerFactoryModule(Module):
     """
@@ -213,6 +237,7 @@ class ServiceConfigurator:
     def __init__(self):
         self._injection_binds = [
             BootstrapLoggerFactoryModule(),
+            MetricsModule(),
             CrystalReceiverClientModule(),
             QueryEnabledStoreModule(),
             StorageClientModule(),

{esd_services_api_client-2.6.2a155.dev2 → esd_services_api_client-2.6.2a158.dev2}/esd_services_api_client/nexus/core/serializers.py

@@ -2,10 +2,10 @@
 from typing import final, Any, TypeVar, Type
 
 import pandas
-from adapta.storage.models.format import (
-    DataFrameParquetSerializationFormat,
+from adapta.storage.models.format import SerializationFormat
+from adapta.storage.models.formatters import (
+    PandasDataFrameParquetSerializationFormat,
     DictJsonSerializationFormat,
-    SerializationFormat,
 )
 
 T = TypeVar("T")  # pylint: disable=C0103
@@ -59,7 +59,7 @@ class TelemetrySerializer(Serializer):
     def __init__(self):
         super().__init__(
             default_serialization_formats={
-                pandas.DataFrame: DataFrameParquetSerializationFormat,
+                pandas.DataFrame: PandasDataFrameParquetSerializationFormat,
                 dict: DictJsonSerializationFormat,
             }
         )
@@ -72,7 +72,7 @@ class ResultSerializer(Serializer):
     def __init__(self):
         super().__init__(
             default_serialization_formats={
-                pandas.DataFrame: DataFrameParquetSerializationFormat,
+                pandas.DataFrame: PandasDataFrameParquetSerializationFormat,
                 dict: DictJsonSerializationFormat,
             }
         )

{esd_services_api_client-2.6.2a155.dev2 → esd_services_api_client-2.6.2a158.dev2}/esd_services_api_client/nexus/input/payload_reader.py

@@ -21,7 +21,7 @@ from dataclasses import dataclass
 
 from typing import final, Optional, Type
 
-from adapta.storage.models.format import DictJsonSerializationFormat
+from adapta.storage.models.formatters import DictJsonSerializationFormat
 from adapta.utils import session_with_retries
 
 from dataclasses_json import DataClassJsonMixin

{esd_services_api_client-2.6.2a155.dev2 → esd_services_api_client-2.6.2a158.dev2}/esd_services_api_client/nexus/telemetry/user_telemetry_recorder.py

@@ -43,7 +43,9 @@ class UserTelemetry:
     """
 
     def __init__(
-        self, telemetry: DataFrame, *telemetry_path_segments: UserTelemetryPathSegment
+        self,
+        telemetry: DataFrame,
+        *telemetry_path_segments: UserTelemetryPathSegment,
     ):
         self._telemetry = telemetry
         self._telemetry_path_segments = telemetry_path_segments
@@ -143,6 +145,8 @@ class UserTelemetryRecorder(Generic[TPayload, TResult], ABC):
             )
             return
 
+        serializer = self._serializer.get_serialization_format(telemetry.telemetry)
+
         self._storage_client.save_data_as_blob(
             data=telemetry.telemetry,
             blob_path=DataSocket(
@@ -152,13 +156,11 @@ class UserTelemetryRecorder(Generic[TPayload, TResult], ABC):
                     "telemetry_group=user",
                     f"recorder_class={self.__class__.alias()}",
                     telemetry.telemetry_path,  # path join eliminates empty segments
-                    run_id,
+                    serializer.get_output_name(output_name=run_id),
                 ),
                 data_format="null",
             ).parse_data_path(),
-            serialization_format=self._serializer.get_serialization_format(
-                telemetry.telemetry
-            ),
+            serialization_format=serializer,
             overwrite=True,
         )
 

{esd_services_api_client-2.6.2a155.dev2 → esd_services_api_client-2.6.2a158.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "esd-services-api-client"
-version = "v2.6.2a155.dev2"
+version = "v2.6.2a158.dev2"
 description = "Python clients for ESD services"
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']
@@ -10,7 +10,7 @@ repository = 'https://github.com/SneaksAndData/esd-services-api-client'
 
 [tool.poetry.dependencies]
 python = ">=3.11,<3.12"
-adapta = { version = "^3.3", extras = ["storage", "datadog"] }
+adapta = { version = "~3.4", extras = ["storage", "datadog"] }
 dataclasses-json = "^0.6.0"
 pycryptodome = "~3.15"
 azure-identity = { version = "~1.7", optional = true }

esd_services_api_client-2.6.2a155.dev2/esd_services_api_client/_version.py

@@ -1 +0,0 @@
-__version__ = 'v2.6.2a155.dev2'

esd_services_api_client-2.6.2a155.dev2/esd_services_api_client/nexus/abstractions/metrics_provider_factory.py

@@ -1,70 +0,0 @@
-"""
- Metrics provider factory.
-"""
-
-#  Copyright (c) 2023-2024. ECCO Sneaks & Data
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-import json
-import os
-from pydoc import locate
-from typing import final
-
-from adapta.metrics import MetricsProvider
-from adapta.metrics.providers.datadog_provider import DatadogMetricsProvider
-
-
-@final
-class MetricsProviderFactory:
-    """
-    Async logger provisioner.
-    """
-
-    def __init__(
-        self,
-        global_tags: dict[str, str] | None = None,
-    ):
-        self._global_tags = global_tags
-        self._metrics_class: type[MetricsProvider] = locate(
-            os.getenv(
-                "NEXUS__METRICS_PROVIDER_CLASS",
-                "adapta.metrics.providers.datadog_provider.DatadogMetricsProvider",
-            )
-        )
-        self._metrics_settings: dict = json.loads(
-            os.getenv("NEXUS__METRICS_PROVIDER_CONFIGURATION")
-        )
-
-    def create_provider(
-        self,
-    ) -> MetricsProvider:
-        """
-        Creates a metrics provider enriched with additional tags for each metric emitted by this algorithm.
-        In case of DatadogMetricsProvider, takes care of UDP/UDS specific initialization.
-        """
-        self._metrics_settings["fixed_tags"] = (
-            self._metrics_settings.get("fixed_tags", {}) | self._global_tags
-        )
-
-        if type(self._metrics_class) is DatadogMetricsProvider:
-            assert isinstance(self._metrics_class, DatadogMetricsProvider)
-
-            if self._metrics_settings["protocol"] == "udp":
-                return self._metrics_class.udp(**self._metrics_settings)
-
-            if self._metrics_settings["protocol"] == "uds":
-                return self._metrics_class.uds(**self._metrics_settings)
-
-        return self._metrics_class(**self._metrics_settings)