esd-services-api-client 2.6.3a150.dev16__tar.gz → 2.6.3a155.dev9__tar.gz

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: esd-services-api-client
-Version: 2.6.3a150.dev16
+Version: 2.6.3a155.dev9
 Summary: Python clients for ESD services
 License: Apache 2.0
 Author: ECCO Sneaks & Data

esd_services_api_client-2.6.3a155.dev9/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = 'v2.6.3a155.dev9'

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/esd_services_api_client/beast/v3/_connector.py

@@ -220,7 +220,6 @@ class BeastConnector:
                 f"Execution failed, please find request's log at: {self.base_url}/job/logs/{request_id}"
             )
 
-    # pylint: disable=too-many-positional-arguments
     @staticmethod
     def _report_backoff_failure(
         target: Any, args: Any, kwargs: Any, tries: int, elapsed: int, wait: int, **_

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/esd_services_api_client/crystal/_connector.py

@@ -151,7 +151,6 @@ class CrystalConnector:
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.dispose()
 
-    # pylint: disable=too-many-positional-arguments
     def create_run(
         self,
         algorithm: str,

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/esd_services_api_client/nexus/README.md

@@ -302,6 +302,10 @@ async def main():
             "(value of y:{y})": {"y": payload.y},
             "(request_id:{request_id})": {"request_id": run_args.request_id}
         }
+    def tag_metrics(payload: MyAlgorithmPayload2, run_args: CrystalEntrypointArguments) -> dict[str, str]:
+        return {
+            "country": payload.y,
+        }
     with ThreadingHTTPServer(("localhost", 9876), MockRequestHandler) as server:
         server_thread = threading.Thread(target=server.serve_forever)
         server_thread.daemon = True
@@ -317,6 +321,7 @@ async def main():
             .inject_configuration(MyAlgorithmConfiguration)
             .inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)
             .with_log_enricher(tags_from_payload, enrich_from_payload)
+            .with_metric_tagger(tag_metrics)
         )
 
         await nexus.activate()

esd_services_api_client-2.6.3a155.dev9/esd_services_api_client/nexus/abstractions/metrics_provider_factory.py

@@ -0,0 +1,106 @@
+"""
+ Metrics provider factory.
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import os
+from dataclasses import dataclass
+from pydoc import locate
+from typing import final
+
+from adapta.metrics import MetricsProvider
+from adapta.metrics.providers.datadog_provider import DatadogMetricsProvider
+from dataclasses_json import DataClassJsonMixin
+
+from esd_services_api_client.nexus.exceptions.startup_error import (
+    FatalStartupConfigurationError,
+)
+
+
+@final
+@dataclass
+class MetricsProviderSettings(DataClassJsonMixin):
+    """
+    Settings model for the metrics provider
+    """
+
+    init_args: dict
+    fixed_tags: dict[str, str] | None = None
+    protocol: str | None = None
+
+    def __post_init__(self):
+        """
+        Force not-null values with default constructor, even if the source provides nulls.
+        """
+        if self.protocol is None:
+            self.protocol = "udp"
+
+        if self.fixed_tags is None:
+            self.fixed_tags = {}
+
+
+@final
+class MetricsProviderFactory:
+    """
+    Async logger provisioner.
+    """
+
+    def __init__(
+        self,
+        global_tags: dict[str, str] | None = None,
+    ):
+        self._global_tags = global_tags
+        self._metrics_class: type[MetricsProvider] = locate(
+            os.getenv(
+                "NEXUS__METRICS_PROVIDER_CLASS",
+                "adapta.metrics.providers.datadog_provider.DatadogMetricsProvider",
+            )
+        )
+
+        if "NEXUS__METRICS_PROVIDER_CONFIGURATION" not in os.environ:
+            raise FatalStartupConfigurationError(
+                "NEXUS__METRICS_PROVIDER_CONFIGURATION is not provided, cannot initialize a metrics provider instance"
+            )
+
+        self._metrics_settings: MetricsProviderSettings = (
+            MetricsProviderSettings.from_json(
+                os.getenv("NEXUS__METRICS_PROVIDER_CONFIGURATION")
+            )
+        )
+
+    def create_provider(
+        self,
+    ) -> MetricsProvider:
+        """
+        Creates a metrics provider enriched with additional tags for each metric emitted by this algorithm.
+        In case of DatadogMetricsProvider, takes care of UDP/UDS specific initialization.
+        """
+        self._metrics_settings.fixed_tags |= self._global_tags
+
+        if self._metrics_class == DatadogMetricsProvider:
+            if self._metrics_settings.protocol == "udp":
+                return self._metrics_class.udp(**self._metrics_settings.init_args)
+
+            if self._metrics_settings.protocol == "uds":
+                return self._metrics_class.uds(**self._metrics_settings.init_args)
+
+        return self._metrics_class(
+            **(
+                self._metrics_settings.init_args
+                | {"fixed_tags": self._metrics_settings.fixed_tags}
+            )
+        )

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/esd_services_api_client/nexus/algorithms/_remote_algorithm.py

@@ -44,7 +44,6 @@ class RemoteAlgorithm(NexusObject[TPayload, AlgorithmResult]):
     Base class for all algorithm implementations.
     """
 
-    # pylint: disable=too-many-positional-arguments
     @inject
     def __init__(
         self,

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/esd_services_api_client/nexus/core/app_core.py

@@ -28,6 +28,7 @@ from typing import final, Type, Optional, Callable
 import backoff
 import urllib3.exceptions
 from adapta.logs import LoggerInterface
+from adapta.metrics import MetricsProvider
 from adapta.process_communication import DataSocket
 from adapta.storage.blob.base import StorageClient
 from adapta.storage.query_enabled_store import QueryEnabledStore
@@ -45,6 +46,9 @@ from esd_services_api_client.nexus.abstractions.logger_factory import (
     LoggerFactory,
     BootstrapLoggerFactory,
 )
+from esd_services_api_client.nexus.abstractions.metrics_provider_factory import (
+    MetricsProviderFactory,
+)
 from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
 from esd_services_api_client.nexus.algorithms import (
     BaselineAlgorithm,
@@ -138,6 +142,14 @@ class Nexus:
         ] | None = None
         self._log_enrichment_delimiter: str = ", "
 
+        self._metric_tagger: Callable[
+            [
+                AlgorithmPayload,
+                CrystalEntrypointArguments,
+            ],
+            dict[str, str],
+        ] | None = None
+
         attach_signal_handlers()
 
     @property
@@ -225,6 +237,23 @@ class Nexus:
         self._log_enrichment_delimiter = delimiter
         return self
 
+    def with_metric_tagger(
+        self,
+        tagger: Callable[
+            [
+                AlgorithmPayload,
+                CrystalEntrypointArguments,
+            ],
+            dict[str, str],
+        ]
+        | None = None,
+    ) -> "Nexus":
+        """
+        Adds a metric `enricher` to be used with injected metrics provider to assign additional tags to emitted metrics.
+        """
+        self._metric_tagger = tagger
+        return self
+
     def with_module(self, module: Type[Module]) -> "Nexus":
         """
         Adds a (custom) DI module into the DI container.
@@ -317,30 +346,66 @@ class Nexus:
 
         bootstrap_logger.start()
 
-        for payload_type in self._payload_types:
-            try:
+        try:
+            logger_fixed_template = {}
+            logger_tags = {}
+            metric_tags = {}
+
+            for payload_type in self._payload_types:
                 payload = await self._get_payload(payload_type=payload_type)
                 self._injector.binder.bind(
                     payload.__class__, to=payload, scope=singleton
                 )
-                logger_factory = LoggerFactory(
-                    fixed_template=None
-                    if not self._log_enricher
-                    else self._log_enricher(payload, self._run_args),
-                    fixed_template_delimiter=self._log_enrichment_delimiter,
-                    global_tags=self._log_tagger(payload, self._run_args)
+                logger_fixed_template |= (
+                    self._log_enricher(payload, self._run_args)
+                    if self._log_enricher
+                    else {}
+                )
+                logger_tags |= (
+                    self._log_tagger(payload, self._run_args)
                     if self._log_tagger
-                    else None,
+                    else {}
                 )
-                # bind app-level LoggerFactory now
-                self._injector.binder.bind(
-                    logger_factory.__class__,
-                    to=logger_factory,
-                    scope=singleton,
+                metric_tags |= (
+                    self._metric_tagger(payload, self._run_args)
+                    if self._metric_tagger
+                    else {}
                 )
-            except BaseException as ex:  # pylint: disable=broad-except
-                bootstrap_logger.error("Error reading algorithm payload", ex)
-                sys.exit(1)
+
+            logger_factory = LoggerFactory(
+                fixed_template=logger_fixed_template,
+                fixed_template_delimiter=self._log_enrichment_delimiter,
+                global_tags=logger_tags,
+            )
+            # bind app-level LoggerFactory now
+            self._injector.binder.bind(
+                logger_factory.__class__,
+                to=logger_factory,
+                scope=singleton,
+            )
+
+            # bind app-level MetricsProvider now
+            metrics_provider = MetricsProviderFactory(
+                global_tags=metric_tags,
+            ).create_provider()
+
+            self._injector.binder.bind(
+                MetricsProvider,
+                to=metrics_provider,
+                scope=singleton,
+            )
+
+            self._injector.binder.bind(
+                metrics_provider.__class__,
+                to=metrics_provider,
+                scope=singleton,
+            )
+        except BaseException as ex:  # pylint: disable=broad-except
+            bootstrap_logger.error("Error reading algorithm payload", ex)
+
+            # ensure we flush bootstrap logger before we exit
+            bootstrap_logger.stop()
+            sys.exit(1)
 
         bootstrap_logger.stop()
 
@@ -355,7 +420,7 @@ class Nexus:
 
         root_logger.info(
             "Running algorithm {algorithm} on Nexus version {version}",
-            algorithm=algorithm.__class__.alias().upper(),
+            algorithm=algorithm.__class__.__name__,
             version=__version__,
         )
 
@@ -374,7 +439,7 @@ class Nexus:
                 root_logger.error(
                     "Algorithm {algorithm} run failed on Nexus version {version}",
                     ex,
-                    algorithm=algorithm.__class__.alias().upper(),
+                    algorithm=algorithm.__class__.__name__,
                     version=__version__,
                 )
 
@@ -402,7 +467,9 @@ class Nexus:
                     for on_complete_task_class in self._on_complete_tasks
                 ]
                 if len(on_complete_tasks) > 0:
-                    done, pending = await asyncio.wait(on_complete_tasks)
+                    done, pending = await asyncio.wait(
+                        on_complete_tasks, return_when=asyncio.FIRST_EXCEPTION
+                    )
                     if len(pending) > 0:
                         root_logger.warning(
                             "Some post-processing operations did not complete or failed. Please review application logs for more information"
@@ -425,9 +492,6 @@ class Nexus:
 
         root_logger.stop()
 
-        if os.getenv("NEXUS__RAISE_ERROR_ON_COMPLETE") and ex is not None:
-            raise ex
-
     @classmethod
     def create(cls) -> "Nexus":
         """

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/esd_services_api_client/nexus/core/app_dependencies.py

@@ -17,13 +17,11 @@
 #  limitations under the License.
 #
 
-import json
 import os
 import re
 from pydoc import locate
 from typing import final, Type, Any
 
-from adapta.metrics import MetricsProvider
 from adapta.storage.blob.base import StorageClient
 from adapta.storage.query_enabled_store import QueryEnabledStore
 from injector import Module, singleton, provider
@@ -51,28 +49,6 @@ from esd_services_api_client.nexus.core.serializers import (
 )
 
 
-@final
-class MetricsModule(Module):
-    """
-    Metrics provider module.
-    """
-
-    @singleton
-    @provider
-    def provide(self) -> MetricsProvider:
-        """
-        DI factory method.
-        """
-        metrics_class: Type[MetricsProvider] = locate(
-            os.getenv(
-                "NEXUS__METRIC_PROVIDER_CLASS",
-                "adapta.metrics.providers.datadog_provider.DatadogMetricsProvider",
-            )
-        )
-        metrics_settings = json.loads(os.getenv("NEXUS__METRIC_PROVIDER_CONFIGURATION"))
-        return metrics_class(**metrics_settings)
-
-
 @final
 class BootstrapLoggerFactoryModule(Module):
     """
@@ -237,7 +213,6 @@ class ServiceConfigurator:
     def __init__(self):
         self._injection_binds = [
             BootstrapLoggerFactoryModule(),
-            MetricsModule(),
             CrystalReceiverClientModule(),
             QueryEnabledStoreModule(),
             StorageClientModule(),

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/esd_services_api_client/nexus/input/payload_reader.py

@@ -1,7 +1,6 @@
 """
  Code infrastructure for manipulating payload received from Crystal SAS URI
 """
-import os
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -63,15 +62,8 @@ class AlgorithmPayloadReader:
         self._http.close()
         self._http = None
 
-    def __init__(
-        self,
-        payload_uri: str,
-        payload_type: Type[AlgorithmPayload],
-    ):
-        self._http = session_with_retries(
-            file_protocol_supported=os.getenv("NEXUS__SUPPORT_FILE_PROTOCOL_PAYLOADS")
-            is not None
-        )
+    def __init__(self, payload_uri: str, payload_type: Type[AlgorithmPayload]):
+        self._http = session_with_retries()
         self._payload: Optional[AlgorithmPayload] = None
         self._payload_uri = payload_uri
         self._payload_type = payload_type

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/esd_services_api_client/nexus/telemetry/user_telemetry_recorder.py

@@ -72,7 +72,6 @@ class UserTelemetryRecorder(Generic[TPayload, TResult], ABC):
     Base class for user-defined telemetry recorders.
     """
 
-    # pylint: disable=too-many-positional-arguments
     @inject
     def __init__(
         self,

{esd_services_api_client-2.6.3a150.dev16 → esd_services_api_client-2.6.3a155.dev9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "esd-services-api-client"
-version = "v2.6.3a150.dev16"
+version = "v2.6.3a155.dev9"
 description = "Python clients for ESD services"
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']

esd_services_api_client-2.6.3a150.dev16/esd_services_api_client/_version.py

@@ -1 +0,0 @@
-__version__ = 'v2.6.3a150.dev16'