esd-services-api-client 2.5.14a148.dev5__tar.gz → 2.6.1a149.dev13__tar.gz

{esd_services_api_client-2.5.14a148.dev5 → esd_services_api_client-2.6.1a149.dev13}/PKG-INFO

@@ -1,21 +1,19 @@
 Metadata-Version: 2.3
 Name: esd-services-api-client
-Version: 2.5.14a148.dev5
+Version: 2.6.1a149.dev13
 Summary: Python clients for ESD services
 License: Apache 2.0
 Author: ECCO Sneaks & Data
 Author-email: esdsupport@ecco.com
 Maintainer: GZU
 Maintainer-email: gzu@ecco.com
-Requires-Python: >=3.9,<3.12
+Requires-Python: >=3.11,<3.12
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: azure
 Provides-Extra: nexus
-Requires-Dist: adapta[azure,datadog,storage] (>=3.2,<4.0)
+Requires-Dist: adapta[datadog,storage] (>=3.3,<4.0)
 Requires-Dist: azure-identity (>=1.7,<1.8) ; extra == "azure"
 Requires-Dist: dataclasses-json (>=0.6.0,<0.7.0)
 Requires-Dist: httpx (>=0.27.0,<0.28.0) ; extra == "nexus"

esd_services_api_client-2.6.1a149.dev13/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = 'v2.6.1a149.dev13'

{esd_services_api_client-2.5.14a148.dev5 → esd_services_api_client-2.6.1a149.dev13}/esd_services_api_client/boxer/_connector.py

@@ -20,7 +20,11 @@ import os
 from functools import reduce
 from typing import Optional, Iterator, final
 
-from adapta.security.clients import AzureClient
+try:
+    from adapta.security.clients import AzureClient
+except ImportError:
+    pass
+
 from adapta.utils import session_with_retries
 from requests import Session, Response
 

{esd_services_api_client-2.5.14a148.dev5 → esd_services_api_client-2.6.1a149.dev13}/esd_services_api_client/nexus/README.md

@@ -22,7 +22,7 @@ import socketserver
 import threading
 from dataclasses import dataclass
 from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
-from typing import Dict, Optional, Any
+from typing import Optional, Any
 
 import pandas
 from adapta.metrics import MetricsProvider
@@ -30,6 +30,7 @@ from adapta.storage.query_enabled_store import QueryEnabledStore
 from dataclasses_json import DataClassJsonMixin
 from injector import inject
 
+from esd_services_api_client.crystal import CrystalEntrypointArguments
 from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
@@ -292,6 +293,15 @@ async def main():
      Mock HTTP Server
     :return:
     """
+    def tags_from_payload(payload: MyAlgorithmPayload, _: CrystalEntrypointArguments) -> dict[str, str]:
+        return {
+            "test_tag": str(payload.x)
+        }
+    def enrich_from_payload(payload: MyAlgorithmPayload2, run_args: CrystalEntrypointArguments) -> dict[str, dict[str, str]]:
+        return {
+            "(value of y:{y})": {"y": payload.y},
+            "(request_id:{request_id})": {"request_id": run_args.request_id}
+        }
     with ThreadingHTTPServer(("localhost", 9876), MockRequestHandler) as server:
         server_thread = threading.Thread(target=server.serve_forever)
         server_thread.daemon = True
@@ -306,6 +316,7 @@ async def main():
             .on_complete(ObjectiveAnalytics)
             .inject_configuration(MyAlgorithmConfiguration)
             .inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)
+            .with_log_enricher(tags_from_payload, enrich_from_payload)
         )
 
         await nexus.activate()

esd_services_api_client-2.6.1a149.dev13/esd_services_api_client/nexus/abstractions/logger_factory.py

@@ -0,0 +1,124 @@
+"""
+ Logger factory for async loggers.
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import json
+import os
+from abc import ABC
+from logging import StreamHandler
+from typing import final, TypeVar
+
+from adapta.logs import LoggerInterface, create_async_logger
+from adapta.logs.handlers.datadog_api_handler import DataDogApiHandler
+from adapta.logs.models import LogLevel
+
+TLogger = TypeVar("TLogger")  # pylint: disable=C0103:
+
+
+@final
+class BootstrapLogger(LoggerInterface, ABC):
+    """
+    Dummy class to separate bootstrap logging from core app loggers
+    """
+
+    pass
+
+
+@final
+class BootstrapLoggerFactory:
+    """
+    Bootstrap logger provisioner.
+    Bootstrap loggers do not use enriched properties since they are initialized before payload is deserialized.
+    """
+
+    def __init__(self):
+        self._log_handlers = [
+            StreamHandler(),
+        ]
+        if "NEXUS__DATADOG_LOGGER_CONFIGURATION" in os.environ:
+            self._log_handlers.append(
+                DataDogApiHandler(
+                    **json.loads(os.getenv("NEXUS__DATADOG_LOGGER_CONFIGURATION"))
+                )
+            )
+
+    def create_logger(self, request_id: str, algorithm_name: str) -> LoggerInterface:
+        """
+        Creates an async-safe logger for the provided class name.
+        """
+        return create_async_logger(
+            logger_type=BootstrapLogger.__class__,
+            log_handlers=self._log_handlers,
+            min_log_level=LogLevel(os.getenv("NEXUS__LOG_LEVEL", "INFO")),
+            global_tags={
+                "request_id": request_id,
+                "algorithm": algorithm_name,
+            },
+        )
+
+
+@final
+class LoggerFactory:
+    """
+    Async logger provisioner.
+    """
+
+    def __init__(
+        self,
+        fixed_template: dict[str, dict[str, str]] | None = None,
+        fixed_template_delimiter: str = None,
+        global_tags: dict[str, str] | None = None,
+    ):
+        self._global_tags = global_tags
+        self._fixed_template = fixed_template
+        self._fixed_template_delimiter = fixed_template_delimiter or ", "
+        self._log_handlers = [
+            StreamHandler(),
+        ]
+        if "NEXUS__DATADOG_LOGGER_CONFIGURATION" in os.environ:
+            self._log_handlers.append(
+                DataDogApiHandler(
+                    **json.loads(os.getenv("NEXUS__DATADOG_LOGGER_CONFIGURATION"))
+                )
+            )
+        if "NEXUS__LOGGER_FIXED_TEMPLATE" in os.environ:
+            self._fixed_template = self._fixed_template | json.loads(
+                os.getenv("NEXUS__LOGGER_FIXED_TEMPLATE")
+            )
+
+        if "NEXUS__LOGGER_FIXED_TEMPLATE_DELIMITER" in os.environ:
+            self._fixed_template_delimiter = (
+                self._fixed_template_delimiter
+                or os.getenv("NEXUS__LOGGER_FIXED_TEMPLATE")
+            )
+
+    def create_logger(
+        self,
+        logger_type: type[TLogger],
+    ) -> LoggerInterface:
+        """
+        Creates an async-safe logger for the provided class name.
+        """
+        return create_async_logger(
+            logger_type=logger_type,
+            log_handlers=self._log_handlers,
+            min_log_level=LogLevel(os.getenv("NEXUS__LOG_LEVEL", "INFO")),
+            fixed_template=self._fixed_template,
+            fixed_template_delimiter=self._fixed_template_delimiter,
+            global_tags=self._global_tags,
+        )

{esd_services_api_client-2.5.14a148.dev5 → esd_services_api_client-2.6.1a149.dev13}/esd_services_api_client/nexus/core/app_core.py

@@ -23,7 +23,7 @@ import platform
 import signal
 import sys
 import traceback
-from typing import final, Type, Optional
+from typing import final, Type, Optional, Callable
 
 import backoff
 import urllib3.exceptions
@@ -41,7 +41,10 @@ from esd_services_api_client.crystal import (
     AlgorithmRunResult,
     CrystalEntrypointArguments,
 )
-from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.abstractions.logger_factory import (
+    LoggerFactory,
+    BootstrapLoggerFactory,
+)
 from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
 from esd_services_api_client.nexus.algorithms import (
     BaselineAlgorithm,
@@ -119,6 +122,21 @@ class Nexus:
         self._algorithm_run_task: Optional[asyncio.Task] = None
         self._on_complete_tasks: list[type[UserTelemetryRecorder]] = []
         self._payload_types: list[type[AlgorithmPayload]] = []
+        self._log_enricher: Callable[
+            [
+                AlgorithmPayload,
+                CrystalEntrypointArguments,
+            ],
+            dict[str, dict[str, str]],
+        ] | None = None
+        self._log_tagger: Callable[
+            [
+                AlgorithmPayload,
+                CrystalEntrypointArguments,
+            ],
+            dict[str, str],
+        ] | None = None
+        self._log_enrichment_delimiter: str = ", "
 
         attach_signal_handlers()
 
@@ -177,6 +195,31 @@ class Nexus:
 
         return self
 
+    def with_log_enricher(
+        self,
+        tagger: Callable[
+            [
+                AlgorithmPayload,
+                CrystalEntrypointArguments,
+            ],
+            dict[str, str],
+        ]
+        | None,
+        enricher: Callable[
+            [
+                AlgorithmPayload,
+                CrystalEntrypointArguments,
+            ],
+            dict[str, dict[str, str]],
+        ]
+        | None = None,
+        delimiter: str = ", ",
+    ) -> "Nexus":
+        self._log_tagger = tagger
+        self._log_enricher = enricher
+        self._log_enrichment_delimiter = delimiter
+        return self
+
     def with_module(self, module: Type[Module]) -> "Nexus":
         """
         Adds a (custom) DI module into the DI container.
@@ -260,11 +303,14 @@ class Nexus:
 
         self._injector = Injector(self._configurator.injection_binds)
 
-        root_logger: LoggerInterface = self._injector.get(LoggerFactory).create_logger(
-            logger_type=self.__class__,
+        bootstrap_logger: LoggerInterface = self._injector.get(
+            BootstrapLoggerFactory
+        ).create_logger(
+            request_id=self._run_args.request_id,
+            algorithm_name=os.getenv("CRYSTAL__ALGORITHM_NAME"),
         )
 
-        root_logger.start()
+        bootstrap_logger.start()
 
         for payload_type in self._payload_types:
             try:
@@ -272,10 +318,30 @@ class Nexus:
                 self._injector.binder.bind(
                     payload.__class__, to=payload, scope=singleton
                 )
+                # bind app-level LoggerFactory now
+                self._injector.binder.bind(
+                    LoggerFactory.__class__,
+                    to=LoggerFactory(
+                        fixed_template=None
+                        if not self._log_enricher
+                        else self._log_enricher(payload, self._run_args),
+                        fixed_template_delimiter=self._log_enrichment_delimiter,
+                        global_tags=self._log_tagger(payload, self._run_args),
+                    ),
+                    scope=singleton,
+                )
             except BaseException as ex:  # pylint: disable=broad-except
-                root_logger.error("Error reading algorithm payload", ex)
+                bootstrap_logger.error("Error reading algorithm payload", ex)
                 sys.exit(1)
 
+        bootstrap_logger.stop()
+
+        root_logger: LoggerInterface = self._injector.get(LoggerFactory).create_logger(
+            logger_type=self.__class__,
+        )
+
+        root_logger.start()
+
         algorithm: BaselineAlgorithm = self._injector.get(self._algorithm_class)
         telemetry_recorder: TelemetryRecorder = self._injector.get(TelemetryRecorder)
 
@@ -289,14 +355,18 @@ class Nexus:
             self._algorithm_run_task = asyncio.create_task(
                 instance.run(**self._run_args.__dict__)
             )
-            await self._algorithm_run_task
+
+            # avoid exception propagation to main thread, since we need to handle it later
+            await asyncio.wait(
+                [self._algorithm_run_task], return_when=asyncio.FIRST_EXCEPTION
+            )
             ex = self._algorithm_run_task.exception()
 
             if ex is not None:
                 root_logger.error(
                     "Algorithm {algorithm} run failed on Nexus version {version}",
                     ex,
-                    algorithm=self._algorithm_class,
+                    algorithm=algorithm.__class__.__name__,
                     version=__version__,
                 )
 

{esd_services_api_client-2.5.14a148.dev5 → esd_services_api_client-2.6.1a149.dev13}/esd_services_api_client/nexus/core/app_dependencies.py

@@ -30,7 +30,9 @@ from injector import Module, singleton, provider
 
 from esd_services_api_client.crystal import CrystalConnector
 from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
-from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.abstractions.logger_factory import (
+    BootstrapLoggerFactory,
+)
 from esd_services_api_client.nexus.abstractions.socket_provider import (
     ExternalSocketProvider,
 )
@@ -72,18 +74,18 @@ class MetricsModule(Module):
 
 
 @final
-class LoggerFactoryModule(Module):
+class BootstrapLoggerFactoryModule(Module):
     """
     Logger factory module.
     """
 
     @singleton
     @provider
-    def provide(self) -> LoggerFactory:
+    def provide(self) -> BootstrapLoggerFactory:
         """
         DI factory method.
         """
-        return LoggerFactory()
+        return BootstrapLoggerFactory()
 
 
 @final
@@ -234,6 +236,7 @@ class ServiceConfigurator:
 
     def __init__(self):
         self._injection_binds = [
+            BootstrapLoggerFactoryModule(),
             MetricsModule(),
             CrystalReceiverClientModule(),
             QueryEnabledStoreModule(),

{esd_services_api_client-2.5.14a148.dev5 → esd_services_api_client-2.6.1a149.dev13}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "esd-services-api-client"
-version = "v2.5.14a148.dev5"
+version = "v2.6.1a149.dev13"
 description = "Python clients for ESD services"
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']
@@ -9,8 +9,8 @@ license = 'Apache 2.0'
 repository = 'https://github.com/SneaksAndData/esd-services-api-client'
 
 [tool.poetry.dependencies]
-python = ">=3.9,<3.12"
-adapta = { version = "^3.2", extras = ["azure", "storage", "datadog"] }
+python = ">=3.11,<3.12"
+adapta = { version = "^3.3", extras = ["storage", "datadog"] }
 dataclasses-json = "^0.6.0"
 pycryptodome = "~3.15"
 azure-identity = { version = "~1.7", optional = true }

esd_services_api_client-2.5.14a148.dev5/esd_services_api_client/_version.py

@@ -1 +0,0 @@
-__version__ = 'v2.5.14a148.dev5'

esd_services_api_client-2.5.14a148.dev5/esd_services_api_client/nexus/abstractions/logger_factory.py

@@ -1,64 +0,0 @@
-"""
- Logger factory for async loggers.
-"""
-
-#  Copyright (c) 2023-2024. ECCO Sneaks & Data
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-import json
-import os
-from logging import StreamHandler
-from typing import final, Type, TypeVar, Optional, Dict
-
-from adapta.logs import LoggerInterface, create_async_logger
-from adapta.logs.handlers.datadog_api_handler import DataDogApiHandler
-from adapta.logs.models import LogLevel
-
-TLogger = TypeVar("TLogger")  # pylint: disable=C0103:
-
-
-@final
-class LoggerFactory:
-    """
-    Async logger provisioner.
-    """
-
-    def __init__(self):
-        self._log_handlers = [
-            StreamHandler(),
-        ]
-        if "NEXUS__DATADOG_LOGGER_CONFIGURATION" in os.environ:
-            self._log_handlers.append(
-                DataDogApiHandler(
-                    **json.loads(os.getenv("NEXUS__DATADOG_LOGGER_CONFIGURATION"))
-                )
-            )
-
-    def create_logger(
-        self,
-        logger_type: Type[TLogger],
-        fixed_template: Optional[Dict[str, Dict[str, str]]] = None,
-        fixed_template_delimiter=", ",
-    ) -> LoggerInterface:
-        """
-        Creates an async-safe logger for the provided class name.
-        """
-        return create_async_logger(
-            logger_type=logger_type,
-            log_handlers=self._log_handlers,
-            min_log_level=LogLevel(os.getenv("NEXUS__LOG_LEVEL", "INFO")),
-            fixed_template=fixed_template,
-            fixed_template_delimiter=fixed_template_delimiter,
-        )