esd-services-api-client 2.5.13__tar.gz → 2.5.14a0.dev0__tar.gz

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/PKG-INFO

@@ -1,21 +1,19 @@
 Metadata-Version: 2.3
 Name: esd-services-api-client
-Version: 2.5.13
+Version: 2.5.14a0.dev0
 Summary: Python clients for ESD services
 License: Apache 2.0
 Author: ECCO Sneaks & Data
 Author-email: esdsupport@ecco.com
 Maintainer: GZU
 Maintainer-email: gzu@ecco.com
-Requires-Python: >=3.9,<3.12
+Requires-Python: >=3.11,<3.12
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: azure
 Provides-Extra: nexus
-Requires-Dist: adapta[azure,datadog,storage] (>=3.2,<4.0)
+Requires-Dist: adapta[azure,datadog,storage] (>=3.3,<4.0)
 Requires-Dist: azure-identity (>=1.7,<1.8) ; extra == "azure"
 Requires-Dist: dataclasses-json (>=0.6.0,<0.7.0)
 Requires-Dist: httpx (>=0.27.0,<0.28.0) ; extra == "nexus"

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/esd_services_api_client/__init__.py

@@ -17,4 +17,4 @@
  Root index.
 """
 
-__version__ = "2.5.13"
+__version__ = "0.0.0"

esd_services_api_client-2.5.14a0.dev0/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = 'v2.5.14a.dev'

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/esd_services_api_client/beast/v3/_connector.py

@@ -220,6 +220,7 @@ class BeastConnector:
                 f"Execution failed, please find request's log at: {self.base_url}/job/logs/{request_id}"
             )
 
+    # pylint: disable=too-many-positional-arguments
     @staticmethod
     def _report_backoff_failure(
         target: Any, args: Any, kwargs: Any, tries: int, elapsed: int, wait: int, **_

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/esd_services_api_client/crystal/_connector.py

@@ -151,6 +151,7 @@ class CrystalConnector:
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.dispose()
 
+    # pylint: disable=too-many-positional-arguments
     def create_run(
         self,
         algorithm: str,

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/esd_services_api_client/nexus/README.md

@@ -297,7 +297,7 @@ async def main():
         server_thread.daemon = True
         server_thread.start()
         nexus = (
-            await Nexus.create()
+            Nexus.create()
             .add_reader(XReader)
             .add_reader(YReader)
             .use_processor(XProcessor)

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/esd_services_api_client/nexus/algorithms/_remote_algorithm.py

@@ -44,6 +44,7 @@ class RemoteAlgorithm(NexusObject[TPayload, AlgorithmResult]):
     Base class for all algorithm implementations.
     """
 
+    # pylint: disable=too-many-positional-arguments
     @inject
     def __init__(
         self,

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/esd_services_api_client/nexus/core/app_core.py

@@ -27,12 +27,11 @@ from typing import final, Type, Optional
 
 import backoff
 import urllib3.exceptions
-import azure.core.exceptions
 from adapta.logs import LoggerInterface
 from adapta.process_communication import DataSocket
 from adapta.storage.blob.base import StorageClient
 from adapta.storage.query_enabled_store import QueryEnabledStore
-from injector import Injector, Module
+from injector import Injector, Module, singleton
 
 import esd_services_api_client.nexus.exceptions
 from esd_services_api_client.crystal import (
@@ -119,6 +118,7 @@ class Nexus:
         self._run_args = args
         self._algorithm_run_task: Optional[asyncio.Task] = None
         self._on_complete_tasks: list[type[UserTelemetryRecorder]] = []
+        self._payload_types: list[type[AlgorithmPayload]] = []
 
         attach_signal_handlers()
 
@@ -157,22 +157,11 @@ class Nexus:
         self._algorithm_class = algorithm
         return self
 
-    async def inject_payload(self, *payload_types: Type[AlgorithmPayload]) -> "Nexus":
+    def inject_payload(self, *payload_types: Type[AlgorithmPayload]) -> "Nexus":
         """
-        Adds payloads processed into the specified types to the DI container
+        Adds payload types to inject to the DI container. Payloads will be deserialized at runtime.
         """
-        for payload_type in payload_types:
-
-            async def get_payload() -> payload_type:  # pylint: disable=W0640
-                async with AlgorithmPayloadReader(
-                    payload_uri=self._run_args.sas_uri,
-                    payload_type=payload_type,  # pylint: disable=W0640
-                ) as reader:
-                    return reader.payload
-
-            # pylint warnings are silenced here because closure is called inside the same loop it is defined, thus each value of a loop variable is used
-
-            self._configurator = self._configurator.with_payload((await get_payload()))
+        self._payload_types = payload_types
         return self
 
     def inject_configuration(
@@ -202,10 +191,7 @@ class Nexus:
     ) -> None:
         @backoff.on_exception(
             wait_gen=backoff.expo,
-            exception=(
-                azure.core.exceptions.HttpResponseError,
-                urllib3.exceptions.HTTPError,
-            ),
+            exception=(urllib3.exceptions.HTTPError,),
             max_time=10,
             raise_on_giveup=True,
         )
@@ -258,6 +244,15 @@ class Nexus:
             case _:
                 sys.exit(1)
 
+    async def _get_payload(
+        self, payload_type: type[AlgorithmPayload]
+    ) -> AlgorithmPayload:
+        async with AlgorithmPayloadReader(
+            payload_uri=self._run_args.sas_uri,
+            payload_type=payload_type,
+        ) as reader:
+            return reader.payload
+
     async def activate(self):
         """
         Activates the run sequence.
@@ -265,14 +260,25 @@ class Nexus:
 
         self._injector = Injector(self._configurator.injection_binds)
 
-        algorithm: BaselineAlgorithm = self._injector.get(self._algorithm_class)
-        telemetry_recorder: TelemetryRecorder = self._injector.get(TelemetryRecorder)
         root_logger: LoggerInterface = self._injector.get(LoggerFactory).create_logger(
             logger_type=self.__class__,
         )
 
         root_logger.start()
 
+        for payload_type in self._payload_types:
+            try:
+                payload = await self._get_payload(payload_type=payload_type)
+                self._injector.binder.bind(
+                    payload.__class__, to=payload, scope=singleton
+                )
+            except BaseException as ex:  # pylint: disable=broad-except
+                root_logger.error("Error reading algorithm payload", ex)
+                sys.exit(1)
+
+        algorithm: BaselineAlgorithm = self._injector.get(self._algorithm_class)
+        telemetry_recorder: TelemetryRecorder = self._injector.get(TelemetryRecorder)
+
         root_logger.info(
             "Running algorithm {algorithm} on Nexus version {version}",
             algorithm=algorithm.__class__.__name__,
@@ -283,9 +289,21 @@ class Nexus:
             self._algorithm_run_task = asyncio.create_task(
                 instance.run(**self._run_args.__dict__)
             )
-            await self._algorithm_run_task
+
+            # avoid exception propagation to main thread, since we need to handle it later
+            await asyncio.wait(
+                [self._algorithm_run_task], return_when=asyncio.FIRST_EXCEPTION
+            )
             ex = self._algorithm_run_task.exception()
 
+            if ex is not None:
+                root_logger.error(
+                    "Algorithm {algorithm} run failed on Nexus version {version}",
+                    ex,
+                    algorithm=self._algorithm_class,
+                    version=__version__,
+                )
+
             await self._submit_result(
                 self._algorithm_run_task.result() if not ex else None,
                 self._algorithm_run_task.exception(),

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/esd_services_api_client/nexus/core/app_dependencies.py

@@ -42,9 +42,6 @@ from esd_services_api_client.nexus.exceptions.startup_error import (
 )
 from esd_services_api_client.nexus.input.input_processor import InputProcessor
 from esd_services_api_client.nexus.input.input_reader import InputReader
-from esd_services_api_client.nexus.input.payload_reader import (
-    AlgorithmPayload,
-)
 from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
 from esd_services_api_client.nexus.core.serializers import (
     TelemetrySerializer,
@@ -247,6 +244,7 @@ class ServiceConfigurator:
             CacheModule(),
             type(f"{TelemetryRecorder.__name__}Module", (Module,), {})(),
         ]
+        self._runtime_injection_binds = []
 
     @property
     def injection_binds(self) -> list:
@@ -255,6 +253,13 @@ class ServiceConfigurator:
         """
         return self._injection_binds
 
+    @property
+    def runtime_injection_binds(self) -> list:
+        """
+        Currently configured injection bindings that are added at runtime
+        """
+        return self._runtime_injection_binds
+
     def with_module(self, module: Type[Module]) -> "ServiceConfigurator":
         """
         Adds a (custom) module into the DI container.
@@ -280,15 +285,6 @@ class ServiceConfigurator:
         )
         return self
 
-    def with_payload(self, payload: AlgorithmPayload) -> "ServiceConfigurator":
-        """
-        Adds the specified payload instance to the DI container.
-        """
-        self._injection_binds.append(
-            lambda binder: binder.bind(payload.__class__, to=payload, scope=singleton)
-        )
-        return self
-
     def with_configuration(self, config: NexusConfiguration) -> "ServiceConfigurator":
         """
         Adds the specified payload instance to the DI container.

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/esd_services_api_client/nexus/input/payload_reader.py

@@ -1,6 +1,7 @@
 """
  Code infrastructure for manipulating payload received from Crystal SAS URI
 """
+import os
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -62,8 +63,15 @@ class AlgorithmPayloadReader:
         self._http.close()
         self._http = None
 
-    def __init__(self, payload_uri: str, payload_type: Type[AlgorithmPayload]):
-        self._http = session_with_retries()
+    def __init__(
+        self,
+        payload_uri: str,
+        payload_type: Type[AlgorithmPayload],
+    ):
+        self._http = session_with_retries(
+            file_protocol_supported=os.getenv("NEXUS__SUPPORT_FILE_PROTOCOL_PAYLOADS")
+            is not None
+        )
         self._payload: Optional[AlgorithmPayload] = None
         self._payload_uri = payload_uri
         self._payload_type = payload_type

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/esd_services_api_client/nexus/telemetry/user_telemetry_recorder.py

@@ -70,6 +70,7 @@ class UserTelemetryRecorder(Generic[TPayload, TResult], ABC):
     Base class for user-defined telemetry recorders.
     """
 
+    # pylint: disable=too-many-positional-arguments
     @inject
     def __init__(
         self,

{esd_services_api_client-2.5.13 → esd_services_api_client-2.5.14a0.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "esd-services-api-client"
-version = "2.5.13"
+version = "v2.5.14a.dev"
 description = "Python clients for ESD services"
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']
@@ -9,8 +9,8 @@ license = 'Apache 2.0'
 repository = 'https://github.com/SneaksAndData/esd-services-api-client'
 
 [tool.poetry.dependencies]
-python = ">=3.9,<3.12"
-adapta = { version = "^3.2", extras = ["azure", "storage", "datadog"] }
+python = ">=3.11,<3.12"
+adapta = { version = "^3.3", extras = ["azure", "storage", "datadog"] }
 dataclasses-json = "^0.6.0"
 pycryptodome = "~3.15"
 azure-identity = { version = "~1.7", optional = true }