esd-services-api-client 2.0.3__tar.gz → 2.0.5__tar.gz

{esd_services_api_client-2.0.3 → esd_services_api_client-2.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.0.3
+Version: 2.0.5
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0

esd_services_api_client-2.0.5/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = '2.0.5'

{esd_services_api_client-2.0.3 → esd_services_api_client-2.0.5}/esd_services_api_client/nexus/README.md

@@ -15,6 +15,7 @@ Example usage:
 ```python
 import asyncio
 import json
+import os
 import socketserver
 import threading
 from dataclasses import dataclass
@@ -31,6 +32,9 @@ from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFact
 from esd_services_api_client.nexus.abstractions.socket_provider import (
     ExternalSocketProvider,
 )
+from esd_services_api_client.nexus.configurations.algorithm_configuration import (
+    NexusConfiguration,
+)
 from esd_services_api_client.nexus.core.app_core import Nexus
 from esd_services_api_client.nexus.algorithms import MinimalisticAlgorithm
 from esd_services_api_client.nexus.input import InputReader, InputProcessor
@@ -47,6 +51,16 @@ async def my_on_complete_func_2(**kwargs):
     pass
 
 
+@dataclass
+class MyAlgorithmConfiguration(NexusConfiguration):
+    @classmethod
+    def from_environment(cls) -> "NexusConfiguration":
+        return MyAlgorithmConfiguration.from_json(os.getenv("NEXUS__MY_CONFIGURATION"))
+
+    c1: str
+    c2: str
+
+
 @dataclass
 class MyAlgorithmPayload(AlgorithmPayload, DataClassJsonMixin):
     x: Optional[list[int]] = None
@@ -130,11 +144,11 @@ class XReader(InputReader[MyAlgorithmPayload]):
         *readers: "InputReader"
     ):
         super().__init__(
-            socket_provider.socket("x"),
-            store,
-            metrics_provider,
-            logger_factory,
-            payload,
+            socket=socket_provider.socket("x"),
+            store=store,
+            metrics_provider=metrics_provider,
+            logger_factory=logger_factory,
+            payload=payload,
             *readers
         )
 
@@ -165,11 +179,11 @@ class YReader(InputReader[MyAlgorithmPayload2]):
         *readers: "InputReader"
     ):
         super().__init__(
-            socket_provider.socket("y"),
-            store,
-            metrics_provider,
-            logger_factory,
-            payload,
+            socket=socket_provider.socket("y"),
+            store=store,
+            metrics_provider=metrics_provider,
+            logger_factory=logger_factory,
+            payload=payload,
             *readers
         )
 
@@ -196,6 +210,7 @@ class MyInputProcessor(InputProcessor):
         y: YReader,
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
+        my_conf: MyAlgorithmConfiguration,
     ):
         super().__init__(
             x,
@@ -205,7 +220,10 @@ class MyInputProcessor(InputProcessor):
             payload=None,
         )
 
+        self.conf = my_conf
+
     async def process_input(self, **_) -> Dict[str, PandasDataFrame]:
+        self._logger.info("Config: {config}", config=self.conf.to_json())
         inputs = await self._read_input()
         return {
             "x_ready": inputs["x"].assign(c=[-1, 1]),
@@ -250,6 +268,7 @@ async def main():
             .add_reader(YReader)
             .use_processor(MyInputProcessor)
             .use_algorithm(MyAlgorithm)
+            .inject_configuration(MyAlgorithmConfiguration)
             .inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)
         )
 

esd_services_api_client-2.0.5/esd_services_api_client/nexus/configurations/algorithm_configuration.py

@@ -0,0 +1,37 @@
+"""
+ Configuration class primitive for injections.
+"""
+from abc import abstractmethod, ABC
+from dataclasses import dataclass
+
+from dataclasses_json import DataClassJsonMixin
+
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+@dataclass
+class NexusConfiguration(DataClassJsonMixin, ABC):
+    """
+    Base class for algorithm configurations
+    """
+
+    @classmethod
+    @abstractmethod
+    def from_environment(cls) -> "NexusConfiguration":
+        """
+        Instantiates this configuration from the environment variable.
+        """

{esd_services_api_client-2.0.3 → esd_services_api_client-2.0.5}/esd_services_api_client/nexus/core/app_core.py

@@ -46,6 +46,9 @@ from esd_services_api_client.crystal import (
 from esd_services_api_client.nexus.algorithms._baseline_algorithm import (
     BaselineAlgorithm,
 )
+from esd_services_api_client.nexus.configurations.algorithm_configuration import (
+    NexusConfiguration,
+)
 from esd_services_api_client.nexus.core.app_dependencies import (
     ServiceConfigurator,
 )
@@ -163,6 +166,19 @@ class Nexus:
             self._configurator = self._configurator.with_payload((await get_payload()))
         return self
 
+    def inject_configuration(
+        self, *configuration_types: Type[NexusConfiguration]
+    ) -> "Nexus":
+        """
+        Adds custom configuration class instances to the DI container.
+        """
+        for config_type in configuration_types:
+            self._configurator = self._configurator.with_configuration(
+                config_type.from_environment()
+            )
+
+        return self
+
     async def _submit_result(
         self,
         result: Optional[PandasDataFrame] = None,

{esd_services_api_client-2.0.3 → esd_services_api_client-2.0.5}/esd_services_api_client/nexus/core/app_dependencies.py

@@ -32,6 +32,9 @@ from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFact
 from esd_services_api_client.nexus.abstractions.socket_provider import (
     ExternalSocketProvider,
 )
+from esd_services_api_client.nexus.configurations.algorithm_configuration import (
+    NexusConfiguration,
+)
 from esd_services_api_client.nexus.exceptions.startup_error import (
     FatalStartupConfigurationError,
 )
@@ -42,6 +45,7 @@ from esd_services_api_client.nexus.input.payload_reader import (
 )
 
 
+@final
 class MetricsModule(Module):
     """
     Metrics provider module.
@@ -63,6 +67,7 @@ class MetricsModule(Module):
         return metrics_class(**metrics_settings)
 
 
+@final
 class LoggerFactoryModule(Module):
     """
     Logger factory module.
@@ -77,6 +82,7 @@ class LoggerFactoryModule(Module):
         return LoggerFactory()
 
 
+@final
 class CrystalReceiverClientModule(Module):
     """
     Crystal receiver module.
@@ -93,6 +99,7 @@ class CrystalReceiverClientModule(Module):
         )
 
 
+@final
 class QueryEnabledStoreModule(Module):
     """
     QES module.
@@ -107,6 +114,7 @@ class QueryEnabledStoreModule(Module):
         return QueryEnabledStore.from_string(os.getenv("NEXUS__QES_CONNECTION_STRING"))
 
 
+@final
 class StorageClientModule(Module):
     """
     Storage client module.
@@ -203,3 +211,12 @@ class ServiceConfigurator:
             lambda binder: binder.bind(payload.__class__, to=payload, scope=singleton)
         )
         return self
+
+    def with_configuration(self, config: NexusConfiguration) -> "ServiceConfigurator":
+        """
+        Adds the specified payload instance to the DI container.
+        """
+        self._injection_binds.append(
+            lambda binder: binder.bind(config.__class__, to=config, scope=singleton)
+        )
+        return self

{esd_services_api_client-2.0.3 → esd_services_api_client-2.0.5}/esd_services_api_client/nexus/input/__init__.py

@@ -20,3 +20,4 @@
 
 from esd_services_api_client.nexus.input.input_processor import *
 from esd_services_api_client.nexus.input.input_reader import *
+from esd_services_api_client.nexus.input._functions import *

esd_services_api_client-2.0.5/esd_services_api_client/nexus/input/_functions.py

@@ -0,0 +1,69 @@
+"""
+ Utility functions to handle input processing.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import asyncio
+from typing import Dict, Union, Type
+import azure.core.exceptions
+import deltalake
+from pandas import DataFrame as PandasDataFrame
+
+from esd_services_api_client.nexus.exceptions.input_reader_error import (
+    FatalInputReaderError,
+    TransientInputReaderError,
+)
+from esd_services_api_client.nexus.input.input_reader import InputReader
+
+
+def resolve_reader_exc_type(
+    ex: BaseException,
+) -> Union[Type[FatalInputReaderError], Type[TransientInputReaderError]]:
+    """
+    Resolve base exception into a specific Nexus exception.
+    """
+    match type(ex):
+        case azure.core.exceptions.HttpResponseError, deltalake.PyDeltaTableError:
+            return TransientInputReaderError
+        case azure.core.exceptions.AzureError, azure.core.exceptions.ClientAuthenticationError:
+            return FatalInputReaderError
+        case _:
+            return FatalInputReaderError
+
+
+async def resolve_readers(*readers: InputReader) -> Dict[str, PandasDataFrame]:
+    """
+    Concurrently resolve `data` property of all readers by invoking their `read` method.
+    """
+
+    def get_result(alias: str, completed_task: asyncio.Task) -> PandasDataFrame:
+        reader_exc = completed_task.exception()
+        if reader_exc:
+            raise resolve_reader_exc_type(reader_exc)(alias, reader_exc)
+
+        return completed_task.result()
+
+    async def _read(input_reader: InputReader):
+        async with input_reader as instance:
+            return await instance.read()
+
+    read_tasks: dict[str, asyncio.Task] = {
+        reader.socket.alias: asyncio.create_task(_read(reader)) for reader in readers
+    }
+    await asyncio.wait(fs=read_tasks.values())
+
+    return {alias: get_result(alias, task) for alias, task in read_tasks.items()}

{esd_services_api_client-2.0.3 → esd_services_api_client-2.0.5}/esd_services_api_client/nexus/input/input_processor.py

@@ -17,15 +17,11 @@
 #  limitations under the License.
 #
 
-import asyncio
 from abc import abstractmethod
-from typing import Dict, Union, Type
+from typing import Dict
 
-import deltalake
 from adapta.metrics import MetricsProvider
 
-import azure.core.exceptions
-
 from pandas import DataFrame as PandasDataFrame
 
 from esd_services_api_client.nexus.abstractions.nexus_object import (
@@ -33,10 +29,7 @@ from esd_services_api_client.nexus.abstractions.nexus_object import (
     TPayload,
 )
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
-from esd_services_api_client.nexus.exceptions.input_reader_error import (
-    FatalInputReaderError,
-    TransientInputReaderError,
-)
+from esd_services_api_client.nexus.input._functions import resolve_readers
 from esd_services_api_client.nexus.input.input_reader import InputReader
 
 
@@ -56,36 +49,8 @@ class InputProcessor(NexusObject[TPayload]):
         self._readers = readers
         self._payload = payload
 
-    def _get_exc_type(
-        self, ex: BaseException
-    ) -> Union[Type[FatalInputReaderError], Type[TransientInputReaderError]]:
-        match type(ex):
-            case azure.core.exceptions.HttpResponseError, deltalake.PyDeltaTableError:
-                return TransientInputReaderError
-            case azure.core.exceptions.AzureError, azure.core.exceptions.ClientAuthenticationError:
-                return FatalInputReaderError
-            case _:
-                return FatalInputReaderError
-
     async def _read_input(self) -> Dict[str, PandasDataFrame]:
-        def get_result(alias: str, completed_task: asyncio.Task) -> PandasDataFrame:
-            reader_exc = completed_task.exception()
-            if reader_exc:
-                raise self._get_exc_type(reader_exc)(alias, reader_exc)
-
-            return completed_task.result()
-
-        async def _read(input_reader: InputReader):
-            async with input_reader as instance:
-                return await instance.read()
-
-        read_tasks: dict[str, asyncio.Task] = {
-            reader.socket.alias: asyncio.create_task(_read(reader))
-            for reader in self._readers
-        }
-        await asyncio.wait(fs=read_tasks.values())
-
-        return {alias: get_result(alias, task) for alias, task in read_tasks.items()}
+        return await resolve_readers(*self._readers)
 
     @abstractmethod
     async def process_input(self, **kwargs) -> Dict[str, PandasDataFrame]:

{esd_services_api_client-2.0.3 → esd_services_api_client-2.0.5}/esd_services_api_client/nexus/input/input_reader.py

@@ -1,6 +1,7 @@
 """
  Input reader.
 """
+import functools
 
 #  Copyright (c) 2023. ECCO Sneaks & Data
 #
@@ -43,12 +44,12 @@ class InputReader(NexusObject[TPayload]):
 
     def __init__(
         self,
-        socket: DataSocket,
         store: QueryEnabledStore,
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
         payload: TPayload,
-        *readers: "InputReader"
+        *readers: "InputReader",
+        socket: Optional[DataSocket] = None,
     ):
         super().__init__(metrics_provider, logger_factory)
         self.socket = socket
@@ -58,6 +59,16 @@ class InputReader(NexusObject[TPayload]):
         self._payload = payload
 
     @property
+    def alias(self) -> str:
+        """
+        Alias to identify this reader's output
+        """
+        if self.socket:
+            return self.socket.alias
+
+        return self._metric_name
+
+    @functools.cached_property
     def data(self) -> Optional[PandasDataFrame]:
         """
         Data read by this reader.
@@ -92,8 +103,10 @@ class InputReader(NexusObject[TPayload]):
             on_finish_message_template="Finished reading {entity} from path {data_path} in {elapsed:.2f}s seconds",
             template_args={
                 "entity": self._metric_name.upper(),
-                "data_path": self.socket.data_path,
-            },
+            }
+            | {"data_path": self.socket.data_path}
+            if self.socket
+            else {},
         )
         async def _read(**_) -> PandasDataFrame:
             if not self._data:

{esd_services_api_client-2.0.3 → esd_services_api_client-2.0.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "esd-services-api-client"
-version = "2.0.3"
+version = "2.0.5"
 description = "Python clients for ESD services"
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']

esd_services_api_client-2.0.3/esd_services_api_client/_version.py

@@ -1 +0,0 @@
-__version__ = '2.0.3'