esd-services-api-client 2.1.3__tar.gz → 2.2.0__tar.gz

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.1.3
+Version: 2.2.0
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0

esd_services_api_client-2.2.0/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = '2.2.0'

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/README.md

@@ -28,7 +28,9 @@ from adapta.storage.query_enabled_store import QueryEnabledStore
 from dataclasses_json import DataClassJsonMixin
 from injector import inject
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
 from esd_services_api_client.nexus.abstractions.socket_provider import (
     ExternalSocketProvider,
 )
@@ -36,7 +38,6 @@ from esd_services_api_client.nexus.configurations.algorithm_configuration import
     NexusConfiguration,
 )
 from esd_services_api_client.nexus.core.app_core import Nexus
-from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
 from esd_services_api_client.nexus.algorithms import MinimalisticAlgorithm
 from esd_services_api_client.nexus.input import InputReader, InputProcessor
 
@@ -127,12 +128,6 @@ class MockRequestHandler(BaseHTTPRequestHandler):
 
 
 class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
-    async def _context_open(self):
-        pass
-
-    async def _context_close(self):
-        pass
-
     @inject
     def __init__(
         self,
@@ -141,7 +136,8 @@ class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
         logger_factory: LoggerFactory,
         payload: MyAlgorithmPayload,
         socket_provider: ExternalSocketProvider,
-        *readers: "InputReader"
+        *readers: "InputReader",
+        cache: InputCache
     ):
         super().__init__(
             socket=socket_provider.socket("x"),
@@ -149,10 +145,11 @@ class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
             metrics_provider=metrics_provider,
             logger_factory=logger_factory,
             payload=payload,
+            cache=cache,
             *readers
         )
 
-    async def _read_input(self) -> pandas.DataFrame:
+    async def _read_input(self, **_) -> pandas.DataFrame:
         self._logger.info(
             "Payload: {payload}; Socket path: {socket_path}",
             payload=self._payload.to_json(),
@@ -162,12 +159,6 @@ class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
 
 
 class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
-    async def _context_open(self):
-        pass
-
-    async def _context_close(self):
-        pass
-
     @inject
     def __init__(
         self,
@@ -176,7 +167,8 @@ class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
         logger_factory: LoggerFactory,
         payload: MyAlgorithmPayload2,
         socket_provider: ExternalSocketProvider,
-        *readers: "InputReader"
+        *readers: "InputReader",
+        cache: InputCache
     ):
         super().__init__(
             socket=socket_provider.socket("y"),
@@ -184,10 +176,11 @@ class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
             metrics_provider=metrics_provider,
             logger_factory=logger_factory,
             payload=payload,
+            cache=cache,
             *readers
         )
 
-    async def _read_input(self) -> pandas.DataFrame:
+    async def _read_input(self, **_) -> pandas.DataFrame:
         self._logger.info(
             "Payload: {payload}; Socket path: {socket_path}",
             payload=self._payload.to_json(),
@@ -196,39 +189,59 @@ class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
         return pandas.DataFrame([{"a": 10, "b": 12}, {"a": 11, "b": 13}])
 
 
-class MyInputProcessor(InputProcessor):
-    async def _context_open(self):
-        pass
+class XProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
+    @inject
+    def __init__(
+        self,
+        x: XReader,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        my_conf: MyAlgorithmConfiguration,
+        cache: InputCache,
+    ):
+        super().__init__(
+            x,
+            metrics_provider=metrics_provider,
+            logger_factory=logger_factory,
+            payload=None,
+            cache=cache,
+        )
+
+        self.conf = my_conf
+
+    async def _process_input(
+        self, x: pandas.DataFrame, **_
+    ) -> pandas.DataFrame:
+        self._logger.info("Config: {config}", config=self.conf.to_json())
+        return x.assign(c=[-1, 1])
 
-    async def _context_close(self):
-        pass
 
+class YProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
     @inject
     def __init__(
         self,
-        x: XReader,
         y: YReader,
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
         my_conf: MyAlgorithmConfiguration,
+        cache: InputCache,
     ):
         super().__init__(
-            x,
             y,
             metrics_provider=metrics_provider,
             logger_factory=logger_factory,
             payload=None,
+            cache=cache,
         )
 
         self.conf = my_conf
 
-    async def process_input(self, **_) -> Dict[str, pandas.DataFrame]:
+    async def _process_input(
+        self, y: pandas.DataFrame, **_
+    ) -> pandas.DataFrame:
         self._logger.info("Config: {config}", config=self.conf.to_json())
-        inputs = await self._read_input()
-        return {
-            "x_ready": inputs["x"].assign(c=[-1, 1]),
-            "y_ready": inputs["y"].assign(c=[-1, 1]),
-        }
+        return y.assign(c=[-1, 1])
+
 
 @dataclass
 class MyResult(AlgorithmResult):
@@ -240,8 +253,8 @@ class MyResult(AlgorithmResult):
 
     def to_kwargs(self) -> dict[str, Any]:
         pass
-    
-    
+
+
 class MyAlgorithm(MinimalisticAlgorithm[MyAlgorithmPayload]):
     async def _context_open(self):
         pass
@@ -250,11 +263,22 @@ class MyAlgorithm(MinimalisticAlgorithm[MyAlgorithmPayload]):
         pass
 
     @inject
-    def __init__(self, metrics_provider: MetricsProvider, logger_factory: LoggerFactory, input_processor: MyInputProcessor):
-        super().__init__(metrics_provider, logger_factory, input_processor)
+    def __init__(
+        self,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        x_processor: XProcessor,
+        y_processor: YProcessor,
+        cache: InputCache,
+    ):
+        super().__init__(
+            metrics_provider, logger_factory, x_processor, y_processor, cache=cache
+        )
 
-    async def _run(self, x_ready: pandas.DataFrame, y_ready: pandas.DataFrame, **kwargs) -> MyResult:
-        return MyResult(x_ready, y_ready)
+    async def _run(
+        self, x: pandas.DataFrame, y: pandas.DataFrame, **kwargs
+    ) -> MyResult:
+        return MyResult(x, y)
 
 
 async def main():
@@ -270,7 +294,8 @@ async def main():
             await Nexus.create()
             .add_reader(XReader)
             .add_reader(YReader)
-            .use_processor(MyInputProcessor)
+            .use_processor(XProcessor)
+            .use_processor(YProcessor)
             .use_algorithm(MyAlgorithm)
             .inject_configuration(MyAlgorithmConfiguration)
             .inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)

esd_services_api_client-2.2.0/esd_services_api_client/nexus/abstractions/algrorithm_cache.py

@@ -0,0 +1,100 @@
+"""
+ Simple in-memory cache for readers and processors
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import asyncio
+from typing import final, Type
+
+import azure.core.exceptions
+import deltalake
+
+from esd_services_api_client.nexus.abstractions.input_object import InputObject
+from esd_services_api_client.nexus.abstractions.nexus_object import TResult, TPayload
+from esd_services_api_client.nexus.exceptions.cache_errors import (
+    FatalCachingError,
+    TransientCachingError,
+)
+
+
+@final
+class InputCache:
+    """
+    In-memory cache for Nexus input readers/processors
+    """
+
+    def __init__(self):
+        self._cache: dict[str, TResult] = {}
+
+    def _resolve_exc_type(
+        self, ex: BaseException
+    ) -> Type[FatalCachingError] | Type[TransientCachingError]:
+        """
+        Resolve base exception into a specific Nexus exception.
+        """
+        match type(ex):
+            case azure.core.exceptions.HttpResponseError, deltalake.PyDeltaTableError:
+                return TransientCachingError
+            case azure.core.exceptions.AzureError, azure.core.exceptions.ClientAuthenticationError:
+                return FatalCachingError
+            case _:
+                return FatalCachingError
+
+    async def resolve(
+        self,
+        *readers_or_processors: InputObject[TPayload, TResult],
+        **kwargs,
+    ) -> dict[str, TResult]:
+        """
+        Concurrently resolve `data` property of all readers by invoking their `read` method.
+        """
+
+        def get_result(alias: str, completed_task: asyncio.Task) -> TResult:
+            object_exc = completed_task.exception()
+            if object_exc:
+                raise self._resolve_exc_type(object_exc)(alias) from object_exc
+
+            return completed_task.result()
+
+        async def _execute(nexus_input: InputObject) -> TResult:
+            async with nexus_input as instance:
+                result = await nexus_input.process(**kwargs)
+
+            self._cache[instance.cache_key()] = result
+
+            return result
+
+        cached = {
+            reader_or_processor.__class__.alias(): reader_or_processor.data
+            for reader_or_processor in readers_or_processors
+            if reader_or_processor.cache_key() in self._cache
+        }
+        if len(cached) == len(readers_or_processors):
+            return cached
+
+        read_tasks: dict[str, asyncio.Task] = {
+            reader.__class__.alias(): asyncio.create_task(_execute(reader))
+            for reader in readers_or_processors
+            if reader.cache_key() not in self._cache
+        }
+
+        if len(read_tasks) > 0:
+            await asyncio.wait(fs=read_tasks.values())
+
+        return {
+            alias: get_result(alias, task) for alias, task in read_tasks.items()
+        } | cached

esd_services_api_client-2.2.0/esd_services_api_client/nexus/abstractions/input_object.py

@@ -0,0 +1,63 @@
+"""
+ Base class for input reading/processing.
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import base64
+import os
+from abc import ABC, abstractmethod
+
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    TPayload,
+    TResult,
+    NexusObject,
+)
+
+
+class InputObject(NexusObject[TPayload, TResult], ABC):
+    """
+    Base class for input processing and reader objects.
+    """
+
+    async def _context_open(self):
+        """
+        Optional actions to perform on context activation.
+        """
+
+    async def _context_close(self):
+        """
+        Optional actions to perform on context closure.
+        """
+
+    def cache_key(self) -> str:
+        """
+        Unique identifier for this Nexus object, can be used to in-memory or external caching.
+        """
+        return f"{base64.b64encode(hex(id(self)).encode('utf-8')).decode('utf-8')}_{os.getpid()}_{self.__class__.__name__}"
+
+    @property
+    def data(self) -> TResult | None:
+        """
+        Data bound to this object.
+        """
+        return None
+
+    @abstractmethod
+    async def process(self, **kwargs) -> TResult:
+        """
+        Executes input processing logic (read or transform)
+        """

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/abstractions/nexus_object.py

@@ -1,8 +1,6 @@
 """
  Base classes for all objects used by Nexus.
 """
-import re
-
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,11 +18,13 @@ import re
 
 
 from abc import ABC, abstractmethod
+import re
 from typing import Generic, TypeVar, Union, Any
 
 import pandas
 import polars
 from adapta.metrics import MetricsProvider
+from dataclasses_json.stringcase import snakecase
 
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 
@@ -92,11 +92,13 @@ class NexusObject(Generic[TPayload, TResult], ABC):
         """
         Alias to identify this reader's output
         """
-        return re.sub(
-            r"(?<!^)(?=[A-Z])",
-            "_",
-            cls.__name__.lower()
-            .replace("reader", "")
-            .replace("processor", "")
-            .replace("algorithm", ""),
+        return snakecase(
+            re.sub(
+                r"(?<!^)(?=[A-Z])",
+                "_",
+                cls.__name__.lower()
+                .replace("reader", "")
+                .replace("processor", "")
+                .replace("algorithm", ""),
+            )
         )

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/algorithms/_baseline_algorithm.py

@@ -19,11 +19,12 @@
 
 
 from abc import abstractmethod
-from functools import reduce, partial
+from functools import partial
 
 from adapta.metrics import MetricsProvider
 from adapta.utils.decorators import run_time_metrics_async
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.nexus_object import (
     NexusObject,
     TPayload,
@@ -32,7 +33,6 @@ from esd_services_api_client.nexus.abstractions.nexus_object import (
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.input.input_processor import (
     InputProcessor,
-    resolve_processors,
 )
 
 
@@ -46,9 +46,11 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
         *input_processors: InputProcessor,
+        cache: InputCache,
     ):
         super().__init__(metrics_provider, logger_factory)
         self._input_processors = input_processors
+        self._cache = cache
 
     @abstractmethod
     async def _run(self, **kwargs) -> AlgorithmResult:
@@ -75,11 +77,11 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         async def _measured_run(**run_args):
             return await self._run(**run_args)
 
-        results = await resolve_processors(*self._input_processors, **kwargs)
+        results = await self._cache.resolve(*self._input_processors, **kwargs)
 
         return await partial(
             _measured_run,
-            **reduce(lambda a, b: a | b, [result for _, result in results.items()]),
+            **results,
             metric_tags=self._metric_tags,
             metrics_provider=self._metrics_provider,
             logger=self._logger,

esd_services_api_client-2.2.0/esd_services_api_client/nexus/algorithms/_remote_algorithm.py

@@ -0,0 +1,118 @@
+"""
+ Remotely executed algorithm
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+from abc import abstractmethod
+from functools import partial
+
+from adapta.metrics import MetricsProvider
+from adapta.utils.decorators import run_time_metrics_async
+
+from esd_services_api_client.crystal import CrystalConnector, AlgorithmConfiguration
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    NexusObject,
+    TPayload,
+    AlgorithmResult,
+)
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.input.input_processor import (
+    InputProcessor,
+)
+from esd_services_api_client.nexus.input.payload_reader import AlgorithmPayload
+
+
+class RemoteAlgorithm(NexusObject[TPayload, AlgorithmResult]):
+    """
+    Base class for all algorithm implementations.
+    """
+
+    def __init__(
+        self,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        remote_client: CrystalConnector,
+        remote_name: str,
+        remote_config: AlgorithmConfiguration,
+        *input_processors: InputProcessor,
+        cache: InputCache,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._input_processors = input_processors
+        self._remote_client = remote_client
+        self._remote_name = remote_name
+        self._remote_config = remote_config
+        self._cache = cache
+
+    @abstractmethod
+    def _generate_tag(self) -> str:
+        """
+        Generates a submission tag.
+        """
+
+    @abstractmethod
+    def _transform_submission_result(
+        self, request_id: str, tag: str
+    ) -> AlgorithmResult:
+        """
+        Called after submitting a remote run. Use this to enrich your output with remote run id and tag.
+        """
+
+    @abstractmethod
+    async def _run(self, **kwargs) -> AlgorithmPayload:
+        """
+        Core logic for this algorithm. Implementing this method is mandatory.
+        """
+
+    @property
+    def _metric_tags(self) -> dict[str, str]:
+        return {"algorithm": self.__class__.alias()}
+
+    async def run(self, **kwargs) -> AlgorithmResult:
+        """
+        Coroutine that executes the algorithm logic.
+        """
+
+        @run_time_metrics_async(
+            metric_name="algorthm_run",
+            on_finish_message_template="Launched a new remote {algorithm} in {elapsed:.2f}s seconds",
+            template_args={
+                "algorithm": self.__class__.alias().upper(),
+            },
+        )
+        async def _measured_run(**run_args) -> AlgorithmResult:
+            payload = await self._run(**run_args)
+            tag = self._generate_tag()
+            request_id = self._remote_client.create_run(
+                algorithm=self._remote_name,
+                payload=payload.to_dict(),
+                custom_config=self._remote_config,
+                tag=tag,
+            )
+            return self._transform_submission_result(request_id, tag)
+
+        results = await self._cache.resolve(*self._input_processors, **kwargs)
+
+        return await partial(
+            _measured_run,
+            **results,
+            metric_tags=self._metric_tags,
+            metrics_provider=self._metrics_provider,
+            logger=self._logger,
+        )()

esd_services_api_client-2.2.0/esd_services_api_client/nexus/algorithms/forked_algorithm.py

@@ -0,0 +1,124 @@
+"""
+ Remotely executed algorithm
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import asyncio
+from abc import abstractmethod
+from functools import partial
+
+from adapta.metrics import MetricsProvider
+from adapta.utils.decorators import run_time_metrics_async
+
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    NexusObject,
+    TPayload,
+    AlgorithmResult,
+)
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.algorithms._remote_algorithm import RemoteAlgorithm
+from esd_services_api_client.nexus.input.input_processor import (
+    InputProcessor,
+)
+
+
+class ForkedAlgorithm(NexusObject[TPayload, AlgorithmResult]):
+    """
+    Forked algorithm is an algorithm that returns a result (main scenario run) and then fires off one or more forked runs
+    with different configurations as specified in fork class implementation.
+
+    Forked algorithm only awaits scheduling of forked runs, but never their results.
+
+     Q: How do I spawn a ForkedAlgorithm run as a remote algorithm w/o ending in an infinite loop?
+     A: Provide class names for forks from your algorithm configuration and construct forks with locate(fork_class)(**kwargs) calls.
+
+     Q: Can I build execution trees with this?
+     A: Yes, they will look like this (F(N) - Forked with N forks):
+
+     graph TB
+        F3["F(3)"] --> F2["F(2)"]
+        F3 --> F0["F(0)"]
+        F3 --> F1["F(1)"]
+        F2 --> F1_1["F(1)"]
+        F2 --> F0_1["F(0)"]
+        F1 --> F0_2["F(0)"]
+        F1_1 --> F0_3["F(0)"]
+    """
+
+    def __init__(
+        self,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        forks: list[RemoteAlgorithm],
+        *input_processors: InputProcessor,
+        cache: InputCache,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._input_processors = input_processors
+        self._forks = forks
+        self._cache = cache
+
+    @abstractmethod
+    async def _run(self, **kwargs) -> AlgorithmResult:
+        """
+        Core logic for this algorithm. Implementing this method is mandatory.
+        """
+
+    @property
+    def _metric_tags(self) -> dict[str, str]:
+        return {"algorithm": self.__class__.alias()}
+
+    async def run(self, **kwargs) -> AlgorithmResult:
+        """
+        Coroutine that executes the algorithm logic.
+        """
+
+        @run_time_metrics_async(
+            metric_name="algorthm_run",
+            on_finish_message_template="Finished running algorithm {algorithm} in {elapsed:.2f}s seconds",
+            template_args={
+                "algorithm": self.__class__.alias().upper(),
+            },
+        )
+        async def _measured_run(**run_args) -> AlgorithmResult:
+            return await self._run(**run_args)
+
+        if len(self._forks) > 0:
+            self._logger.info(
+                "This algorithm has forks attached: {forks}. They will be executed after the main run",
+                forks=",".join([fork.alias() for fork in self._forks]),
+            )
+        else:
+            self._logger.info(
+                "This algorithm supports forks but none were injected. Proceeding with a main run only"
+            )
+
+        results = await self._cache.resolve(*self._input_processors, **kwargs)
+
+        run_result = await partial(
+            _measured_run,
+            **results,
+            metric_tags=self._metric_tags,
+            metrics_provider=self._metrics_provider,
+            logger=self._logger,
+        )()
+
+        # now await callback scheduling
+        await asyncio.wait([fork.run(**kwargs) for fork in self._forks])
+
+        return run_result

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/algorithms/minimalistic.py

@@ -22,6 +22,7 @@ from abc import ABC
 from adapta.metrics import MetricsProvider
 from injector import inject
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.abstractions.nexus_object import TPayload
 from esd_services_api_client.nexus.algorithms._baseline_algorithm import (
@@ -41,5 +42,8 @@ class MinimalisticAlgorithm(BaselineAlgorithm[TPayload], ABC):
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
         *input_processors: InputProcessor,
+        cache: InputCache,
     ):
-        super().__init__(metrics_provider, logger_factory, *input_processors)
+        super().__init__(
+            metrics_provider, logger_factory, *input_processors, cache=cache
+        )

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/algorithms/recursive.py

@@ -23,6 +23,7 @@ from abc import abstractmethod
 from adapta.metrics import MetricsProvider
 from injector import inject
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.abstractions.nexus_object import (
     TPayload,
@@ -45,8 +46,11 @@ class RecursiveAlgorithm(BaselineAlgorithm[TPayload]):
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
         *input_processors: InputProcessor,
+        cache: InputCache,
     ):
-        super().__init__(metrics_provider, logger_factory, *input_processors)
+        super().__init__(
+            metrics_provider, logger_factory, *input_processors, cache=cache
+        )
 
     @abstractmethod
     async def _is_finished(self, **kwargs) -> bool:

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/core/app_dependencies.py

@@ -28,6 +28,7 @@ from adapta.storage.query_enabled_store import QueryEnabledStore
 from injector import Module, singleton, provider
 
 from esd_services_api_client.crystal import CrystalConnector
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.abstractions.socket_provider import (
     ExternalSocketProvider,
@@ -165,6 +166,21 @@ class ExternalSocketsModule(Module):
         )
 
 
+@final
+class CacheModule(Module):
+    """
+    Storage client module.
+    """
+
+    @singleton
+    @provider
+    def provide(self) -> InputCache:
+        """
+        Dependency provider.
+        """
+        return InputCache()
+
+
 @final
 class ServiceConfigurator:
     """
@@ -178,6 +194,7 @@ class ServiceConfigurator:
             QueryEnabledStoreModule(),
             StorageClientModule(),
             ExternalSocketsModule(),
+            CacheModule(),
         ]
 
     @property

esd_services_api_client-2.2.0/esd_services_api_client/nexus/exceptions/cache_errors.py

@@ -0,0 +1,49 @@
+"""
+ Cache module exceptions.
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from esd_services_api_client.nexus.exceptions._nexus_error import (
+    FatalNexusError,
+    TransientNexusError,
+)
+
+
+class FatalCachingError(FatalNexusError):
+    """
+    Cache-level exception that shuts down the Nexus.
+    """
+
+    def __init__(self, failed_object: str):
+        super().__init__()
+        self._failed_object = failed_object
+
+    def __str__(self) -> str:
+        return f"Nexus object with alias '{self._failed_object}' failed the caching operation that cannot be retried. Review traceback for more information"
+
+
+class TransientCachingError(TransientNexusError):
+    """
+    Cache-level exception that will initiate a retry with the Nexus re-activation.
+    """
+
+    def __init__(self, failed_object):
+        super().__init__()
+        self._failed_object = failed_object
+
+    def __str__(self) -> str:
+        return f"Nexus object with alias '{self._failed_object}' failed the caching operation and it will be retried. Review traceback for more information"

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/exceptions/startup_error.py

@@ -1,6 +1,7 @@
 """
  App startup exceptions.
 """
+from typing import Type
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -46,3 +47,17 @@ class FatalStartupConfigurationError(FatalNexusError):
 
     def __str__(self) -> str:
         return f"Algorithm initialization failed due to a missing configuration entry: {self._missing_entry}."
+
+
+class FatalAlgorithmConfigurationError(FatalNexusError):
+    """
+    Service configuration error that shuts down the Nexus.
+    """
+
+    def __init__(self, message: str, algorithm_class: Type):
+        super().__init__()
+        self._message = message
+        self._type_name = str(algorithm_class)
+
+    def __str__(self) -> str:
+        return f"Algorithm {self._type_name} misconfigured: {self._message}."

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/input/__init__.py

@@ -20,4 +20,3 @@
 
 from esd_services_api_client.nexus.input.input_processor import *
 from esd_services_api_client.nexus.input.input_reader import *
-from esd_services_api_client.nexus.input._functions import *

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/input/input_processor.py

@@ -1,7 +1,6 @@
 """
  Input processing.
 """
-import asyncio
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -25,22 +24,17 @@ from typing import Optional
 from adapta.metrics import MetricsProvider
 from adapta.utils.decorators import run_time_metrics_async
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
+from esd_services_api_client.nexus.abstractions.input_object import InputObject
 from esd_services_api_client.nexus.abstractions.nexus_object import (
-    NexusObject,
     TPayload,
     TResult,
 )
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
-from esd_services_api_client.nexus.input._functions import (
-    resolve_readers,
-    resolve_reader_exc_type,
-)
 from esd_services_api_client.nexus.input.input_reader import InputReader
 
-_processor_cache = {}
-
 
-class InputProcessor(NexusObject[TPayload, TResult]):
+class InputProcessor(InputObject[TPayload, TResult]):
     """
     Base class for raw data processing into algorithm input.
     """
@@ -51,24 +45,23 @@ class InputProcessor(NexusObject[TPayload, TResult]):
         payload: TPayload,
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
+        cache: InputCache
     ):
         super().__init__(metrics_provider, logger_factory)
         self._readers = readers
         self._payload = payload
         self._result: Optional[TResult] = None
-
-    async def _read_input(self) -> dict[str, TResult]:
-        return await resolve_readers(*self._readers)
+        self._cache = cache
 
     @property
-    def result(self) -> dict[str, TResult]:
+    def data(self) -> Optional[TResult]:
         """
         Data returned by this processor
         """
         return self._result
 
     @abstractmethod
-    async def _process_input(self, **kwargs) -> dict[str, TResult]:
+    async def _process_input(self, **kwargs) -> TResult:
         """
         Input processing logic. Implement this method to prepare data for your algorithm code.
         """
@@ -77,7 +70,7 @@ class InputProcessor(NexusObject[TPayload, TResult]):
     def _metric_tags(self) -> dict[str, str]:
         return {"processor": self.__class__.alias()}
 
-    async def process_input(self, **kwargs) -> dict[str, TResult]:
+    async def process(self, **kwargs) -> TResult:
         """
         Input processing coroutine. Do not override this method.
         """
@@ -89,8 +82,9 @@ class InputProcessor(NexusObject[TPayload, TResult]):
                 "processor": self.__class__.alias().upper(),
             },
         )
-        async def _process(**_) -> dict[str, TResult]:
-            return await self._process_input(**kwargs)
+        async def _process(**_) -> TResult:
+            readers = await self._cache.resolve(*self._readers)
+            return await self._process_input(**(kwargs | readers))
 
         if self._result is None:
             self._result = await partial(
@@ -101,44 +95,3 @@ class InputProcessor(NexusObject[TPayload, TResult]):
             )()
 
         return self._result
-
-
-async def resolve_processors(
-    *processors: InputProcessor[TPayload, TResult], **kwargs
-) -> dict[str, dict[str, TResult]]:
-    """
-    Concurrently resolve `result` property of all processors by invoking their `process_input` method.
-    """
-
-    def get_result(alias: str, completed_task: asyncio.Task) -> dict[str, TResult]:
-        reader_exc = completed_task.exception()
-        if reader_exc:
-            raise resolve_reader_exc_type(reader_exc)(alias, reader_exc) from reader_exc
-
-        return completed_task.result()
-
-    async def _process(input_processor: InputProcessor):
-        async with input_processor as instance:
-            result = await instance.process_input(**kwargs)
-            _processor_cache[input_processor.__class__.alias()] = result
-        return result
-
-    cached = {
-        processor.__class__.alias(): processor.result
-        for processor in processors
-        if processor.__class__.alias() in _processor_cache
-    }
-    if len(cached) == len(processors):
-        return cached
-
-    process_tasks: dict[str, asyncio.Task] = {
-        processor.__class__.alias(): asyncio.create_task(_process(processor))
-        for processor in processors
-        if processor.__class__.alias() not in _processor_cache
-    }
-    if len(process_tasks) > 0:
-        await asyncio.wait(fs=process_tasks.values())
-
-    return {
-        alias: get_result(alias, task) for alias, task in process_tasks.items()
-    } | cached

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/esd_services_api_client/nexus/input/input_reader.py

@@ -26,15 +26,16 @@ from adapta.process_communication import DataSocket
 from adapta.storage.query_enabled_store import QueryEnabledStore
 from adapta.utils.decorators import run_time_metrics_async
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
+from esd_services_api_client.nexus.abstractions.input_object import InputObject
 from esd_services_api_client.nexus.abstractions.nexus_object import (
-    NexusObject,
     TPayload,
     TResult,
 )
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 
 
-class InputReader(NexusObject[TPayload, TResult]):
+class InputReader(InputObject[TPayload, TResult]):
     """
     Base class for a raw data reader.
     """
@@ -47,6 +48,7 @@ class InputReader(NexusObject[TPayload, TResult]):
         payload: TPayload,
         *readers: "InputReader",
         socket: Optional[DataSocket] = None,
+        cache: InputCache
     ):
         super().__init__(metrics_provider, logger_factory)
         self.socket = socket
@@ -54,6 +56,7 @@ class InputReader(NexusObject[TPayload, TResult]):
         self._data: Optional[TResult] = None
         self._readers = readers
         self._payload = payload
+        self._cache = cache
 
     @property
     def data(self) -> Optional[TResult]:
@@ -63,7 +66,7 @@ class InputReader(NexusObject[TPayload, TResult]):
         return self._data
 
     @abstractmethod
-    async def _read_input(self) -> TResult:
+    async def _read_input(self, **kwargs) -> TResult:
         """
         Actual data reader logic. Implementing this method is mandatory for the reader to work
         """
@@ -72,7 +75,7 @@ class InputReader(NexusObject[TPayload, TResult]):
     def _metric_tags(self) -> dict[str, str]:
         return {"entity": self.__class__.alias()}
 
-    async def read(self) -> TResult:
+    async def process(self, **_) -> TResult:
         """
         Coroutine that reads the data from external store and converts it to a dataframe, or generates data locally. Do not override this method.
         """
@@ -88,7 +91,8 @@ class InputReader(NexusObject[TPayload, TResult]):
             | ({"data_path": self.socket.data_path} if self.socket else {}),
         )
         async def _read(**_) -> TResult:
-            return await self._read_input()
+            readers = await self._cache.resolve(*self._readers)
+            return await self._read_input(**readers)
 
         if self._data is None:
             self._data = await partial(

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "esd-services-api-client"
-version = "2.1.3"
+version = "2.2.0"
 description = "Python clients for ESD services"
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']

esd_services_api_client-2.1.3/esd_services_api_client/_version.py

@@ -1 +0,0 @@
-__version__ = '2.1.3'

esd_services_api_client-2.1.3/esd_services_api_client/nexus/input/_functions.py

@@ -1,89 +0,0 @@
-"""
- Utility functions to handle input processing.
-"""
-
-#  Copyright (c) 2023-2024. ECCO Sneaks & Data
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-import asyncio
-from typing import Union, Type
-import azure.core.exceptions
-import deltalake
-
-from esd_services_api_client.nexus.abstractions.nexus_object import TResult, TPayload
-from esd_services_api_client.nexus.exceptions.input_reader_error import (
-    FatalInputReaderError,
-    TransientInputReaderError,
-)
-from esd_services_api_client.nexus.input.input_reader import InputReader
-
-
-_reader_cache = {}
-
-
-def resolve_reader_exc_type(
-    ex: BaseException,
-) -> Union[Type[FatalInputReaderError], Type[TransientInputReaderError]]:
-    """
-    Resolve base exception into a specific Nexus exception.
-    """
-    match type(ex):
-        case azure.core.exceptions.HttpResponseError, deltalake.PyDeltaTableError:
-            return TransientInputReaderError
-        case azure.core.exceptions.AzureError, azure.core.exceptions.ClientAuthenticationError:
-            return FatalInputReaderError
-        case _:
-            return FatalInputReaderError
-
-
-async def resolve_readers(
-    *readers: InputReader[TPayload, TResult]
-) -> dict[str, TResult]:
-    """
-    Concurrently resolve `data` property of all readers by invoking their `read` method.
-    """
-
-    def get_result(alias: str, completed_task: asyncio.Task) -> TResult:
-        reader_exc = completed_task.exception()
-        if reader_exc:
-            raise resolve_reader_exc_type(reader_exc)(alias, reader_exc) from reader_exc
-
-        return completed_task.result()
-
-    async def _read(input_reader: InputReader):
-        async with input_reader as instance:
-            result = await instance.read()
-            _reader_cache[input_reader.__class__.alias()] = result
-        return result
-
-    cached = {
-        reader.__class__.alias(): reader.data
-        for reader in readers
-        if reader.__class__.alias() in _reader_cache
-    }
-    if len(cached) == len(readers):
-        return cached
-
-    read_tasks: dict[str, asyncio.Task] = {
-        reader.__class__.alias(): asyncio.create_task(_read(reader))
-        for reader in readers
-        if reader.__class__.alias() not in _reader_cache
-    }
-    if len(read_tasks) > 0:
-        await asyncio.wait(fs=read_tasks.values())
-
-    return {
-        alias: get_result(alias, task) for alias, task in read_tasks.items()
-    } | cached