esd-services-api-client 2.1.3__tar.gz → 2.2.1__tar.gz

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.1.3
+Version: 2.2.1
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0

esd_services_api_client-2.2.1/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = '2.2.1'

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.1}/esd_services_api_client/nexus/README.md

@@ -28,7 +28,9 @@ from adapta.storage.query_enabled_store import QueryEnabledStore
 from dataclasses_json import DataClassJsonMixin
 from injector import inject
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
 from esd_services_api_client.nexus.abstractions.socket_provider import (
     ExternalSocketProvider,
 )
@@ -36,7 +38,6 @@ from esd_services_api_client.nexus.configurations.algorithm_configuration import
     NexusConfiguration,
 )
 from esd_services_api_client.nexus.core.app_core import Nexus
-from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
 from esd_services_api_client.nexus.algorithms import MinimalisticAlgorithm
 from esd_services_api_client.nexus.input import InputReader, InputProcessor
 
@@ -127,12 +128,6 @@ class MockRequestHandler(BaseHTTPRequestHandler):
 
 
 class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
-    async def _context_open(self):
-        pass
-
-    async def _context_close(self):
-        pass
-
     @inject
     def __init__(
         self,
@@ -141,7 +136,8 @@ class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
         logger_factory: LoggerFactory,
         payload: MyAlgorithmPayload,
         socket_provider: ExternalSocketProvider,
-        *readers: "InputReader"
+        *readers: "InputReader",
+        cache: InputCache
     ):
         super().__init__(
             socket=socket_provider.socket("x"),
@@ -149,10 +145,11 @@ class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
             metrics_provider=metrics_provider,
             logger_factory=logger_factory,
             payload=payload,
+            cache=cache,
             *readers
         )
 
-    async def _read_input(self) -> pandas.DataFrame:
+    async def _read_input(self, **_) -> pandas.DataFrame:
         self._logger.info(
             "Payload: {payload}; Socket path: {socket_path}",
             payload=self._payload.to_json(),
@@ -162,12 +159,6 @@ class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
 
 
 class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
-    async def _context_open(self):
-        pass
-
-    async def _context_close(self):
-        pass
-
     @inject
     def __init__(
         self,
@@ -176,7 +167,8 @@ class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
         logger_factory: LoggerFactory,
         payload: MyAlgorithmPayload2,
         socket_provider: ExternalSocketProvider,
-        *readers: "InputReader"
+        *readers: "InputReader",
+        cache: InputCache
     ):
         super().__init__(
             socket=socket_provider.socket("y"),
@@ -184,10 +176,11 @@ class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
             metrics_provider=metrics_provider,
             logger_factory=logger_factory,
             payload=payload,
+            cache=cache,
             *readers
         )
 
-    async def _read_input(self) -> pandas.DataFrame:
+    async def _read_input(self, **_) -> pandas.DataFrame:
         self._logger.info(
             "Payload: {payload}; Socket path: {socket_path}",
             payload=self._payload.to_json(),
@@ -196,39 +189,59 @@ class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
         return pandas.DataFrame([{"a": 10, "b": 12}, {"a": 11, "b": 13}])
 
 
-class MyInputProcessor(InputProcessor):
-    async def _context_open(self):
-        pass
+class XProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
+    @inject
+    def __init__(
+        self,
+        x: XReader,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        my_conf: MyAlgorithmConfiguration,
+        cache: InputCache,
+    ):
+        super().__init__(
+            x,
+            metrics_provider=metrics_provider,
+            logger_factory=logger_factory,
+            payload=None,
+            cache=cache,
+        )
+
+        self.conf = my_conf
+
+    async def _process_input(
+        self, x: pandas.DataFrame, **_
+    ) -> pandas.DataFrame:
+        self._logger.info("Config: {config}", config=self.conf.to_json())
+        return x.assign(c=[-1, 1])
 
-    async def _context_close(self):
-        pass
 
+class YProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
     @inject
     def __init__(
         self,
-        x: XReader,
         y: YReader,
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
         my_conf: MyAlgorithmConfiguration,
+        cache: InputCache,
     ):
         super().__init__(
-            x,
             y,
             metrics_provider=metrics_provider,
             logger_factory=logger_factory,
             payload=None,
+            cache=cache,
         )
 
         self.conf = my_conf
 
-    async def process_input(self, **_) -> Dict[str, pandas.DataFrame]:
+    async def _process_input(
+        self, y: pandas.DataFrame, **_
+    ) -> pandas.DataFrame:
         self._logger.info("Config: {config}", config=self.conf.to_json())
-        inputs = await self._read_input()
-        return {
-            "x_ready": inputs["x"].assign(c=[-1, 1]),
-            "y_ready": inputs["y"].assign(c=[-1, 1]),
-        }
+        return y.assign(c=[-1, 1])
+
 
 @dataclass
 class MyResult(AlgorithmResult):
@@ -240,8 +253,8 @@ class MyResult(AlgorithmResult):
 
     def to_kwargs(self) -> dict[str, Any]:
         pass
-    
-    
+
+
 class MyAlgorithm(MinimalisticAlgorithm[MyAlgorithmPayload]):
     async def _context_open(self):
         pass
@@ -250,11 +263,22 @@ class MyAlgorithm(MinimalisticAlgorithm[MyAlgorithmPayload]):
         pass
 
     @inject
-    def __init__(self, metrics_provider: MetricsProvider, logger_factory: LoggerFactory, input_processor: MyInputProcessor):
-        super().__init__(metrics_provider, logger_factory, input_processor)
+    def __init__(
+        self,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        x_processor: XProcessor,
+        y_processor: YProcessor,
+        cache: InputCache,
+    ):
+        super().__init__(
+            metrics_provider, logger_factory, x_processor, y_processor, cache=cache
+        )
 
-    async def _run(self, x_ready: pandas.DataFrame, y_ready: pandas.DataFrame, **kwargs) -> MyResult:
-        return MyResult(x_ready, y_ready)
+    async def _run(
+        self, x: pandas.DataFrame, y: pandas.DataFrame, **kwargs
+    ) -> MyResult:
+        return MyResult(x, y)
 
 
 async def main():
@@ -270,7 +294,8 @@ async def main():
             await Nexus.create()
             .add_reader(XReader)
             .add_reader(YReader)
-            .use_processor(MyInputProcessor)
+            .use_processor(XProcessor)
+            .use_processor(YProcessor)
             .use_algorithm(MyAlgorithm)
             .inject_configuration(MyAlgorithmConfiguration)
             .inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)

esd_services_api_client-2.2.1/esd_services_api_client/nexus/abstractions/algrorithm_cache.py

@@ -0,0 +1,100 @@
+"""
+ Simple in-memory cache for readers and processors
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import asyncio
+from typing import final, Type
+
+import azure.core.exceptions
+import deltalake
+
+from esd_services_api_client.nexus.abstractions.input_object import InputObject
+from esd_services_api_client.nexus.abstractions.nexus_object import TResult, TPayload
+from esd_services_api_client.nexus.exceptions.cache_errors import (
+    FatalCachingError,
+    TransientCachingError,
+)
+
+
+@final
+class InputCache:
+    """
+    In-memory cache for Nexus input readers/processors
+    """
+
+    def __init__(self):
+        self._cache: dict[str, TResult] = {}
+
+    def _resolve_exc_type(
+        self, ex: BaseException
+    ) -> Type[FatalCachingError] | Type[TransientCachingError]:
+        """
+        Resolve base exception into a specific Nexus exception.
+        """
+        match type(ex):
+            case azure.core.exceptions.HttpResponseError, deltalake.PyDeltaTableError:
+                return TransientCachingError
+            case azure.core.exceptions.AzureError, azure.core.exceptions.ClientAuthenticationError:
+                return FatalCachingError
+            case _:
+                return FatalCachingError
+
+    async def resolve(
+        self,
+        *readers_or_processors: InputObject[TPayload, TResult],
+        **kwargs,
+    ) -> dict[str, TResult]:
+        """
+        Concurrently resolve `data` property of all readers by invoking their `read` method.
+        """
+
+        def get_result(alias: str, completed_task: asyncio.Task) -> TResult:
+            object_exc = completed_task.exception()
+            if object_exc:
+                raise self._resolve_exc_type(object_exc)(alias) from object_exc
+
+            return completed_task.result()
+
+        async def _execute(nexus_input: InputObject) -> TResult:
+            async with nexus_input as instance:
+                result = await nexus_input.process(**kwargs)
+
+            self._cache[instance.cache_key()] = result
+
+            return result
+
+        cached = {
+            reader_or_processor.__class__.alias(): reader_or_processor.data
+            for reader_or_processor in readers_or_processors
+            if reader_or_processor.cache_key() in self._cache
+        }
+        if len(cached) == len(readers_or_processors):
+            return cached
+
+        read_tasks: dict[str, asyncio.Task] = {
+            reader.__class__.alias(): asyncio.create_task(_execute(reader))
+            for reader in readers_or_processors
+            if reader.cache_key() not in self._cache
+        }
+
+        if len(read_tasks) > 0:
+            await asyncio.wait(fs=read_tasks.values())
+
+        return {
+            alias: get_result(alias, task) for alias, task in read_tasks.items()
+        } | cached

esd_services_api_client-2.2.1/esd_services_api_client/nexus/abstractions/input_object.py

@@ -0,0 +1,63 @@
+"""
+ Base class for input reading/processing.
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import base64
+import os
+from abc import ABC, abstractmethod
+
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    TPayload,
+    TResult,
+    NexusObject,
+)
+
+
+class InputObject(NexusObject[TPayload, TResult], ABC):
+    """
+    Base class for input processing and reader objects.
+    """
+
+    async def _context_open(self):
+        """
+        Optional actions to perform on context activation.
+        """
+
+    async def _context_close(self):
+        """
+        Optional actions to perform on context closure.
+        """
+
+    def cache_key(self) -> str:
+        """
+        Unique identifier for this Nexus object, can be used to in-memory or external caching.
+        """
+        return f"{base64.b64encode(hex(id(self)).encode('utf-8')).decode('utf-8')}_{os.getpid()}_{self.__class__.__name__}"
+
+    @property
+    def data(self) -> TResult | None:
+        """
+        Data bound to this object.
+        """
+        return None
+
+    @abstractmethod
+    async def process(self, **kwargs) -> TResult:
+        """
+        Executes input processing logic (read or transform)
+        """

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.1}/esd_services_api_client/nexus/abstractions/nexus_object.py

@@ -1,8 +1,6 @@
 """
  Base classes for all objects used by Nexus.
 """
-import re
-
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,11 +18,13 @@ import re
 
 
 from abc import ABC, abstractmethod
+import re
 from typing import Generic, TypeVar, Union, Any
 
 import pandas
 import polars
 from adapta.metrics import MetricsProvider
+from dataclasses_json.stringcase import snakecase
 
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 
@@ -53,7 +53,7 @@ TResult = TypeVar(  # pylint: disable=C0103
 )
 
 
-class NexusObject(Generic[TPayload, TResult], ABC):
+class NexusCoreObject(ABC):
     """
     Base class for all Nexus objects.
     """
@@ -87,16 +87,24 @@ class NexusObject(Generic[TPayload, TResult], ABC):
         Optional actions to perform on context closure.
         """
 
+
+class NexusObject(Generic[TPayload, TResult], NexusCoreObject, ABC):
+    """
+    Base class for all Nexus objects.
+    """
+
     @classmethod
     def alias(cls) -> str:
         """
         Alias to identify this reader's output
         """
-        return re.sub(
-            r"(?<!^)(?=[A-Z])",
-            "_",
-            cls.__name__.lower()
-            .replace("reader", "")
-            .replace("processor", "")
-            .replace("algorithm", ""),
+        return snakecase(
+            re.sub(
+                r"(?<!^)(?=[A-Z])",
+                "_",
+                cls.__name__.lower()
+                .replace("reader", "")
+                .replace("processor", "")
+                .replace("algorithm", ""),
+            )
         )

{esd_services_api_client-2.1.3 → esd_services_api_client-2.2.1}/esd_services_api_client/nexus/algorithms/_baseline_algorithm.py

@@ -1,7 +1,6 @@
 """
  Base algorithm
 """
-
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,13 +16,13 @@
 #  limitations under the License.
 #
 
-
 from abc import abstractmethod
-from functools import reduce, partial
+from functools import partial
 
 from adapta.metrics import MetricsProvider
 from adapta.utils.decorators import run_time_metrics_async
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.nexus_object import (
     NexusObject,
     TPayload,
@@ -32,7 +31,6 @@ from esd_services_api_client.nexus.abstractions.nexus_object import (
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.input.input_processor import (
     InputProcessor,
-    resolve_processors,
 )
 
 
@@ -46,9 +44,19 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
         *input_processors: InputProcessor,
+        cache: InputCache,
     ):
         super().__init__(metrics_provider, logger_factory)
         self._input_processors = input_processors
+        self._cache = cache
+        self._inputs: dict = {}
+
+    @property
+    def inputs(self) -> dict:
+        """
+        Inputs generated for this algorithm run.
+        """
+        return self._inputs
 
     @abstractmethod
     async def _run(self, **kwargs) -> AlgorithmResult:
@@ -75,11 +83,11 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         async def _measured_run(**run_args):
             return await self._run(**run_args)
 
-        results = await resolve_processors(*self._input_processors, **kwargs)
+        self._inputs = await self._cache.resolve(*self._input_processors, **kwargs)
 
         return await partial(
             _measured_run,
-            **reduce(lambda a, b: a | b, [result for _, result in results.items()]),
+            **self._inputs,
             metric_tags=self._metric_tags,
             metrics_provider=self._metrics_provider,
             logger=self._logger,

esd_services_api_client-2.2.1/esd_services_api_client/nexus/algorithms/_remote_algorithm.py

@@ -0,0 +1,118 @@
+"""
+ Remotely executed algorithm
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+from abc import abstractmethod
+from functools import partial
+
+from adapta.metrics import MetricsProvider
+from adapta.utils.decorators import run_time_metrics_async
+
+from esd_services_api_client.crystal import CrystalConnector, AlgorithmConfiguration
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    NexusObject,
+    TPayload,
+    AlgorithmResult,
+)
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.input.input_processor import (
+    InputProcessor,
+)
+from esd_services_api_client.nexus.input.payload_reader import AlgorithmPayload
+
+
+class RemoteAlgorithm(NexusObject[TPayload, AlgorithmResult]):
+    """
+    Base class for all algorithm implementations.
+    """
+
+    def __init__(
+        self,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        remote_client: CrystalConnector,
+        remote_name: str,
+        remote_config: AlgorithmConfiguration,
+        *input_processors: InputProcessor,
+        cache: InputCache,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._input_processors = input_processors
+        self._remote_client = remote_client
+        self._remote_name = remote_name
+        self._remote_config = remote_config
+        self._cache = cache
+
+    @abstractmethod
+    def _generate_tag(self) -> str:
+        """
+        Generates a submission tag.
+        """
+
+    @abstractmethod
+    def _transform_submission_result(
+        self, request_id: str, tag: str
+    ) -> AlgorithmResult:
+        """
+        Called after submitting a remote run. Use this to enrich your output with remote run id and tag.
+        """
+
+    @abstractmethod
+    async def _run(self, **kwargs) -> AlgorithmPayload:
+        """
+        Core logic for this algorithm. Implementing this method is mandatory.
+        """
+
+    @property
+    def _metric_tags(self) -> dict[str, str]:
+        return {"algorithm": self.__class__.alias()}
+
+    async def run(self, **kwargs) -> AlgorithmResult:
+        """
+        Coroutine that executes the algorithm logic.
+        """
+
+        @run_time_metrics_async(
+            metric_name="algorthm_run",
+            on_finish_message_template="Launched a new remote {algorithm} in {elapsed:.2f}s seconds",
+            template_args={
+                "algorithm": self.__class__.alias().upper(),
+            },
+        )
+        async def _measured_run(**run_args) -> AlgorithmResult:
+            payload = await self._run(**run_args)
+            tag = self._generate_tag()
+            request_id = self._remote_client.create_run(
+                algorithm=self._remote_name,
+                payload=payload.to_dict(),
+                custom_config=self._remote_config,
+                tag=tag,
+            )
+            return self._transform_submission_result(request_id, tag)
+
+        results = await self._cache.resolve(*self._input_processors, **kwargs)
+
+        return await partial(
+            _measured_run,
+            **results,
+            metric_tags=self._metric_tags,
+            metrics_provider=self._metrics_provider,
+            logger=self._logger,
+        )()