esd-services-api-client 2.1.1__tar.gz → 2.1.2__tar.gz

{esd_services_api_client-2.1.1 → esd_services_api_client-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.1.1
+Version: 2.1.2
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0

esd_services_api_client-2.1.2/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = '2.1.2'

{esd_services_api_client-2.1.1 → esd_services_api_client-2.1.2}/esd_services_api_client/nexus/abstractions/nexus_object.py

@@ -95,5 +95,8 @@ class NexusObject(Generic[TPayload, TResult], ABC):
         return re.sub(
             r"(?<!^)(?=[A-Z])",
             "_",
-            cls.__name__.lower().replace("reader", "").replace("processor", ""),
+            cls.__name__.lower()
+            .replace("reader", "")
+            .replace("processor", "")
+            .replace("algorithm", ""),
         )

{esd_services_api_client-2.1.1 → esd_services_api_client-2.1.2}/esd_services_api_client/nexus/abstractions/socket_provider.py

@@ -1,7 +1,6 @@
 """
  Socket provider for all data sockets used by algorithms.
 """
-import json
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -18,10 +17,15 @@ import json
 #  limitations under the License.
 #
 
-from typing import final, Optional
+import json
+from typing import final
 
 from adapta.process_communication import DataSocket
 
+from esd_services_api_client.nexus.exceptions.startup_error import (
+    FatalStartupConfigurationError,
+)
+
 
 @final
 class ExternalSocketProvider:
@@ -32,11 +36,16 @@ class ExternalSocketProvider:
     def __init__(self, *sockets: DataSocket):
         self._sockets = {socket.alias: socket for socket in sockets}
 
-    def socket(self, name: str) -> Optional[DataSocket]:
+    def socket(self, name: str) -> DataSocket:
         """
         Retrieve a socket if it exists.
         """
-        return self._sockets.get(name, None)
+        if name in self._sockets:
+            return self._sockets[name]
+
+        raise FatalStartupConfigurationError(
+            missing_entry=f"socket with alias `{name}`"
+        )
 
     @classmethod
     def from_serialized(cls, socket_list_ser: str) -> "ExternalSocketProvider":

{esd_services_api_client-2.1.1 → esd_services_api_client-2.1.2}/esd_services_api_client/nexus/algorithms/_baseline_algorithm.py

@@ -1,7 +1,6 @@
 """
  Base algorithm
 """
-import asyncio
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -20,18 +19,21 @@ import asyncio
 
 
 from abc import abstractmethod
-from functools import reduce
+from functools import reduce, partial
 
 from adapta.metrics import MetricsProvider
+from adapta.utils.decorators import run_time_metrics_async
 
 from esd_services_api_client.nexus.abstractions.nexus_object import (
     NexusObject,
     TPayload,
-    TResult,
     AlgorithmResult,
 )
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
-from esd_services_api_client.nexus.input.input_processor import InputProcessor
+from esd_services_api_client.nexus.input.input_processor import (
+    InputProcessor,
+    resolve_processors,
+)
 
 
 class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
@@ -54,24 +56,31 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         Core logic for this algorithm. Implementing this method is mandatory.
         """
 
+    @property
+    def _metric_tags(self) -> dict[str, str]:
+        return {"algorithm": self.__class__.alias()}
+
     async def run(self, **kwargs) -> AlgorithmResult:
         """
         Coroutine that executes the algorithm logic.
         """
 
-        async def _process(
-            processor: InputProcessor[TPayload, TResult]
-        ) -> dict[str, TResult]:
-            async with processor as instance:
-                return await instance.process_input(**kwargs)
+        @run_time_metrics_async(
+            metric_name="algorthm_run",
+            on_finish_message_template="Finished running {algorithm} in {elapsed:.2f}s seconds",
+            template_args={
+                "algorithm": self.__class__.alias().upper(),
+            },
+        )
+        async def _measured_run(**run_args):
+            return await self._run(**run_args)
 
-        process_tasks: dict[str, asyncio.Task] = {
-            input_processor.__class__.__name__.lower(): asyncio.create_task(
-                _process(input_processor)
-            )
-            for input_processor in self._input_processors
-        }
-        await asyncio.wait(fs=process_tasks.values())
-        results = [task.result() for task in process_tasks.values()]
+        results = await resolve_processors(*self._input_processors, **kwargs)
 
-        return await self._run(**reduce(lambda a, b: a | b, results))
+        return await partial(
+            _measured_run,
+            **reduce(lambda a, b: a | b, [result for _, result in results.items()]),
+            metric_tags=self._metric_tags,
+            metrics_provider=self._metrics_provider,
+            logger=self._logger,
+        )()

{esd_services_api_client-2.1.1 → esd_services_api_client-2.1.2}/esd_services_api_client/nexus/exceptions/startup_error.py

@@ -1,5 +1,5 @@
 """
- Custom exceptions.
+ App startup exceptions.
 """
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data

{esd_services_api_client-2.1.1 → esd_services_api_client-2.1.2}/esd_services_api_client/nexus/input/_functions.py

@@ -30,6 +30,9 @@ from esd_services_api_client.nexus.exceptions.input_reader_error import (
 from esd_services_api_client.nexus.input.input_reader import InputReader
 
 
+_reader_cache = {}
+
+
 def resolve_reader_exc_type(
     ex: BaseException,
 ) -> Union[Type[FatalInputReaderError], Type[TransientInputReaderError]]:
@@ -61,12 +64,26 @@ async def resolve_readers(
 
     async def _read(input_reader: InputReader):
         async with input_reader as instance:
-            return await instance.read()
+            result = await instance.read()
+            _reader_cache[input_reader.__class__.alias()] = result
+        return result
+
+    cached = {
+        reader.__class__.alias(): reader.data
+        for reader in readers
+        if reader.__class__.alias() in _reader_cache
+    }
+    if len(cached) == len(readers):
+        return cached
 
     read_tasks: dict[str, asyncio.Task] = {
         reader.__class__.alias(): asyncio.create_task(_read(reader))
         for reader in readers
+        if reader.__class__.alias() not in _reader_cache
     }
-    await asyncio.wait(fs=read_tasks.values())
+    if len(read_tasks) > 0:
+        await asyncio.wait(fs=read_tasks.values())
 
-    return {alias: get_result(alias, task) for alias, task in read_tasks.items()}
+    return {
+        alias: get_result(alias, task) for alias, task in read_tasks.items()
+    } | cached

esd_services_api_client-2.1.2/esd_services_api_client/nexus/input/input_processor.py

@@ -0,0 +1,144 @@
+"""
+ Input processing.
+"""
+import asyncio
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from abc import abstractmethod
+from functools import partial
+from typing import Optional
+
+from adapta.metrics import MetricsProvider
+from adapta.utils.decorators import run_time_metrics_async
+
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    NexusObject,
+    TPayload,
+    TResult,
+)
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.input._functions import (
+    resolve_readers,
+    resolve_reader_exc_type,
+)
+from esd_services_api_client.nexus.input.input_reader import InputReader
+
+_processor_cache = {}
+
+
+class InputProcessor(NexusObject[TPayload, TResult]):
+    """
+    Base class for raw data processing into algorithm input.
+    """
+
+    def __init__(
+        self,
+        *readers: InputReader,
+        payload: TPayload,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._readers = readers
+        self._payload = payload
+        self._result: Optional[TResult] = None
+
+    async def _read_input(self) -> dict[str, TResult]:
+        return await resolve_readers(*self._readers)
+
+    @property
+    def result(self) -> dict[str, TResult]:
+        """
+        Data returned by this processor
+        """
+        return self._result
+
+    @abstractmethod
+    async def _process_input(self, **kwargs) -> dict[str, TResult]:
+        """
+        Input processing logic. Implement this method to prepare data for your algorithm code.
+        """
+
+    @property
+    def _metric_tags(self) -> dict[str, str]:
+        return {"processor": self.__class__.alias()}
+
+    async def process_input(self, **kwargs) -> dict[str, TResult]:
+        """
+        Input processing coroutine. Do not override this method.
+        """
+
+        @run_time_metrics_async(
+            metric_name="input_process",
+            on_finish_message_template="Finished processing {processor} in {elapsed:.2f}s seconds",
+            template_args={
+                "processor": self.__class__.alias().upper(),
+            },
+        )
+        async def _process(**_) -> dict[str, TResult]:
+            return await self._process_input(**kwargs)
+
+        if self._result is None:
+            self._result = await partial(
+                _process,
+                metric_tags=self._metric_tags,
+                metrics_provider=self._metrics_provider,
+                logger=self._logger,
+            )()
+
+        return self._result
+
+
+async def resolve_processors(
+    *processors: InputProcessor[TPayload, TResult], **kwargs
+) -> dict[str, dict[str, TResult]]:
+    """
+    Concurrently resolve `result` property of all processors by invoking their `process_input` method.
+    """
+
+    def get_result(alias: str, completed_task: asyncio.Task) -> dict[str, TResult]:
+        reader_exc = completed_task.exception()
+        if reader_exc:
+            raise resolve_reader_exc_type(reader_exc)(alias, reader_exc) from reader_exc
+
+        return completed_task.result()
+
+    async def _process(input_processor: InputProcessor):
+        async with input_processor as instance:
+            result = await instance.process_input(**kwargs)
+            _processor_cache[input_processor.__class__.alias()] = result
+        return result
+
+    cached = {
+        processor.__class__.alias(): processor.result
+        for processor in processors
+        if processor.__class__.alias() in _processor_cache
+    }
+    if len(cached) == len(processors):
+        return cached
+
+    process_tasks: dict[str, asyncio.Task] = {
+        processor.__class__.alias(): asyncio.create_task(_process(processor))
+        for processor in processors
+        if processor.__class__.alias() not in _processor_cache
+    }
+    if len(process_tasks) > 0:
+        await asyncio.wait(fs=process_tasks.values())
+
+    return {
+        alias: get_result(alias, task) for alias, task in process_tasks.items()
+    } | cached

{esd_services_api_client-2.1.1 → esd_services_api_client-2.1.2}/esd_services_api_client/nexus/input/input_reader.py

@@ -55,6 +55,13 @@ class InputReader(NexusObject[TPayload, TResult]):
         self._readers = readers
         self._payload = payload
 
+    @property
+    def data(self) -> Optional[TResult]:
+        """
+        Data returned by this reader
+        """
+        return self._data
+
     @abstractmethod
     async def _read_input(self) -> TResult:
         """
@@ -67,11 +74,11 @@ class InputReader(NexusObject[TPayload, TResult]):
 
     async def read(self) -> TResult:
         """
-        Coroutine that reads the data from external store and converts it to a dataframe.
+        Coroutine that reads the data from external store and converts it to a dataframe, or generates data locally. Do not override this method.
         """
 
         @run_time_metrics_async(
-            metric_name="read_input",
+            metric_name="input_read",
             on_finish_message_template="Finished reading {entity} from path {data_path} in {elapsed:.2f}s seconds"
             if self.socket
             else "Finished reading {entity} in {elapsed:.2f}s seconds",

{esd_services_api_client-2.1.1 → esd_services_api_client-2.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "esd-services-api-client"
-version = "2.1.1"
+version = "2.1.2"
 description = "Python clients for ESD services"
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']

esd_services_api_client-2.1.1/esd_services_api_client/_version.py

@@ -1 +0,0 @@
-__version__ = '2.1.1'

esd_services_api_client-2.1.1/esd_services_api_client/nexus/input/input_processor.py

@@ -1,57 +0,0 @@
-"""
- Input processing.
-"""
-
-#  Copyright (c) 2023-2024. ECCO Sneaks & Data
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-from abc import abstractmethod
-
-from adapta.metrics import MetricsProvider
-
-from esd_services_api_client.nexus.abstractions.nexus_object import (
-    NexusObject,
-    TPayload,
-    TResult,
-)
-from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
-from esd_services_api_client.nexus.input._functions import resolve_readers
-from esd_services_api_client.nexus.input.input_reader import InputReader
-
-
-class InputProcessor(NexusObject[TPayload, TResult]):
-    """
-    Base class for raw data processing into algorithm input.
-    """
-
-    def __init__(
-        self,
-        *readers: InputReader,
-        payload: TPayload,
-        metrics_provider: MetricsProvider,
-        logger_factory: LoggerFactory,
-    ):
-        super().__init__(metrics_provider, logger_factory)
-        self._readers = readers
-        self._payload = payload
-
-    async def _read_input(self) -> dict[str, TResult]:
-        return await resolve_readers(*self._readers)
-
-    @abstractmethod
-    async def process_input(self, **kwargs) -> dict[str, TResult]:
-        """
-        Input processing logic. Implement this method to prepare data for your algorithm code.
-        """