esd-services-api-client 2.1.1__py3-none-any.whl → 2.1.2__py3-none-any.whl

esd_services_api_client/_version.py

@@ -1 +1 @@
-__version__ = '2.1.1'
+__version__ = '2.1.2'

esd_services_api_client/nexus/abstractions/nexus_object.py

@@ -95,5 +95,8 @@ class NexusObject(Generic[TPayload, TResult], ABC):
         return re.sub(
             r"(?<!^)(?=[A-Z])",
             "_",
-            cls.__name__.lower().replace("reader", "").replace("processor", ""),
+            cls.__name__.lower()
+            .replace("reader", "")
+            .replace("processor", "")
+            .replace("algorithm", ""),
         )

esd_services_api_client/nexus/abstractions/socket_provider.py

@@ -1,7 +1,6 @@
 """
  Socket provider for all data sockets used by algorithms.
 """
-import json
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -18,10 +17,15 @@ import json
 #  limitations under the License.
 #
 
-from typing import final, Optional
+import json
+from typing import final
 
 from adapta.process_communication import DataSocket
 
+from esd_services_api_client.nexus.exceptions.startup_error import (
+    FatalStartupConfigurationError,
+)
+
 
 @final
 class ExternalSocketProvider:
@@ -32,11 +36,16 @@ class ExternalSocketProvider:
     def __init__(self, *sockets: DataSocket):
         self._sockets = {socket.alias: socket for socket in sockets}
 
-    def socket(self, name: str) -> Optional[DataSocket]:
+    def socket(self, name: str) -> DataSocket:
         """
         Retrieve a socket if it exists.
         """
-        return self._sockets.get(name, None)
+        if name in self._sockets:
+            return self._sockets[name]
+
+        raise FatalStartupConfigurationError(
+            missing_entry=f"socket with alias `{name}`"
+        )
 
     @classmethod
     def from_serialized(cls, socket_list_ser: str) -> "ExternalSocketProvider":

esd_services_api_client/nexus/algorithms/_baseline_algorithm.py

@@ -1,7 +1,6 @@
 """
  Base algorithm
 """
-import asyncio
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -20,18 +19,21 @@ import asyncio
 
 
 from abc import abstractmethod
-from functools import reduce
+from functools import reduce, partial
 
 from adapta.metrics import MetricsProvider
+from adapta.utils.decorators import run_time_metrics_async
 
 from esd_services_api_client.nexus.abstractions.nexus_object import (
     NexusObject,
     TPayload,
-    TResult,
     AlgorithmResult,
 )
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
-from esd_services_api_client.nexus.input.input_processor import InputProcessor
+from esd_services_api_client.nexus.input.input_processor import (
+    InputProcessor,
+    resolve_processors,
+)
 
 
 class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
@@ -54,24 +56,31 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         Core logic for this algorithm. Implementing this method is mandatory.
         """
 
+    @property
+    def _metric_tags(self) -> dict[str, str]:
+        return {"algorithm": self.__class__.alias()}
+
     async def run(self, **kwargs) -> AlgorithmResult:
         """
         Coroutine that executes the algorithm logic.
         """
 
-        async def _process(
-            processor: InputProcessor[TPayload, TResult]
-        ) -> dict[str, TResult]:
-            async with processor as instance:
-                return await instance.process_input(**kwargs)
+        @run_time_metrics_async(
+            metric_name="algorthm_run",
+            on_finish_message_template="Finished running {algorithm} in {elapsed:.2f}s seconds",
+            template_args={
+                "algorithm": self.__class__.alias().upper(),
+            },
+        )
+        async def _measured_run(**run_args):
+            return await self._run(**run_args)
 
-        process_tasks: dict[str, asyncio.Task] = {
-            input_processor.__class__.__name__.lower(): asyncio.create_task(
-                _process(input_processor)
-            )
-            for input_processor in self._input_processors
-        }
-        await asyncio.wait(fs=process_tasks.values())
-        results = [task.result() for task in process_tasks.values()]
+        results = await resolve_processors(*self._input_processors, **kwargs)
 
-        return await self._run(**reduce(lambda a, b: a | b, results))
+        return await partial(
+            _measured_run,
+            **reduce(lambda a, b: a | b, [result for _, result in results.items()]),
+            metric_tags=self._metric_tags,
+            metrics_provider=self._metrics_provider,
+            logger=self._logger,
+        )()

esd_services_api_client/nexus/exceptions/startup_error.py

@@ -1,5 +1,5 @@
 """
- Custom exceptions.
+ App startup exceptions.
 """
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data

esd_services_api_client/nexus/input/_functions.py

@@ -30,6 +30,9 @@ from esd_services_api_client.nexus.exceptions.input_reader_error import (
 from esd_services_api_client.nexus.input.input_reader import InputReader
 
 
+_reader_cache = {}
+
+
 def resolve_reader_exc_type(
     ex: BaseException,
 ) -> Union[Type[FatalInputReaderError], Type[TransientInputReaderError]]:
@@ -61,12 +64,26 @@ async def resolve_readers(
 
     async def _read(input_reader: InputReader):
         async with input_reader as instance:
-            return await instance.read()
+            result = await instance.read()
+            _reader_cache[input_reader.__class__.alias()] = result
+        return result
+
+    cached = {
+        reader.__class__.alias(): reader.data
+        for reader in readers
+        if reader.__class__.alias() in _reader_cache
+    }
+    if len(cached) == len(readers):
+        return cached
 
     read_tasks: dict[str, asyncio.Task] = {
         reader.__class__.alias(): asyncio.create_task(_read(reader))
         for reader in readers
+        if reader.__class__.alias() not in _reader_cache
     }
-    await asyncio.wait(fs=read_tasks.values())
+    if len(read_tasks) > 0:
+        await asyncio.wait(fs=read_tasks.values())
 
-    return {alias: get_result(alias, task) for alias, task in read_tasks.items()}
+    return {
+        alias: get_result(alias, task) for alias, task in read_tasks.items()
+    } | cached

esd_services_api_client/nexus/input/input_processor.py

@@ -1,6 +1,7 @@
 """
  Input processing.
 """
+import asyncio
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -18,8 +19,11 @@
 #
 
 from abc import abstractmethod
+from functools import partial
+from typing import Optional
 
 from adapta.metrics import MetricsProvider
+from adapta.utils.decorators import run_time_metrics_async
 
 from esd_services_api_client.nexus.abstractions.nexus_object import (
     NexusObject,
@@ -27,9 +31,14 @@ from esd_services_api_client.nexus.abstractions.nexus_object import (
     TResult,
 )
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
-from esd_services_api_client.nexus.input._functions import resolve_readers
+from esd_services_api_client.nexus.input._functions import (
+    resolve_readers,
+    resolve_reader_exc_type,
+)
 from esd_services_api_client.nexus.input.input_reader import InputReader
 
+_processor_cache = {}
+
 
 class InputProcessor(NexusObject[TPayload, TResult]):
     """
@@ -46,12 +55,90 @@ class InputProcessor(NexusObject[TPayload, TResult]):
         super().__init__(metrics_provider, logger_factory)
         self._readers = readers
         self._payload = payload
+        self._result: Optional[TResult] = None
 
     async def _read_input(self) -> dict[str, TResult]:
         return await resolve_readers(*self._readers)
 
+    @property
+    def result(self) -> dict[str, TResult]:
+        """
+        Data returned by this processor
+        """
+        return self._result
+
     @abstractmethod
-    async def process_input(self, **kwargs) -> dict[str, TResult]:
+    async def _process_input(self, **kwargs) -> dict[str, TResult]:
         """
         Input processing logic. Implement this method to prepare data for your algorithm code.
         """
+
+    @property
+    def _metric_tags(self) -> dict[str, str]:
+        return {"processor": self.__class__.alias()}
+
+    async def process_input(self, **kwargs) -> dict[str, TResult]:
+        """
+        Input processing coroutine. Do not override this method.
+        """
+
+        @run_time_metrics_async(
+            metric_name="input_process",
+            on_finish_message_template="Finished processing {processor} in {elapsed:.2f}s seconds",
+            template_args={
+                "processor": self.__class__.alias().upper(),
+            },
+        )
+        async def _process(**_) -> dict[str, TResult]:
+            return await self._process_input(**kwargs)
+
+        if self._result is None:
+            self._result = await partial(
+                _process,
+                metric_tags=self._metric_tags,
+                metrics_provider=self._metrics_provider,
+                logger=self._logger,
+            )()
+
+        return self._result
+
+
+async def resolve_processors(
+    *processors: InputProcessor[TPayload, TResult], **kwargs
+) -> dict[str, dict[str, TResult]]:
+    """
+    Concurrently resolve `result` property of all processors by invoking their `process_input` method.
+    """
+
+    def get_result(alias: str, completed_task: asyncio.Task) -> dict[str, TResult]:
+        reader_exc = completed_task.exception()
+        if reader_exc:
+            raise resolve_reader_exc_type(reader_exc)(alias, reader_exc) from reader_exc
+
+        return completed_task.result()
+
+    async def _process(input_processor: InputProcessor):
+        async with input_processor as instance:
+            result = await instance.process_input(**kwargs)
+            _processor_cache[input_processor.__class__.alias()] = result
+        return result
+
+    cached = {
+        processor.__class__.alias(): processor.result
+        for processor in processors
+        if processor.__class__.alias() in _processor_cache
+    }
+    if len(cached) == len(processors):
+        return cached
+
+    process_tasks: dict[str, asyncio.Task] = {
+        processor.__class__.alias(): asyncio.create_task(_process(processor))
+        for processor in processors
+        if processor.__class__.alias() not in _processor_cache
+    }
+    if len(process_tasks) > 0:
+        await asyncio.wait(fs=process_tasks.values())
+
+    return {
+        alias: get_result(alias, task) for alias, task in process_tasks.items()
+    } | cached

esd_services_api_client/nexus/input/input_reader.py

@@ -55,6 +55,13 @@ class InputReader(NexusObject[TPayload, TResult]):
         self._readers = readers
         self._payload = payload
 
+    @property
+    def data(self) -> Optional[TResult]:
+        """
+        Data returned by this reader
+        """
+        return self._data
+
     @abstractmethod
     async def _read_input(self) -> TResult:
         """
@@ -67,11 +74,11 @@ class InputReader(NexusObject[TPayload, TResult]):
 
     async def read(self) -> TResult:
         """
-        Coroutine that reads the data from external store and converts it to a dataframe.
+        Coroutine that reads the data from external store and converts it to a dataframe, or generates data locally. Do not override this method.
         """
 
         @run_time_metrics_async(
-            metric_name="read_input",
+            metric_name="input_read",
             on_finish_message_template="Finished reading {entity} from path {data_path} in {elapsed:.2f}s seconds"
             if self.socket
             else "Finished reading {entity} in {elapsed:.2f}s seconds",

{esd_services_api_client-2.1.1.dist-info → esd_services_api_client-2.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.1.1
+Version: 2.1.2
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0

{esd_services_api_client-2.1.1.dist-info → esd_services_api_client-2.1.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 esd_services_api_client/__init__.py,sha256=L-cEW1mVbnTJLCLG5V6Ucw7zBgx1zf0t1bYcQC1heyw,603
-esd_services_api_client/_version.py,sha256=Aht2295j8FswZ-nPYofCYr3fBZ6Uyf0thTfl5Oc2mWA,22
+esd_services_api_client/_version.py,sha256=m5qImnzcnIhayvILFVqEnXPYsN-vE0vxokygykKhRfw,22
 esd_services_api_client/beast/__init__.py,sha256=zNhXcHSP5w4P9quM1XP4oXVJEccvC_VScG41TZ0GzZ8,723
 esd_services_api_client/beast/v3/__init__.py,sha256=FtumtInoDyCCRE424Llqv8QZLRuwXzj-smyfu1od1nc,754
 esd_services_api_client/beast/v3/_connector.py,sha256=WNmCiTXFRb3q56mrr7ZbqBHWDUxbfyWhiWlBFLUIOnc,11478
@@ -19,10 +19,10 @@ esd_services_api_client/nexus/README.md,sha256=Q3laWkuCxqVBFsfP58mGAys-jVZeesroI
 esd_services_api_client/nexus/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
 esd_services_api_client/nexus/abstractions/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
 esd_services_api_client/nexus/abstractions/logger_factory.py,sha256=9biONvCqNrP__yrmeRkoDL05TMA5v-LyrcKwgiKG59U,2019
-esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=h6mxolrfBF1BEz4FH2HRqiTcLU8oAteIVUgYAsoxI6U,2764
-esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=pF7vWZNNE26B9ftNbAUeSEvYIE2JBIrvpzAhznr1_q8,1500
+esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=Be90h8iVDp2o1recnx8-Ousm3OjaVjIjthPqE5u5vik,2828
+esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=Rwa_aPErI4Es5AdyCd3EoGze7mg2D70u8kuc2UGEBaI,1729
 esd_services_api_client/nexus/algorithms/__init__.py,sha256=yMvLFSqg5eUKOXI0zMFX69Ni0ibKQHOqAnrZsxQqhOo,903
-esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=dvoIcDj6c1AZW5_THrTS2qKWylRfTM89laIypijzUhU,2441
+esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=UxqGFzpl-8w4SLJX0GMxLzRinRnFVoBT2zBKzHOapBM,2694
 esd_services_api_client/nexus/algorithms/distributed.py,sha256=vkKSCsd480RKwrtu3uZ2iU1bh593fkgBcOBrcb9cLjA,1702
 esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=PjLs_xhpd3rTViaLbWUEvJ1LWDPTKEZFj57iPk8wswo,1462
 esd_services_api_client/nexus/algorithms/recursive.py,sha256=MAyfj3gQZ0zQB5hU3RHIOfH889v_Wtp4GHBhB-trr2w,1862
@@ -34,13 +34,13 @@ esd_services_api_client/nexus/core/app_dependencies.py,sha256=GChNBRE-9CGqZ41xQO
 esd_services_api_client/nexus/exceptions/__init__.py,sha256=feN33VdqB5-2bD9aJesJl_OlsKrNNo3hZCnQgKuaU9k,696
 esd_services_api_client/nexus/exceptions/_nexus_error.py,sha256=QvtY38mNoIA6t26dUN6UIsaPfljhtVNsbQVS7ksMb-Q,895
 esd_services_api_client/nexus/exceptions/input_reader_error.py,sha256=Chy8XW6Ien4-bkZZ1CmP8CWU49mi2hobS6L_R59ONs8,1765
-esd_services_api_client/nexus/exceptions/startup_error.py,sha256=TJRRwDff3wQP5RkZPEfgqguuiC9q1PAqoHX6UiJslxQ,1551
+esd_services_api_client/nexus/exceptions/startup_error.py,sha256=HPTlrg2voRbGXZsakWcVT7lcGSaccSR-GsBIo08SJ4A,1556
 esd_services_api_client/nexus/input/__init__.py,sha256=ahJ7yUuRTCNIYB5GitBRGAPZZX82ghISzF6lm_CkyEA,819
-esd_services_api_client/nexus/input/_functions.py,sha256=HeTu3JcUPudfWrfZRmNAaejVb6oPb8gZufEEq56eT-4,2485
-esd_services_api_client/nexus/input/input_processor.py,sha256=kNTmjZkUMbrb-8mpVyhIUbqKKwZaO0fOGoG5EQOLL_Q,1808
-esd_services_api_client/nexus/input/input_reader.py,sha256=tC7r5dXnovMtSOFos3Bv0tYTNyFOVNCvHrdAQXcVLTM,3015
+esd_services_api_client/nexus/input/_functions.py,sha256=9bySbwcXGV7JIsQbSqg3YuW7avJUErsvsS2ZDHRWfT0,2918
+esd_services_api_client/nexus/input/input_processor.py,sha256=ZUdpz2M6PfsF-99-DpCMDtD9Id0SQYhC-HbjosMz8O8,4609
+esd_services_api_client/nexus/input/input_reader.py,sha256=MtZNdxjWLjoSqlSlwa6f7BIqsu-_GoQav5tCz2g6WrA,3214
 esd_services_api_client/nexus/input/payload_reader.py,sha256=Kq0xN1Shyqv71v6YkcrqVTDbmsEjZc8ithsXYpyu87M,2516
-esd_services_api_client-2.1.1.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
-esd_services_api_client-2.1.1.dist-info/METADATA,sha256=oS20SFp1_sFJLcsyW5bqrcRdPbARdNlpfzA2JC13ja0,1292
-esd_services_api_client-2.1.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-esd_services_api_client-2.1.1.dist-info/RECORD,,
+esd_services_api_client-2.1.2.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
+esd_services_api_client-2.1.2.dist-info/METADATA,sha256=xr1Dw96DI1t-qKxCb1aq48Dyxt-co-BzwRePv-Jgoz0,1292
+esd_services_api_client-2.1.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+esd_services_api_client-2.1.2.dist-info/RECORD,,