esd-services-api-client 2.1.3__py3-none-any.whl → 2.2.1__py3-none-any.whl

esd_services_api_client/nexus/algorithms/forked_algorithm.py

@@ -0,0 +1,124 @@
+"""
+ Remotely executed algorithm
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import asyncio
+from abc import abstractmethod
+from functools import partial
+
+from adapta.metrics import MetricsProvider
+from adapta.utils.decorators import run_time_metrics_async
+
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    NexusObject,
+    TPayload,
+    AlgorithmResult,
+)
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.algorithms._remote_algorithm import RemoteAlgorithm
+from esd_services_api_client.nexus.input.input_processor import (
+    InputProcessor,
+)
+
+
+class ForkedAlgorithm(NexusObject[TPayload, AlgorithmResult]):
+    """
+    Forked algorithm is an algorithm that returns a result (main scenario run) and then fires off one or more forked runs
+    with different configurations as specified in fork class implementation.
+
+    Forked algorithm only awaits scheduling of forked runs, but never their results.
+
+     Q: How do I spawn a ForkedAlgorithm run as a remote algorithm w/o ending in an infinite loop?
+     A: Provide class names for forks from your algorithm configuration and construct forks with locate(fork_class)(**kwargs) calls.
+
+     Q: Can I build execution trees with this?
+     A: Yes, they will look like this (F(N) - Forked with N forks):
+
+     graph TB
+        F3["F(3)"] --> F2["F(2)"]
+        F3 --> F0["F(0)"]
+        F3 --> F1["F(1)"]
+        F2 --> F1_1["F(1)"]
+        F2 --> F0_1["F(0)"]
+        F1 --> F0_2["F(0)"]
+        F1_1 --> F0_3["F(0)"]
+    """
+
+    def __init__(
+        self,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        forks: list[RemoteAlgorithm],
+        *input_processors: InputProcessor,
+        cache: InputCache,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._input_processors = input_processors
+        self._forks = forks
+        self._cache = cache
+
+    @abstractmethod
+    async def _run(self, **kwargs) -> AlgorithmResult:
+        """
+        Core logic for this algorithm. Implementing this method is mandatory.
+        """
+
+    @property
+    def _metric_tags(self) -> dict[str, str]:
+        return {"algorithm": self.__class__.alias()}
+
+    async def run(self, **kwargs) -> AlgorithmResult:
+        """
+        Coroutine that executes the algorithm logic.
+        """
+
+        @run_time_metrics_async(
+            metric_name="algorthm_run",
+            on_finish_message_template="Finished running algorithm {algorithm} in {elapsed:.2f}s seconds",
+            template_args={
+                "algorithm": self.__class__.alias().upper(),
+            },
+        )
+        async def _measured_run(**run_args) -> AlgorithmResult:
+            return await self._run(**run_args)
+
+        if len(self._forks) > 0:
+            self._logger.info(
+                "This algorithm has forks attached: {forks}. They will be executed after the main run",
+                forks=",".join([fork.alias() for fork in self._forks]),
+            )
+        else:
+            self._logger.info(
+                "This algorithm supports forks but none were injected. Proceeding with a main run only"
+            )
+
+        results = await self._cache.resolve(*self._input_processors, **kwargs)
+
+        run_result = await partial(
+            _measured_run,
+            **results,
+            metric_tags=self._metric_tags,
+            metrics_provider=self._metrics_provider,
+            logger=self._logger,
+        )()
+
+        # now await callback scheduling
+        await asyncio.wait([fork.run(**kwargs) for fork in self._forks])
+
+        return run_result

esd_services_api_client/nexus/algorithms/minimalistic.py

@@ -22,6 +22,7 @@ from abc import ABC
 from adapta.metrics import MetricsProvider
 from injector import inject
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.abstractions.nexus_object import TPayload
 from esd_services_api_client.nexus.algorithms._baseline_algorithm import (
@@ -41,5 +42,11 @@ class MinimalisticAlgorithm(BaselineAlgorithm[TPayload], ABC):
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
         *input_processors: InputProcessor,
+        cache: InputCache,
     ):
-        super().__init__(metrics_provider, logger_factory, *input_processors)
+        super().__init__(
+            metrics_provider,
+            logger_factory,
+            *input_processors,
+            cache=cache,
+        )

esd_services_api_client/nexus/algorithms/recursive.py

@@ -23,6 +23,7 @@ from abc import abstractmethod
 from adapta.metrics import MetricsProvider
 from injector import inject
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.abstractions.nexus_object import (
     TPayload,
@@ -45,8 +46,11 @@ class RecursiveAlgorithm(BaselineAlgorithm[TPayload]):
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
         *input_processors: InputProcessor,
+        cache: InputCache,
     ):
-        super().__init__(metrics_provider, logger_factory, *input_processors)
+        super().__init__(
+            metrics_provider, logger_factory, *input_processors, cache=cache
+        )
 
     @abstractmethod
     async def _is_finished(self, **kwargs) -> bool:

esd_services_api_client/nexus/core/app_core.py

@@ -58,6 +58,7 @@ from esd_services_api_client.nexus.input.payload_reader import (
     AlgorithmPayloadReader,
     AlgorithmPayload,
 )
+from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
 
 
 def is_transient_exception(exception: Optional[BaseException]) -> Optional[bool]:
@@ -244,9 +245,11 @@ class Nexus:
         """
         Activates the run sequence.
         """
+
         self._injector = Injector(self._configurator.injection_binds)
 
         algorithm: BaselineAlgorithm = self._injector.get(self._algorithm_class)
+        telemetry_recorder: TelemetryRecorder = self._injector.get(TelemetryRecorder)
 
         async with algorithm as instance:
             self._algorithm_run_task = asyncio.create_task(
@@ -266,6 +269,12 @@ class Nexus:
             if len(on_complete_tasks) > 0:
                 await asyncio.wait(on_complete_tasks)
 
+            # record telemetry
+            async with telemetry_recorder as recorder:
+                await recorder.record(
+                    run_id=self._run_args.request_id, **algorithm.inputs
+                )
+
             # dispose of QES instance gracefully as it might hold open connections
             qes = self._injector.get(QueryEnabledStore)
             qes.close()

esd_services_api_client/nexus/core/app_dependencies.py

@@ -28,6 +28,7 @@ from adapta.storage.query_enabled_store import QueryEnabledStore
 from injector import Module, singleton, provider
 
 from esd_services_api_client.crystal import CrystalConnector
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.abstractions.socket_provider import (
     ExternalSocketProvider,
@@ -43,6 +44,7 @@ from esd_services_api_client.nexus.input.input_reader import InputReader
 from esd_services_api_client.nexus.input.payload_reader import (
     AlgorithmPayload,
 )
+from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
 
 
 @final
@@ -165,6 +167,21 @@ class ExternalSocketsModule(Module):
         )
 
 
+@final
+class CacheModule(Module):
+    """
+    Storage client module.
+    """
+
+    @singleton
+    @provider
+    def provide(self) -> InputCache:
+        """
+        Dependency provider.
+        """
+        return InputCache()
+
+
 @final
 class ServiceConfigurator:
     """
@@ -178,6 +195,8 @@ class ServiceConfigurator:
             QueryEnabledStoreModule(),
             StorageClientModule(),
             ExternalSocketsModule(),
+            CacheModule(),
+            type(f"{TelemetryRecorder.__name__}Module", (Module,), {})(),
         ]
 
     @property

esd_services_api_client/nexus/exceptions/cache_errors.py

@@ -0,0 +1,49 @@
+"""
+ Cache module exceptions.
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from esd_services_api_client.nexus.exceptions._nexus_error import (
+    FatalNexusError,
+    TransientNexusError,
+)
+
+
+class FatalCachingError(FatalNexusError):
+    """
+    Cache-level exception that shuts down the Nexus.
+    """
+
+    def __init__(self, failed_object: str):
+        super().__init__()
+        self._failed_object = failed_object
+
+    def __str__(self) -> str:
+        return f"Nexus object with alias '{self._failed_object}' failed the caching operation that cannot be retried. Review traceback for more information"
+
+
+class TransientCachingError(TransientNexusError):
+    """
+    Cache-level exception that will initiate a retry with the Nexus re-activation.
+    """
+
+    def __init__(self, failed_object):
+        super().__init__()
+        self._failed_object = failed_object
+
+    def __str__(self) -> str:
+        return f"Nexus object with alias '{self._failed_object}' failed the caching operation and it will be retried. Review traceback for more information"

esd_services_api_client/nexus/exceptions/startup_error.py

@@ -1,6 +1,7 @@
 """
  App startup exceptions.
 """
+from typing import Type
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -46,3 +47,17 @@ class FatalStartupConfigurationError(FatalNexusError):
 
     def __str__(self) -> str:
         return f"Algorithm initialization failed due to a missing configuration entry: {self._missing_entry}."
+
+
+class FatalAlgorithmConfigurationError(FatalNexusError):
+    """
+    Service configuration error that shuts down the Nexus.
+    """
+
+    def __init__(self, message: str, algorithm_class: Type):
+        super().__init__()
+        self._message = message
+        self._type_name = str(algorithm_class)
+
+    def __str__(self) -> str:
+        return f"Algorithm {self._type_name} misconfigured: {self._message}."

esd_services_api_client/nexus/input/__init__.py

@@ -20,4 +20,3 @@
 
 from esd_services_api_client.nexus.input.input_processor import *
 from esd_services_api_client.nexus.input.input_reader import *
-from esd_services_api_client.nexus.input._functions import *

esd_services_api_client/nexus/input/input_processor.py

@@ -1,7 +1,6 @@
 """
  Input processing.
 """
-import asyncio
 
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
@@ -25,22 +24,17 @@ from typing import Optional
 from adapta.metrics import MetricsProvider
 from adapta.utils.decorators import run_time_metrics_async
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
+from esd_services_api_client.nexus.abstractions.input_object import InputObject
 from esd_services_api_client.nexus.abstractions.nexus_object import (
-    NexusObject,
     TPayload,
     TResult,
 )
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
-from esd_services_api_client.nexus.input._functions import (
-    resolve_readers,
-    resolve_reader_exc_type,
-)
 from esd_services_api_client.nexus.input.input_reader import InputReader
 
-_processor_cache = {}
-
 
-class InputProcessor(NexusObject[TPayload, TResult]):
+class InputProcessor(InputObject[TPayload, TResult]):
     """
     Base class for raw data processing into algorithm input.
     """
@@ -51,24 +45,23 @@ class InputProcessor(NexusObject[TPayload, TResult]):
         payload: TPayload,
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
+        cache: InputCache
     ):
         super().__init__(metrics_provider, logger_factory)
         self._readers = readers
         self._payload = payload
         self._result: Optional[TResult] = None
-
-    async def _read_input(self) -> dict[str, TResult]:
-        return await resolve_readers(*self._readers)
+        self._cache = cache
 
     @property
-    def result(self) -> dict[str, TResult]:
+    def data(self) -> Optional[TResult]:
         """
         Data returned by this processor
         """
         return self._result
 
     @abstractmethod
-    async def _process_input(self, **kwargs) -> dict[str, TResult]:
+    async def _process_input(self, **kwargs) -> TResult:
         """
         Input processing logic. Implement this method to prepare data for your algorithm code.
         """
@@ -77,7 +70,7 @@ class InputProcessor(NexusObject[TPayload, TResult]):
     def _metric_tags(self) -> dict[str, str]:
         return {"processor": self.__class__.alias()}
 
-    async def process_input(self, **kwargs) -> dict[str, TResult]:
+    async def process(self, **kwargs) -> TResult:
         """
         Input processing coroutine. Do not override this method.
         """
@@ -89,8 +82,9 @@ class InputProcessor(NexusObject[TPayload, TResult]):
                 "processor": self.__class__.alias().upper(),
             },
         )
-        async def _process(**_) -> dict[str, TResult]:
-            return await self._process_input(**kwargs)
+        async def _process(**_) -> TResult:
+            readers = await self._cache.resolve(*self._readers)
+            return await self._process_input(**(kwargs | readers))
 
         if self._result is None:
             self._result = await partial(
@@ -101,44 +95,3 @@ class InputProcessor(NexusObject[TPayload, TResult]):
             )()
 
         return self._result
-
-
-async def resolve_processors(
-    *processors: InputProcessor[TPayload, TResult], **kwargs
-) -> dict[str, dict[str, TResult]]:
-    """
-    Concurrently resolve `result` property of all processors by invoking their `process_input` method.
-    """
-
-    def get_result(alias: str, completed_task: asyncio.Task) -> dict[str, TResult]:
-        reader_exc = completed_task.exception()
-        if reader_exc:
-            raise resolve_reader_exc_type(reader_exc)(alias, reader_exc) from reader_exc
-
-        return completed_task.result()
-
-    async def _process(input_processor: InputProcessor):
-        async with input_processor as instance:
-            result = await instance.process_input(**kwargs)
-            _processor_cache[input_processor.__class__.alias()] = result
-        return result
-
-    cached = {
-        processor.__class__.alias(): processor.result
-        for processor in processors
-        if processor.__class__.alias() in _processor_cache
-    }
-    if len(cached) == len(processors):
-        return cached
-
-    process_tasks: dict[str, asyncio.Task] = {
-        processor.__class__.alias(): asyncio.create_task(_process(processor))
-        for processor in processors
-        if processor.__class__.alias() not in _processor_cache
-    }
-    if len(process_tasks) > 0:
-        await asyncio.wait(fs=process_tasks.values())
-
-    return {
-        alias: get_result(alias, task) for alias, task in process_tasks.items()
-    } | cached

esd_services_api_client/nexus/input/input_reader.py

@@ -26,15 +26,16 @@ from adapta.process_communication import DataSocket
 from adapta.storage.query_enabled_store import QueryEnabledStore
 from adapta.utils.decorators import run_time_metrics_async
 
+from esd_services_api_client.nexus.abstractions.algrorithm_cache import InputCache
+from esd_services_api_client.nexus.abstractions.input_object import InputObject
 from esd_services_api_client.nexus.abstractions.nexus_object import (
-    NexusObject,
     TPayload,
     TResult,
 )
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 
 
-class InputReader(NexusObject[TPayload, TResult]):
+class InputReader(InputObject[TPayload, TResult]):
     """
     Base class for a raw data reader.
     """
@@ -47,6 +48,7 @@ class InputReader(NexusObject[TPayload, TResult]):
         payload: TPayload,
         *readers: "InputReader",
         socket: Optional[DataSocket] = None,
+        cache: InputCache
     ):
         super().__init__(metrics_provider, logger_factory)
         self.socket = socket
@@ -54,6 +56,7 @@ class InputReader(NexusObject[TPayload, TResult]):
         self._data: Optional[TResult] = None
         self._readers = readers
         self._payload = payload
+        self._cache = cache
 
     @property
     def data(self) -> Optional[TResult]:
@@ -63,7 +66,7 @@ class InputReader(NexusObject[TPayload, TResult]):
         return self._data
 
     @abstractmethod
-    async def _read_input(self) -> TResult:
+    async def _read_input(self, **kwargs) -> TResult:
         """
         Actual data reader logic. Implementing this method is mandatory for the reader to work
         """
@@ -72,7 +75,7 @@ class InputReader(NexusObject[TPayload, TResult]):
     def _metric_tags(self) -> dict[str, str]:
         return {"entity": self.__class__.alias()}
 
-    async def read(self) -> TResult:
+    async def process(self, **_) -> TResult:
         """
         Coroutine that reads the data from external store and converts it to a dataframe, or generates data locally. Do not override this method.
         """
@@ -88,7 +91,8 @@ class InputReader(NexusObject[TPayload, TResult]):
             | ({"data_path": self.socket.data_path} if self.socket else {}),
         )
         async def _read(**_) -> TResult:
-            return await self._read_input()
+            readers = await self._cache.resolve(*self._readers)
+            return await self._read_input(**readers)
 
         if self._data is None:
             self._data = await partial(

esd_services_api_client/nexus/telemetry/recorder.py

@@ -0,0 +1,97 @@
+"""
+ Telemetry recording module.
+"""
+import asyncio
+import os
+from functools import partial
+from typing import final
+
+import pandas as pd
+from adapta.metrics import MetricsProvider
+from adapta.process_communication import DataSocket
+from adapta.storage.blob.base import StorageClient
+from adapta.storage.models.format import (
+    DictJsonSerializationFormat,
+    DataFrameParquetSerializationFormat,
+)
+from injector import inject, singleton
+
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.abstractions.nexus_object import NexusCoreObject
+
+
+@final
+@singleton
+class TelemetryRecorder(NexusCoreObject):
+    """
+    Class for instantiating a telemetry recorder that will save all algorithm inputs (run method arguments) to a persistent location.
+    """
+
+    async def _context_open(self):
+        pass
+
+    async def _context_close(self):
+        pass
+
+    @inject
+    def __init__(
+        self,
+        storage_client: StorageClient,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._storage_client = storage_client
+        self._telemetry_base_path = os.getenv("NEXUS__TELEMETRY_PATH")
+
+    async def record(self, run_id: str, **telemetry_args):
+        """
+        Record all data in telemetry args for the provided run_id.
+        """
+
+        async def _record(
+            entity_to_record: pd.DataFrame | dict,
+            entity_name: str,
+            **_,
+        ) -> None:
+            self._logger.debug(
+                "Recording telemetry for {entity_name} in the run {run_id}",
+                entity_name=entity_name,
+                run_id=run_id,
+            )
+            self._storage_client.save_data_as_blob(
+                data=entity_to_record,
+                blob_path=DataSocket(
+                    alias="telemetry",
+                    data_path=f"{self._telemetry_base_path}/{entity_name}/{run_id}",
+                    data_format="null",
+                ).parse_data_path(),
+                serialization_format=DictJsonSerializationFormat
+                if isinstance(entity_to_record, dict)
+                else DataFrameParquetSerializationFormat,
+                overwrite=True,
+            )
+
+        telemetry_tasks = [
+            asyncio.create_task(
+                partial(
+                    _record,
+                    entity_to_record=telemetry_value,
+                    entity_name=telemetry_key,
+                    run_id=run_id,
+                )()
+            )
+            for telemetry_key, telemetry_value in telemetry_args.items()
+        ]
+
+        done, pending = await asyncio.wait(telemetry_tasks)
+        if len(pending) > 0:
+            self._logger.warning(
+                "Some telemetry recording operations did not complete within specified time. This run might lack observability coverage."
+            )
+        for done_telemetry_task in done:
+            telemetry_exc = done_telemetry_task.exception()
+            if telemetry_exc:
+                self._logger.warning(
+                    "Telemetry recoding failed", exception=telemetry_exc
+                )

{esd_services_api_client-2.1.3.dist-info → esd_services_api_client-2.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.1.3
+Version: 2.2.1
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0