esd-services-api-client 2.2.0__py3-none-any.whl → 2.2.2__py3-none-any.whl

esd_services_api_client/_version.py

@@ -1 +1 @@
-__version__ = '2.2.0'
+__version__ = '2.2.2'

esd_services_api_client/crystal/_connector.py

@@ -37,6 +37,7 @@ from esd_services_api_client.crystal._models import (
     AlgorithmRequest,
     AlgorithmConfiguration,
     RequestLifeCycleStage,
+    ParentRequest,
 )
 
 T = TypeVar("T")  # pylint: disable=C0103
@@ -154,6 +155,7 @@ class CrystalConnector:
         algorithm: str,
         payload: Dict,
         custom_config: Optional[AlgorithmConfiguration] = None,
+        parent_request: Optional[ParentRequest] = None,
         tag: Optional[str] = None,
     ) -> str:
         """
@@ -162,6 +164,7 @@ class CrystalConnector:
         :param algorithm: Name of a connected algorithm.
         :param payload: Algorithm payload.
         :param custom_config: Customized config for this run.
+        :param parent_request: Parent request for this run.
         :param tag: Client-side submission identifier.
         :return: Request identifier assigned to the job by Crystal.
         """
@@ -176,6 +179,7 @@ class CrystalConnector:
             algorithm_name=algorithm,
             algorithm_parameters=payload,
             custom_configuration=custom_config,
+            parent_request=parent_request,
             tag=tag,
         ).to_dict()
 

esd_services_api_client/crystal/_models.py

@@ -133,6 +133,17 @@ class AlgorithmConfiguration(DataClassJsonMixin):
     speculative_attempts: Optional[int] = None
 
 
+@dataclass_json(letter_case=LetterCase.CAMEL)
+@dataclass
+class ParentRequest(DataClassJsonMixin):
+    """
+    Used to specify crystal parent job for a new crystal job.
+    """
+
+    request_id: Optional[str] = None
+    algorithm_name: Optional[str] = None
+
+
 @dataclass_json(letter_case=LetterCase.CAMEL)
 @dataclass
 class AlgorithmRequest(DataClassJsonMixin):
@@ -143,4 +154,5 @@ class AlgorithmRequest(DataClassJsonMixin):
     algorithm_parameters: Dict
     algorithm_name: Optional[str] = None
     custom_configuration: Optional[AlgorithmConfiguration] = None
+    parent_request: Optional[ParentRequest] = None
     tag: Optional[str] = None

esd_services_api_client/nexus/abstractions/nexus_object.py

@@ -53,7 +53,7 @@ TResult = TypeVar(  # pylint: disable=C0103
 )
 
 
-class NexusObject(Generic[TPayload, TResult], ABC):
+class NexusCoreObject(ABC):
     """
     Base class for all Nexus objects.
     """
@@ -87,6 +87,12 @@ class NexusObject(Generic[TPayload, TResult], ABC):
         Optional actions to perform on context closure.
         """
 
+
+class NexusObject(Generic[TPayload, TResult], NexusCoreObject, ABC):
+    """
+    Base class for all Nexus objects.
+    """
+
     @classmethod
     def alias(cls) -> str:
         """

esd_services_api_client/nexus/algorithms/_baseline_algorithm.py

@@ -1,7 +1,6 @@
 """
  Base algorithm
 """
-
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +16,6 @@
 #  limitations under the License.
 #
 
-
 from abc import abstractmethod
 from functools import partial
 
@@ -51,6 +49,14 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         super().__init__(metrics_provider, logger_factory)
         self._input_processors = input_processors
         self._cache = cache
+        self._inputs: dict = {}
+
+    @property
+    def inputs(self) -> dict:
+        """
+        Inputs generated for this algorithm run.
+        """
+        return self._inputs
 
     @abstractmethod
     async def _run(self, **kwargs) -> AlgorithmResult:
@@ -77,11 +83,11 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         async def _measured_run(**run_args):
             return await self._run(**run_args)
 
-        results = await self._cache.resolve(*self._input_processors, **kwargs)
+        self._inputs = await self._cache.resolve(*self._input_processors, **kwargs)
 
         return await partial(
             _measured_run,
-            **results,
+            **self._inputs,
             metric_tags=self._metric_tags,
             metrics_provider=self._metrics_provider,
             logger=self._logger,

esd_services_api_client/nexus/algorithms/minimalistic.py

@@ -45,5 +45,8 @@ class MinimalisticAlgorithm(BaselineAlgorithm[TPayload], ABC):
         cache: InputCache,
     ):
         super().__init__(
-            metrics_provider, logger_factory, *input_processors, cache=cache
+            metrics_provider,
+            logger_factory,
+            *input_processors,
+            cache=cache,
         )

esd_services_api_client/nexus/core/app_core.py

@@ -58,6 +58,7 @@ from esd_services_api_client.nexus.input.payload_reader import (
     AlgorithmPayloadReader,
     AlgorithmPayload,
 )
+from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
 
 
 def is_transient_exception(exception: Optional[BaseException]) -> Optional[bool]:
@@ -244,9 +245,11 @@ class Nexus:
         """
         Activates the run sequence.
         """
+
         self._injector = Injector(self._configurator.injection_binds)
 
         algorithm: BaselineAlgorithm = self._injector.get(self._algorithm_class)
+        telemetry_recorder: TelemetryRecorder = self._injector.get(TelemetryRecorder)
 
         async with algorithm as instance:
             self._algorithm_run_task = asyncio.create_task(
@@ -266,6 +269,12 @@ class Nexus:
             if len(on_complete_tasks) > 0:
                 await asyncio.wait(on_complete_tasks)
 
+            # record telemetry
+            async with telemetry_recorder as recorder:
+                await recorder.record(
+                    run_id=self._run_args.request_id, **algorithm.inputs
+                )
+
             # dispose of QES instance gracefully as it might hold open connections
             qes = self._injector.get(QueryEnabledStore)
             qes.close()

esd_services_api_client/nexus/core/app_dependencies.py

@@ -44,6 +44,7 @@ from esd_services_api_client.nexus.input.input_reader import InputReader
 from esd_services_api_client.nexus.input.payload_reader import (
     AlgorithmPayload,
 )
+from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
 
 
 @final
@@ -195,6 +196,7 @@ class ServiceConfigurator:
             StorageClientModule(),
             ExternalSocketsModule(),
             CacheModule(),
+            type(f"{TelemetryRecorder.__name__}Module", (Module,), {})(),
         ]
 
     @property

esd_services_api_client/nexus/telemetry/recorder.py

@@ -0,0 +1,105 @@
+"""
+ Telemetry recording module.
+"""
+import asyncio
+import os
+from functools import partial
+from typing import final
+
+import pandas as pd
+from adapta.metrics import MetricsProvider
+from adapta.process_communication import DataSocket
+from adapta.storage.blob.base import StorageClient
+from adapta.storage.models.format import (
+    DictJsonSerializationFormat,
+    DataFrameParquetSerializationFormat,
+)
+from injector import inject, singleton
+
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.abstractions.nexus_object import NexusCoreObject
+
+
+@final
+@singleton
+class TelemetryRecorder(NexusCoreObject):
+    """
+    Class for instantiating a telemetry recorder that will save all algorithm inputs (run method arguments) to a persistent location.
+    """
+
+    async def _context_open(self):
+        pass
+
+    async def _context_close(self):
+        pass
+
+    @inject
+    def __init__(
+        self,
+        storage_client: StorageClient,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._storage_client = storage_client
+        self._telemetry_base_path = os.getenv("NEXUS__TELEMETRY_PATH")
+
+    async def record(self, run_id: str, **telemetry_args):
+        """
+        Record all data in telemetry args for the provided run_id.
+        """
+
+        async def _record(
+            entity_to_record: pd.DataFrame | dict,
+            entity_name: str,
+            **_,
+        ) -> None:
+            self._logger.debug(
+                "Recording telemetry for {entity_name} in the run {run_id}",
+                entity_name=entity_name,
+                run_id=run_id,
+            )
+            if not isinstance(entity_to_record, dict) and not isinstance(
+                entity_to_record, pd.DataFrame
+            ):
+                self._logger.warning(
+                    "Unsupported data type: {telemetry_entity_type}. Telemetry recording skipped.",
+                    telemetry_entity_type=type(entity_to_record),
+                )
+            else:
+                self._storage_client.save_data_as_blob(
+                    data=entity_to_record,
+                    blob_path=DataSocket(
+                        alias="telemetry",
+                        data_path=f"{self._telemetry_base_path}/{entity_name}/{run_id}",
+                        data_format="null",
+                    ).parse_data_path(),
+                    serialization_format=DictJsonSerializationFormat
+                    if isinstance(entity_to_record, dict)
+                    else DataFrameParquetSerializationFormat,
+                    overwrite=True,
+                )
+
+        telemetry_tasks = [
+            asyncio.create_task(
+                partial(
+                    _record,
+                    entity_to_record=telemetry_value,
+                    entity_name=telemetry_key,
+                    run_id=run_id,
+                )()
+            )
+            for telemetry_key, telemetry_value in telemetry_args.items()
+        ]
+
+        done, pending = await asyncio.wait(telemetry_tasks)
+        if len(pending) > 0:
+            self._logger.warning(
+                "Some telemetry recording operations did not complete within specified time. This run might lack observability coverage."
+            )
+        for done_telemetry_task in done:
+            telemetry_exc = done_telemetry_task.exception()
+            if telemetry_exc:
+                self._logger.warning(
+                    "Telemetry recoding failed", exception=telemetry_exc
+                )

{esd_services_api_client-2.2.0.dist-info → esd_services_api_client-2.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.2.0
+Version: 2.2.2
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0

{esd_services_api_client-2.2.0.dist-info → esd_services_api_client-2.2.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 esd_services_api_client/__init__.py,sha256=L-cEW1mVbnTJLCLG5V6Ucw7zBgx1zf0t1bYcQC1heyw,603
-esd_services_api_client/_version.py,sha256=Vyf6P6UCZKFeQtRzYujPmFfdlqSfnc01VEMWE3O0ZrA,22
+esd_services_api_client/_version.py,sha256=jsJ9CNIuUt8dDFB4i0PiBf07nzBU0RtG1CVRQ7TdoQ0,22
 esd_services_api_client/beast/__init__.py,sha256=zNhXcHSP5w4P9quM1XP4oXVJEccvC_VScG41TZ0GzZ8,723
 esd_services_api_client/beast/v3/__init__.py,sha256=FtumtInoDyCCRE424Llqv8QZLRuwXzj-smyfu1od1nc,754
 esd_services_api_client/beast/v3/_connector.py,sha256=WNmCiTXFRb3q56mrr7ZbqBHWDUxbfyWhiWlBFLUIOnc,11478
@@ -13,28 +13,28 @@ esd_services_api_client/boxer/_models.py,sha256=ursQIR_c9jcVfRKc0LH1OuVL2KFD6kqo
 esd_services_api_client/common/__init__.py,sha256=L-cEW1mVbnTJLCLG5V6Ucw7zBgx1zf0t1bYcQC1heyw,603
 esd_services_api_client/crystal/__init__.py,sha256=oeyJjdQ9EpTnIq6XnjPq5v0DWPdHqi4uEfRIcD1mNZA,792
 esd_services_api_client/crystal/_api_versions.py,sha256=GHbmV_5lP9fP72TZE0j_ZeQSeJjMRcRaBRxNJbz-MWQ,837
-esd_services_api_client/crystal/_connector.py,sha256=wT8SahCkkRWPoHcUSLz0I-sjeK_9OSYtt07zJKie0CU,12875
-esd_services_api_client/crystal/_models.py,sha256=OCaidMqipl-TA8VLs7v12BppyBR3QpRA5jU2KjZxU_Q,4030
+esd_services_api_client/crystal/_connector.py,sha256=U2cmgmOhXtKWlDljg8G20W4pTBOjgfb-F4gH7AgXBYg,13053
+esd_services_api_client/crystal/_models.py,sha256=4ZizNNfMij31eV87cOog724gJWGWgfQAiKJ1nHKlNw0,4338
 esd_services_api_client/nexus/README.md,sha256=QQgvkhRwZtktxlzTmHkbp8KNnUvDvFGEVqvSM1QgpU8,9393
 esd_services_api_client/nexus/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
 esd_services_api_client/nexus/abstractions/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
 esd_services_api_client/nexus/abstractions/algrorithm_cache.py,sha256=3Umb9bKsl8Yo5a3FMrdO_7JTk2mrYJf9MLR-_C0yzFo,3338
 esd_services_api_client/nexus/abstractions/input_object.py,sha256=RUKnhekuZwd_RVvnLGAxHa4wYDFJf6wEwWQI9f-o0lM,1761
 esd_services_api_client/nexus/abstractions/logger_factory.py,sha256=9biONvCqNrP__yrmeRkoDL05TMA5v-LyrcKwgiKG59U,2019
-esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=P5lQ5jhIk4nTLESseBy-G5HPILpBd75PWykD12jn6eQ,2938
+esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=TFRsiqfrIRGdoZsUDrMIWSHL9LsNdrcCXO-dEliRaAA,3039
 esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=Rwa_aPErI4Es5AdyCd3EoGze7mg2D70u8kuc2UGEBaI,1729
 esd_services_api_client/nexus/algorithms/__init__.py,sha256=yMvLFSqg5eUKOXI0zMFX69Ni0ibKQHOqAnrZsxQqhOo,903
-esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=hGj_qNNNtz8DRhbyUDOG7ouuv8HmqAZsM-ccJKFXTH4,2739
+esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=24ALLx4Bxlgi0EwZB1a0SJeEwBUWKj7CGad-CpIygU0,2925
 esd_services_api_client/nexus/algorithms/_remote_algorithm.py,sha256=nQDQ2si-_-B2QdtBC8IwSM8YyNwfIhrCMto6g87BcnQ,3900
 esd_services_api_client/nexus/algorithms/distributed.py,sha256=vkKSCsd480RKwrtu3uZ2iU1bh593fkgBcOBrcb9cLjA,1702
 esd_services_api_client/nexus/algorithms/forked_algorithm.py,sha256=Y1BFCbEMmLFmQlvq0Ot_8RAlvSbFqvZFWa_RLIYvb2Y,4310
-esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=te8h2SQiAB8xn9OsGciZl51b_oROOodgHIX6408Lz2s,1607
+esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=tSYXodIW-_Aje-_ZyYUoWAThcZIeE4_kMvMINsT4Lb8,1644
 esd_services_api_client/nexus/algorithms/recursive.py,sha256=uaCCl4q-st_KqbcmkdOJedJ0nAjbJvn6jdZEdW0_0ss,2007
 esd_services_api_client/nexus/configurations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 esd_services_api_client/nexus/configurations/algorithm_configuration.py,sha256=eE7diX2PATCGkmqhvFOcZwXrr6vns4fqnJGmgNvhhZM,1091
 esd_services_api_client/nexus/core/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
-esd_services_api_client/nexus/core/app_core.py,sha256=gs1oIwc9KEog46vOZp6g_JVzTa8LLfcCpuauTLGhYVM,9654
-esd_services_api_client/nexus/core/app_dependencies.py,sha256=BVihH0gqfAGY851hzv4GLajC8eJx3zn8uYnOzW6-2_8,6521
+esd_services_api_client/nexus/core/app_core.py,sha256=jW6eISWBboaDe94Sy9r4w-K2-TcL9uFGJv5JOym6Brg,10037
+esd_services_api_client/nexus/core/app_dependencies.py,sha256=6jrFR5x7lLSDL1rc6YlSxMpwe641rb1YAvDE455zRbQ,6674
 esd_services_api_client/nexus/exceptions/__init__.py,sha256=feN33VdqB5-2bD9aJesJl_OlsKrNNo3hZCnQgKuaU9k,696
 esd_services_api_client/nexus/exceptions/_nexus_error.py,sha256=QvtY38mNoIA6t26dUN6UIsaPfljhtVNsbQVS7ksMb-Q,895
 esd_services_api_client/nexus/exceptions/cache_errors.py,sha256=IO_rBQKXfIRHHXQuC8kAHejgZZw9yvSJk5BPYBnDYbc,1622
@@ -44,7 +44,9 @@ esd_services_api_client/nexus/input/__init__.py,sha256=ODYhZ791tPC4-eVxSRRlh8FLD
 esd_services_api_client/nexus/input/input_processor.py,sha256=vqzeQrtRFqBKTPSEiWX_JZJTF9itMwwvWjPnJVLrSwQ,3132
 esd_services_api_client/nexus/input/input_reader.py,sha256=aXNMGxrdUX5RDYR666GSGkcZqYMFYoZ8zGVDuUFFFZQ,3505
 esd_services_api_client/nexus/input/payload_reader.py,sha256=Kq0xN1Shyqv71v6YkcrqVTDbmsEjZc8ithsXYpyu87M,2516
-esd_services_api_client-2.2.0.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
-esd_services_api_client-2.2.0.dist-info/METADATA,sha256=-FllzJEL9xB4GA_iM0zRfW3tmwuaAGbL_YhqQ7WfPys,1292
-esd_services_api_client-2.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-esd_services_api_client-2.2.0.dist-info/RECORD,,
+esd_services_api_client/nexus/telemetry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+esd_services_api_client/nexus/telemetry/recorder.py,sha256=P6DDr6cQ9FYTZhgY1YMzez0Y6c6ac2FnYktapNssyJo,3644
+esd_services_api_client-2.2.2.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
+esd_services_api_client-2.2.2.dist-info/METADATA,sha256=IQRjdfLWt5AZ89_7w-iHJuvglGwPcFhUYwtMewJVL64,1292
+esd_services_api_client-2.2.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+esd_services_api_client-2.2.2.dist-info/RECORD,,