esd-services-api-client 2.2.0__tar.gz → 2.2.2__tar.gz

{esd_services_api_client-2.2.0 → esd_services_api_client-2.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.2.0
+Version: 2.2.2
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0

esd_services_api_client-2.2.2/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = '2.2.2'

{esd_services_api_client-2.2.0 → esd_services_api_client-2.2.2}/esd_services_api_client/crystal/_connector.py

@@ -37,6 +37,7 @@ from esd_services_api_client.crystal._models import (
     AlgorithmRequest,
     AlgorithmConfiguration,
     RequestLifeCycleStage,
+    ParentRequest,
 )
 
 T = TypeVar("T")  # pylint: disable=C0103
@@ -154,6 +155,7 @@ class CrystalConnector:
         algorithm: str,
         payload: Dict,
         custom_config: Optional[AlgorithmConfiguration] = None,
+        parent_request: Optional[ParentRequest] = None,
         tag: Optional[str] = None,
     ) -> str:
         """
@@ -162,6 +164,7 @@ class CrystalConnector:
         :param algorithm: Name of a connected algorithm.
         :param payload: Algorithm payload.
         :param custom_config: Customized config for this run.
+        :param parent_request: Parent request for this run.
         :param tag: Client-side submission identifier.
         :return: Request identifier assigned to the job by Crystal.
         """
@@ -176,6 +179,7 @@ class CrystalConnector:
             algorithm_name=algorithm,
             algorithm_parameters=payload,
             custom_configuration=custom_config,
+            parent_request=parent_request,
             tag=tag,
         ).to_dict()
 

{esd_services_api_client-2.2.0 → esd_services_api_client-2.2.2}/esd_services_api_client/crystal/_models.py

@@ -133,6 +133,17 @@ class AlgorithmConfiguration(DataClassJsonMixin):
     speculative_attempts: Optional[int] = None
 
 
+@dataclass_json(letter_case=LetterCase.CAMEL)
+@dataclass
+class ParentRequest(DataClassJsonMixin):
+    """
+    Used to specify crystal parent job for a new crystal job.
+    """
+
+    request_id: Optional[str] = None
+    algorithm_name: Optional[str] = None
+
+
 @dataclass_json(letter_case=LetterCase.CAMEL)
 @dataclass
 class AlgorithmRequest(DataClassJsonMixin):
@@ -143,4 +154,5 @@ class AlgorithmRequest(DataClassJsonMixin):
     algorithm_parameters: Dict
     algorithm_name: Optional[str] = None
     custom_configuration: Optional[AlgorithmConfiguration] = None
+    parent_request: Optional[ParentRequest] = None
     tag: Optional[str] = None

{esd_services_api_client-2.2.0 → esd_services_api_client-2.2.2}/esd_services_api_client/nexus/abstractions/nexus_object.py

@@ -53,7 +53,7 @@ TResult = TypeVar(  # pylint: disable=C0103
 )
 
 
-class NexusObject(Generic[TPayload, TResult], ABC):
+class NexusCoreObject(ABC):
     """
     Base class for all Nexus objects.
     """
@@ -87,6 +87,12 @@ class NexusObject(Generic[TPayload, TResult], ABC):
         Optional actions to perform on context closure.
         """
 
+
+class NexusObject(Generic[TPayload, TResult], NexusCoreObject, ABC):
+    """
+    Base class for all Nexus objects.
+    """
+
     @classmethod
     def alias(cls) -> str:
         """

{esd_services_api_client-2.2.0 → esd_services_api_client-2.2.2}/esd_services_api_client/nexus/algorithms/_baseline_algorithm.py

@@ -1,7 +1,6 @@
 """
  Base algorithm
 """
-
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +16,6 @@
 #  limitations under the License.
 #
 
-
 from abc import abstractmethod
 from functools import partial
 
@@ -51,6 +49,14 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         super().__init__(metrics_provider, logger_factory)
         self._input_processors = input_processors
         self._cache = cache
+        self._inputs: dict = {}
+
+    @property
+    def inputs(self) -> dict:
+        """
+        Inputs generated for this algorithm run.
+        """
+        return self._inputs
 
     @abstractmethod
     async def _run(self, **kwargs) -> AlgorithmResult:
@@ -77,11 +83,11 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
         async def _measured_run(**run_args):
             return await self._run(**run_args)
 
-        results = await self._cache.resolve(*self._input_processors, **kwargs)
+        self._inputs = await self._cache.resolve(*self._input_processors, **kwargs)
 
         return await partial(
             _measured_run,
-            **results,
+            **self._inputs,
             metric_tags=self._metric_tags,
             metrics_provider=self._metrics_provider,
             logger=self._logger,

{esd_services_api_client-2.2.0 → esd_services_api_client-2.2.2}/esd_services_api_client/nexus/algorithms/minimalistic.py

@@ -45,5 +45,8 @@ class MinimalisticAlgorithm(BaselineAlgorithm[TPayload], ABC):
         cache: InputCache,
     ):
         super().__init__(
-            metrics_provider, logger_factory, *input_processors, cache=cache
+            metrics_provider,
+            logger_factory,
+            *input_processors,
+            cache=cache,
         )

{esd_services_api_client-2.2.0 → esd_services_api_client-2.2.2}/esd_services_api_client/nexus/core/app_core.py

@@ -58,6 +58,7 @@ from esd_services_api_client.nexus.input.payload_reader import (
     AlgorithmPayloadReader,
     AlgorithmPayload,
 )
+from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
 
 
 def is_transient_exception(exception: Optional[BaseException]) -> Optional[bool]:
@@ -244,9 +245,11 @@ class Nexus:
         """
         Activates the run sequence.
         """
+
         self._injector = Injector(self._configurator.injection_binds)
 
         algorithm: BaselineAlgorithm = self._injector.get(self._algorithm_class)
+        telemetry_recorder: TelemetryRecorder = self._injector.get(TelemetryRecorder)
 
         async with algorithm as instance:
             self._algorithm_run_task = asyncio.create_task(
@@ -266,6 +269,12 @@ class Nexus:
             if len(on_complete_tasks) > 0:
                 await asyncio.wait(on_complete_tasks)
 
+            # record telemetry
+            async with telemetry_recorder as recorder:
+                await recorder.record(
+                    run_id=self._run_args.request_id, **algorithm.inputs
+                )
+
             # dispose of QES instance gracefully as it might hold open connections
             qes = self._injector.get(QueryEnabledStore)
             qes.close()

{esd_services_api_client-2.2.0 → esd_services_api_client-2.2.2}/esd_services_api_client/nexus/core/app_dependencies.py

@@ -44,6 +44,7 @@ from esd_services_api_client.nexus.input.input_reader import InputReader
 from esd_services_api_client.nexus.input.payload_reader import (
     AlgorithmPayload,
 )
+from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
 
 
 @final
@@ -195,6 +196,7 @@ class ServiceConfigurator:
             StorageClientModule(),
             ExternalSocketsModule(),
             CacheModule(),
+            type(f"{TelemetryRecorder.__name__}Module", (Module,), {})(),
         ]
 
     @property

esd_services_api_client-2.2.2/esd_services_api_client/nexus/telemetry/recorder.py

@@ -0,0 +1,105 @@
+"""
+ Telemetry recording module.
+"""
+import asyncio
+import os
+from functools import partial
+from typing import final
+
+import pandas as pd
+from adapta.metrics import MetricsProvider
+from adapta.process_communication import DataSocket
+from adapta.storage.blob.base import StorageClient
+from adapta.storage.models.format import (
+    DictJsonSerializationFormat,
+    DataFrameParquetSerializationFormat,
+)
+from injector import inject, singleton
+
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.abstractions.nexus_object import NexusCoreObject
+
+
+@final
+@singleton
+class TelemetryRecorder(NexusCoreObject):
+    """
+    Class for instantiating a telemetry recorder that will save all algorithm inputs (run method arguments) to a persistent location.
+    """
+
+    async def _context_open(self):
+        pass
+
+    async def _context_close(self):
+        pass
+
+    @inject
+    def __init__(
+        self,
+        storage_client: StorageClient,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._storage_client = storage_client
+        self._telemetry_base_path = os.getenv("NEXUS__TELEMETRY_PATH")
+
+    async def record(self, run_id: str, **telemetry_args):
+        """
+        Record all data in telemetry args for the provided run_id.
+        """
+
+        async def _record(
+            entity_to_record: pd.DataFrame | dict,
+            entity_name: str,
+            **_,
+        ) -> None:
+            self._logger.debug(
+                "Recording telemetry for {entity_name} in the run {run_id}",
+                entity_name=entity_name,
+                run_id=run_id,
+            )
+            if not isinstance(entity_to_record, dict) and not isinstance(
+                entity_to_record, pd.DataFrame
+            ):
+                self._logger.warning(
+                    "Unsupported data type: {telemetry_entity_type}. Telemetry recording skipped.",
+                    telemetry_entity_type=type(entity_to_record),
+                )
+            else:
+                self._storage_client.save_data_as_blob(
+                    data=entity_to_record,
+                    blob_path=DataSocket(
+                        alias="telemetry",
+                        data_path=f"{self._telemetry_base_path}/{entity_name}/{run_id}",
+                        data_format="null",
+                    ).parse_data_path(),
+                    serialization_format=DictJsonSerializationFormat
+                    if isinstance(entity_to_record, dict)
+                    else DataFrameParquetSerializationFormat,
+                    overwrite=True,
+                )
+
+        telemetry_tasks = [
+            asyncio.create_task(
+                partial(
+                    _record,
+                    entity_to_record=telemetry_value,
+                    entity_name=telemetry_key,
+                    run_id=run_id,
+                )()
+            )
+            for telemetry_key, telemetry_value in telemetry_args.items()
+        ]
+
+        done, pending = await asyncio.wait(telemetry_tasks)
+        if len(pending) > 0:
+            self._logger.warning(
+                "Some telemetry recording operations did not complete within specified time. This run might lack observability coverage."
+            )
+        for done_telemetry_task in done:
+            telemetry_exc = done_telemetry_task.exception()
+            if telemetry_exc:
+                self._logger.warning(
+                    "Telemetry recoding failed", exception=telemetry_exc
+                )

{esd_services_api_client-2.2.0 → esd_services_api_client-2.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "esd-services-api-client"
-version = "2.2.0"
+version = "2.2.2"
 description = "Python clients for ESD services"
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']

esd_services_api_client-2.2.0/esd_services_api_client/_version.py

@@ -1 +0,0 @@
-__version__ = '2.2.0'