esd-services-api-client 2.2.3__tar.gz → 2.2.5__tar.gz

{esd_services_api_client-2.2.3 → esd_services_api_client-2.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.2.3
+Version: 2.2.5
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0

esd_services_api_client-2.2.5/esd_services_api_client/_version.py

@@ -0,0 +1 @@
+__version__ = '2.2.5'

{esd_services_api_client-2.2.3 → esd_services_api_client-2.2.5}/esd_services_api_client/nexus/core/app_core.py

@@ -30,7 +30,6 @@ import urllib3.exceptions
 import azure.core.exceptions
 from adapta.process_communication import DataSocket
 from adapta.storage.blob.base import StorageClient
-from adapta.storage.models.format import DataFrameJsonSerializationFormat
 from adapta.storage.query_enabled_store import QueryEnabledStore
 from injector import Injector
 
@@ -52,6 +51,9 @@ from esd_services_api_client.nexus.configurations.algorithm_configuration import
 from esd_services_api_client.nexus.core.app_dependencies import (
     ServiceConfigurator,
 )
+from esd_services_api_client.nexus.core.serializers import (
+    ResultSerializer,
+)
 from esd_services_api_client.nexus.input.input_processor import InputProcessor
 from esd_services_api_client.nexus.input.input_reader import InputReader
 from esd_services_api_client.nexus.input.payload_reader import (
@@ -204,15 +206,19 @@ class Nexus:
 
             :return: blob uri
             """
+            dataframe_data = data.dataframe()
+            serializer = self._injector.get(ResultSerializer)
             storage_client = self._injector.get(StorageClient)
             output_path = f"{os.getenv('NEXUS__ALGORITHM_OUTPUT_PATH')}/{self._run_args.request_id}.json"
             blob_path = DataSocket(
                 data_path=output_path, alias="output", data_format="null"
             ).parse_data_path()
             storage_client.save_data_as_blob(
-                data=data.dataframe(),
+                data=dataframe_data,
                 blob_path=blob_path,
-                serialization_format=DataFrameJsonSerializationFormat,
+                serialization_format=serializer.get_serialization_format(
+                    dataframe_data
+                ),
                 overwrite=True,
             )
             return storage_client.get_blob_uri(blob_path=blob_path)

{esd_services_api_client-2.2.3 → esd_services_api_client-2.2.5}/esd_services_api_client/nexus/core/app_dependencies.py

@@ -19,8 +19,9 @@
 
 import json
 import os
+import re
 from pydoc import locate
-from typing import final, Type
+from typing import final, Type, Any
 
 from adapta.metrics import MetricsProvider
 from adapta.storage.blob.base import StorageClient
@@ -45,6 +46,10 @@ from esd_services_api_client.nexus.input.payload_reader import (
     AlgorithmPayload,
 )
 from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
+from esd_services_api_client.nexus.core.serializers import (
+    TelemetrySerializer,
+    ResultSerializer,
+)
 
 
 @final
@@ -167,6 +172,48 @@ class ExternalSocketsModule(Module):
         )
 
 
+@final
+class ResultSerializerModule(Module):
+    """
+    Serialization format module for results.
+    """
+
+    @singleton
+    @provider
+    def provide(self) -> ResultSerializer:
+        """
+        DI factory method.
+        """
+        serializer = ResultSerializer()
+        for serialization_format in locate_classes(
+            re.compile(r"NEXUS__RESULT_SERIALIZATION_FORMAT_(.+)_CLASS")
+        ):
+            serializer = serializer.with_format(serialization_format)
+
+        return serializer
+
+
+@final
+class TelemetrySerializerModule(Module):
+    """
+    Serialization format module for telemetry.
+    """
+
+    @singleton
+    @provider
+    def provide(self) -> TelemetrySerializer:
+        """
+        DI factory method.
+        """
+        serializer = TelemetrySerializer()
+        for serialization_format in locate_classes(
+            re.compile(r"NEXUS__TELEMETRY_SERIALIZATION_FORMAT_(.+)_CLASS")
+        ):
+            serializer = serializer.with_format(serialization_format)
+
+        return serializer
+
+
 @final
 class CacheModule(Module):
     """
@@ -195,6 +242,8 @@ class ServiceConfigurator:
             QueryEnabledStoreModule(),
             StorageClientModule(),
             ExternalSocketsModule(),
+            TelemetrySerializerModule(),
+            ResultSerializerModule(),
             CacheModule(),
             type(f"{TelemetryRecorder.__name__}Module", (Module,), {})(),
         ]
@@ -241,3 +290,24 @@ class ServiceConfigurator:
             lambda binder: binder.bind(config.__class__, to=config, scope=singleton)
         )
         return self
+
+
+def locate_classes(pattern: re.Pattern) -> list[Type[Any]]:
+    """
+    Locates all classes matching the pattern in the environment. Throws a start-up error if any class is not found.
+    """
+    classes = {
+        (var_name, class_path): locate(class_path)
+        for var_name, class_path in os.environ.items()
+        if pattern.match(var_name)
+    }
+
+    non_located_classes = [
+        name_and_path for name_and_path, class_ in classes.items() if class_ is None
+    ]
+    if non_located_classes:
+        raise FatalStartupConfigurationError(
+            f"Failed to locate classes: {non_located_classes}"
+        )
+
+    return list(classes.values())

esd_services_api_client-2.2.5/esd_services_api_client/nexus/core/serializers.py

@@ -0,0 +1,78 @@
+"""Serialization format module."""
+from typing import final, Any, TypeVar, Type
+
+import pandas
+from adapta.storage.models.format import (
+    DataFrameParquetSerializationFormat,
+    DictJsonSerializationFormat,
+    SerializationFormat,
+)
+
+T = TypeVar("T")  # pylint: disable=C0103
+
+
+class Serializer:
+    """
+    Serializer that dynamically infers serialization format. The format to use is determined at runtime
+    by the type of the data.
+    """
+
+    def __init__(
+        self,
+        default_serialization_formats: dict[
+            Type[T], Type[SerializationFormat[T]]
+        ] = None,
+    ):
+        self._serialization_formats = (
+            {}
+            if default_serialization_formats is None
+            else default_serialization_formats
+        )
+
+    def get_serialization_format(self, data: Any) -> Type[SerializationFormat]:
+        """
+        Get the serializer for the data.
+        """
+        return self._serialization_formats[type(data)]
+
+    def with_format(
+        self, serialization_format: Type[SerializationFormat]
+    ) -> "Serializer":
+        """Add a serialization format to the supported formats. Note that only 1 serialization format is allowed per
+        type."""
+        serialization_target_type = serialization_format.__orig_bases__[0].__args__[0]
+        self._serialization_formats[serialization_target_type] = serialization_format
+
+        return self
+
+    def serialize(self, data) -> bytes:
+        """
+        Serialize data.
+        """
+        return self.get_serialization_format(data)().serialize(data)
+
+
+@final
+class TelemetrySerializer(Serializer):
+    """Telemetry serialization format"""
+
+    def __init__(self):
+        super().__init__(
+            default_serialization_formats={
+                pandas.DataFrame: DataFrameParquetSerializationFormat,
+                dict: DictJsonSerializationFormat,
+            }
+        )
+
+
+@final
+class ResultSerializer(Serializer):
+    """Result serialization format"""
+
+    def __init__(self):
+        super().__init__(
+            default_serialization_formats={
+                pandas.DataFrame: DataFrameParquetSerializationFormat,
+                dict: DictJsonSerializationFormat,
+            }
+        )

{esd_services_api_client-2.2.3 → esd_services_api_client-2.2.5}/esd_services_api_client/nexus/telemetry/recorder.py

@@ -10,14 +10,13 @@ import pandas as pd
 from adapta.metrics import MetricsProvider
 from adapta.process_communication import DataSocket
 from adapta.storage.blob.base import StorageClient
-from adapta.storage.models.format import (
-    DictJsonSerializationFormat,
-    DataFrameParquetSerializationFormat,
-)
 from injector import inject, singleton
 
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.abstractions.nexus_object import NexusCoreObject
+from esd_services_api_client.nexus.core.serializers import (
+    TelemetrySerializer,
+)
 
 
 @final
@@ -37,12 +36,14 @@ class TelemetryRecorder(NexusCoreObject):
     def __init__(
         self,
         storage_client: StorageClient,
+        serializer: TelemetrySerializer,
         metrics_provider: MetricsProvider,
         logger_factory: LoggerFactory,
     ):
         super().__init__(metrics_provider, logger_factory)
         self._storage_client = storage_client
         self._telemetry_base_path = os.getenv("NEXUS__TELEMETRY_PATH")
+        self._serializer = serializer
 
     async def record(self, run_id: str, **telemetry_args):
         """
@@ -74,9 +75,9 @@ class TelemetryRecorder(NexusCoreObject):
                         data_path=f"{self._telemetry_base_path}/{entity_name}/{run_id}",
                         data_format="null",
                     ).parse_data_path(),
-                    serialization_format=DictJsonSerializationFormat
-                    if isinstance(entity_to_record, dict)
-                    else DataFrameParquetSerializationFormat,
+                    serialization_format=self._serializer.get_serialization_format(
+                        entity_to_record
+                    ),
                     overwrite=True,
                 )
 
@@ -91,6 +92,8 @@ class TelemetryRecorder(NexusCoreObject):
             )
             for telemetry_key, telemetry_value in telemetry_args.items()
         ]
+        if not telemetry_tasks:
+            return
 
         done, pending = await asyncio.wait(telemetry_tasks)
         if len(pending) > 0:

{esd_services_api_client-2.2.3 → esd_services_api_client-2.2.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "esd-services-api-client"
-version = "2.2.3"
+version = "2.2.5"
 description = "Python clients for ESD services"
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']

esd_services_api_client-2.2.3/esd_services_api_client/_version.py

@@ -1 +0,0 @@
-__version__ = '2.2.3'