esd-services-api-client 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

esd_services_api_client/nexus/exceptions/startup_error.py

@@ -0,0 +1,48 @@
+"""
+ Custom exceptions.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+from esd_services_api_client.nexus.exceptions import FatalNexusError
+
+
+class FatalServiceStartupError(FatalNexusError):
+    """
+    Service startup error that shuts down the Nexus.
+    """
+
+    def __init__(self, service: str, underlying: BaseException):
+        super().__init__()
+        self.with_traceback(underlying.__traceback__)
+        self._service = service
+
+    def __str__(self) -> str:
+        return f"Algorithm initialization failed on service {self._service} startup. Review the traceback for more information"
+
+
+class FatalStartupConfigurationError(FatalNexusError):
+    """
+    Service configuration error that shuts down the Nexus.
+    """
+
+    def __init__(self, missing_entry: str):
+        super().__init__()
+        self._missing_entry = missing_entry
+
+    def __str__(self) -> str:
+        return f"Algorithm initialization failed due to a missing configuration entry: {self._missing_entry}."

esd_services_api_client/nexus/input/__init__.py

@@ -0,0 +1,22 @@
+"""
+ Import index.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+from esd_services_api_client.nexus.input.input_processor import *
+from esd_services_api_client.nexus.input.input_reader import *

esd_services_api_client/nexus/input/input_processor.py

@@ -0,0 +1,94 @@
+"""
+ Input processing.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import asyncio
+from abc import abstractmethod
+from typing import Dict, Union, Type
+
+import deltalake
+from adapta.metrics import MetricsProvider
+
+import azure.core.exceptions
+
+from pandas import DataFrame as PandasDataFrame
+
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    NexusObject,
+    TPayload,
+)
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.exceptions.input_reader_error import (
+    FatalInputReaderError,
+    TransientInputReaderError,
+)
+from esd_services_api_client.nexus.input.input_reader import InputReader
+
+
+class InputProcessor(NexusObject[TPayload]):
+    """
+    Base class for raw data processing into algorithm input.
+    """
+
+    def __init__(
+        self,
+        *readers: InputReader,
+        payload: TPayload,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._readers = readers
+        self._payload = payload
+
+    def _get_exc_type(
+        self, ex: BaseException
+    ) -> Union[Type[FatalInputReaderError], Type[TransientInputReaderError]]:
+        match type(ex):
+            case azure.core.exceptions.HttpResponseError, deltalake.PyDeltaTableError:
+                return TransientInputReaderError
+            case azure.core.exceptions.AzureError, azure.core.exceptions.ClientAuthenticationError:
+                return FatalInputReaderError
+            case _:
+                return FatalInputReaderError
+
+    async def _read_input(self) -> Dict[str, PandasDataFrame]:
+        def get_result(alias: str, completed_task: asyncio.Task) -> PandasDataFrame:
+            reader_exc = completed_task.exception()
+            if reader_exc:
+                raise self._get_exc_type(reader_exc)(alias, reader_exc)
+
+            return completed_task.result()
+
+        async def _read(input_reader: InputReader):
+            async with input_reader as instance:
+                return await instance.read()
+
+        read_tasks: dict[str, asyncio.Task] = {
+            reader.socket.alias: asyncio.create_task(_read(reader))
+            for reader in self._readers
+        }
+        await asyncio.wait(fs=read_tasks.values())
+
+        return {alias: get_result(alias, task) for alias, task in read_tasks.items()}
+
+    @abstractmethod
+    async def process_input(self, **kwargs) -> Dict[str, PandasDataFrame]:
+        """
+        Input processing logic. Implement this method to prepare data for your algorithm code.
+        """

esd_services_api_client/nexus/input/input_reader.py

@@ -0,0 +1,109 @@
+"""
+ Input reader.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import re
+from abc import abstractmethod
+from functools import partial
+from typing import Optional
+
+from adapta.metrics import MetricsProvider
+from adapta.process_communication import DataSocket
+from adapta.storage.query_enabled_store import QueryEnabledStore
+from adapta.utils.decorators._logging import run_time_metrics_async
+
+from pandas import DataFrame as PandasDataFrame
+
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    NexusObject,
+    TPayload,
+)
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+
+
+class InputReader(NexusObject[TPayload]):
+    """
+    Base class for a raw data reader.
+    """
+
+    def __init__(
+        self,
+        socket: DataSocket,
+        store: QueryEnabledStore,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        payload: TPayload,
+        *readers: "InputReader"
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self.socket = socket
+        self._store = store
+        self._data: Optional[PandasDataFrame] = None
+        self._readers = readers
+        self._payload = payload
+
+    @property
+    def data(self) -> Optional[PandasDataFrame]:
+        """
+        Data read by this reader.
+        """
+        return self._data
+
+    @abstractmethod
+    async def _read_input(self) -> PandasDataFrame:
+        """
+        Actual data reader logic. Implementing this method is mandatory for the reader to work
+        """
+
+    @property
+    def _metric_name(self) -> str:
+        return re.sub(
+            r"(?<!^)(?=[A-Z])",
+            "_",
+            self.__class__.__name__.lower().replace("reader", ""),
+        )
+
+    @property
+    def _metric_tags(self) -> dict[str, str]:
+        return {"entity": self._metric_name}
+
+    async def read(self) -> PandasDataFrame:
+        """
+        Coroutine that reads the data from external store and converts it to a dataframe.
+        """
+
+        @run_time_metrics_async(
+            metric_name="read_input",
+            on_finish_message_template="Finished reading {entity} from path {data_path} in {elapsed:.2f}s seconds",
+            template_args={
+                "entity": self._metric_name.upper(),
+                "data_path": self.socket.data_path,
+            },
+        )
+        async def _read(**_) -> PandasDataFrame:
+            if not self._data:
+                self._data = await self._read_input()
+
+            return self._data
+
+        return await partial(
+            _read,
+            metric_tags=self._metric_tags,
+            metrics_provider=self._metrics_provider,
+            logger=self._logger,
+        )()

esd_services_api_client/nexus/input/payload_reader.py

@@ -0,0 +1,83 @@
+"""
+ Code infrastructure for manipulating payload received from Crystal SAS URI
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from dataclasses import dataclass
+
+from typing import final, Optional, Type
+
+from adapta.storage.models.format import DictJsonSerializationFormat
+from adapta.utils import session_with_retries
+
+from dataclasses_json import DataClassJsonMixin
+
+
+@dataclass
+class AlgorithmPayload(DataClassJsonMixin):
+    """
+    Base class for algorithm payload
+    """
+
+    def validate(self):
+        """
+        Optional post-validation of the data. Define this method to analyze class contents after payload has been read and deserialized.
+        """
+
+    def __post_init__(self):
+        self.validate()
+
+
+@final
+class AlgorithmPayloadReader:
+    """
+    Crystal Payload Reader - receives the payload from the URI and deserializes it into the specified type
+    """
+
+    async def __aenter__(self):
+        if not self._http:
+            self._http = session_with_retries()
+        http_response = self._http.get(url=self._payload_uri)
+        http_response.raise_for_status()
+        self._payload = self._payload_type.from_dict(
+            DictJsonSerializationFormat().deserialize(http_response.content)
+        )
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        self._http.close()
+        self._http = None
+
+    def __init__(self, payload_uri: str, payload_type: Type[AlgorithmPayload]):
+        self._http = session_with_retries()
+        self._payload: Optional[AlgorithmPayload] = None
+        self._payload_uri = payload_uri
+        self._payload_type = payload_type
+
+    @property
+    def payload_uri(self) -> str:
+        """
+        Uri of the paylod for the algorithm
+        """
+        return self._payload_uri
+
+    @property
+    def payload(self) -> Optional[AlgorithmPayload]:
+        """
+        Payload data deserialized into the user class.
+        """
+        return self._payload

{esd_services_api_client-2.0.0.dist-info → esd_services_api_client-2.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.0.0
+Version: 2.0.2
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0
@@ -15,9 +15,12 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: azure
-Requires-Dist: adapta[azure] (>=2.2.0,<3.0.0)
+Provides-Extra: nexus
+Requires-Dist: adapta[azure,datadog,storage] (>=2.6.6,<3.0.0)
 Requires-Dist: azure-identity (>=1.7,<1.8) ; extra == "azure"
 Requires-Dist: dataclasses-json (>=0.6.0,<0.7.0)
+Requires-Dist: httpx (>=0.26.0,<0.27.0) ; extra == "nexus"
+Requires-Dist: injector (>=0.21.0,<0.22.0) ; extra == "nexus"
 Requires-Dist: pycryptodome (>=3.15,<3.16)
 Requires-Dist: pyjwt (>=2.4.0,<2.5.0)
 Project-URL: Repository, https://github.com/SneaksAndData/esd-services-api-client

esd_services_api_client-2.0.2.dist-info/RECORD

@@ -0,0 +1,43 @@
+esd_services_api_client/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
+esd_services_api_client/_version.py,sha256=kumiGImhzOTlTrRM-6jDo2mNnVHGO_2vxtrhB0nzAiw,22
+esd_services_api_client/beast/__init__.py,sha256=NTaz_7YoLPK8MCLwbwqH7rW1zDWLxXu2T7fGmMmRxyg,718
+esd_services_api_client/beast/v3/__init__.py,sha256=TRjB4-T6eIORpMvdylb32_GinrIpYNFmAdshSC1HqHg,749
+esd_services_api_client/beast/v3/_connector.py,sha256=oPizDQ1KOKOfiyh-jAofKodlpRzrRiELv-rmP_o_oio,11473
+esd_services_api_client/beast/v3/_models.py,sha256=hvh8QlwYLCfdBJxUKPutOq0xFn9rUU1q6UgHEpoZaFM,6761
+esd_services_api_client/boxer/README.md,sha256=-MAhYUPvmEMcgx_lo_2PlH_gZI1lndGv8fnDQYIpurU,3618
+esd_services_api_client/boxer/__init__.py,sha256=OYsWvdnLan0kmjUcH4I2-m1rbPeARKp5iqhp8uyudPk,780
+esd_services_api_client/boxer/_auth.py,sha256=vA7T9y0oZV2f17UWQ2or9CK8vAsNnHB10G5HNQe1l1I,7440
+esd_services_api_client/boxer/_base.py,sha256=PSU08QzTKFkXfzx7fF5FIHBOZXshxNEd1J_qKGo0Rd0,976
+esd_services_api_client/boxer/_connector.py,sha256=kATr7IyQKtvsGlvgVInO_DsGGC4yCC1aW3RxoXt-QIM,8672
+esd_services_api_client/boxer/_models.py,sha256=q_xRHOXlRroKrDrNUxQJUFGSotxYvUbRY1rYmx8UfJg,1697
+esd_services_api_client/common/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
+esd_services_api_client/crystal/__init__.py,sha256=afSGQRkDic0ECsJfgu3b291kX8CyU57sjreqxj5cx-s,787
+esd_services_api_client/crystal/_api_versions.py,sha256=2BMiQRS0D8IEpWCCys3dge5alVBRCZrOuCR1QAn8UIM,832
+esd_services_api_client/crystal/_connector.py,sha256=WjfMezWXia41Z8aiNupaT577fk9Sx6uy6V23O6y9hfI,12870
+esd_services_api_client/crystal/_models.py,sha256=eRhGAl8LjglCyIFwf1bcFBhjbpSuRYucuF2LO388L2E,4025
+esd_services_api_client/nexus/README.md,sha256=6fGBYTvzkmZH9Yk9VcPqWCtqU_D2zleG1rv1U3GA3IM,7721
+esd_services_api_client/nexus/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
+esd_services_api_client/nexus/abstractions/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
+esd_services_api_client/nexus/abstractions/logger_factory.py,sha256=JHl_t0d0ra_k-EixZlkw-s746wHUdBhSU6preqoARtk,2031
+esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=E9iYmjYjkmS0Uv-VS1ixRhi3zdtQfcSNx4gxad5_aZ0,1780
+esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=Mv9BWdxw8VY4Gi4EOrdxWK1zsR3-fqIbpyF1xHWchbE,1495
+esd_services_api_client/nexus/algorithms/__init__.py,sha256=uWX24NHUnUcnOJN2IIp1kaaozCC0rOmB36WxHBCapCY,823
+esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=4Gqp8qV5nunZ_DwZyHAu0vLuF1rls8nnq2zoHy2orME,1825
+esd_services_api_client/nexus/algorithms/distributed.py,sha256=iWjx9D6g-ASwTWPkQ9GmInTLymVlxl7UkfEBcEfnkmc,1628
+esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=9KvIXXsiOZ9wAOvrIZuOlNYcJLB-SjlQoNpICJu-qeQ,1366
+esd_services_api_client/nexus/algorithms/recursive.py,sha256=2YuAeYZ6-K5hcY8NEAAJVyGhtVVQMTZ7wjkWPPo8qeE,1829
+esd_services_api_client/nexus/core/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
+esd_services_api_client/nexus/core/app_core.py,sha256=bwbFzbjx8PwZWHfHsEYDCGAjJuTzmVKuunSeMU54tHs,8877
+esd_services_api_client/nexus/core/app_dependencies.py,sha256=afX7QrUEuaLnFssaCTohXBt7QCeGJ0MuovF62CZTZ34,5672
+esd_services_api_client/nexus/exceptions/__init__.py,sha256=JgPXhrvBIi0U1QOF90TYHS8jv_SBQEoRLsEg6rrtRoc,691
+esd_services_api_client/nexus/exceptions/_nexus_error.py,sha256=b3L8JnNvV2jdxNfuFWh9-j4kVb_VX7gNH5WHKcC-R78,890
+esd_services_api_client/nexus/exceptions/input_reader_error.py,sha256=D-xYTKRNREQ2-NGhc88GHOmXCvLNsIVQsH8wf0LLC_0,1760
+esd_services_api_client/nexus/exceptions/startup_error.py,sha256=f2PIOSdLgT-42eKD6ec8p7nROADshMawCsDGDUbxO_w,1546
+esd_services_api_client/nexus/input/__init__.py,sha256=0k_HMIP4NPC5O2ixKJPgKsLzYeHS14DhibF_MUtez1c,753
+esd_services_api_client/nexus/input/input_processor.py,sha256=MiXXd_APrG85Pi-Ke68_UHNEV7T_QHN1hU1WAPWoTsw,3187
+esd_services_api_client/nexus/input/input_reader.py,sha256=uxTAGX5xNhjTFpEsVQnr8BkVgpIH_U_om54hh3pvJ3s,3269
+esd_services_api_client/nexus/input/payload_reader.py,sha256=__r_QjIFRAWwx56X5WUK1qensJUae0vZEb422dzOgSY,2511
+esd_services_api_client-2.0.2.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
+esd_services_api_client-2.0.2.dist-info/METADATA,sha256=rNOu2tUM-mWp4kQbt13EpUweANcI808ZNCFmMdcqQRM,1236
+esd_services_api_client-2.0.2.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+esd_services_api_client-2.0.2.dist-info/RECORD,,

esd_services_api_client-2.0.0.dist-info/RECORD

@@ -1,21 +0,0 @@
-esd_services_api_client/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
-esd_services_api_client/_version.py,sha256=hwOJuFUEKtsgiOt4l6op9s-hviovM6TK1uCdtRNWY4E,22
-esd_services_api_client/beast/__init__.py,sha256=NTaz_7YoLPK8MCLwbwqH7rW1zDWLxXu2T7fGmMmRxyg,718
-esd_services_api_client/beast/v3/__init__.py,sha256=TRjB4-T6eIORpMvdylb32_GinrIpYNFmAdshSC1HqHg,749
-esd_services_api_client/beast/v3/_connector.py,sha256=oPizDQ1KOKOfiyh-jAofKodlpRzrRiELv-rmP_o_oio,11473
-esd_services_api_client/beast/v3/_models.py,sha256=hvh8QlwYLCfdBJxUKPutOq0xFn9rUU1q6UgHEpoZaFM,6761
-esd_services_api_client/boxer/README.md,sha256=-MAhYUPvmEMcgx_lo_2PlH_gZI1lndGv8fnDQYIpurU,3618
-esd_services_api_client/boxer/__init__.py,sha256=OYsWvdnLan0kmjUcH4I2-m1rbPeARKp5iqhp8uyudPk,780
-esd_services_api_client/boxer/_auth.py,sha256=vA7T9y0oZV2f17UWQ2or9CK8vAsNnHB10G5HNQe1l1I,7440
-esd_services_api_client/boxer/_base.py,sha256=PSU08QzTKFkXfzx7fF5FIHBOZXshxNEd1J_qKGo0Rd0,976
-esd_services_api_client/boxer/_connector.py,sha256=kATr7IyQKtvsGlvgVInO_DsGGC4yCC1aW3RxoXt-QIM,8672
-esd_services_api_client/boxer/_models.py,sha256=q_xRHOXlRroKrDrNUxQJUFGSotxYvUbRY1rYmx8UfJg,1697
-esd_services_api_client/common/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
-esd_services_api_client/crystal/__init__.py,sha256=afSGQRkDic0ECsJfgu3b291kX8CyU57sjreqxj5cx-s,787
-esd_services_api_client/crystal/_api_versions.py,sha256=2BMiQRS0D8IEpWCCys3dge5alVBRCZrOuCR1QAn8UIM,832
-esd_services_api_client/crystal/_connector.py,sha256=yJObx9kHtaUmjry_B7EaJqbxn6t9aR0ypgrCGsCO5pI,12854
-esd_services_api_client/crystal/_models.py,sha256=eRhGAl8LjglCyIFwf1bcFBhjbpSuRYucuF2LO388L2E,4025
-esd_services_api_client-2.0.0.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
-esd_services_api_client-2.0.0.dist-info/METADATA,sha256=p6kY-uup2AS5oQ66wcPuMpfWPxogN0FbzKcfEA4XPX4,1077
-esd_services_api_client-2.0.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-esd_services_api_client-2.0.0.dist-info/RECORD,,