esd-services-api-client 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

esd_services_api_client/_version.py

@@ -1 +1 @@
-__version__ = '2.0.0'
+__version__ = '2.0.2'

esd_services_api_client/crystal/_connector.py

@@ -282,19 +282,19 @@ class CrystalConnector:
             "sasUri": result.sas_uri,
         }
 
-        if debug and self._logger is not None:
+        if not debug:
+            run_response = self._http.post(url=get_api_path(), json=payload)
+            # raise if not successful
+            run_response.raise_for_status()
+            return
+
+        if self._logger is not None:
             self._logger.debug(
                 "Submitting result to {submission_url}, payload {payload}",
                 submission_url=get_api_path(),
                 payload=json.dumps(payload),
             )
 
-        else:
-            run_response = self._http.post(url=get_api_path(), json=payload)
-
-            # raise if not successful
-            run_response.raise_for_status()
-
     @staticmethod
     def read_input(
         *,

esd_services_api_client/nexus/README.md

@@ -0,0 +1,280 @@
+## Nexus
+Set the following environment variables for Azure:
+```
+IS_LOCAL_RUN=1
+NEXUS__ALGORITHM_OUTPUT_PATH=abfss://container@account.dfs.core.windows.net/path/to/result
+NEXUS__METRIC_PROVIDER_CONFIGURATION={"metric_namespace": "test"}
+NEXUS__QES_CONNECTION_STRING=qes://engine\=DELTA\;plaintext_credentials\={"auth_client_class":"adapta.security.clients.AzureClient"}\;settings\={}
+NEXUS__STORAGE_CLIENT_CLASS=adapta.storage.blob.azure_storage_client.AzureStorageClient
+NEXUS__ALGORITHM_INPUT_EXTERNAL_DATA_SOCKETS=[{"alias": "x", "data_path": "test/x", "data_format": "test"}, {"alias": "y", "data_path": "test/y", "data_format": "test"}]
+PROTEUS__USE_AZURE_CREDENTIAL=1
+```
+
+Example usage:
+
+```python
+import asyncio
+import json
+import socketserver
+import threading
+from dataclasses import dataclass
+from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
+from typing import Dict, Optional
+
+import pandas
+from adapta.metrics import MetricsProvider
+from adapta.storage.query_enabled_store import QueryEnabledStore
+from dataclasses_json import DataClassJsonMixin
+from injector import inject
+
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.abstractions.socket_provider import (
+    ExternalSocketProvider,
+)
+from esd_services_api_client.nexus.core.app_core import Nexus
+from esd_services_api_client.nexus.algorithms import MinimalisticAlgorithm
+from esd_services_api_client.nexus.input import InputReader, InputProcessor
+from pandas import DataFrame as PandasDataFrame
+
+from esd_services_api_client.nexus.input.payload_reader import AlgorithmPayload
+
+
+async def my_on_complete_func_1(**kwargs):
+    pass
+
+
+async def my_on_complete_func_2(**kwargs):
+    pass
+
+
+@dataclass
+class MyAlgorithmPayload(AlgorithmPayload, DataClassJsonMixin):
+    x: Optional[list[int]] = None
+    y: Optional[list[int]] = None
+
+
+@dataclass
+class MyAlgorithmPayload2(AlgorithmPayload, DataClassJsonMixin):
+    z: list[int]
+    x: Optional[list[int]] = None
+    y: Optional[list[int]] = None
+
+
+class MockRequestHandler(BaseHTTPRequestHandler):
+    """
+    HTTPServer Mock Request handler
+    """
+
+    def __init__(
+        self,
+        request: bytes,
+        client_address: tuple[str, int],
+        server: socketserver.BaseServer,
+    ):
+        """
+         Initialize request handler
+        :param request:
+        :param client_address:
+        :param server:
+        """
+        self._responses = {
+            "some/payload": (
+                {
+                    # "x": [-1, 0, 2],
+                    # "y": [10, 11, 12],
+                    "z": [1, 2, 3]
+                },
+                200,
+            )
+        }
+        super().__init__(request, client_address, server)
+
+    def do_GET(self):  # pylint: disable=invalid-name
+        """Handle POST requests"""
+        current_url = self.path.removeprefix("/")
+
+        if current_url not in self._responses:
+            self.send_response(500, "Unknown URL")
+            return
+
+        self.send_response(self._responses[current_url][1])
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        self.wfile.write(json.dumps(self._responses[current_url][0]).encode("utf-8"))
+
+    def log_request(self, code=None, size=None):
+        """
+         Don't log anything
+        :param code:
+        :param size:
+        :return:
+        """
+        pass
+
+
+class XReader(InputReader[MyAlgorithmPayload]):
+    async def _context_open(self):
+        pass
+
+    async def _context_close(self):
+        pass
+
+    @inject
+    def __init__(
+        self,
+        store: QueryEnabledStore,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        payload: MyAlgorithmPayload,
+        socket_provider: ExternalSocketProvider,
+        *readers: "InputReader"
+    ):
+        super().__init__(
+            socket_provider.socket("x"),
+            store,
+            metrics_provider,
+            logger_factory,
+            payload,
+            *readers
+        )
+
+    async def _read_input(self) -> PandasDataFrame:
+        self._logger.info(
+            "Payload: {payload}; Socket path: {socket_path}",
+            payload=self._payload.to_json(),
+            socket_path=self.socket.data_path,
+        )
+        return pandas.DataFrame([{"a": 1, "b": 2}, {"a": 2, "b": 3}])
+
+
+class YReader(InputReader[MyAlgorithmPayload2]):
+    async def _context_open(self):
+        pass
+
+    async def _context_close(self):
+        pass
+
+    @inject
+    def __init__(
+        self,
+        store: QueryEnabledStore,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+        payload: MyAlgorithmPayload2,
+        socket_provider: ExternalSocketProvider,
+        *readers: "InputReader"
+    ):
+        super().__init__(
+            socket_provider.socket("y"),
+            store,
+            metrics_provider,
+            logger_factory,
+            payload,
+            *readers
+        )
+
+    async def _read_input(self) -> PandasDataFrame:
+        self._logger.info(
+            "Payload: {payload}; Socket path: {socket_path}",
+            payload=self._payload.to_json(),
+            socket_path=self.socket.data_path,
+        )
+        return pandas.DataFrame([{"a": 10, "b": 12}, {"a": 11, "b": 13}])
+
+
+class MyInputProcessor(InputProcessor):
+    async def _context_open(self):
+        pass
+
+    async def _context_close(self):
+        pass
+
+    @inject
+    def __init__(
+        self,
+        x: XReader,
+        y: YReader,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+    ):
+        super().__init__(
+            x,
+            y,
+            metrics_provider=metrics_provider,
+            logger_factory=logger_factory,
+            payload=None,
+        )
+
+    async def process_input(self, **_) -> Dict[str, PandasDataFrame]:
+        inputs = await self._read_input()
+        return {
+            "x_ready": inputs["x"].assign(c=[-1, 1]),
+            "y_ready": inputs["y"].assign(c=[-1, 1]),
+        }
+
+
+class MyAlgorithm(MinimalisticAlgorithm):
+    async def _context_open(self):
+        pass
+
+    async def _context_close(self):
+        pass
+
+    @inject
+    def __init__(
+        self,
+        input_processor: MyInputProcessor,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+    ):
+        super().__init__(input_processor, metrics_provider, logger_factory)
+
+    async def _run(
+        self, x_ready: PandasDataFrame, y_ready: PandasDataFrame, **kwargs
+    ) -> PandasDataFrame:
+        return pandas.concat([x_ready, y_ready])
+
+
+async def main():
+    """
+     Mock HTTP Server
+    :return:
+    """
+    with ThreadingHTTPServer(("localhost", 9876), MockRequestHandler) as server:
+        server_thread = threading.Thread(target=server.serve_forever)
+        server_thread.daemon = True
+        server_thread.start()
+        nexus = (
+            await Nexus.create()
+            .add_reader(XReader)
+            .add_reader(YReader)
+            .use_processor(MyInputProcessor)
+            .use_algorithm(MyAlgorithm)
+            .inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)
+        )
+
+        await nexus.activate()
+        server.shutdown()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+
+```
+
+Run this code as `sample.py`:
+
+```shell
+python3 sample.py --sas-uri http://localhost:9876/some/payload --request-id test
+```
+
+Produces the following:
+
+```
+Running _read
+Payload: {"x": null, "y": null}; Socket path: test/x
+Finished reading X from path test/x in 0.00s seconds
+Running _read
+Payload: {"z": [1, 2, 3], "x": null, "y": null}; Socket path: test/y
+Finished reading Y from path test/y in 0.00s seconds
+```

esd_services_api_client/nexus/__init__.py

@@ -0,0 +1,18 @@
+"""
+ Import index.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#

esd_services_api_client/nexus/abstractions/__init__.py

@@ -0,0 +1,18 @@
+"""
+ Import index.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#

esd_services_api_client/nexus/abstractions/logger_factory.py

@@ -0,0 +1,64 @@
+"""
+ Logger factory for async loggers.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import json
+import os
+from logging import StreamHandler
+from typing import final, Type, TypeVar, Optional, Dict
+
+from adapta.logs._async_logger import _AsyncLogger, create_async_logger
+from adapta.logs.handlers.datadog_api_handler import DataDogApiHandler
+from adapta.logs.models import LogLevel
+
+TLogger = TypeVar("TLogger")  # pylint: disable=C0103:
+
+
+@final
+class LoggerFactory:
+    """
+    Async logger provisioner.
+    """
+
+    def __init__(self):
+        self._log_handlers = [
+            StreamHandler(),
+        ]
+        if "NEXUS__DATADOG_LOGGER_CONFIGURATION" in os.environ:
+            self._log_handlers.append(
+                DataDogApiHandler(
+                    **json.loads(os.getenv("NEXUS__DATADOG_LOGGER_CONFIGURATION"))
+                )
+            )
+
+    def create_logger(
+        self,
+        logger_type: Type[TLogger],
+        fixed_template: Optional[Dict[str, Dict[str, str]]] = None,
+        fixed_template_delimiter=", ",
+    ) -> _AsyncLogger[TLogger]:
+        """
+        Creates an async-safe logger for the provided class name.
+        """
+        return create_async_logger(
+            logger_type=logger_type,
+            log_handlers=self._log_handlers,
+            min_log_level=LogLevel(os.getenv("NEXUS__LOG_LEVEL", "INFO")),
+            fixed_template=fixed_template,
+            fixed_template_delimiter=fixed_template_delimiter,
+        )

esd_services_api_client/nexus/abstractions/nexus_object.py

@@ -0,0 +1,64 @@
+"""
+ Base classes for all objects used by Nexus.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+from adapta.metrics import MetricsProvider
+
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+
+
+TPayload = TypeVar("TPayload")  # pylint: disable=C0103
+
+
+class NexusObject(Generic[TPayload], ABC):
+    """
+    Base class for all Nexus objects.
+    """
+
+    def __init__(
+        self,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+    ):
+        self._metrics_provider = metrics_provider
+        self._logger = logger_factory.create_logger(logger_type=self.__class__)
+
+    async def __aenter__(self):
+        self._logger.start()
+        await self._context_open()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        self._logger.stop()
+        await self._context_close()
+
+    @abstractmethod
+    async def _context_open(self):
+        """
+        Optional actions to perform on context activation.
+        """
+
+    @abstractmethod
+    async def _context_close(self):
+        """
+        Optional actions to perform on context closure.
+        """

esd_services_api_client/nexus/abstractions/socket_provider.py

@@ -0,0 +1,51 @@
+"""
+ Socket provider for all data sockets used by algorithms.
+"""
+import json
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from typing import final, Optional
+
+from adapta.process_communication import DataSocket
+
+
+@final
+class ExternalSocketProvider:
+    """
+    Wraps a socket collection
+    """
+
+    def __init__(self, *sockets: DataSocket):
+        self._sockets = {socket.alias: socket for socket in sockets}
+
+    def socket(self, name: str) -> Optional[DataSocket]:
+        """
+        Retrieve a socket if it exists.
+        """
+        return self._sockets.get(name, None)
+
+    @classmethod
+    def from_serialized(cls, socket_list_ser: str) -> "ExternalSocketProvider":
+        """
+        Creates a SocketProvider from a list of serialized sockets
+        """
+        return cls(
+            *[
+                DataSocket.from_dict(socket_dict)
+                for socket_dict in json.loads(socket_list_ser)
+            ]
+        )

esd_services_api_client/nexus/algorithms/__init__.py

@@ -0,0 +1,22 @@
+"""
+ Import index.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from esd_services_api_client.nexus.algorithms.minimalistic import *
+from esd_services_api_client.nexus.algorithms.recursive import *
+from esd_services_api_client.nexus.algorithms.distributed import *

esd_services_api_client/nexus/algorithms/_baseline_algorithm.py

@@ -0,0 +1,56 @@
+"""
+ Base algorithm
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+from abc import abstractmethod
+
+from adapta.metrics import MetricsProvider
+from pandas import DataFrame as PandasDataFrame
+
+from esd_services_api_client.nexus.abstractions.nexus_object import NexusObject
+from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
+from esd_services_api_client.nexus.input.input_processor import InputProcessor
+
+
+class BaselineAlgorithm(NexusObject):
+    """
+    Base class for all algorithm implementations.
+    """
+
+    def __init__(
+        self,
+        input_processor: InputProcessor,
+        metrics_provider: MetricsProvider,
+        logger_factory: LoggerFactory,
+    ):
+        super().__init__(metrics_provider, logger_factory)
+        self._input_processor = input_processor
+
+    @abstractmethod
+    async def _run(self, **kwargs) -> PandasDataFrame:
+        """
+        Core logic for this algorithm. Implementing this method is mandatory.
+        """
+
+    async def run(self, **kwargs) -> PandasDataFrame:
+        """
+        Coroutine that executes the algorithm logic.
+        """
+        async with self._input_processor as input_processor:
+            return await self._run(**(await input_processor.process_input(**kwargs)))

esd_services_api_client/nexus/algorithms/distributed.py

@@ -0,0 +1,53 @@
+"""
+ Algorithm that supports splitting the problem into sub-problems and combining the results.
+ Sub-problems can be Distributed, Minimalistic or Recursive as well.
+"""
+
+#  Copyright (c) 2023. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import asyncio
+from abc import ABC, abstractmethod
+
+from pandas import DataFrame as PandasDataFrame
+from esd_services_api_client.nexus.algorithms._baseline_algorithm import (
+    BaselineAlgorithm,
+)
+
+
+class DistributedAlgorithm(BaselineAlgorithm, ABC):
+    """
+    Distributed algorithm base class.
+    """
+
+    @abstractmethod
+    async def _split(self, **_) -> list[BaselineAlgorithm]:
+        """
+        Sub-problem generator.
+        """
+
+    @abstractmethod
+    async def _fold(self, *split_tasks: asyncio.Task) -> PandasDataFrame:
+        """
+        Sub-problem result aggregator.
+        """
+
+    async def _run(self, **kwargs) -> PandasDataFrame:
+        splits = await self._split(**kwargs)
+        tasks = [asyncio.create_task(split.run(**kwargs)) for split in splits]
+
+        await asyncio.wait(*tasks)
+
+        return await self._fold(*tasks)