esd-services-api-client 2.0.7__py3-none-any.whl → 2.0.9__py3-none-any.whl

esd_services_api_client/_version.py

@@ -1 +1 @@
-__version__ = '2.0.7'
+__version__ = '2.0.9'

esd_services_api_client/nexus/README.md

@@ -20,7 +20,7 @@ import socketserver
 import threading
 from dataclasses import dataclass
 from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
-from typing import Dict, Optional
+from typing import Dict, Optional, Any
 
 import pandas
 from adapta.metrics import MetricsProvider
@@ -36,6 +36,7 @@ from esd_services_api_client.nexus.configurations.algorithm_configuration import
     NexusConfiguration,
 )
 from esd_services_api_client.nexus.core.app_core import Nexus
+from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
 from esd_services_api_client.nexus.algorithms import MinimalisticAlgorithm
 from esd_services_api_client.nexus.input import InputReader, InputProcessor
 
@@ -229,8 +230,19 @@ class MyInputProcessor(InputProcessor):
             "y_ready": inputs["y"].assign(c=[-1, 1]),
         }
 
+@dataclass
+class MyResult(AlgorithmResult):
+    x: pandas.DataFrame
+    y: pandas.DataFrame
+
+    def dataframe(self) -> pandas.DataFrame:
+        return pandas.concat([self.x, self.y])
 
-class MyAlgorithm(MinimalisticAlgorithm[MyAlgorithmPayload, pandas.DataFrame]):
+    def to_kwargs(self) -> dict[str, Any]:
+        pass
+    
+    
+class MyAlgorithm(MinimalisticAlgorithm[MyAlgorithmPayload]):
     async def _context_open(self):
         pass
 
@@ -241,8 +253,8 @@ class MyAlgorithm(MinimalisticAlgorithm[MyAlgorithmPayload, pandas.DataFrame]):
     def __init__(self, metrics_provider: MetricsProvider, logger_factory: LoggerFactory, input_processor: MyInputProcessor):
         super().__init__(metrics_provider, logger_factory, input_processor)
 
-    async def _run(self, x_ready: pandas.DataFrame, y_ready: pandas.DataFrame, **kwargs) -> pandas.DataFrame:
-        return pandas.concat([x_ready, y_ready])
+    async def _run(self, x_ready: pandas.DataFrame, y_ready: pandas.DataFrame, **kwargs) -> MyResult:
+        return MyResult(x_ready, y_ready)
 
 
 async def main():

esd_services_api_client/nexus/abstractions/nexus_object.py

@@ -1,6 +1,7 @@
 """
  Base classes for all objects used by Nexus.
 """
+import re
 
 #  Copyright (c) 2023. ECCO Sneaks & Data
 #
@@ -19,7 +20,7 @@
 
 
 from abc import ABC, abstractmethod
-from typing import Generic, TypeVar
+from typing import Generic, TypeVar, Union, Any
 
 import pandas
 import polars
@@ -28,6 +29,24 @@ from adapta.metrics import MetricsProvider
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 
 
+class AlgorithmResult(ABC):
+    """
+    Interface for algorithm run result. You can store arbitrary data here, but `dataframe` method must be implemented.
+    """
+
+    @abstractmethod
+    def dataframe(self) -> Union[pandas.DataFrame, polars.DataFrame]:
+        """
+        Returns the main result dataframe. This will be written to the linked output storage.
+        """
+
+    @abstractmethod
+    def to_kwargs(self) -> dict[str, Any]:
+        """
+        Convert result to kwargs for the next iteration (for recursive algorithms)
+        """
+
+
 TPayload = TypeVar("TPayload")  # pylint: disable=C0103
 TResult = TypeVar(  # pylint: disable=C0103
     "TResult", pandas.DataFrame, polars.DataFrame
@@ -67,3 +86,14 @@ class NexusObject(Generic[TPayload, TResult], ABC):
         """
         Optional actions to perform on context closure.
         """
+
+    @classmethod
+    def alias(cls) -> str:
+        """
+        Alias to identify this reader's output
+        """
+        return re.sub(
+            r"(?<!^)(?=[A-Z])",
+            "_",
+            cls.__name__.lower().replace("reader", "").replace("processor", ""),
+        )

esd_services_api_client/nexus/algorithms/__init__.py

@@ -20,3 +20,4 @@
 from esd_services_api_client.nexus.algorithms.minimalistic import *
 from esd_services_api_client.nexus.algorithms.recursive import *
 from esd_services_api_client.nexus.algorithms.distributed import *
+from esd_services_api_client.nexus.algorithms._baseline_algorithm import *

esd_services_api_client/nexus/algorithms/_baseline_algorithm.py

@@ -28,12 +28,13 @@ from esd_services_api_client.nexus.abstractions.nexus_object import (
     NexusObject,
     TPayload,
     TResult,
+    AlgorithmResult,
 )
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
 from esd_services_api_client.nexus.input.input_processor import InputProcessor
 
 
-class BaselineAlgorithm(NexusObject[TPayload, TResult]):
+class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
     """
     Base class for all algorithm implementations.
     """
@@ -48,17 +49,19 @@ class BaselineAlgorithm(NexusObject[TPayload, TResult]):
         self._input_processors = input_processors
 
     @abstractmethod
-    async def _run(self, **kwargs) -> TResult:
+    async def _run(self, **kwargs) -> AlgorithmResult:
         """
         Core logic for this algorithm. Implementing this method is mandatory.
         """
 
-    async def run(self, **kwargs) -> TResult:
+    async def run(self, **kwargs) -> AlgorithmResult:
         """
         Coroutine that executes the algorithm logic.
         """
 
-        async def _process(processor: InputProcessor) -> dict[str, TResult]:
+        async def _process(
+            processor: InputProcessor[TPayload, TResult]
+        ) -> dict[str, TResult]:
             async with processor as instance:
                 return await instance.process_input(**kwargs)
 

esd_services_api_client/nexus/algorithms/distributed.py

@@ -21,13 +21,16 @@
 import asyncio
 from abc import ABC, abstractmethod
 
-from esd_services_api_client.nexus.abstractions.nexus_object import TResult, TPayload
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    TPayload,
+    AlgorithmResult,
+)
 from esd_services_api_client.nexus.algorithms._baseline_algorithm import (
     BaselineAlgorithm,
 )
 
 
-class DistributedAlgorithm(BaselineAlgorithm[TPayload, TResult], ABC):
+class DistributedAlgorithm(BaselineAlgorithm[TPayload], ABC):
     """
     Distributed algorithm base class.
     """
@@ -39,12 +42,12 @@ class DistributedAlgorithm(BaselineAlgorithm[TPayload, TResult], ABC):
         """
 
     @abstractmethod
-    async def _fold(self, *split_tasks: asyncio.Task) -> TResult:
+    async def _fold(self, *split_tasks: asyncio.Task) -> AlgorithmResult:
         """
         Sub-problem result aggregator.
         """
 
-    async def _run(self, **kwargs) -> TResult:
+    async def _run(self, **kwargs) -> AlgorithmResult:
         splits = await self._split(**kwargs)
         tasks = [asyncio.create_task(split.run(**kwargs)) for split in splits]
 

esd_services_api_client/nexus/algorithms/minimalistic.py

@@ -23,14 +23,14 @@ from adapta.metrics import MetricsProvider
 from injector import inject
 
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
-from esd_services_api_client.nexus.abstractions.nexus_object import TResult, TPayload
+from esd_services_api_client.nexus.abstractions.nexus_object import TPayload
 from esd_services_api_client.nexus.algorithms._baseline_algorithm import (
     BaselineAlgorithm,
 )
 from esd_services_api_client.nexus.input import InputProcessor
 
 
-class MinimalisticAlgorithm(BaselineAlgorithm[TPayload, TResult], ABC):
+class MinimalisticAlgorithm(BaselineAlgorithm[TPayload], ABC):
     """
     Simple algorithm base class.
     """

esd_services_api_client/nexus/algorithms/recursive.py

@@ -24,14 +24,17 @@ from adapta.metrics import MetricsProvider
 from injector import inject
 
 from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
-from esd_services_api_client.nexus.abstractions.nexus_object import TResult, TPayload
+from esd_services_api_client.nexus.abstractions.nexus_object import (
+    TPayload,
+    AlgorithmResult,
+)
 from esd_services_api_client.nexus.algorithms._baseline_algorithm import (
     BaselineAlgorithm,
 )
 from esd_services_api_client.nexus.input import InputProcessor
 
 
-class RecursiveAlgorithm(BaselineAlgorithm[TPayload, TResult]):
+class RecursiveAlgorithm(BaselineAlgorithm[TPayload]):
     """
     Recursive algorithm base class.
     """
@@ -49,8 +52,8 @@ class RecursiveAlgorithm(BaselineAlgorithm[TPayload, TResult]):
     async def _is_finished(self, **kwargs) -> bool:
         """ """
 
-    async def run(self, **kwargs) -> TResult:
+    async def run(self, **kwargs) -> AlgorithmResult:
         result = await self._run(**kwargs)
-        if self._is_finished(**result):
+        if self._is_finished(**result.to_kwargs()):
             return result
-        return await self.run(**result)
+        return await self.run(**result.to_kwargs())

esd_services_api_client/nexus/core/app_core.py

@@ -33,8 +33,6 @@ from adapta.storage.blob.base import StorageClient
 from adapta.storage.models.format import DataFrameJsonSerializationFormat
 from injector import Injector
 
-from pandas import DataFrame as PandasDataFrame
-
 import esd_services_api_client.nexus.exceptions
 from esd_services_api_client.crystal import (
     add_crystal_args,
@@ -43,7 +41,8 @@ from esd_services_api_client.crystal import (
     AlgorithmRunResult,
     CrystalEntrypointArguments,
 )
-from esd_services_api_client.nexus.algorithms._baseline_algorithm import (
+from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
+from esd_services_api_client.nexus.algorithms import (
     BaselineAlgorithm,
 )
 from esd_services_api_client.nexus.configurations.algorithm_configuration import (
@@ -181,7 +180,7 @@ class Nexus:
 
     async def _submit_result(
         self,
-        result: Optional[PandasDataFrame] = None,
+        result: Optional[AlgorithmResult] = None,
         ex: Optional[BaseException] = None,
     ) -> None:
         @backoff.on_exception(
@@ -193,7 +192,7 @@ class Nexus:
             max_time=10,
             raise_on_giveup=True,
         )
-        def save_result(data: PandasDataFrame) -> str:
+        def save_result(data: AlgorithmResult) -> str:
             """
             Saves blob and returns the uri
 
@@ -209,7 +208,7 @@ class Nexus:
                 data_path=output_path, alias="output", data_format="null"
             ).parse_data_path()
             storage_client.save_data_as_blob(
-                data=data,
+                data=data.dataframe(),
                 blob_path=blob_path,
                 serialization_format=DataFrameJsonSerializationFormat,
                 overwrite=True,

esd_services_api_client/nexus/input/_functions.py

@@ -64,7 +64,8 @@ async def resolve_readers(
             return await instance.read()
 
     read_tasks: dict[str, asyncio.Task] = {
-        reader.alias: asyncio.create_task(_read(reader)) for reader in readers
+        reader.__class__.alias(): asyncio.create_task(_read(reader))
+        for reader in readers
     }
     await asyncio.wait(fs=read_tasks.values())
 

esd_services_api_client/nexus/input/input_reader.py

@@ -18,7 +18,6 @@ import functools
 #  limitations under the License.
 #
 
-import re
 from abc import abstractmethod
 from functools import partial
 from typing import Optional
@@ -57,16 +56,6 @@ class InputReader(NexusObject[TPayload, TResult]):
         self._readers = readers
         self._payload = payload
 
-    @property
-    def alias(self) -> str:
-        """
-        Alias to identify this reader's output
-        """
-        if self.socket:
-            return self.socket.alias
-
-        return self._metric_name
-
     @functools.cached_property
     def data(self) -> Optional[TResult]:
         """
@@ -80,17 +69,9 @@ class InputReader(NexusObject[TPayload, TResult]):
         Actual data reader logic. Implementing this method is mandatory for the reader to work
         """
 
-    @property
-    def _metric_name(self) -> str:
-        return re.sub(
-            r"(?<!^)(?=[A-Z])",
-            "_",
-            self.__class__.__name__.lower().replace("reader", ""),
-        )
-
     @property
     def _metric_tags(self) -> dict[str, str]:
-        return {"entity": self._metric_name}
+        return {"entity": self.__class__.alias()}
 
     async def read(self) -> TResult:
         """
@@ -101,7 +82,7 @@ class InputReader(NexusObject[TPayload, TResult]):
             metric_name="read_input",
             on_finish_message_template="Finished reading {entity} from path {data_path} in {elapsed:.2f}s seconds",
             template_args={
-                "entity": self._metric_name.upper(),
+                "entity": self.__class__.alias().upper(),
             }
             | {"data_path": self.socket.data_path}
             if self.socket

{esd_services_api_client-2.0.7.dist-info → esd_services_api_client-2.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 2.0.7
+Version: 2.0.9
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0
@@ -17,7 +17,7 @@ Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: azure
 Provides-Extra: nexus
 Provides-Extra: nexus-polars
-Requires-Dist: adapta[azure,datadog,storage] (>=2.6.6,<3.0.0)
+Requires-Dist: adapta[azure,datadog,storage] (>=2.7.1,<3.0.0)
 Requires-Dist: azure-identity (>=1.7,<1.8) ; extra == "azure"
 Requires-Dist: dataclasses-json (>=0.6.0,<0.7.0)
 Requires-Dist: httpx (>=0.26.0,<0.27.0) ; extra == "nexus"

{esd_services_api_client-2.0.7.dist-info → esd_services_api_client-2.0.9.dist-info}/RECORD

@@ -1,5 +1,5 @@
 esd_services_api_client/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
-esd_services_api_client/_version.py,sha256=Eqd_jrp6aA0AOwkbRY6r2MSGXmOOkqUkdUqKDmktZJM,22
+esd_services_api_client/_version.py,sha256=YHfSoqxCUyNphrTJsW3Ed3e2IVM5_H1IYKii1J5IND4,22
 esd_services_api_client/beast/__init__.py,sha256=NTaz_7YoLPK8MCLwbwqH7rW1zDWLxXu2T7fGmMmRxyg,718
 esd_services_api_client/beast/v3/__init__.py,sha256=TRjB4-T6eIORpMvdylb32_GinrIpYNFmAdshSC1HqHg,749
 esd_services_api_client/beast/v3/_connector.py,sha256=oPizDQ1KOKOfiyh-jAofKodlpRzrRiELv-rmP_o_oio,11473
@@ -15,32 +15,32 @@ esd_services_api_client/crystal/__init__.py,sha256=afSGQRkDic0ECsJfgu3b291kX8CyU
 esd_services_api_client/crystal/_api_versions.py,sha256=2BMiQRS0D8IEpWCCys3dge5alVBRCZrOuCR1QAn8UIM,832
 esd_services_api_client/crystal/_connector.py,sha256=WjfMezWXia41Z8aiNupaT577fk9Sx6uy6V23O6y9hfI,12870
 esd_services_api_client/crystal/_models.py,sha256=eRhGAl8LjglCyIFwf1bcFBhjbpSuRYucuF2LO388L2E,4025
-esd_services_api_client/nexus/README.md,sha256=e3rqD2023wFGoumhXU2q68QLwbJEVbODAwznQ6e9VO4,8381
+esd_services_api_client/nexus/README.md,sha256=Q3laWkuCxqVBFsfP58mGAys-jVZeesroIRRMiaQMntc,8688
 esd_services_api_client/nexus/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
 esd_services_api_client/nexus/abstractions/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
 esd_services_api_client/nexus/abstractions/logger_factory.py,sha256=jw1xnrU2dEWSUhqNut4RzjRPZFmeNx6czOU_8-q4kuo,2014
-esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=BrE8fIyOz6gAz24yMRnBcB-px7dtzSJteRQN-kXEOys,1913
+esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=Y2jwpSa0cLyMtGxOSEpg3F8evvSMxYHx5t5KD92KotE,2759
 esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=Mv9BWdxw8VY4Gi4EOrdxWK1zsR3-fqIbpyF1xHWchbE,1495
-esd_services_api_client/nexus/algorithms/__init__.py,sha256=uWX24NHUnUcnOJN2IIp1kaaozCC0rOmB36WxHBCapCY,823
-esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=oQPEMFfWVYIYIyDzL5Trev9KMtIMapjkHGwdIUGTNPU,2350
-esd_services_api_client/nexus/algorithms/distributed.py,sha256=HM_uGr2V4tvRnDOn8Cd44IfnjcMDr1fwWlPEhQbtDM0,1669
-esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=HMemgm6Ogp8DW6HDrkMQZQikrE0ogUSCVVimTsAUos8,1475
-esd_services_api_client/nexus/algorithms/recursive.py,sha256=wfhoiZ9D_NJFRGHd7gDVo4Kq1wrqKKPX6w8_l_YnPlg,1813
+esd_services_api_client/nexus/algorithms/__init__.py,sha256=ySnLfuYuxi28EaqT18oW2Ka50ZR7WOvJlLhmIts-Xgw,898
+esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=_3QHWmBDnfByEJTftf4jKzje5GNqftb0_aCfp-8ksCU,2436
+esd_services_api_client/nexus/algorithms/distributed.py,sha256=kZS4InM4GoT4T6UrXRnfoDn4zxE2iGuqLvxBrzBetZc,1697
+esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=DOU52Me_W0luUPWmIRBFkJdfapyEkPqVbbvy4V5yQZ8,1457
+esd_services_api_client/nexus/algorithms/recursive.py,sha256=r4_LIlC_hhIBSrKtL99xE9md3SahctF3R3GT1tIwlg4,1857
 esd_services_api_client/nexus/configurations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 esd_services_api_client/nexus/configurations/algorithm_configuration.py,sha256=sg3ul-Vqd39li42cfMsdWtFJIBNfQ52XLmqKO2sryDg,1086
 esd_services_api_client/nexus/core/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
-esd_services_api_client/nexus/core/app_core.py,sha256=7cT6ZEzKzq3lfio6YNxREpTTeXt1wjhHUOXxaezuFiQ,9393
+esd_services_api_client/nexus/core/app_core.py,sha256=XvPWnr6wJckjNoM4vWg3CYBnrJOaWZFxiIil1sK-qNc,9420
 esd_services_api_client/nexus/core/app_dependencies.py,sha256=rIk6R-InJeF1P0DC4bhj4kJTNNxYg39Vhuz8AXfV6pk,6140
 esd_services_api_client/nexus/exceptions/__init__.py,sha256=JgPXhrvBIi0U1QOF90TYHS8jv_SBQEoRLsEg6rrtRoc,691
 esd_services_api_client/nexus/exceptions/_nexus_error.py,sha256=b3L8JnNvV2jdxNfuFWh9-j4kVb_VX7gNH5WHKcC-R78,890
 esd_services_api_client/nexus/exceptions/input_reader_error.py,sha256=D-xYTKRNREQ2-NGhc88GHOmXCvLNsIVQsH8wf0LLC_0,1760
 esd_services_api_client/nexus/exceptions/startup_error.py,sha256=f2PIOSdLgT-42eKD6ec8p7nROADshMawCsDGDUbxO_w,1546
 esd_services_api_client/nexus/input/__init__.py,sha256=DEdzkK43xjNl3XPQy3Q8xHGzXYsMwxCMs1QdEAsS4FI,814
-esd_services_api_client/nexus/input/_functions.py,sha256=UsjTV0kbErS_uYWusCMZrXYhiqgMMQjYJtFa8oQZAmo,2450
+esd_services_api_client/nexus/input/_functions.py,sha256=SonIImDN9Eiw_Eb2tKXXtLtBffPYdhdZKzWiPQPcjW8,2470
 esd_services_api_client/nexus/input/input_processor.py,sha256=lRtxrvxGXXmDWHsg4mTWCgM8ecu4kroMDqC3UQMGKYk,1827
-esd_services_api_client/nexus/input/input_reader.py,sha256=-ZEMXLtPqOm1OTbkGIJ7YGOLc_8erBMC87-L1YrRmGw,3500
+esd_services_api_client/nexus/input/input_reader.py,sha256=zVv_qYmf-kHfPmyo_mYyPO6HFk-lX8dItLkHtUGYxZQ,3092
 esd_services_api_client/nexus/input/payload_reader.py,sha256=__r_QjIFRAWwx56X5WUK1qensJUae0vZEb422dzOgSY,2511
-esd_services_api_client-2.0.7.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
-esd_services_api_client-2.0.7.dist-info/METADATA,sha256=M486vwf2pKSxZSE3cOfH1lDWqhs9SveNUtYN9aNVpwA,1332
-esd_services_api_client-2.0.7.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-esd_services_api_client-2.0.7.dist-info/RECORD,,
+esd_services_api_client-2.0.9.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
+esd_services_api_client-2.0.9.dist-info/METADATA,sha256=8wldbRYSsxJWGT8P_Lkl_sIvq9nrf9qt6e5O7xhoRWQ,1332
+esd_services_api_client-2.0.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+esd_services_api_client-2.0.9.dist-info/RECORD,,

{esd_services_api_client-2.0.7.dist-info → esd_services_api_client-2.0.9.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.8.1
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any