esd-services-api-client 2.0.1__py3-none-any.whl → 2.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- esd_services_api_client/_version.py +1 -1
- esd_services_api_client/nexus/README.md +182 -26
- esd_services_api_client/nexus/abstractions/nexus_object.py +5 -1
- esd_services_api_client/nexus/abstractions/socket_provider.py +51 -0
- esd_services_api_client/nexus/core/app_core.py +22 -0
- esd_services_api_client/nexus/core/app_dependencies.py +38 -0
- esd_services_api_client/nexus/input/input_processor.py +7 -4
- esd_services_api_client/nexus/input/input_reader.py +15 -3
- esd_services_api_client/nexus/input/payload_reader.py +83 -0
- {esd_services_api_client-2.0.1.dist-info → esd_services_api_client-2.0.2.dist-info}/METADATA +2 -2
- {esd_services_api_client-2.0.1.dist-info → esd_services_api_client-2.0.2.dist-info}/RECORD +13 -11
- {esd_services_api_client-2.0.1.dist-info → esd_services_api_client-2.0.2.dist-info}/LICENSE +0 -0
- {esd_services_api_client-2.0.1.dist-info → esd_services_api_client-2.0.2.dist-info}/WHEEL +0 -0
@@ -1 +1 @@
|
|
1
|
-
__version__ = '2.0.
|
1
|
+
__version__ = '2.0.2'
|
@@ -6,6 +6,7 @@ NEXUS__ALGORITHM_OUTPUT_PATH=abfss://container@account.dfs.core.windows.net/path
|
|
6
6
|
NEXUS__METRIC_PROVIDER_CONFIGURATION={"metric_namespace": "test"}
|
7
7
|
NEXUS__QES_CONNECTION_STRING=qes://engine\=DELTA\;plaintext_credentials\={"auth_client_class":"adapta.security.clients.AzureClient"}\;settings\={}
|
8
8
|
NEXUS__STORAGE_CLIENT_CLASS=adapta.storage.blob.azure_storage_client.AzureStorageClient
|
9
|
+
NEXUS__ALGORITHM_INPUT_EXTERNAL_DATA_SOCKETS=[{"alias": "x", "data_path": "test/x", "data_format": "test"}, {"alias": "y", "data_path": "test/y", "data_format": "test"}]
|
9
10
|
PROTEUS__USE_AZURE_CREDENTIAL=1
|
10
11
|
```
|
11
12
|
|
@@ -13,20 +14,30 @@ Example usage:
|
|
13
14
|
|
14
15
|
```python
|
15
16
|
import asyncio
|
16
|
-
|
17
|
+
import json
|
18
|
+
import socketserver
|
19
|
+
import threading
|
20
|
+
from dataclasses import dataclass
|
21
|
+
from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
|
22
|
+
from typing import Dict, Optional
|
17
23
|
|
18
24
|
import pandas
|
19
25
|
from adapta.metrics import MetricsProvider
|
20
|
-
from adapta.process_communication import DataSocket
|
21
26
|
from adapta.storage.query_enabled_store import QueryEnabledStore
|
27
|
+
from dataclasses_json import DataClassJsonMixin
|
22
28
|
from injector import inject
|
23
29
|
|
24
30
|
from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
|
31
|
+
from esd_services_api_client.nexus.abstractions.socket_provider import (
|
32
|
+
ExternalSocketProvider,
|
33
|
+
)
|
25
34
|
from esd_services_api_client.nexus.core.app_core import Nexus
|
26
35
|
from esd_services_api_client.nexus.algorithms import MinimalisticAlgorithm
|
27
36
|
from esd_services_api_client.nexus.input import InputReader, InputProcessor
|
28
37
|
from pandas import DataFrame as PandasDataFrame
|
29
38
|
|
39
|
+
from esd_services_api_client.nexus.input.payload_reader import AlgorithmPayload
|
40
|
+
|
30
41
|
|
31
42
|
async def my_on_complete_func_1(**kwargs):
|
32
43
|
pass
|
@@ -36,7 +47,72 @@ async def my_on_complete_func_2(**kwargs):
|
|
36
47
|
pass
|
37
48
|
|
38
49
|
|
39
|
-
|
50
|
+
@dataclass
|
51
|
+
class MyAlgorithmPayload(AlgorithmPayload, DataClassJsonMixin):
|
52
|
+
x: Optional[list[int]] = None
|
53
|
+
y: Optional[list[int]] = None
|
54
|
+
|
55
|
+
|
56
|
+
@dataclass
|
57
|
+
class MyAlgorithmPayload2(AlgorithmPayload, DataClassJsonMixin):
|
58
|
+
z: list[int]
|
59
|
+
x: Optional[list[int]] = None
|
60
|
+
y: Optional[list[int]] = None
|
61
|
+
|
62
|
+
|
63
|
+
class MockRequestHandler(BaseHTTPRequestHandler):
|
64
|
+
"""
|
65
|
+
HTTPServer Mock Request handler
|
66
|
+
"""
|
67
|
+
|
68
|
+
def __init__(
|
69
|
+
self,
|
70
|
+
request: bytes,
|
71
|
+
client_address: tuple[str, int],
|
72
|
+
server: socketserver.BaseServer,
|
73
|
+
):
|
74
|
+
"""
|
75
|
+
Initialize request handler
|
76
|
+
:param request:
|
77
|
+
:param client_address:
|
78
|
+
:param server:
|
79
|
+
"""
|
80
|
+
self._responses = {
|
81
|
+
"some/payload": (
|
82
|
+
{
|
83
|
+
# "x": [-1, 0, 2],
|
84
|
+
# "y": [10, 11, 12],
|
85
|
+
"z": [1, 2, 3]
|
86
|
+
},
|
87
|
+
200,
|
88
|
+
)
|
89
|
+
}
|
90
|
+
super().__init__(request, client_address, server)
|
91
|
+
|
92
|
+
def do_GET(self): # pylint: disable=invalid-name
|
93
|
+
"""Handle POST requests"""
|
94
|
+
current_url = self.path.removeprefix("/")
|
95
|
+
|
96
|
+
if current_url not in self._responses:
|
97
|
+
self.send_response(500, "Unknown URL")
|
98
|
+
return
|
99
|
+
|
100
|
+
self.send_response(self._responses[current_url][1])
|
101
|
+
self.send_header("Content-Type", "application/json")
|
102
|
+
self.end_headers()
|
103
|
+
self.wfile.write(json.dumps(self._responses[current_url][0]).encode("utf-8"))
|
104
|
+
|
105
|
+
def log_request(self, code=None, size=None):
|
106
|
+
"""
|
107
|
+
Don't log anything
|
108
|
+
:param code:
|
109
|
+
:param size:
|
110
|
+
:return:
|
111
|
+
"""
|
112
|
+
pass
|
113
|
+
|
114
|
+
|
115
|
+
class XReader(InputReader[MyAlgorithmPayload]):
|
40
116
|
async def _context_open(self):
|
41
117
|
pass
|
42
118
|
|
@@ -44,15 +120,34 @@ class XReader(InputReader):
|
|
44
120
|
pass
|
45
121
|
|
46
122
|
@inject
|
47
|
-
def __init__(
|
48
|
-
|
49
|
-
|
123
|
+
def __init__(
|
124
|
+
self,
|
125
|
+
store: QueryEnabledStore,
|
126
|
+
metrics_provider: MetricsProvider,
|
127
|
+
logger_factory: LoggerFactory,
|
128
|
+
payload: MyAlgorithmPayload,
|
129
|
+
socket_provider: ExternalSocketProvider,
|
130
|
+
*readers: "InputReader"
|
131
|
+
):
|
132
|
+
super().__init__(
|
133
|
+
socket_provider.socket("x"),
|
134
|
+
store,
|
135
|
+
metrics_provider,
|
136
|
+
logger_factory,
|
137
|
+
payload,
|
138
|
+
*readers
|
139
|
+
)
|
50
140
|
|
51
141
|
async def _read_input(self) -> PandasDataFrame:
|
52
|
-
|
142
|
+
self._logger.info(
|
143
|
+
"Payload: {payload}; Socket path: {socket_path}",
|
144
|
+
payload=self._payload.to_json(),
|
145
|
+
socket_path=self.socket.data_path,
|
146
|
+
)
|
147
|
+
return pandas.DataFrame([{"a": 1, "b": 2}, {"a": 2, "b": 3}])
|
53
148
|
|
54
149
|
|
55
|
-
class YReader(InputReader):
|
150
|
+
class YReader(InputReader[MyAlgorithmPayload2]):
|
56
151
|
async def _context_open(self):
|
57
152
|
pass
|
58
153
|
|
@@ -60,12 +155,31 @@ class YReader(InputReader):
|
|
60
155
|
pass
|
61
156
|
|
62
157
|
@inject
|
63
|
-
def __init__(
|
64
|
-
|
65
|
-
|
158
|
+
def __init__(
|
159
|
+
self,
|
160
|
+
store: QueryEnabledStore,
|
161
|
+
metrics_provider: MetricsProvider,
|
162
|
+
logger_factory: LoggerFactory,
|
163
|
+
payload: MyAlgorithmPayload2,
|
164
|
+
socket_provider: ExternalSocketProvider,
|
165
|
+
*readers: "InputReader"
|
166
|
+
):
|
167
|
+
super().__init__(
|
168
|
+
socket_provider.socket("y"),
|
169
|
+
store,
|
170
|
+
metrics_provider,
|
171
|
+
logger_factory,
|
172
|
+
payload,
|
173
|
+
*readers
|
174
|
+
)
|
66
175
|
|
67
176
|
async def _read_input(self) -> PandasDataFrame:
|
68
|
-
|
177
|
+
self._logger.info(
|
178
|
+
"Payload: {payload}; Socket path: {socket_path}",
|
179
|
+
payload=self._payload.to_json(),
|
180
|
+
socket_path=self.socket.data_path,
|
181
|
+
)
|
182
|
+
return pandas.DataFrame([{"a": 10, "b": 12}, {"a": 11, "b": 13}])
|
69
183
|
|
70
184
|
|
71
185
|
class MyInputProcessor(InputProcessor):
|
@@ -76,14 +190,26 @@ class MyInputProcessor(InputProcessor):
|
|
76
190
|
pass
|
77
191
|
|
78
192
|
@inject
|
79
|
-
def __init__(
|
80
|
-
|
193
|
+
def __init__(
|
194
|
+
self,
|
195
|
+
x: XReader,
|
196
|
+
y: YReader,
|
197
|
+
metrics_provider: MetricsProvider,
|
198
|
+
logger_factory: LoggerFactory,
|
199
|
+
):
|
200
|
+
super().__init__(
|
201
|
+
x,
|
202
|
+
y,
|
203
|
+
metrics_provider=metrics_provider,
|
204
|
+
logger_factory=logger_factory,
|
205
|
+
payload=None,
|
206
|
+
)
|
81
207
|
|
82
208
|
async def process_input(self, **_) -> Dict[str, PandasDataFrame]:
|
83
209
|
inputs = await self._read_input()
|
84
210
|
return {
|
85
|
-
|
86
|
-
|
211
|
+
"x_ready": inputs["x"].assign(c=[-1, 1]),
|
212
|
+
"y_ready": inputs["y"].assign(c=[-1, 1]),
|
87
213
|
}
|
88
214
|
|
89
215
|
|
@@ -95,22 +221,41 @@ class MyAlgorithm(MinimalisticAlgorithm):
|
|
95
221
|
pass
|
96
222
|
|
97
223
|
@inject
|
98
|
-
def __init__(
|
224
|
+
def __init__(
|
225
|
+
self,
|
226
|
+
input_processor: MyInputProcessor,
|
227
|
+
metrics_provider: MetricsProvider,
|
228
|
+
logger_factory: LoggerFactory,
|
229
|
+
):
|
99
230
|
super().__init__(input_processor, metrics_provider, logger_factory)
|
100
231
|
|
101
|
-
async def _run(
|
232
|
+
async def _run(
|
233
|
+
self, x_ready: PandasDataFrame, y_ready: PandasDataFrame, **kwargs
|
234
|
+
) -> PandasDataFrame:
|
102
235
|
return pandas.concat([x_ready, y_ready])
|
103
236
|
|
104
237
|
|
105
238
|
async def main():
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
239
|
+
"""
|
240
|
+
Mock HTTP Server
|
241
|
+
:return:
|
242
|
+
"""
|
243
|
+
with ThreadingHTTPServer(("localhost", 9876), MockRequestHandler) as server:
|
244
|
+
server_thread = threading.Thread(target=server.serve_forever)
|
245
|
+
server_thread.daemon = True
|
246
|
+
server_thread.start()
|
247
|
+
nexus = (
|
248
|
+
await Nexus.create()
|
249
|
+
.add_reader(XReader)
|
250
|
+
.add_reader(YReader)
|
251
|
+
.use_processor(MyInputProcessor)
|
252
|
+
.use_algorithm(MyAlgorithm)
|
253
|
+
.inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)
|
254
|
+
)
|
255
|
+
|
256
|
+
await nexus.activate()
|
257
|
+
server.shutdown()
|
111
258
|
|
112
|
-
await nexus.activate()
|
113
|
-
|
114
259
|
|
115
260
|
if __name__ == "__main__":
|
116
261
|
asyncio.run(main())
|
@@ -120,5 +265,16 @@ if __name__ == "__main__":
|
|
120
265
|
Run this code as `sample.py`:
|
121
266
|
|
122
267
|
```shell
|
123
|
-
python3 sample.py --sas-uri
|
268
|
+
python3 sample.py --sas-uri http://localhost:9876/some/payload --request-id test
|
269
|
+
```
|
270
|
+
|
271
|
+
Produces the following:
|
272
|
+
|
273
|
+
```
|
274
|
+
Running _read
|
275
|
+
Payload: {"x": null, "y": null}; Socket path: test/x
|
276
|
+
Finished reading X from path test/x in 0.00s seconds
|
277
|
+
Running _read
|
278
|
+
Payload: {"z": [1, 2, 3], "x": null, "y": null}; Socket path: test/y
|
279
|
+
Finished reading Y from path test/y in 0.00s seconds
|
124
280
|
```
|
@@ -19,13 +19,17 @@
|
|
19
19
|
|
20
20
|
|
21
21
|
from abc import ABC, abstractmethod
|
22
|
+
from typing import Generic, TypeVar
|
22
23
|
|
23
24
|
from adapta.metrics import MetricsProvider
|
24
25
|
|
25
26
|
from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
|
26
27
|
|
27
28
|
|
28
|
-
|
29
|
+
TPayload = TypeVar("TPayload") # pylint: disable=C0103
|
30
|
+
|
31
|
+
|
32
|
+
class NexusObject(Generic[TPayload], ABC):
|
29
33
|
"""
|
30
34
|
Base class for all Nexus objects.
|
31
35
|
"""
|
@@ -0,0 +1,51 @@
|
|
1
|
+
"""
|
2
|
+
Socket provider for all data sockets used by algorithms.
|
3
|
+
"""
|
4
|
+
import json
|
5
|
+
|
6
|
+
# Copyright (c) 2023. ECCO Sneaks & Data
|
7
|
+
#
|
8
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
|
+
# you may not use this file except in compliance with the License.
|
10
|
+
# You may obtain a copy of the License at
|
11
|
+
#
|
12
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
# See the License for the specific language governing permissions and
|
18
|
+
# limitations under the License.
|
19
|
+
#
|
20
|
+
|
21
|
+
from typing import final, Optional
|
22
|
+
|
23
|
+
from adapta.process_communication import DataSocket
|
24
|
+
|
25
|
+
|
26
|
+
@final
|
27
|
+
class ExternalSocketProvider:
|
28
|
+
"""
|
29
|
+
Wraps a socket collection
|
30
|
+
"""
|
31
|
+
|
32
|
+
def __init__(self, *sockets: DataSocket):
|
33
|
+
self._sockets = {socket.alias: socket for socket in sockets}
|
34
|
+
|
35
|
+
def socket(self, name: str) -> Optional[DataSocket]:
|
36
|
+
"""
|
37
|
+
Retrieve a socket if it exists.
|
38
|
+
"""
|
39
|
+
return self._sockets.get(name, None)
|
40
|
+
|
41
|
+
@classmethod
|
42
|
+
def from_serialized(cls, socket_list_ser: str) -> "ExternalSocketProvider":
|
43
|
+
"""
|
44
|
+
Creates a SocketProvider from a list of serialized sockets
|
45
|
+
"""
|
46
|
+
return cls(
|
47
|
+
*[
|
48
|
+
DataSocket.from_dict(socket_dict)
|
49
|
+
for socket_dict in json.loads(socket_list_ser)
|
50
|
+
]
|
51
|
+
)
|
@@ -51,6 +51,10 @@ from esd_services_api_client.nexus.core.app_dependencies import (
|
|
51
51
|
)
|
52
52
|
from esd_services_api_client.nexus.input.input_processor import InputProcessor
|
53
53
|
from esd_services_api_client.nexus.input.input_reader import InputReader
|
54
|
+
from esd_services_api_client.nexus.input.payload_reader import (
|
55
|
+
AlgorithmPayloadReader,
|
56
|
+
AlgorithmPayload,
|
57
|
+
)
|
54
58
|
|
55
59
|
|
56
60
|
def is_transient_exception(exception: Optional[BaseException]) -> Optional[bool]:
|
@@ -141,6 +145,24 @@ class Nexus:
|
|
141
145
|
self._algorithm_class = algorithm
|
142
146
|
return self
|
143
147
|
|
148
|
+
async def inject_payload(self, *payload_types: Type[AlgorithmPayload]) -> "Nexus":
|
149
|
+
"""
|
150
|
+
Adds payloads processed into the specified types to the DI container
|
151
|
+
"""
|
152
|
+
for payload_type in payload_types:
|
153
|
+
|
154
|
+
async def get_payload() -> payload_type: # pylint: disable=W0640
|
155
|
+
async with AlgorithmPayloadReader(
|
156
|
+
payload_uri=self._run_args.sas_uri,
|
157
|
+
payload_type=payload_type, # pylint: disable=W0640
|
158
|
+
) as reader:
|
159
|
+
return reader.payload
|
160
|
+
|
161
|
+
# pylint warnings are silenced here because closure is called inside the same loop it is defined, thus each value of a loop variable is used
|
162
|
+
|
163
|
+
self._configurator = self._configurator.with_payload((await get_payload()))
|
164
|
+
return self
|
165
|
+
|
144
166
|
async def _submit_result(
|
145
167
|
self,
|
146
168
|
result: Optional[PandasDataFrame] = None,
|
@@ -29,11 +29,17 @@ from injector import Module, singleton, provider
|
|
29
29
|
|
30
30
|
from esd_services_api_client.crystal import CrystalConnector
|
31
31
|
from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
|
32
|
+
from esd_services_api_client.nexus.abstractions.socket_provider import (
|
33
|
+
ExternalSocketProvider,
|
34
|
+
)
|
32
35
|
from esd_services_api_client.nexus.exceptions.startup_error import (
|
33
36
|
FatalStartupConfigurationError,
|
34
37
|
)
|
35
38
|
from esd_services_api_client.nexus.input.input_processor import InputProcessor
|
36
39
|
from esd_services_api_client.nexus.input.input_reader import InputReader
|
40
|
+
from esd_services_api_client.nexus.input.payload_reader import (
|
41
|
+
AlgorithmPayload,
|
42
|
+
)
|
37
43
|
|
38
44
|
|
39
45
|
class MetricsModule(Module):
|
@@ -127,6 +133,28 @@ class StorageClientModule(Module):
|
|
127
133
|
)
|
128
134
|
|
129
135
|
|
136
|
+
@final
|
137
|
+
class ExternalSocketsModule(Module):
|
138
|
+
"""
|
139
|
+
Storage client module.
|
140
|
+
"""
|
141
|
+
|
142
|
+
@singleton
|
143
|
+
@provider
|
144
|
+
def provide(self) -> ExternalSocketProvider:
|
145
|
+
"""
|
146
|
+
Dependency provider.
|
147
|
+
"""
|
148
|
+
if "NEXUS__ALGORITHM_INPUT_EXTERNAL_DATA_SOCKETS" not in os.environ:
|
149
|
+
raise FatalStartupConfigurationError(
|
150
|
+
"NEXUS__ALGORITHM_INPUT_EXTERNAL_DATA_SOCKETS"
|
151
|
+
)
|
152
|
+
|
153
|
+
return ExternalSocketProvider.from_serialized(
|
154
|
+
os.getenv("NEXUS__ALGORITHM_INPUT_EXTERNAL_DATA_SOCKETS")
|
155
|
+
)
|
156
|
+
|
157
|
+
|
130
158
|
@final
|
131
159
|
class ServiceConfigurator:
|
132
160
|
"""
|
@@ -139,6 +167,7 @@ class ServiceConfigurator:
|
|
139
167
|
CrystalReceiverClientModule(),
|
140
168
|
QueryEnabledStoreModule(),
|
141
169
|
StorageClientModule(),
|
170
|
+
ExternalSocketsModule(),
|
142
171
|
]
|
143
172
|
|
144
173
|
@property
|
@@ -165,3 +194,12 @@ class ServiceConfigurator:
|
|
165
194
|
type(f"{input_processor.__name__}Module", (Module,), {})()
|
166
195
|
)
|
167
196
|
return self
|
197
|
+
|
198
|
+
def with_payload(self, payload: AlgorithmPayload) -> "ServiceConfigurator":
|
199
|
+
"""
|
200
|
+
Adds the specified payload instance to the DI container.
|
201
|
+
"""
|
202
|
+
self._injection_binds.append(
|
203
|
+
lambda binder: binder.bind(payload.__class__, to=payload, scope=singleton)
|
204
|
+
)
|
205
|
+
return self
|
@@ -26,10 +26,12 @@ from adapta.metrics import MetricsProvider
|
|
26
26
|
|
27
27
|
import azure.core.exceptions
|
28
28
|
|
29
|
-
from injector import inject
|
30
29
|
from pandas import DataFrame as PandasDataFrame
|
31
30
|
|
32
|
-
from esd_services_api_client.nexus.abstractions.nexus_object import
|
31
|
+
from esd_services_api_client.nexus.abstractions.nexus_object import (
|
32
|
+
NexusObject,
|
33
|
+
TPayload,
|
34
|
+
)
|
33
35
|
from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
|
34
36
|
from esd_services_api_client.nexus.exceptions.input_reader_error import (
|
35
37
|
FatalInputReaderError,
|
@@ -38,20 +40,21 @@ from esd_services_api_client.nexus.exceptions.input_reader_error import (
|
|
38
40
|
from esd_services_api_client.nexus.input.input_reader import InputReader
|
39
41
|
|
40
42
|
|
41
|
-
class InputProcessor(NexusObject):
|
43
|
+
class InputProcessor(NexusObject[TPayload]):
|
42
44
|
"""
|
43
45
|
Base class for raw data processing into algorithm input.
|
44
46
|
"""
|
45
47
|
|
46
|
-
@inject
|
47
48
|
def __init__(
|
48
49
|
self,
|
49
50
|
*readers: InputReader,
|
51
|
+
payload: TPayload,
|
50
52
|
metrics_provider: MetricsProvider,
|
51
53
|
logger_factory: LoggerFactory,
|
52
54
|
):
|
53
55
|
super().__init__(metrics_provider, logger_factory)
|
54
56
|
self._readers = readers
|
57
|
+
self._payload = payload
|
55
58
|
|
56
59
|
def _get_exc_type(
|
57
60
|
self, ex: BaseException
|
@@ -29,11 +29,14 @@ from adapta.utils.decorators._logging import run_time_metrics_async
|
|
29
29
|
|
30
30
|
from pandas import DataFrame as PandasDataFrame
|
31
31
|
|
32
|
-
from esd_services_api_client.nexus.abstractions.nexus_object import
|
32
|
+
from esd_services_api_client.nexus.abstractions.nexus_object import (
|
33
|
+
NexusObject,
|
34
|
+
TPayload,
|
35
|
+
)
|
33
36
|
from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
|
34
37
|
|
35
38
|
|
36
|
-
class InputReader(NexusObject):
|
39
|
+
class InputReader(NexusObject[TPayload]):
|
37
40
|
"""
|
38
41
|
Base class for a raw data reader.
|
39
42
|
"""
|
@@ -44,6 +47,7 @@ class InputReader(NexusObject):
|
|
44
47
|
store: QueryEnabledStore,
|
45
48
|
metrics_provider: MetricsProvider,
|
46
49
|
logger_factory: LoggerFactory,
|
50
|
+
payload: TPayload,
|
47
51
|
*readers: "InputReader"
|
48
52
|
):
|
49
53
|
super().__init__(metrics_provider, logger_factory)
|
@@ -51,6 +55,7 @@ class InputReader(NexusObject):
|
|
51
55
|
self._store = store
|
52
56
|
self._data: Optional[PandasDataFrame] = None
|
53
57
|
self._readers = readers
|
58
|
+
self._payload = payload
|
54
59
|
|
55
60
|
@property
|
56
61
|
def data(self) -> Optional[PandasDataFrame]:
|
@@ -82,7 +87,14 @@ class InputReader(NexusObject):
|
|
82
87
|
Coroutine that reads the data from external store and converts it to a dataframe.
|
83
88
|
"""
|
84
89
|
|
85
|
-
@run_time_metrics_async(
|
90
|
+
@run_time_metrics_async(
|
91
|
+
metric_name="read_input",
|
92
|
+
on_finish_message_template="Finished reading {entity} from path {data_path} in {elapsed:.2f}s seconds",
|
93
|
+
template_args={
|
94
|
+
"entity": self._metric_name.upper(),
|
95
|
+
"data_path": self.socket.data_path,
|
96
|
+
},
|
97
|
+
)
|
86
98
|
async def _read(**_) -> PandasDataFrame:
|
87
99
|
if not self._data:
|
88
100
|
self._data = await self._read_input()
|
@@ -0,0 +1,83 @@
|
|
1
|
+
"""
|
2
|
+
Code infrastructure for manipulating payload received from Crystal SAS URI
|
3
|
+
"""
|
4
|
+
|
5
|
+
# Copyright (c) 2023. ECCO Sneaks & Data
|
6
|
+
#
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
|
+
# you may not use this file except in compliance with the License.
|
9
|
+
# You may obtain a copy of the License at
|
10
|
+
#
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
12
|
+
#
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
16
|
+
# See the License for the specific language governing permissions and
|
17
|
+
# limitations under the License.
|
18
|
+
#
|
19
|
+
|
20
|
+
from dataclasses import dataclass
|
21
|
+
|
22
|
+
from typing import final, Optional, Type
|
23
|
+
|
24
|
+
from adapta.storage.models.format import DictJsonSerializationFormat
|
25
|
+
from adapta.utils import session_with_retries
|
26
|
+
|
27
|
+
from dataclasses_json import DataClassJsonMixin
|
28
|
+
|
29
|
+
|
30
|
+
@dataclass
|
31
|
+
class AlgorithmPayload(DataClassJsonMixin):
|
32
|
+
"""
|
33
|
+
Base class for algorithm payload
|
34
|
+
"""
|
35
|
+
|
36
|
+
def validate(self):
|
37
|
+
"""
|
38
|
+
Optional post-validation of the data. Define this method to analyze class contents after payload has been read and deserialized.
|
39
|
+
"""
|
40
|
+
|
41
|
+
def __post_init__(self):
|
42
|
+
self.validate()
|
43
|
+
|
44
|
+
|
45
|
+
@final
|
46
|
+
class AlgorithmPayloadReader:
|
47
|
+
"""
|
48
|
+
Crystal Payload Reader - receives the payload from the URI and deserializes it into the specified type
|
49
|
+
"""
|
50
|
+
|
51
|
+
async def __aenter__(self):
|
52
|
+
if not self._http:
|
53
|
+
self._http = session_with_retries()
|
54
|
+
http_response = self._http.get(url=self._payload_uri)
|
55
|
+
http_response.raise_for_status()
|
56
|
+
self._payload = self._payload_type.from_dict(
|
57
|
+
DictJsonSerializationFormat().deserialize(http_response.content)
|
58
|
+
)
|
59
|
+
return self
|
60
|
+
|
61
|
+
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
62
|
+
self._http.close()
|
63
|
+
self._http = None
|
64
|
+
|
65
|
+
def __init__(self, payload_uri: str, payload_type: Type[AlgorithmPayload]):
|
66
|
+
self._http = session_with_retries()
|
67
|
+
self._payload: Optional[AlgorithmPayload] = None
|
68
|
+
self._payload_uri = payload_uri
|
69
|
+
self._payload_type = payload_type
|
70
|
+
|
71
|
+
@property
|
72
|
+
def payload_uri(self) -> str:
|
73
|
+
"""
|
74
|
+
Uri of the paylod for the algorithm
|
75
|
+
"""
|
76
|
+
return self._payload_uri
|
77
|
+
|
78
|
+
@property
|
79
|
+
def payload(self) -> Optional[AlgorithmPayload]:
|
80
|
+
"""
|
81
|
+
Payload data deserialized into the user class.
|
82
|
+
"""
|
83
|
+
return self._payload
|
{esd_services_api_client-2.0.1.dist-info → esd_services_api_client-2.0.2.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: esd-services-api-client
|
3
|
-
Version: 2.0.
|
3
|
+
Version: 2.0.2
|
4
4
|
Summary: Python clients for ESD services
|
5
5
|
Home-page: https://github.com/SneaksAndData/esd-services-api-client
|
6
6
|
License: Apache 2.0
|
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.11
|
17
17
|
Provides-Extra: azure
|
18
18
|
Provides-Extra: nexus
|
19
|
-
Requires-Dist: adapta[azure,datadog,storage] (>=2.6.
|
19
|
+
Requires-Dist: adapta[azure,datadog,storage] (>=2.6.6,<3.0.0)
|
20
20
|
Requires-Dist: azure-identity (>=1.7,<1.8) ; extra == "azure"
|
21
21
|
Requires-Dist: dataclasses-json (>=0.6.0,<0.7.0)
|
22
22
|
Requires-Dist: httpx (>=0.26.0,<0.27.0) ; extra == "nexus"
|
@@ -1,5 +1,5 @@
|
|
1
1
|
esd_services_api_client/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
|
2
|
-
esd_services_api_client/_version.py,sha256=
|
2
|
+
esd_services_api_client/_version.py,sha256=kumiGImhzOTlTrRM-6jDo2mNnVHGO_2vxtrhB0nzAiw,22
|
3
3
|
esd_services_api_client/beast/__init__.py,sha256=NTaz_7YoLPK8MCLwbwqH7rW1zDWLxXu2T7fGmMmRxyg,718
|
4
4
|
esd_services_api_client/beast/v3/__init__.py,sha256=TRjB4-T6eIORpMvdylb32_GinrIpYNFmAdshSC1HqHg,749
|
5
5
|
esd_services_api_client/beast/v3/_connector.py,sha256=oPizDQ1KOKOfiyh-jAofKodlpRzrRiELv-rmP_o_oio,11473
|
@@ -15,27 +15,29 @@ esd_services_api_client/crystal/__init__.py,sha256=afSGQRkDic0ECsJfgu3b291kX8CyU
|
|
15
15
|
esd_services_api_client/crystal/_api_versions.py,sha256=2BMiQRS0D8IEpWCCys3dge5alVBRCZrOuCR1QAn8UIM,832
|
16
16
|
esd_services_api_client/crystal/_connector.py,sha256=WjfMezWXia41Z8aiNupaT577fk9Sx6uy6V23O6y9hfI,12870
|
17
17
|
esd_services_api_client/crystal/_models.py,sha256=eRhGAl8LjglCyIFwf1bcFBhjbpSuRYucuF2LO388L2E,4025
|
18
|
-
esd_services_api_client/nexus/README.md,sha256=
|
18
|
+
esd_services_api_client/nexus/README.md,sha256=6fGBYTvzkmZH9Yk9VcPqWCtqU_D2zleG1rv1U3GA3IM,7721
|
19
19
|
esd_services_api_client/nexus/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
|
20
20
|
esd_services_api_client/nexus/abstractions/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
|
21
21
|
esd_services_api_client/nexus/abstractions/logger_factory.py,sha256=JHl_t0d0ra_k-EixZlkw-s746wHUdBhSU6preqoARtk,2031
|
22
|
-
esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=
|
22
|
+
esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=E9iYmjYjkmS0Uv-VS1ixRhi3zdtQfcSNx4gxad5_aZ0,1780
|
23
|
+
esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=Mv9BWdxw8VY4Gi4EOrdxWK1zsR3-fqIbpyF1xHWchbE,1495
|
23
24
|
esd_services_api_client/nexus/algorithms/__init__.py,sha256=uWX24NHUnUcnOJN2IIp1kaaozCC0rOmB36WxHBCapCY,823
|
24
25
|
esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=4Gqp8qV5nunZ_DwZyHAu0vLuF1rls8nnq2zoHy2orME,1825
|
25
26
|
esd_services_api_client/nexus/algorithms/distributed.py,sha256=iWjx9D6g-ASwTWPkQ9GmInTLymVlxl7UkfEBcEfnkmc,1628
|
26
27
|
esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=9KvIXXsiOZ9wAOvrIZuOlNYcJLB-SjlQoNpICJu-qeQ,1366
|
27
28
|
esd_services_api_client/nexus/algorithms/recursive.py,sha256=2YuAeYZ6-K5hcY8NEAAJVyGhtVVQMTZ7wjkWPPo8qeE,1829
|
28
29
|
esd_services_api_client/nexus/core/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
|
29
|
-
esd_services_api_client/nexus/core/app_core.py,sha256=
|
30
|
-
esd_services_api_client/nexus/core/app_dependencies.py,sha256=
|
30
|
+
esd_services_api_client/nexus/core/app_core.py,sha256=bwbFzbjx8PwZWHfHsEYDCGAjJuTzmVKuunSeMU54tHs,8877
|
31
|
+
esd_services_api_client/nexus/core/app_dependencies.py,sha256=afX7QrUEuaLnFssaCTohXBt7QCeGJ0MuovF62CZTZ34,5672
|
31
32
|
esd_services_api_client/nexus/exceptions/__init__.py,sha256=JgPXhrvBIi0U1QOF90TYHS8jv_SBQEoRLsEg6rrtRoc,691
|
32
33
|
esd_services_api_client/nexus/exceptions/_nexus_error.py,sha256=b3L8JnNvV2jdxNfuFWh9-j4kVb_VX7gNH5WHKcC-R78,890
|
33
34
|
esd_services_api_client/nexus/exceptions/input_reader_error.py,sha256=D-xYTKRNREQ2-NGhc88GHOmXCvLNsIVQsH8wf0LLC_0,1760
|
34
35
|
esd_services_api_client/nexus/exceptions/startup_error.py,sha256=f2PIOSdLgT-42eKD6ec8p7nROADshMawCsDGDUbxO_w,1546
|
35
36
|
esd_services_api_client/nexus/input/__init__.py,sha256=0k_HMIP4NPC5O2ixKJPgKsLzYeHS14DhibF_MUtez1c,753
|
36
|
-
esd_services_api_client/nexus/input/input_processor.py,sha256=
|
37
|
-
esd_services_api_client/nexus/input/input_reader.py,sha256=
|
38
|
-
esd_services_api_client
|
39
|
-
esd_services_api_client-2.0.
|
40
|
-
esd_services_api_client-2.0.
|
41
|
-
esd_services_api_client-2.0.
|
37
|
+
esd_services_api_client/nexus/input/input_processor.py,sha256=MiXXd_APrG85Pi-Ke68_UHNEV7T_QHN1hU1WAPWoTsw,3187
|
38
|
+
esd_services_api_client/nexus/input/input_reader.py,sha256=uxTAGX5xNhjTFpEsVQnr8BkVgpIH_U_om54hh3pvJ3s,3269
|
39
|
+
esd_services_api_client/nexus/input/payload_reader.py,sha256=__r_QjIFRAWwx56X5WUK1qensJUae0vZEb422dzOgSY,2511
|
40
|
+
esd_services_api_client-2.0.2.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
|
41
|
+
esd_services_api_client-2.0.2.dist-info/METADATA,sha256=rNOu2tUM-mWp4kQbt13EpUweANcI808ZNCFmMdcqQRM,1236
|
42
|
+
esd_services_api_client-2.0.2.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
|
43
|
+
esd_services_api_client-2.0.2.dist-info/RECORD,,
|
File without changes
|
File without changes
|