esd-services-api-client 2.4.0__tar.gz → 2.5.1a122.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/PKG-INFO +4 -4
- esd_services_api_client-2.5.1a122.dev1/esd_services_api_client/__init__.py +20 -0
- esd_services_api_client-2.5.1a122.dev1/esd_services_api_client/_version.py +1 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/README.md +56 -49
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/abstractions/nexus_object.py +2 -2
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/core/app_core.py +49 -10
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/telemetry/recorder.py +44 -5
- esd_services_api_client-2.5.1a122.dev1/esd_services_api_client/nexus/telemetry/user_telemetry_recorder.py +169 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/pyproject.toml +6 -6
- esd_services_api_client-2.4.0/esd_services_api_client/_version.py +0 -1
- esd_services_api_client-2.4.0/esd_services_api_client/common/__init__.py +0 -14
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/LICENSE +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/README.md +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/beast/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/beast/v3/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/beast/v3/_connector.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/beast/v3/_models.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/boxer/README.md +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/boxer/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/boxer/_auth.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/boxer/_base.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/boxer/_connector.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/boxer/_models.py +0 -0
- {esd_services_api_client-2.4.0/esd_services_api_client → esd_services_api_client-2.5.1a122.dev1/esd_services_api_client/common}/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/crystal/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/crystal/_api_versions.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/crystal/_connector.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/crystal/_models.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/abstractions/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/abstractions/algrorithm_cache.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/abstractions/input_object.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/abstractions/logger_factory.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/abstractions/socket_provider.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/algorithms/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/algorithms/_baseline_algorithm.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/algorithms/_remote_algorithm.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/algorithms/distributed.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/algorithms/forked_algorithm.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/algorithms/minimalistic.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/algorithms/recursive.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/configurations/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/configurations/algorithm_configuration.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/core/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/core/app_dependencies.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/core/serializers.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/exceptions/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/exceptions/_nexus_error.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/exceptions/cache_errors.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/exceptions/input_reader_error.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/exceptions/startup_error.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/input/__init__.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/input/input_processor.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/input/input_reader.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/input/payload_reader.py +0 -0
- {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.1a122.dev1}/esd_services_api_client/nexus/telemetry/__init__.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: esd-services-api-client
|
3
|
-
Version: 2.
|
3
|
+
Version: 2.5.1a122.dev1
|
4
4
|
Summary: Python clients for ESD services
|
5
5
|
Home-page: https://github.com/SneaksAndData/esd-services-api-client
|
6
6
|
License: Apache 2.0
|
@@ -16,11 +16,11 @@ Classifier: Programming Language :: Python :: 3.10
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.11
|
17
17
|
Provides-Extra: azure
|
18
18
|
Provides-Extra: nexus
|
19
|
-
Requires-Dist: adapta[azure,datadog,storage] (>=3.
|
19
|
+
Requires-Dist: adapta[azure,datadog,storage] (>=3.2,<4.0)
|
20
20
|
Requires-Dist: azure-identity (>=1.7,<1.8) ; extra == "azure"
|
21
21
|
Requires-Dist: dataclasses-json (>=0.6.0,<0.7.0)
|
22
|
-
Requires-Dist: httpx (>=0.
|
23
|
-
Requires-Dist: injector (>=0.
|
22
|
+
Requires-Dist: httpx (>=0.27.0,<0.28.0) ; extra == "nexus"
|
23
|
+
Requires-Dist: injector (>=0.22.0,<0.23.0) ; extra == "nexus"
|
24
24
|
Requires-Dist: pycryptodome (>=3.15,<3.16)
|
25
25
|
Requires-Dist: pyjwt (>=2.4.0,<2.5.0)
|
26
26
|
Project-URL: Repository, https://github.com/SneaksAndData/esd-services-api-client
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
#
|
15
|
+
|
16
|
+
"""
|
17
|
+
Root index.
|
18
|
+
"""
|
19
|
+
|
20
|
+
__version__ = "0.0.0"
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__ = 'v2.5.1a122.dev1'
|
@@ -1,3 +1,5 @@
|
|
1
|
+
from pandas import DataFramefrom pandas import DataFramefrom pandas import DataFrame
|
2
|
+
|
1
3
|
## Nexus
|
2
4
|
Set the following environment variables for Azure:
|
3
5
|
```
|
@@ -42,14 +44,7 @@ from esd_services_api_client.nexus.algorithms import MinimalisticAlgorithm
|
|
42
44
|
from esd_services_api_client.nexus.input import InputReader, InputProcessor
|
43
45
|
|
44
46
|
from esd_services_api_client.nexus.input.payload_reader import AlgorithmPayload
|
45
|
-
|
46
|
-
|
47
|
-
async def my_on_complete_func_1(**kwargs):
|
48
|
-
pass
|
49
|
-
|
50
|
-
|
51
|
-
async def my_on_complete_func_2(**kwargs):
|
52
|
-
pass
|
47
|
+
from esd_services_api_client.nexus.telemetry.user_telemetry_recorder import UserTelemetryRecorder, UserTelemetry
|
53
48
|
|
54
49
|
|
55
50
|
@dataclass
|
@@ -81,10 +76,10 @@ class MockRequestHandler(BaseHTTPRequestHandler):
|
|
81
76
|
"""
|
82
77
|
|
83
78
|
def __init__(
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
79
|
+
self,
|
80
|
+
request: bytes,
|
81
|
+
client_address: tuple[str, int],
|
82
|
+
server: socketserver.BaseServer,
|
88
83
|
):
|
89
84
|
"""
|
90
85
|
Initialize request handler
|
@@ -130,14 +125,14 @@ class MockRequestHandler(BaseHTTPRequestHandler):
|
|
130
125
|
class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
|
131
126
|
@inject
|
132
127
|
def __init__(
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
128
|
+
self,
|
129
|
+
store: QueryEnabledStore,
|
130
|
+
metrics_provider: MetricsProvider,
|
131
|
+
logger_factory: LoggerFactory,
|
132
|
+
payload: MyAlgorithmPayload,
|
133
|
+
socket_provider: ExternalSocketProvider,
|
134
|
+
*readers: "InputReader",
|
135
|
+
cache: InputCache
|
141
136
|
):
|
142
137
|
super().__init__(
|
143
138
|
socket=socket_provider.socket("x"),
|
@@ -161,14 +156,14 @@ class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
|
|
161
156
|
class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
|
162
157
|
@inject
|
163
158
|
def __init__(
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
159
|
+
self,
|
160
|
+
store: QueryEnabledStore,
|
161
|
+
metrics_provider: MetricsProvider,
|
162
|
+
logger_factory: LoggerFactory,
|
163
|
+
payload: MyAlgorithmPayload2,
|
164
|
+
socket_provider: ExternalSocketProvider,
|
165
|
+
*readers: "InputReader",
|
166
|
+
cache: InputCache
|
172
167
|
):
|
173
168
|
super().__init__(
|
174
169
|
socket=socket_provider.socket("y"),
|
@@ -192,12 +187,12 @@ class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
|
|
192
187
|
class XProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
|
193
188
|
@inject
|
194
189
|
def __init__(
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
190
|
+
self,
|
191
|
+
x: XReader,
|
192
|
+
metrics_provider: MetricsProvider,
|
193
|
+
logger_factory: LoggerFactory,
|
194
|
+
my_conf: MyAlgorithmConfiguration,
|
195
|
+
cache: InputCache,
|
201
196
|
):
|
202
197
|
super().__init__(
|
203
198
|
x,
|
@@ -210,7 +205,7 @@ class XProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
|
|
210
205
|
self.conf = my_conf
|
211
206
|
|
212
207
|
async def _process_input(
|
213
|
-
|
208
|
+
self, x: pandas.DataFrame, **_
|
214
209
|
) -> pandas.DataFrame:
|
215
210
|
self._logger.info("Config: {config}", config=self.conf.to_json())
|
216
211
|
return x.assign(c=[-1, 1])
|
@@ -219,12 +214,12 @@ class XProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
|
|
219
214
|
class YProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
|
220
215
|
@inject
|
221
216
|
def __init__(
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
217
|
+
self,
|
218
|
+
y: YReader,
|
219
|
+
metrics_provider: MetricsProvider,
|
220
|
+
logger_factory: LoggerFactory,
|
221
|
+
my_conf: MyAlgorithmConfiguration,
|
222
|
+
cache: InputCache,
|
228
223
|
):
|
229
224
|
super().__init__(
|
230
225
|
y,
|
@@ -237,7 +232,7 @@ class YProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
|
|
237
232
|
self.conf = my_conf
|
238
233
|
|
239
234
|
async def _process_input(
|
240
|
-
|
235
|
+
self, y: pandas.DataFrame, **_
|
241
236
|
) -> pandas.DataFrame:
|
242
237
|
self._logger.info("Config: {config}", config=self.conf.to_json())
|
243
238
|
return y.assign(c=[-1, 1])
|
@@ -264,23 +259,34 @@ class MyAlgorithm(MinimalisticAlgorithm[MyAlgorithmPayload]):
|
|
264
259
|
|
265
260
|
@inject
|
266
261
|
def __init__(
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
262
|
+
self,
|
263
|
+
metrics_provider: MetricsProvider,
|
264
|
+
logger_factory: LoggerFactory,
|
265
|
+
x_processor: XProcessor,
|
266
|
+
y_processor: YProcessor,
|
267
|
+
cache: InputCache,
|
273
268
|
):
|
274
269
|
super().__init__(
|
275
270
|
metrics_provider, logger_factory, x_processor, y_processor, cache=cache
|
276
271
|
)
|
277
272
|
|
278
273
|
async def _run(
|
279
|
-
|
274
|
+
self, x: pandas.DataFrame, y: pandas.DataFrame, **kwargs
|
280
275
|
) -> MyResult:
|
281
276
|
return MyResult(x, y)
|
282
277
|
|
283
278
|
|
279
|
+
class ObjectiveAnalytics(UserTelemetryRecorder):
|
280
|
+
|
281
|
+
async def _compute(self,
|
282
|
+
algorithm_payload:
|
283
|
+
AlgorithmPayload,
|
284
|
+
algorithm_result: AlgorithmResult,
|
285
|
+
run_id: str,
|
286
|
+
**inputs: pandas.DataFrame) -> UserTelemetry:
|
287
|
+
pass
|
288
|
+
|
289
|
+
|
284
290
|
async def main():
|
285
291
|
"""
|
286
292
|
Mock HTTP Server
|
@@ -297,6 +303,7 @@ async def main():
|
|
297
303
|
.use_processor(XProcessor)
|
298
304
|
.use_processor(YProcessor)
|
299
305
|
.use_algorithm(MyAlgorithm)
|
306
|
+
.on_complete(ObjectiveAnalytics)
|
300
307
|
.inject_configuration(MyAlgorithmConfiguration)
|
301
308
|
.inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)
|
302
309
|
)
|
@@ -90,13 +90,13 @@ class NexusCoreObject(ABC):
|
|
90
90
|
|
91
91
|
class NexusObject(Generic[TPayload, TResult], NexusCoreObject, ABC):
|
92
92
|
"""
|
93
|
-
Base class for all Nexus objects.
|
93
|
+
Base class for all Nexus objects that perform operations on the algorithm payload.
|
94
94
|
"""
|
95
95
|
|
96
96
|
@classmethod
|
97
97
|
def alias(cls) -> str:
|
98
98
|
"""
|
99
|
-
Alias to identify this
|
99
|
+
Alias to identify this class instances when passed through kwargs.
|
100
100
|
"""
|
101
101
|
return snakecase(
|
102
102
|
re.sub(
|
@@ -23,11 +23,12 @@ import platform
|
|
23
23
|
import signal
|
24
24
|
import sys
|
25
25
|
import traceback
|
26
|
-
from typing import final, Type, Optional
|
26
|
+
from typing import final, Type, Optional
|
27
27
|
|
28
28
|
import backoff
|
29
29
|
import urllib3.exceptions
|
30
30
|
import azure.core.exceptions
|
31
|
+
from adapta.logs import LoggerInterface
|
31
32
|
from adapta.process_communication import DataSocket
|
32
33
|
from adapta.storage.blob.base import StorageClient
|
33
34
|
from adapta.storage.query_enabled_store import QueryEnabledStore
|
@@ -41,6 +42,7 @@ from esd_services_api_client.crystal import (
|
|
41
42
|
AlgorithmRunResult,
|
42
43
|
CrystalEntrypointArguments,
|
43
44
|
)
|
45
|
+
from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
|
44
46
|
from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
|
45
47
|
from esd_services_api_client.nexus.algorithms import (
|
46
48
|
BaselineAlgorithm,
|
@@ -61,6 +63,10 @@ from esd_services_api_client.nexus.input.payload_reader import (
|
|
61
63
|
AlgorithmPayload,
|
62
64
|
)
|
63
65
|
from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
|
66
|
+
from esd_services_api_client.nexus.telemetry.user_telemetry_recorder import (
|
67
|
+
UserTelemetryRecorder,
|
68
|
+
)
|
69
|
+
from esd_services_api_client import __version__
|
64
70
|
|
65
71
|
|
66
72
|
def is_transient_exception(exception: Optional[BaseException]) -> Optional[bool]:
|
@@ -112,7 +118,7 @@ class Nexus:
|
|
112
118
|
self._algorithm_class: Optional[Type[BaselineAlgorithm]] = None
|
113
119
|
self._run_args = args
|
114
120
|
self._algorithm_run_task: Optional[asyncio.Task] = None
|
115
|
-
self._on_complete_tasks: list[
|
121
|
+
self._on_complete_tasks: list[type[UserTelemetryRecorder]] = []
|
116
122
|
|
117
123
|
attach_signal_handlers()
|
118
124
|
|
@@ -123,11 +129,11 @@ class Nexus:
|
|
123
129
|
"""
|
124
130
|
return self._algorithm_class
|
125
131
|
|
126
|
-
def on_complete(self,
|
132
|
+
def on_complete(self, *post_processors: type[UserTelemetryRecorder]) -> "Nexus":
|
127
133
|
"""
|
128
134
|
Attaches a coroutine to run on algorithm completion.
|
129
135
|
"""
|
130
|
-
self._on_complete_tasks.
|
136
|
+
self._on_complete_tasks.extend(post_processors)
|
131
137
|
return self
|
132
138
|
|
133
139
|
def add_reader(self, reader: Type[InputReader]) -> "Nexus":
|
@@ -254,6 +260,15 @@ class Nexus:
|
|
254
260
|
|
255
261
|
algorithm: BaselineAlgorithm = self._injector.get(self._algorithm_class)
|
256
262
|
telemetry_recorder: TelemetryRecorder = self._injector.get(TelemetryRecorder)
|
263
|
+
root_logger: LoggerInterface = self._injector.get(LoggerFactory).create_logger(
|
264
|
+
logger_type=self.__class__,
|
265
|
+
)
|
266
|
+
|
267
|
+
root_logger.info(
|
268
|
+
"Running algorithm {algorithm} on Nexus version {version}",
|
269
|
+
algorithm=algorithm.__class__.__name__,
|
270
|
+
version=__version__,
|
271
|
+
)
|
257
272
|
|
258
273
|
async with algorithm as instance:
|
259
274
|
self._algorithm_run_task = asyncio.create_task(
|
@@ -261,23 +276,47 @@ class Nexus:
|
|
261
276
|
)
|
262
277
|
await self._algorithm_run_task
|
263
278
|
ex = self._algorithm_run_task.exception()
|
264
|
-
on_complete_tasks = [
|
265
|
-
asyncio.create_task(on_complete_task)
|
266
|
-
for on_complete_task in self._on_complete_tasks
|
267
|
-
]
|
268
279
|
|
269
280
|
await self._submit_result(
|
270
281
|
self._algorithm_run_task.result() if not ex else None,
|
271
282
|
self._algorithm_run_task.exception(),
|
272
283
|
)
|
273
|
-
if len(on_complete_tasks) > 0:
|
274
|
-
await asyncio.wait(on_complete_tasks)
|
275
284
|
|
276
285
|
# record telemetry
|
286
|
+
root_logger.info(
|
287
|
+
"Recording telemetry for the run {request_id}",
|
288
|
+
request_id=self._run_args.request_id,
|
289
|
+
)
|
277
290
|
async with telemetry_recorder as recorder:
|
278
291
|
await recorder.record(
|
279
292
|
run_id=self._run_args.request_id, **algorithm.inputs
|
280
293
|
)
|
294
|
+
on_complete_tasks = [
|
295
|
+
recorder.record_user_telemetry(
|
296
|
+
user_recorder_type=on_complete_task_class,
|
297
|
+
run_id=self._run_args.request_id,
|
298
|
+
result=self._algorithm_run_task.result(),
|
299
|
+
**algorithm.inputs,
|
300
|
+
)
|
301
|
+
for on_complete_task_class in self._on_complete_tasks
|
302
|
+
]
|
303
|
+
if len(on_complete_tasks) > 0:
|
304
|
+
done, pending = await asyncio.wait(on_complete_tasks)
|
305
|
+
if len(pending) > 0:
|
306
|
+
root_logger.warning(
|
307
|
+
"Some post-processing operations did not complete or failed. Please review application logs for more information"
|
308
|
+
)
|
309
|
+
for done_on_complete_task in done:
|
310
|
+
on_complete_task_exc = done_on_complete_task.exception()
|
311
|
+
if on_complete_task_exc:
|
312
|
+
root_logger.warning(
|
313
|
+
"Post processing task failed",
|
314
|
+
exception=on_complete_task_exc,
|
315
|
+
)
|
316
|
+
else:
|
317
|
+
root_logger.info(
|
318
|
+
"No post processing tasks were defined for this run."
|
319
|
+
)
|
281
320
|
|
282
321
|
# dispose of QES instance gracefully as it might hold open connections
|
283
322
|
qes = self._injector.get(QueryEnabledStore)
|
@@ -3,20 +3,27 @@
|
|
3
3
|
"""
|
4
4
|
import asyncio
|
5
5
|
import os
|
6
|
+
from asyncio import Task
|
6
7
|
from functools import partial
|
7
8
|
from typing import final
|
8
9
|
|
9
|
-
|
10
|
+
from pandas import DataFrame
|
10
11
|
from adapta.metrics import MetricsProvider
|
11
12
|
from adapta.process_communication import DataSocket
|
12
13
|
from adapta.storage.blob.base import StorageClient
|
13
14
|
from injector import inject, singleton
|
14
15
|
|
15
16
|
from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
|
16
|
-
from esd_services_api_client.nexus.abstractions.nexus_object import
|
17
|
+
from esd_services_api_client.nexus.abstractions.nexus_object import (
|
18
|
+
NexusCoreObject,
|
19
|
+
AlgorithmResult,
|
20
|
+
)
|
17
21
|
from esd_services_api_client.nexus.core.serializers import (
|
18
22
|
TelemetrySerializer,
|
19
23
|
)
|
24
|
+
from esd_services_api_client.nexus.telemetry.user_telemetry_recorder import (
|
25
|
+
UserTelemetryRecorder,
|
26
|
+
)
|
20
27
|
|
21
28
|
|
22
29
|
@final
|
@@ -51,7 +58,7 @@ class TelemetryRecorder(NexusCoreObject):
|
|
51
58
|
"""
|
52
59
|
|
53
60
|
async def _record(
|
54
|
-
entity_to_record:
|
61
|
+
entity_to_record: DataFrame | dict,
|
55
62
|
entity_name: str,
|
56
63
|
**_,
|
57
64
|
) -> None:
|
@@ -61,7 +68,7 @@ class TelemetryRecorder(NexusCoreObject):
|
|
61
68
|
run_id=run_id,
|
62
69
|
)
|
63
70
|
if not isinstance(entity_to_record, dict) and not isinstance(
|
64
|
-
entity_to_record,
|
71
|
+
entity_to_record, DataFrame
|
65
72
|
):
|
66
73
|
self._logger.warning(
|
67
74
|
"Unsupported data type: {telemetry_entity_type}. Telemetry recording skipped.",
|
@@ -72,7 +79,13 @@ class TelemetryRecorder(NexusCoreObject):
|
|
72
79
|
data=entity_to_record,
|
73
80
|
blob_path=DataSocket(
|
74
81
|
alias="telemetry",
|
75
|
-
data_path=
|
82
|
+
data_path=os.path.join(
|
83
|
+
self._telemetry_base_path,
|
84
|
+
"telemetry_group=inputs",
|
85
|
+
f"entity_name={entity_name}",
|
86
|
+
f"request_id={run_id}",
|
87
|
+
run_id,
|
88
|
+
),
|
76
89
|
data_format="null",
|
77
90
|
).parse_data_path(),
|
78
91
|
serialization_format=self._serializer.get_serialization_format(
|
@@ -106,3 +119,29 @@ class TelemetryRecorder(NexusCoreObject):
|
|
106
119
|
self._logger.warning(
|
107
120
|
"Telemetry recoding failed", exception=telemetry_exc
|
108
121
|
)
|
122
|
+
|
123
|
+
def record_user_telemetry(
|
124
|
+
self,
|
125
|
+
user_recorder_type: type[UserTelemetryRecorder],
|
126
|
+
run_id: str,
|
127
|
+
result: AlgorithmResult,
|
128
|
+
**inputs: DataFrame,
|
129
|
+
) -> Task:
|
130
|
+
"""
|
131
|
+
Creates an awaitable task that records user telemetry using provided recorder type.
|
132
|
+
|
133
|
+
:param user_recorder_type: Recorder type to record user telemetry.
|
134
|
+
:param run_id: The request_id to record user telemetry for.
|
135
|
+
:param result: Result of the algorithm.
|
136
|
+
:param inputs: Algorithm input data.
|
137
|
+
"""
|
138
|
+
return asyncio.create_task(
|
139
|
+
user_recorder_type(
|
140
|
+
run_id=run_id,
|
141
|
+
metrics_provider=self._metrics_provider,
|
142
|
+
logger=self._logger,
|
143
|
+
storage_client=self._storage_client,
|
144
|
+
serializer=self._serializer,
|
145
|
+
telemetry_base_path=self._telemetry_base_path,
|
146
|
+
).record(run_id=run_id, algorithm_result=result, **inputs)
|
147
|
+
)
|
@@ -0,0 +1,169 @@
|
|
1
|
+
"""
|
2
|
+
User-defined telemetry.
|
3
|
+
"""
|
4
|
+
import os.path
|
5
|
+
import re
|
6
|
+
from abc import ABC, abstractmethod
|
7
|
+
from dataclasses import dataclass
|
8
|
+
from functools import partial
|
9
|
+
from typing import final
|
10
|
+
|
11
|
+
from pandas import DataFrame
|
12
|
+
|
13
|
+
from adapta.process_communication import DataSocket
|
14
|
+
from adapta.storage.blob.base import StorageClient
|
15
|
+
from adapta.logs import LoggerInterface
|
16
|
+
from adapta.metrics import MetricsProvider
|
17
|
+
from adapta.utils.decorators import run_time_metrics_async
|
18
|
+
from dataclasses_json.stringcase import snakecase
|
19
|
+
from injector import inject
|
20
|
+
|
21
|
+
from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
|
22
|
+
from esd_services_api_client.nexus.core.serializers import TelemetrySerializer
|
23
|
+
from esd_services_api_client.nexus.input.payload_reader import AlgorithmPayload
|
24
|
+
|
25
|
+
|
26
|
+
@final
|
27
|
+
@dataclass
|
28
|
+
class UserTelemetryPathSegment:
|
29
|
+
"""
|
30
|
+
Path segment for user telemetry.
|
31
|
+
"""
|
32
|
+
|
33
|
+
segment: str
|
34
|
+
segment_header: str
|
35
|
+
|
36
|
+
def __str__(self):
|
37
|
+
return "=".join([self.segment_header, self.segment])
|
38
|
+
|
39
|
+
|
40
|
+
@final
|
41
|
+
class UserTelemetry:
|
42
|
+
"""
|
43
|
+
Base class for user-defined telemetry types.
|
44
|
+
"""
|
45
|
+
|
46
|
+
def __init__(
|
47
|
+
self, telemetry: DataFrame, *telemetry_path_segments: UserTelemetryPathSegment
|
48
|
+
):
|
49
|
+
self._telemetry = telemetry
|
50
|
+
self._telemetry_path_segments = telemetry_path_segments
|
51
|
+
|
52
|
+
@property
|
53
|
+
def telemetry(self) -> DataFrame:
|
54
|
+
"""
|
55
|
+
User telemetry data
|
56
|
+
"""
|
57
|
+
return self._telemetry
|
58
|
+
|
59
|
+
@property
|
60
|
+
def telemetry_path(self) -> str:
|
61
|
+
"""
|
62
|
+
Path segment for user telemetry data to include when writing it out.
|
63
|
+
"""
|
64
|
+
if len(self._telemetry_path_segments) == 0:
|
65
|
+
return ""
|
66
|
+
return "/".join([str(t_path) for t_path in self._telemetry_path_segments])
|
67
|
+
|
68
|
+
|
69
|
+
class UserTelemetryRecorder(ABC):
|
70
|
+
"""
|
71
|
+
Base class for user-defined telemetry recorders.
|
72
|
+
"""
|
73
|
+
|
74
|
+
@inject
|
75
|
+
def __init__(
|
76
|
+
self,
|
77
|
+
algorithm_payload: AlgorithmPayload,
|
78
|
+
metrics_provider: MetricsProvider,
|
79
|
+
logger: LoggerInterface,
|
80
|
+
storage_client: StorageClient,
|
81
|
+
serializer: TelemetrySerializer,
|
82
|
+
telemetry_base_path: str,
|
83
|
+
run_id: str,
|
84
|
+
):
|
85
|
+
self._metrics_provider = metrics_provider
|
86
|
+
self._logger = logger
|
87
|
+
self._payload = algorithm_payload
|
88
|
+
self._storage_client = storage_client
|
89
|
+
self._serializer = serializer
|
90
|
+
self._telemetry_base_path = telemetry_base_path
|
91
|
+
self._run_id = run_id
|
92
|
+
|
93
|
+
@property
|
94
|
+
def _metric_tags(self) -> dict[str, str]:
|
95
|
+
return {"recorder": self.__class__.alias().upper()}
|
96
|
+
|
97
|
+
@abstractmethod
|
98
|
+
async def _compute(
|
99
|
+
self,
|
100
|
+
algorithm_payload: AlgorithmPayload,
|
101
|
+
algorithm_result: AlgorithmResult,
|
102
|
+
**inputs: DataFrame,
|
103
|
+
) -> UserTelemetry:
|
104
|
+
"""
|
105
|
+
Produces the dataframe to record as user-level telemetry data.
|
106
|
+
"""
|
107
|
+
|
108
|
+
async def record(
|
109
|
+
self, algorithm_result: AlgorithmResult, **inputs: DataFrame
|
110
|
+
):
|
111
|
+
"""
|
112
|
+
Record user-defined telemetry data.
|
113
|
+
"""
|
114
|
+
|
115
|
+
@run_time_metrics_async(
|
116
|
+
metric_name="user_telemetry_recording",
|
117
|
+
on_finish_message_template="Finished recording telemetry from {recorder} in {elapsed:.2f}s seconds",
|
118
|
+
template_args={
|
119
|
+
"recorder": self.__class__.alias().upper(),
|
120
|
+
},
|
121
|
+
)
|
122
|
+
async def _measured_recording(**run_args) -> UserTelemetry:
|
123
|
+
return await self._compute(**run_args)
|
124
|
+
|
125
|
+
telemetry: UserTelemetry = await partial(
|
126
|
+
_measured_recording,
|
127
|
+
**(
|
128
|
+
{
|
129
|
+
"algorithm_payload": self._payload,
|
130
|
+
"algorithm_result": algorithm_result,
|
131
|
+
"run_id": self._run_id,
|
132
|
+
}
|
133
|
+
| inputs
|
134
|
+
),
|
135
|
+
metric_tags=self._metric_tags,
|
136
|
+
metrics_provider=self._metrics_provider,
|
137
|
+
logger=self._logger,
|
138
|
+
)()
|
139
|
+
|
140
|
+
self._storage_client.save_data_as_blob(
|
141
|
+
data=telemetry.telemetry,
|
142
|
+
blob_path=DataSocket(
|
143
|
+
alias="user_telemetry",
|
144
|
+
data_path=os.path.join(
|
145
|
+
self._telemetry_base_path,
|
146
|
+
"telemetry_group=user",
|
147
|
+
f"recorder_class={self.__class__.alias()}",
|
148
|
+
telemetry.telemetry_path, # path join eliminates empty segments
|
149
|
+
f"request_id={self._run_id}",
|
150
|
+
self._run_id,
|
151
|
+
),
|
152
|
+
data_format="null",
|
153
|
+
).parse_data_path(),
|
154
|
+
serialization_format=self._serializer.get_serialization_format(telemetry),
|
155
|
+
overwrite=True,
|
156
|
+
)
|
157
|
+
|
158
|
+
@classmethod
|
159
|
+
def alias(cls) -> str:
|
160
|
+
"""
|
161
|
+
Alias to identify this recorder in logging and metrics data.
|
162
|
+
"""
|
163
|
+
return snakecase(
|
164
|
+
re.sub(
|
165
|
+
r"(?<!^)(?=[A-Z])",
|
166
|
+
"_",
|
167
|
+
cls.__name__.lower(),
|
168
|
+
)
|
169
|
+
)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[tool.poetry]
|
2
2
|
name = "esd-services-api-client"
|
3
|
-
version = "
|
3
|
+
version = "v2.5.1a122.dev1"
|
4
4
|
description = "Python clients for ESD services"
|
5
5
|
authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
|
6
6
|
maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']
|
@@ -10,19 +10,19 @@ repository = 'https://github.com/SneaksAndData/esd-services-api-client'
|
|
10
10
|
|
11
11
|
[tool.poetry.dependencies]
|
12
12
|
python = ">=3.9,<3.12"
|
13
|
-
adapta = { version = "^3.
|
13
|
+
adapta = { version = "^3.2", extras = ["azure", "storage", "datadog"] }
|
14
14
|
dataclasses-json = "^0.6.0"
|
15
15
|
pycryptodome = "~3.15"
|
16
16
|
azure-identity = { version = "~1.7", optional = true }
|
17
|
-
injector = { version = "~0.
|
18
|
-
httpx = { version = "^0.
|
17
|
+
injector = { version = "~0.22.0", optional = true }
|
18
|
+
httpx = { version = "^0.27.0", optional = true }
|
19
19
|
pyjwt = "~2.4.0"
|
20
20
|
|
21
21
|
[tool.poetry.group.dev.dependencies]
|
22
22
|
pytest = "^7.2"
|
23
|
-
pylint = "^
|
23
|
+
pylint = "^3"
|
24
24
|
pytest-mock = "^3.6.1"
|
25
|
-
pytest-cov = "^
|
25
|
+
pytest-cov = "^3"
|
26
26
|
requests = "^2.27"
|
27
27
|
cryptography = "~36.0"
|
28
28
|
requests-mock = "^1.10"
|
@@ -1 +0,0 @@
|
|
1
|
-
__version__ = '2.4.0'
|
@@ -1,14 +0,0 @@
|
|
1
|
-
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
|
-
#
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|