esd-services-api-client 2.4.0__tar.gz → 2.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/PKG-INFO +4 -4
  2. esd_services_api_client-2.5.0/esd_services_api_client/__init__.py +20 -0
  3. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/README.md +56 -49
  4. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/abstractions/nexus_object.py +2 -2
  5. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/core/app_core.py +49 -10
  6. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/telemetry/recorder.py +44 -5
  7. esd_services_api_client-2.5.0/esd_services_api_client/nexus/telemetry/user_telemetry_recorder.py +168 -0
  8. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/pyproject.toml +6 -6
  9. esd_services_api_client-2.4.0/esd_services_api_client/_version.py +0 -1
  10. esd_services_api_client-2.4.0/esd_services_api_client/common/__init__.py +0 -14
  11. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/LICENSE +0 -0
  12. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/README.md +0 -0
  13. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/beast/__init__.py +0 -0
  14. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/beast/v3/__init__.py +0 -0
  15. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/beast/v3/_connector.py +0 -0
  16. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/beast/v3/_models.py +0 -0
  17. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/boxer/README.md +0 -0
  18. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/boxer/__init__.py +0 -0
  19. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/boxer/_auth.py +0 -0
  20. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/boxer/_base.py +0 -0
  21. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/boxer/_connector.py +0 -0
  22. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/boxer/_models.py +0 -0
  23. {esd_services_api_client-2.4.0/esd_services_api_client → esd_services_api_client-2.5.0/esd_services_api_client/common}/__init__.py +0 -0
  24. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/crystal/__init__.py +0 -0
  25. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/crystal/_api_versions.py +0 -0
  26. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/crystal/_connector.py +0 -0
  27. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/crystal/_models.py +0 -0
  28. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/__init__.py +0 -0
  29. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/abstractions/__init__.py +0 -0
  30. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/abstractions/algrorithm_cache.py +0 -0
  31. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/abstractions/input_object.py +0 -0
  32. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/abstractions/logger_factory.py +0 -0
  33. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/abstractions/socket_provider.py +0 -0
  34. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/algorithms/__init__.py +0 -0
  35. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/algorithms/_baseline_algorithm.py +0 -0
  36. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/algorithms/_remote_algorithm.py +0 -0
  37. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/algorithms/distributed.py +0 -0
  38. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/algorithms/forked_algorithm.py +0 -0
  39. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/algorithms/minimalistic.py +0 -0
  40. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/algorithms/recursive.py +0 -0
  41. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/configurations/__init__.py +0 -0
  42. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/configurations/algorithm_configuration.py +0 -0
  43. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/core/__init__.py +0 -0
  44. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/core/app_dependencies.py +0 -0
  45. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/core/serializers.py +0 -0
  46. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/exceptions/__init__.py +0 -0
  47. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/exceptions/_nexus_error.py +0 -0
  48. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/exceptions/cache_errors.py +0 -0
  49. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/exceptions/input_reader_error.py +0 -0
  50. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/exceptions/startup_error.py +0 -0
  51. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/input/__init__.py +0 -0
  52. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/input/input_processor.py +0 -0
  53. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/input/input_reader.py +0 -0
  54. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/input/payload_reader.py +0 -0
  55. {esd_services_api_client-2.4.0 → esd_services_api_client-2.5.0}/esd_services_api_client/nexus/telemetry/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: esd-services-api-client
3
- Version: 2.4.0
3
+ Version: 2.5.0
4
4
  Summary: Python clients for ESD services
5
5
  Home-page: https://github.com/SneaksAndData/esd-services-api-client
6
6
  License: Apache 2.0
@@ -16,11 +16,11 @@ Classifier: Programming Language :: Python :: 3.10
16
16
  Classifier: Programming Language :: Python :: 3.11
17
17
  Provides-Extra: azure
18
18
  Provides-Extra: nexus
19
- Requires-Dist: adapta[azure,datadog,storage] (>=3.0,<4.0)
19
+ Requires-Dist: adapta[azure,datadog,storage] (>=3.2,<4.0)
20
20
  Requires-Dist: azure-identity (>=1.7,<1.8) ; extra == "azure"
21
21
  Requires-Dist: dataclasses-json (>=0.6.0,<0.7.0)
22
- Requires-Dist: httpx (>=0.26.0,<0.27.0) ; extra == "nexus"
23
- Requires-Dist: injector (>=0.21.0,<0.22.0) ; extra == "nexus"
22
+ Requires-Dist: httpx (>=0.27.0,<0.28.0) ; extra == "nexus"
23
+ Requires-Dist: injector (>=0.22.0,<0.23.0) ; extra == "nexus"
24
24
  Requires-Dist: pycryptodome (>=3.15,<3.16)
25
25
  Requires-Dist: pyjwt (>=2.4.0,<2.5.0)
26
26
  Project-URL: Repository, https://github.com/SneaksAndData/esd-services-api-client
@@ -0,0 +1,20 @@
1
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ """
17
+ Root index.
18
+ """
19
+
20
+ __version__ = "2.5.0"
@@ -1,3 +1,5 @@
1
+ from pandas import DataFramefrom pandas import DataFramefrom pandas import DataFrame
2
+
1
3
  ## Nexus
2
4
  Set the following environment variables for Azure:
3
5
  ```
@@ -42,14 +44,7 @@ from esd_services_api_client.nexus.algorithms import MinimalisticAlgorithm
42
44
  from esd_services_api_client.nexus.input import InputReader, InputProcessor
43
45
 
44
46
  from esd_services_api_client.nexus.input.payload_reader import AlgorithmPayload
45
-
46
-
47
- async def my_on_complete_func_1(**kwargs):
48
- pass
49
-
50
-
51
- async def my_on_complete_func_2(**kwargs):
52
- pass
47
+ from esd_services_api_client.nexus.telemetry.user_telemetry_recorder import UserTelemetryRecorder, UserTelemetry
53
48
 
54
49
 
55
50
  @dataclass
@@ -81,10 +76,10 @@ class MockRequestHandler(BaseHTTPRequestHandler):
81
76
  """
82
77
 
83
78
  def __init__(
84
- self,
85
- request: bytes,
86
- client_address: tuple[str, int],
87
- server: socketserver.BaseServer,
79
+ self,
80
+ request: bytes,
81
+ client_address: tuple[str, int],
82
+ server: socketserver.BaseServer,
88
83
  ):
89
84
  """
90
85
  Initialize request handler
@@ -130,14 +125,14 @@ class MockRequestHandler(BaseHTTPRequestHandler):
130
125
  class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
131
126
  @inject
132
127
  def __init__(
133
- self,
134
- store: QueryEnabledStore,
135
- metrics_provider: MetricsProvider,
136
- logger_factory: LoggerFactory,
137
- payload: MyAlgorithmPayload,
138
- socket_provider: ExternalSocketProvider,
139
- *readers: "InputReader",
140
- cache: InputCache
128
+ self,
129
+ store: QueryEnabledStore,
130
+ metrics_provider: MetricsProvider,
131
+ logger_factory: LoggerFactory,
132
+ payload: MyAlgorithmPayload,
133
+ socket_provider: ExternalSocketProvider,
134
+ *readers: "InputReader",
135
+ cache: InputCache
141
136
  ):
142
137
  super().__init__(
143
138
  socket=socket_provider.socket("x"),
@@ -161,14 +156,14 @@ class XReader(InputReader[MyAlgorithmPayload, pandas.DataFrame]):
161
156
  class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
162
157
  @inject
163
158
  def __init__(
164
- self,
165
- store: QueryEnabledStore,
166
- metrics_provider: MetricsProvider,
167
- logger_factory: LoggerFactory,
168
- payload: MyAlgorithmPayload2,
169
- socket_provider: ExternalSocketProvider,
170
- *readers: "InputReader",
171
- cache: InputCache
159
+ self,
160
+ store: QueryEnabledStore,
161
+ metrics_provider: MetricsProvider,
162
+ logger_factory: LoggerFactory,
163
+ payload: MyAlgorithmPayload2,
164
+ socket_provider: ExternalSocketProvider,
165
+ *readers: "InputReader",
166
+ cache: InputCache
172
167
  ):
173
168
  super().__init__(
174
169
  socket=socket_provider.socket("y"),
@@ -192,12 +187,12 @@ class YReader(InputReader[MyAlgorithmPayload2, pandas.DataFrame]):
192
187
  class XProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
193
188
  @inject
194
189
  def __init__(
195
- self,
196
- x: XReader,
197
- metrics_provider: MetricsProvider,
198
- logger_factory: LoggerFactory,
199
- my_conf: MyAlgorithmConfiguration,
200
- cache: InputCache,
190
+ self,
191
+ x: XReader,
192
+ metrics_provider: MetricsProvider,
193
+ logger_factory: LoggerFactory,
194
+ my_conf: MyAlgorithmConfiguration,
195
+ cache: InputCache,
201
196
  ):
202
197
  super().__init__(
203
198
  x,
@@ -210,7 +205,7 @@ class XProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
210
205
  self.conf = my_conf
211
206
 
212
207
  async def _process_input(
213
- self, x: pandas.DataFrame, **_
208
+ self, x: pandas.DataFrame, **_
214
209
  ) -> pandas.DataFrame:
215
210
  self._logger.info("Config: {config}", config=self.conf.to_json())
216
211
  return x.assign(c=[-1, 1])
@@ -219,12 +214,12 @@ class XProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
219
214
  class YProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
220
215
  @inject
221
216
  def __init__(
222
- self,
223
- y: YReader,
224
- metrics_provider: MetricsProvider,
225
- logger_factory: LoggerFactory,
226
- my_conf: MyAlgorithmConfiguration,
227
- cache: InputCache,
217
+ self,
218
+ y: YReader,
219
+ metrics_provider: MetricsProvider,
220
+ logger_factory: LoggerFactory,
221
+ my_conf: MyAlgorithmConfiguration,
222
+ cache: InputCache,
228
223
  ):
229
224
  super().__init__(
230
225
  y,
@@ -237,7 +232,7 @@ class YProcessor(InputProcessor[MyAlgorithmPayload, pandas.DataFrame]):
237
232
  self.conf = my_conf
238
233
 
239
234
  async def _process_input(
240
- self, y: pandas.DataFrame, **_
235
+ self, y: pandas.DataFrame, **_
241
236
  ) -> pandas.DataFrame:
242
237
  self._logger.info("Config: {config}", config=self.conf.to_json())
243
238
  return y.assign(c=[-1, 1])
@@ -264,23 +259,34 @@ class MyAlgorithm(MinimalisticAlgorithm[MyAlgorithmPayload]):
264
259
 
265
260
  @inject
266
261
  def __init__(
267
- self,
268
- metrics_provider: MetricsProvider,
269
- logger_factory: LoggerFactory,
270
- x_processor: XProcessor,
271
- y_processor: YProcessor,
272
- cache: InputCache,
262
+ self,
263
+ metrics_provider: MetricsProvider,
264
+ logger_factory: LoggerFactory,
265
+ x_processor: XProcessor,
266
+ y_processor: YProcessor,
267
+ cache: InputCache,
273
268
  ):
274
269
  super().__init__(
275
270
  metrics_provider, logger_factory, x_processor, y_processor, cache=cache
276
271
  )
277
272
 
278
273
  async def _run(
279
- self, x: pandas.DataFrame, y: pandas.DataFrame, **kwargs
274
+ self, x: pandas.DataFrame, y: pandas.DataFrame, **kwargs
280
275
  ) -> MyResult:
281
276
  return MyResult(x, y)
282
277
 
283
278
 
279
+ class ObjectiveAnalytics(UserTelemetryRecorder):
280
+
281
+ async def _compute(self,
282
+ algorithm_payload:
283
+ AlgorithmPayload,
284
+ algorithm_result: AlgorithmResult,
285
+ run_id: str,
286
+ **inputs: pandas.DataFrame) -> UserTelemetry:
287
+ pass
288
+
289
+
284
290
  async def main():
285
291
  """
286
292
  Mock HTTP Server
@@ -297,6 +303,7 @@ async def main():
297
303
  .use_processor(XProcessor)
298
304
  .use_processor(YProcessor)
299
305
  .use_algorithm(MyAlgorithm)
306
+ .on_complete(ObjectiveAnalytics)
300
307
  .inject_configuration(MyAlgorithmConfiguration)
301
308
  .inject_payload(MyAlgorithmPayload, MyAlgorithmPayload2)
302
309
  )
@@ -90,13 +90,13 @@ class NexusCoreObject(ABC):
90
90
 
91
91
  class NexusObject(Generic[TPayload, TResult], NexusCoreObject, ABC):
92
92
  """
93
- Base class for all Nexus objects.
93
+ Base class for all Nexus objects that perform operations on the algorithm payload.
94
94
  """
95
95
 
96
96
  @classmethod
97
97
  def alias(cls) -> str:
98
98
  """
99
- Alias to identify this reader's output
99
+ Alias to identify this class instances when passed through kwargs.
100
100
  """
101
101
  return snakecase(
102
102
  re.sub(
@@ -23,11 +23,12 @@ import platform
23
23
  import signal
24
24
  import sys
25
25
  import traceback
26
- from typing import final, Type, Optional, Coroutine
26
+ from typing import final, Type, Optional
27
27
 
28
28
  import backoff
29
29
  import urllib3.exceptions
30
30
  import azure.core.exceptions
31
+ from adapta.logs import LoggerInterface
31
32
  from adapta.process_communication import DataSocket
32
33
  from adapta.storage.blob.base import StorageClient
33
34
  from adapta.storage.query_enabled_store import QueryEnabledStore
@@ -41,6 +42,7 @@ from esd_services_api_client.crystal import (
41
42
  AlgorithmRunResult,
42
43
  CrystalEntrypointArguments,
43
44
  )
45
+ from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
44
46
  from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
45
47
  from esd_services_api_client.nexus.algorithms import (
46
48
  BaselineAlgorithm,
@@ -61,6 +63,10 @@ from esd_services_api_client.nexus.input.payload_reader import (
61
63
  AlgorithmPayload,
62
64
  )
63
65
  from esd_services_api_client.nexus.telemetry.recorder import TelemetryRecorder
66
+ from esd_services_api_client.nexus.telemetry.user_telemetry_recorder import (
67
+ UserTelemetryRecorder,
68
+ )
69
+ from esd_services_api_client import __version__
64
70
 
65
71
 
66
72
  def is_transient_exception(exception: Optional[BaseException]) -> Optional[bool]:
@@ -112,7 +118,7 @@ class Nexus:
112
118
  self._algorithm_class: Optional[Type[BaselineAlgorithm]] = None
113
119
  self._run_args = args
114
120
  self._algorithm_run_task: Optional[asyncio.Task] = None
115
- self._on_complete_tasks: list[Coroutine] = []
121
+ self._on_complete_tasks: list[type[UserTelemetryRecorder]] = []
116
122
 
117
123
  attach_signal_handlers()
118
124
 
@@ -123,11 +129,11 @@ class Nexus:
123
129
  """
124
130
  return self._algorithm_class
125
131
 
126
- def on_complete(self, coro: Coroutine) -> "Nexus":
132
+ def on_complete(self, *post_processors: type[UserTelemetryRecorder]) -> "Nexus":
127
133
  """
128
134
  Attaches a coroutine to run on algorithm completion.
129
135
  """
130
- self._on_complete_tasks.append(coro)
136
+ self._on_complete_tasks.extend(post_processors)
131
137
  return self
132
138
 
133
139
  def add_reader(self, reader: Type[InputReader]) -> "Nexus":
@@ -254,6 +260,15 @@ class Nexus:
254
260
 
255
261
  algorithm: BaselineAlgorithm = self._injector.get(self._algorithm_class)
256
262
  telemetry_recorder: TelemetryRecorder = self._injector.get(TelemetryRecorder)
263
+ root_logger: LoggerInterface = self._injector.get(LoggerFactory).create_logger(
264
+ logger_type=self.__class__,
265
+ )
266
+
267
+ root_logger.info(
268
+ "Running algorithm {algorithm} on Nexus version {version}",
269
+ algorithm=algorithm.__class__.__name__,
270
+ version=__version__,
271
+ )
257
272
 
258
273
  async with algorithm as instance:
259
274
  self._algorithm_run_task = asyncio.create_task(
@@ -261,23 +276,47 @@ class Nexus:
261
276
  )
262
277
  await self._algorithm_run_task
263
278
  ex = self._algorithm_run_task.exception()
264
- on_complete_tasks = [
265
- asyncio.create_task(on_complete_task)
266
- for on_complete_task in self._on_complete_tasks
267
- ]
268
279
 
269
280
  await self._submit_result(
270
281
  self._algorithm_run_task.result() if not ex else None,
271
282
  self._algorithm_run_task.exception(),
272
283
  )
273
- if len(on_complete_tasks) > 0:
274
- await asyncio.wait(on_complete_tasks)
275
284
 
276
285
  # record telemetry
286
+ root_logger.info(
287
+ "Recording telemetry for the run {request_id}",
288
+ request_id=self._run_args.request_id,
289
+ )
277
290
  async with telemetry_recorder as recorder:
278
291
  await recorder.record(
279
292
  run_id=self._run_args.request_id, **algorithm.inputs
280
293
  )
294
+ on_complete_tasks = [
295
+ recorder.record_user_telemetry(
296
+ user_recorder_type=on_complete_task_class,
297
+ run_id=self._run_args.request_id,
298
+ result=self._algorithm_run_task.result(),
299
+ **algorithm.inputs,
300
+ )
301
+ for on_complete_task_class in self._on_complete_tasks
302
+ ]
303
+ if len(on_complete_tasks) > 0:
304
+ done, pending = await asyncio.wait(on_complete_tasks)
305
+ if len(pending) > 0:
306
+ root_logger.warning(
307
+ "Some post-processing operations did not complete or failed. Please review application logs for more information"
308
+ )
309
+ for done_on_complete_task in done:
310
+ on_complete_task_exc = done_on_complete_task.exception()
311
+ if on_complete_task_exc:
312
+ root_logger.warning(
313
+ "Post processing task failed",
314
+ exception=on_complete_task_exc,
315
+ )
316
+ else:
317
+ root_logger.info(
318
+ "No post processing tasks were defined for this run."
319
+ )
281
320
 
282
321
  # dispose of QES instance gracefully as it might hold open connections
283
322
  qes = self._injector.get(QueryEnabledStore)
@@ -3,20 +3,27 @@
3
3
  """
4
4
  import asyncio
5
5
  import os
6
+ from asyncio import Task
6
7
  from functools import partial
7
8
  from typing import final
8
9
 
9
- import pandas as pd
10
+ from pandas import DataFrame
10
11
  from adapta.metrics import MetricsProvider
11
12
  from adapta.process_communication import DataSocket
12
13
  from adapta.storage.blob.base import StorageClient
13
14
  from injector import inject, singleton
14
15
 
15
16
  from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
16
- from esd_services_api_client.nexus.abstractions.nexus_object import NexusCoreObject
17
+ from esd_services_api_client.nexus.abstractions.nexus_object import (
18
+ NexusCoreObject,
19
+ AlgorithmResult,
20
+ )
17
21
  from esd_services_api_client.nexus.core.serializers import (
18
22
  TelemetrySerializer,
19
23
  )
24
+ from esd_services_api_client.nexus.telemetry.user_telemetry_recorder import (
25
+ UserTelemetryRecorder,
26
+ )
20
27
 
21
28
 
22
29
  @final
@@ -51,7 +58,7 @@ class TelemetryRecorder(NexusCoreObject):
51
58
  """
52
59
 
53
60
  async def _record(
54
- entity_to_record: pd.DataFrame | dict,
61
+ entity_to_record: DataFrame | dict,
55
62
  entity_name: str,
56
63
  **_,
57
64
  ) -> None:
@@ -61,7 +68,7 @@ class TelemetryRecorder(NexusCoreObject):
61
68
  run_id=run_id,
62
69
  )
63
70
  if not isinstance(entity_to_record, dict) and not isinstance(
64
- entity_to_record, pd.DataFrame
71
+ entity_to_record, DataFrame
65
72
  ):
66
73
  self._logger.warning(
67
74
  "Unsupported data type: {telemetry_entity_type}. Telemetry recording skipped.",
@@ -72,7 +79,13 @@ class TelemetryRecorder(NexusCoreObject):
72
79
  data=entity_to_record,
73
80
  blob_path=DataSocket(
74
81
  alias="telemetry",
75
- data_path=f"{self._telemetry_base_path}/{entity_name}/{run_id}",
82
+ data_path=os.path.join(
83
+ self._telemetry_base_path,
84
+ "telemetry_group=inputs",
85
+ f"entity_name={entity_name}",
86
+ f"request_id={run_id}",
87
+ run_id,
88
+ ),
76
89
  data_format="null",
77
90
  ).parse_data_path(),
78
91
  serialization_format=self._serializer.get_serialization_format(
@@ -106,3 +119,29 @@ class TelemetryRecorder(NexusCoreObject):
106
119
  self._logger.warning(
107
120
  "Telemetry recoding failed", exception=telemetry_exc
108
121
  )
122
+
123
+ def record_user_telemetry(
124
+ self,
125
+ user_recorder_type: type[UserTelemetryRecorder],
126
+ run_id: str,
127
+ result: AlgorithmResult,
128
+ **inputs: DataFrame,
129
+ ) -> Task:
130
+ """
131
+ Creates an awaitable task that records user telemetry using provided recorder type.
132
+
133
+ :param user_recorder_type: Recorder type to record user telemetry.
134
+ :param run_id: The request_id to record user telemetry for.
135
+ :param result: Result of the algorithm.
136
+ :param inputs: Algorithm input data.
137
+ """
138
+ return asyncio.create_task(
139
+ user_recorder_type(
140
+ run_id=run_id,
141
+ metrics_provider=self._metrics_provider,
142
+ logger=self._logger,
143
+ storage_client=self._storage_client,
144
+ serializer=self._serializer,
145
+ telemetry_base_path=self._telemetry_base_path,
146
+ ).record(run_id=run_id, algorithm_result=result, **inputs)
147
+ )
@@ -0,0 +1,168 @@
1
+ """
2
+ User-defined telemetry.
3
+ """
4
+ import os.path
5
+ import re
6
+ from abc import ABC, abstractmethod
7
+ from dataclasses import dataclass
8
+ from functools import partial
9
+ from typing import final
10
+
11
+ from pandas import DataFrame
12
+
13
+ from adapta.process_communication import DataSocket
14
+ from adapta.storage.blob.base import StorageClient
15
+ from adapta.logs import LoggerInterface
16
+ from adapta.metrics import MetricsProvider
17
+ from adapta.utils.decorators import run_time_metrics_async
18
+ from dataclasses_json.stringcase import snakecase
19
+ from injector import inject
20
+
21
+ from esd_services_api_client.nexus.abstractions.nexus_object import AlgorithmResult
22
+ from esd_services_api_client.nexus.core.serializers import TelemetrySerializer
23
+ from esd_services_api_client.nexus.input.payload_reader import AlgorithmPayload
24
+
25
+
26
+ @final
27
+ @dataclass
28
+ class UserTelemetryPathSegment:
29
+ """
30
+ Path segment for user telemetry.
31
+ """
32
+
33
+ segment: str
34
+ segment_header: str
35
+
36
+ def __str__(self):
37
+ return "=".join([self.segment_header, self.segment])
38
+
39
+
40
+ @final
41
+ class UserTelemetry:
42
+ """
43
+ Base class for user-defined telemetry types.
44
+ """
45
+
46
+ def __init__(
47
+ self, telemetry: DataFrame, *telemetry_path_segments: UserTelemetryPathSegment
48
+ ):
49
+ self._telemetry = telemetry
50
+ self._telemetry_path_segments = telemetry_path_segments
51
+
52
+ @property
53
+ def telemetry(self) -> DataFrame:
54
+ """
55
+ User telemetry data
56
+ """
57
+ return self._telemetry
58
+
59
+ @property
60
+ def telemetry_path(self) -> str:
61
+ """
62
+ Path segment for user telemetry data to include when writing it out.
63
+ """
64
+ if len(self._telemetry_path_segments) == 0:
65
+ return ""
66
+ return "/".join([str(t_path) for t_path in self._telemetry_path_segments])
67
+
68
+
69
+ class UserTelemetryRecorder(ABC):
70
+ """
71
+ Base class for user-defined telemetry recorders.
72
+ """
73
+
74
+ @inject
75
+ def __init__(
76
+ self,
77
+ algorithm_payload: AlgorithmPayload,
78
+ metrics_provider: MetricsProvider,
79
+ logger: LoggerInterface,
80
+ storage_client: StorageClient,
81
+ serializer: TelemetrySerializer,
82
+ telemetry_base_path: str,
83
+ ):
84
+ self._metrics_provider = metrics_provider
85
+ self._logger = logger
86
+ self._payload = algorithm_payload
87
+ self._storage_client = storage_client
88
+ self._serializer = serializer
89
+ self._telemetry_base_path = telemetry_base_path
90
+
91
+ @property
92
+ def _metric_tags(self) -> dict[str, str]:
93
+ return {"recorder": self.__class__.alias().upper()}
94
+
95
+ @abstractmethod
96
+ async def _compute(
97
+ self,
98
+ algorithm_payload: AlgorithmPayload,
99
+ algorithm_result: AlgorithmResult,
100
+ run_id: str,
101
+ **inputs: DataFrame,
102
+ ) -> UserTelemetry:
103
+ """
104
+ Produces the dataframe to record as user-level telemetry data.
105
+ """
106
+
107
+ async def record(
108
+ self, algorithm_result: AlgorithmResult, run_id: str, **inputs: DataFrame
109
+ ):
110
+ """
111
+ Record user-defined telemetry data.
112
+ """
113
+
114
+ @run_time_metrics_async(
115
+ metric_name="user_telemetry_recording",
116
+ on_finish_message_template="Finished recording telemetry from {recorder} in {elapsed:.2f}s seconds",
117
+ template_args={
118
+ "recorder": self.__class__.alias().upper(),
119
+ },
120
+ )
121
+ async def _measured_recording(**run_args) -> UserTelemetry:
122
+ return await self._compute(**run_args)
123
+
124
+ telemetry: UserTelemetry = await partial(
125
+ _measured_recording,
126
+ **(
127
+ {
128
+ "algorithm_payload": self._payload,
129
+ "algorithm_result": algorithm_result,
130
+ "run_id": run_id,
131
+ }
132
+ | inputs
133
+ ),
134
+ metric_tags=self._metric_tags,
135
+ metrics_provider=self._metrics_provider,
136
+ logger=self._logger,
137
+ )()
138
+
139
+ self._storage_client.save_data_as_blob(
140
+ data=telemetry.telemetry,
141
+ blob_path=DataSocket(
142
+ alias="user_telemetry",
143
+ data_path=os.path.join(
144
+ self._telemetry_base_path,
145
+ "telemetry_group=user",
146
+ f"recorder_class={self.__class__.alias()}",
147
+ telemetry.telemetry_path, # path join eliminates empty segments
148
+ f"request_id={run_id}",
149
+ run_id,
150
+ ),
151
+ data_format="null",
152
+ ).parse_data_path(),
153
+ serialization_format=self._serializer.get_serialization_format(telemetry),
154
+ overwrite=True,
155
+ )
156
+
157
+ @classmethod
158
+ def alias(cls) -> str:
159
+ """
160
+ Alias to identify this recorder in logging and metrics data.
161
+ """
162
+ return snakecase(
163
+ re.sub(
164
+ r"(?<!^)(?=[A-Z])",
165
+ "_",
166
+ cls.__name__.lower(),
167
+ )
168
+ )
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "esd-services-api-client"
3
- version = "2.4.0"
3
+ version = "2.5.0"
4
4
  description = "Python clients for ESD services"
5
5
  authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
6
6
  maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>', 'VISA <visa@ecco.com>']
@@ -10,19 +10,19 @@ repository = 'https://github.com/SneaksAndData/esd-services-api-client'
10
10
 
11
11
  [tool.poetry.dependencies]
12
12
  python = ">=3.9,<3.12"
13
- adapta = { version = "^3.0", extras = ["azure", "storage", "datadog"] }
13
+ adapta = { version = "^3.2", extras = ["azure", "storage", "datadog"] }
14
14
  dataclasses-json = "^0.6.0"
15
15
  pycryptodome = "~3.15"
16
16
  azure-identity = { version = "~1.7", optional = true }
17
- injector = { version = "~0.21.0", optional = true }
18
- httpx = { version = "^0.26.0", optional = true }
17
+ injector = { version = "~0.22.0", optional = true }
18
+ httpx = { version = "^0.27.0", optional = true }
19
19
  pyjwt = "~2.4.0"
20
20
 
21
21
  [tool.poetry.group.dev.dependencies]
22
22
  pytest = "^7.2"
23
- pylint = "^2.12"
23
+ pylint = "^3"
24
24
  pytest-mock = "^3.6.1"
25
- pytest-cov = "^2.12"
25
+ pytest-cov = "^3"
26
26
  requests = "^2.27"
27
27
  cryptography = "~36.0"
28
28
  requests-mock = "^1.10"
@@ -1 +0,0 @@
1
- __version__ = '2.4.0'
@@ -1,14 +0,0 @@
1
- # Copyright (c) 2023-2024. ECCO Sneaks & Data
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- #