esd-services-api-client 2.1.0__py3-none-any.whl → 2.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. esd_services_api_client/__init__.py +1 -1
  2. esd_services_api_client/_version.py +1 -1
  3. esd_services_api_client/beast/__init__.py +1 -1
  4. esd_services_api_client/beast/v3/__init__.py +1 -1
  5. esd_services_api_client/beast/v3/_connector.py +1 -1
  6. esd_services_api_client/beast/v3/_models.py +1 -1
  7. esd_services_api_client/boxer/__init__.py +1 -1
  8. esd_services_api_client/boxer/_auth.py +1 -1
  9. esd_services_api_client/boxer/_base.py +1 -1
  10. esd_services_api_client/boxer/_connector.py +1 -1
  11. esd_services_api_client/boxer/_models.py +1 -1
  12. esd_services_api_client/common/__init__.py +1 -1
  13. esd_services_api_client/crystal/__init__.py +1 -1
  14. esd_services_api_client/crystal/_api_versions.py +1 -1
  15. esd_services_api_client/crystal/_connector.py +1 -1
  16. esd_services_api_client/crystal/_models.py +1 -1
  17. esd_services_api_client/nexus/__init__.py +1 -1
  18. esd_services_api_client/nexus/abstractions/__init__.py +1 -1
  19. esd_services_api_client/nexus/abstractions/logger_factory.py +1 -1
  20. esd_services_api_client/nexus/abstractions/nexus_object.py +5 -2
  21. esd_services_api_client/nexus/abstractions/socket_provider.py +14 -5
  22. esd_services_api_client/nexus/algorithms/__init__.py +1 -1
  23. esd_services_api_client/nexus/algorithms/_baseline_algorithm.py +28 -19
  24. esd_services_api_client/nexus/algorithms/distributed.py +1 -1
  25. esd_services_api_client/nexus/algorithms/minimalistic.py +1 -1
  26. esd_services_api_client/nexus/algorithms/recursive.py +1 -1
  27. esd_services_api_client/nexus/configurations/algorithm_configuration.py +1 -1
  28. esd_services_api_client/nexus/core/__init__.py +1 -1
  29. esd_services_api_client/nexus/core/app_core.py +1 -1
  30. esd_services_api_client/nexus/core/app_dependencies.py +1 -1
  31. esd_services_api_client/nexus/exceptions/__init__.py +1 -1
  32. esd_services_api_client/nexus/exceptions/_nexus_error.py +1 -1
  33. esd_services_api_client/nexus/exceptions/input_reader_error.py +1 -1
  34. esd_services_api_client/nexus/exceptions/startup_error.py +2 -2
  35. esd_services_api_client/nexus/input/__init__.py +1 -1
  36. esd_services_api_client/nexus/input/_functions.py +21 -4
  37. esd_services_api_client/nexus/input/input_processor.py +90 -3
  38. esd_services_api_client/nexus/input/input_reader.py +14 -15
  39. esd_services_api_client/nexus/input/payload_reader.py +1 -1
  40. {esd_services_api_client-2.1.0.dist-info → esd_services_api_client-2.1.2.dist-info}/METADATA +1 -1
  41. esd_services_api_client-2.1.2.dist-info/RECORD +46 -0
  42. esd_services_api_client-2.1.0.dist-info/RECORD +0 -46
  43. {esd_services_api_client-2.1.0.dist-info → esd_services_api_client-2.1.2.dist-info}/LICENSE +0 -0
  44. {esd_services_api_client-2.1.0.dist-info → esd_services_api_client-2.1.2.dist-info}/WHEEL +0 -0
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2023. ECCO Sneaks & Data
1
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -1 +1 @@
1
- __version__ = '2.1.0'
1
+ __version__ = '2.1.2'
@@ -2,7 +2,7 @@
2
2
  Import index.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Import index.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -1,7 +1,7 @@
1
1
  """
2
2
  Connector for Beast Workload Manager (Spark AKS)
3
3
  """
4
- # Copyright (c) 2023. ECCO Sneaks & Data
4
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -1,7 +1,7 @@
1
1
  """
2
2
  Models for Beast connector
3
3
  """
4
- # Copyright (c) 2023. ECCO Sneaks & Data
4
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Import index.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Boxer Auth classes
3
3
  Based on https://docs.python-requests.org/en/master/user/advanced/#custom-authentication
4
4
  """
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -1,7 +1,7 @@
1
1
  """
2
2
  Interfaces for boxer API
3
3
  """
4
- # Copyright (c) 2023. ECCO Sneaks & Data
4
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -1,7 +1,7 @@
1
1
  """
2
2
  Connector for Boxer Auth API.
3
3
  """
4
- # Copyright (c) 2023. ECCO Sneaks & Data
4
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -1,7 +1,7 @@
1
1
  """
2
2
  Models for Boxer
3
3
  """
4
- # Copyright (c) 2023. ECCO Sneaks & Data
4
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2023. ECCO Sneaks & Data
1
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -1,6 +1,6 @@
1
1
  """init file"""
2
2
 
3
- # Copyright (c) 2023. ECCO Sneaks & Data
3
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
4
4
  #
5
5
  # Licensed under the Apache License, Version 2.0 (the "License");
6
6
  # you may not use this file except in compliance with the License.
@@ -1,7 +1,7 @@
1
1
  """
2
2
  API versioning for arcane connector
3
3
  """
4
- # Copyright (c) 2023. ECCO Sneaks & Data
4
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -1,7 +1,7 @@
1
1
  """
2
2
  Connector for Crystal Job Runtime (AKS)
3
3
  """
4
- # Copyright (c) 2023. ECCO Sneaks & Data
4
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -1,7 +1,7 @@
1
1
  """
2
2
  Models for Crystal connector
3
3
  """
4
- # Copyright (c) 2023. ECCO Sneaks & Data
4
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Import index.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Import index.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Logger factory for async loggers.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -3,7 +3,7 @@
3
3
  """
4
4
  import re
5
5
 
6
- # Copyright (c) 2023. ECCO Sneaks & Data
6
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
7
7
  #
8
8
  # Licensed under the Apache License, Version 2.0 (the "License");
9
9
  # you may not use this file except in compliance with the License.
@@ -95,5 +95,8 @@ class NexusObject(Generic[TPayload, TResult], ABC):
95
95
  return re.sub(
96
96
  r"(?<!^)(?=[A-Z])",
97
97
  "_",
98
- cls.__name__.lower().replace("reader", "").replace("processor", ""),
98
+ cls.__name__.lower()
99
+ .replace("reader", "")
100
+ .replace("processor", "")
101
+ .replace("algorithm", ""),
99
102
  )
@@ -1,9 +1,8 @@
1
1
  """
2
2
  Socket provider for all data sockets used by algorithms.
3
3
  """
4
- import json
5
4
 
6
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
7
6
  #
8
7
  # Licensed under the Apache License, Version 2.0 (the "License");
9
8
  # you may not use this file except in compliance with the License.
@@ -18,10 +17,15 @@ import json
18
17
  # limitations under the License.
19
18
  #
20
19
 
21
- from typing import final, Optional
20
+ import json
21
+ from typing import final
22
22
 
23
23
  from adapta.process_communication import DataSocket
24
24
 
25
+ from esd_services_api_client.nexus.exceptions.startup_error import (
26
+ FatalStartupConfigurationError,
27
+ )
28
+
25
29
 
26
30
  @final
27
31
  class ExternalSocketProvider:
@@ -32,11 +36,16 @@ class ExternalSocketProvider:
32
36
  def __init__(self, *sockets: DataSocket):
33
37
  self._sockets = {socket.alias: socket for socket in sockets}
34
38
 
35
- def socket(self, name: str) -> Optional[DataSocket]:
39
+ def socket(self, name: str) -> DataSocket:
36
40
  """
37
41
  Retrieve a socket if it exists.
38
42
  """
39
- return self._sockets.get(name, None)
43
+ if name in self._sockets:
44
+ return self._sockets[name]
45
+
46
+ raise FatalStartupConfigurationError(
47
+ missing_entry=f"socket with alias `{name}`"
48
+ )
40
49
 
41
50
  @classmethod
42
51
  def from_serialized(cls, socket_list_ser: str) -> "ExternalSocketProvider":
@@ -2,7 +2,7 @@
2
2
  Import index.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -1,9 +1,8 @@
1
1
  """
2
2
  Base algorithm
3
3
  """
4
- import asyncio
5
4
 
6
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
7
6
  #
8
7
  # Licensed under the Apache License, Version 2.0 (the "License");
9
8
  # you may not use this file except in compliance with the License.
@@ -20,18 +19,21 @@ import asyncio
20
19
 
21
20
 
22
21
  from abc import abstractmethod
23
- from functools import reduce
22
+ from functools import reduce, partial
24
23
 
25
24
  from adapta.metrics import MetricsProvider
25
+ from adapta.utils.decorators import run_time_metrics_async
26
26
 
27
27
  from esd_services_api_client.nexus.abstractions.nexus_object import (
28
28
  NexusObject,
29
29
  TPayload,
30
- TResult,
31
30
  AlgorithmResult,
32
31
  )
33
32
  from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
34
- from esd_services_api_client.nexus.input.input_processor import InputProcessor
33
+ from esd_services_api_client.nexus.input.input_processor import (
34
+ InputProcessor,
35
+ resolve_processors,
36
+ )
35
37
 
36
38
 
37
39
  class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
@@ -54,24 +56,31 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
54
56
  Core logic for this algorithm. Implementing this method is mandatory.
55
57
  """
56
58
 
59
+ @property
60
+ def _metric_tags(self) -> dict[str, str]:
61
+ return {"algorithm": self.__class__.alias()}
62
+
57
63
  async def run(self, **kwargs) -> AlgorithmResult:
58
64
  """
59
65
  Coroutine that executes the algorithm logic.
60
66
  """
61
67
 
62
- async def _process(
63
- processor: InputProcessor[TPayload, TResult]
64
- ) -> dict[str, TResult]:
65
- async with processor as instance:
66
- return await instance.process_input(**kwargs)
68
+ @run_time_metrics_async(
69
+ metric_name="algorthm_run",
70
+ on_finish_message_template="Finished running {algorithm} in {elapsed:.2f}s seconds",
71
+ template_args={
72
+ "algorithm": self.__class__.alias().upper(),
73
+ },
74
+ )
75
+ async def _measured_run(**run_args):
76
+ return await self._run(**run_args)
67
77
 
68
- process_tasks: dict[str, asyncio.Task] = {
69
- input_processor.__class__.__name__.lower(): asyncio.create_task(
70
- _process(input_processor)
71
- )
72
- for input_processor in self._input_processors
73
- }
74
- await asyncio.wait(fs=process_tasks.values())
75
- results = [task.result() for task in process_tasks.values()]
78
+ results = await resolve_processors(*self._input_processors, **kwargs)
76
79
 
77
- return await self._run(**reduce(lambda a, b: a | b, results))
80
+ return await partial(
81
+ _measured_run,
82
+ **reduce(lambda a, b: a | b, [result for _, result in results.items()]),
83
+ metric_tags=self._metric_tags,
84
+ metrics_provider=self._metrics_provider,
85
+ logger=self._logger,
86
+ )()
@@ -3,7 +3,7 @@
3
3
  Sub-problems can be Distributed, Minimalistic or Recursive as well.
4
4
  """
5
5
 
6
- # Copyright (c) 2023. ECCO Sneaks & Data
6
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
7
7
  #
8
8
  # Licensed under the Apache License, Version 2.0 (the "License");
9
9
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Simple algorithm with a single train/predict/solve iteration.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Algorithm that uses its output to alter the input for the next iteration, until a certain condition is met.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -7,7 +7,7 @@ from dataclasses import dataclass
7
7
  from dataclasses_json import DataClassJsonMixin
8
8
 
9
9
 
10
- # Copyright (c) 2023. ECCO Sneaks & Data
10
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
11
11
  #
12
12
  # Licensed under the Apache License, Version 2.0 (the "License");
13
13
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Import index.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Nexus Core.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Dependency injections.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Import index.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Base classes for custom exceptions.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Input reader exceptions.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -1,8 +1,8 @@
1
1
  """
2
- Custom exceptions.
2
+ App startup exceptions.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Import index.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -2,7 +2,7 @@
2
2
  Utility functions to handle input processing.
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -30,6 +30,9 @@ from esd_services_api_client.nexus.exceptions.input_reader_error import (
30
30
  from esd_services_api_client.nexus.input.input_reader import InputReader
31
31
 
32
32
 
33
+ _reader_cache = {}
34
+
35
+
33
36
  def resolve_reader_exc_type(
34
37
  ex: BaseException,
35
38
  ) -> Union[Type[FatalInputReaderError], Type[TransientInputReaderError]]:
@@ -61,12 +64,26 @@ async def resolve_readers(
61
64
 
62
65
  async def _read(input_reader: InputReader):
63
66
  async with input_reader as instance:
64
- return await instance.read()
67
+ result = await instance.read()
68
+ _reader_cache[input_reader.__class__.alias()] = result
69
+ return result
70
+
71
+ cached = {
72
+ reader.__class__.alias(): reader.data
73
+ for reader in readers
74
+ if reader.__class__.alias() in _reader_cache
75
+ }
76
+ if len(cached) == len(readers):
77
+ return cached
65
78
 
66
79
  read_tasks: dict[str, asyncio.Task] = {
67
80
  reader.__class__.alias(): asyncio.create_task(_read(reader))
68
81
  for reader in readers
82
+ if reader.__class__.alias() not in _reader_cache
69
83
  }
70
- await asyncio.wait(fs=read_tasks.values())
84
+ if len(read_tasks) > 0:
85
+ await asyncio.wait(fs=read_tasks.values())
71
86
 
72
- return {alias: get_result(alias, task) for alias, task in read_tasks.items()}
87
+ return {
88
+ alias: get_result(alias, task) for alias, task in read_tasks.items()
89
+ } | cached
@@ -1,8 +1,9 @@
1
1
  """
2
2
  Input processing.
3
3
  """
4
+ import asyncio
4
5
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
6
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
7
  #
7
8
  # Licensed under the Apache License, Version 2.0 (the "License");
8
9
  # you may not use this file except in compliance with the License.
@@ -18,8 +19,11 @@
18
19
  #
19
20
 
20
21
  from abc import abstractmethod
22
+ from functools import partial
23
+ from typing import Optional
21
24
 
22
25
  from adapta.metrics import MetricsProvider
26
+ from adapta.utils.decorators import run_time_metrics_async
23
27
 
24
28
  from esd_services_api_client.nexus.abstractions.nexus_object import (
25
29
  NexusObject,
@@ -27,9 +31,14 @@ from esd_services_api_client.nexus.abstractions.nexus_object import (
27
31
  TResult,
28
32
  )
29
33
  from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
30
- from esd_services_api_client.nexus.input._functions import resolve_readers
34
+ from esd_services_api_client.nexus.input._functions import (
35
+ resolve_readers,
36
+ resolve_reader_exc_type,
37
+ )
31
38
  from esd_services_api_client.nexus.input.input_reader import InputReader
32
39
 
40
+ _processor_cache = {}
41
+
33
42
 
34
43
  class InputProcessor(NexusObject[TPayload, TResult]):
35
44
  """
@@ -46,12 +55,90 @@ class InputProcessor(NexusObject[TPayload, TResult]):
46
55
  super().__init__(metrics_provider, logger_factory)
47
56
  self._readers = readers
48
57
  self._payload = payload
58
+ self._result: Optional[TResult] = None
49
59
 
50
60
  async def _read_input(self) -> dict[str, TResult]:
51
61
  return await resolve_readers(*self._readers)
52
62
 
63
+ @property
64
+ def result(self) -> dict[str, TResult]:
65
+ """
66
+ Data returned by this processor
67
+ """
68
+ return self._result
69
+
53
70
  @abstractmethod
54
- async def process_input(self, **kwargs) -> dict[str, TResult]:
71
+ async def _process_input(self, **kwargs) -> dict[str, TResult]:
55
72
  """
56
73
  Input processing logic. Implement this method to prepare data for your algorithm code.
57
74
  """
75
+
76
+ @property
77
+ def _metric_tags(self) -> dict[str, str]:
78
+ return {"processor": self.__class__.alias()}
79
+
80
+ async def process_input(self, **kwargs) -> dict[str, TResult]:
81
+ """
82
+ Input processing coroutine. Do not override this method.
83
+ """
84
+
85
+ @run_time_metrics_async(
86
+ metric_name="input_process",
87
+ on_finish_message_template="Finished processing {processor} in {elapsed:.2f}s seconds",
88
+ template_args={
89
+ "processor": self.__class__.alias().upper(),
90
+ },
91
+ )
92
+ async def _process(**_) -> dict[str, TResult]:
93
+ return await self._process_input(**kwargs)
94
+
95
+ if self._result is None:
96
+ self._result = await partial(
97
+ _process,
98
+ metric_tags=self._metric_tags,
99
+ metrics_provider=self._metrics_provider,
100
+ logger=self._logger,
101
+ )()
102
+
103
+ return self._result
104
+
105
+
106
+ async def resolve_processors(
107
+ *processors: InputProcessor[TPayload, TResult], **kwargs
108
+ ) -> dict[str, dict[str, TResult]]:
109
+ """
110
+ Concurrently resolve `result` property of all processors by invoking their `process_input` method.
111
+ """
112
+
113
+ def get_result(alias: str, completed_task: asyncio.Task) -> dict[str, TResult]:
114
+ reader_exc = completed_task.exception()
115
+ if reader_exc:
116
+ raise resolve_reader_exc_type(reader_exc)(alias, reader_exc) from reader_exc
117
+
118
+ return completed_task.result()
119
+
120
+ async def _process(input_processor: InputProcessor):
121
+ async with input_processor as instance:
122
+ result = await instance.process_input(**kwargs)
123
+ _processor_cache[input_processor.__class__.alias()] = result
124
+ return result
125
+
126
+ cached = {
127
+ processor.__class__.alias(): processor.result
128
+ for processor in processors
129
+ if processor.__class__.alias() in _processor_cache
130
+ }
131
+ if len(cached) == len(processors):
132
+ return cached
133
+
134
+ process_tasks: dict[str, asyncio.Task] = {
135
+ processor.__class__.alias(): asyncio.create_task(_process(processor))
136
+ for processor in processors
137
+ if processor.__class__.alias() not in _processor_cache
138
+ }
139
+ if len(process_tasks) > 0:
140
+ await asyncio.wait(fs=process_tasks.values())
141
+
142
+ return {
143
+ alias: get_result(alias, task) for alias, task in process_tasks.items()
144
+ } | cached
@@ -1,9 +1,8 @@
1
1
  """
2
2
  Input reader.
3
3
  """
4
- import functools
5
4
 
6
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
7
6
  #
8
7
  # Licensed under the Apache License, Version 2.0 (the "License");
9
8
  # you may not use this file except in compliance with the License.
@@ -56,10 +55,10 @@ class InputReader(NexusObject[TPayload, TResult]):
56
55
  self._readers = readers
57
56
  self._payload = payload
58
57
 
59
- @functools.cached_property
58
+ @property
60
59
  def data(self) -> Optional[TResult]:
61
60
  """
62
- Data read by this reader.
61
+ Data returned by this reader
63
62
  """
64
63
  return self._data
65
64
 
@@ -75,11 +74,11 @@ class InputReader(NexusObject[TPayload, TResult]):
75
74
 
76
75
  async def read(self) -> TResult:
77
76
  """
78
- Coroutine that reads the data from external store and converts it to a dataframe.
77
+ Coroutine that reads the data from external store and converts it to a dataframe, or generates data locally. Do not override this method.
79
78
  """
80
79
 
81
80
  @run_time_metrics_async(
82
- metric_name="read_input",
81
+ metric_name="input_read",
83
82
  on_finish_message_template="Finished reading {entity} from path {data_path} in {elapsed:.2f}s seconds"
84
83
  if self.socket
85
84
  else "Finished reading {entity} in {elapsed:.2f}s seconds",
@@ -89,14 +88,14 @@ class InputReader(NexusObject[TPayload, TResult]):
89
88
  | ({"data_path": self.socket.data_path} if self.socket else {}),
90
89
  )
91
90
  async def _read(**_) -> TResult:
92
- if self._data is None:
93
- self._data = await self._read_input()
91
+ return await self._read_input()
94
92
 
95
- return self._data
93
+ if self._data is None:
94
+ self._data = await partial(
95
+ _read,
96
+ metric_tags=self._metric_tags,
97
+ metrics_provider=self._metrics_provider,
98
+ logger=self._logger,
99
+ )()
96
100
 
97
- return await partial(
98
- _read,
99
- metric_tags=self._metric_tags,
100
- metrics_provider=self._metrics_provider,
101
- logger=self._logger,
102
- )()
101
+ return self._data
@@ -2,7 +2,7 @@
2
2
  Code infrastructure for manipulating payload received from Crystal SAS URI
3
3
  """
4
4
 
5
- # Copyright (c) 2023. ECCO Sneaks & Data
5
+ # Copyright (c) 2023-2024. ECCO Sneaks & Data
6
6
  #
7
7
  # Licensed under the Apache License, Version 2.0 (the "License");
8
8
  # you may not use this file except in compliance with the License.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: esd-services-api-client
3
- Version: 2.1.0
3
+ Version: 2.1.2
4
4
  Summary: Python clients for ESD services
5
5
  Home-page: https://github.com/SneaksAndData/esd-services-api-client
6
6
  License: Apache 2.0
@@ -0,0 +1,46 @@
1
+ esd_services_api_client/__init__.py,sha256=L-cEW1mVbnTJLCLG5V6Ucw7zBgx1zf0t1bYcQC1heyw,603
2
+ esd_services_api_client/_version.py,sha256=m5qImnzcnIhayvILFVqEnXPYsN-vE0vxokygykKhRfw,22
3
+ esd_services_api_client/beast/__init__.py,sha256=zNhXcHSP5w4P9quM1XP4oXVJEccvC_VScG41TZ0GzZ8,723
4
+ esd_services_api_client/beast/v3/__init__.py,sha256=FtumtInoDyCCRE424Llqv8QZLRuwXzj-smyfu1od1nc,754
5
+ esd_services_api_client/beast/v3/_connector.py,sha256=WNmCiTXFRb3q56mrr7ZbqBHWDUxbfyWhiWlBFLUIOnc,11478
6
+ esd_services_api_client/beast/v3/_models.py,sha256=UyBAEtJhDtF_qPE6TB9QWoffvNk1nY-AXm9bMTVBT2U,6766
7
+ esd_services_api_client/boxer/README.md,sha256=-MAhYUPvmEMcgx_lo_2PlH_gZI1lndGv8fnDQYIpurU,3618
8
+ esd_services_api_client/boxer/__init__.py,sha256=yDEcfL-9q0H91UWr-al-1iMJtVLv-oIXky6hUmFZs0g,785
9
+ esd_services_api_client/boxer/_auth.py,sha256=q9vU4agH-s6HPpVTEdkXFA_RvO92hogOfjurfjHBmBg,7445
10
+ esd_services_api_client/boxer/_base.py,sha256=LMKDArELZlWVKJH10coUFSSN8MFH6KfHCBncEDjsiIU,981
11
+ esd_services_api_client/boxer/_connector.py,sha256=gUplRNAxcOoZfMPee0ZUJmhbzYmreys9cMeIMHmIyKM,8677
12
+ esd_services_api_client/boxer/_models.py,sha256=ursQIR_c9jcVfRKc0LH1OuVL2KFD6kqojyGBadRfRPI,1702
13
+ esd_services_api_client/common/__init__.py,sha256=L-cEW1mVbnTJLCLG5V6Ucw7zBgx1zf0t1bYcQC1heyw,603
14
+ esd_services_api_client/crystal/__init__.py,sha256=oeyJjdQ9EpTnIq6XnjPq5v0DWPdHqi4uEfRIcD1mNZA,792
15
+ esd_services_api_client/crystal/_api_versions.py,sha256=GHbmV_5lP9fP72TZE0j_ZeQSeJjMRcRaBRxNJbz-MWQ,837
16
+ esd_services_api_client/crystal/_connector.py,sha256=wT8SahCkkRWPoHcUSLz0I-sjeK_9OSYtt07zJKie0CU,12875
17
+ esd_services_api_client/crystal/_models.py,sha256=OCaidMqipl-TA8VLs7v12BppyBR3QpRA5jU2KjZxU_Q,4030
18
+ esd_services_api_client/nexus/README.md,sha256=Q3laWkuCxqVBFsfP58mGAys-jVZeesroIRRMiaQMntc,8688
19
+ esd_services_api_client/nexus/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
20
+ esd_services_api_client/nexus/abstractions/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
21
+ esd_services_api_client/nexus/abstractions/logger_factory.py,sha256=9biONvCqNrP__yrmeRkoDL05TMA5v-LyrcKwgiKG59U,2019
22
+ esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=Be90h8iVDp2o1recnx8-Ousm3OjaVjIjthPqE5u5vik,2828
23
+ esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=Rwa_aPErI4Es5AdyCd3EoGze7mg2D70u8kuc2UGEBaI,1729
24
+ esd_services_api_client/nexus/algorithms/__init__.py,sha256=yMvLFSqg5eUKOXI0zMFX69Ni0ibKQHOqAnrZsxQqhOo,903
25
+ esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=UxqGFzpl-8w4SLJX0GMxLzRinRnFVoBT2zBKzHOapBM,2694
26
+ esd_services_api_client/nexus/algorithms/distributed.py,sha256=vkKSCsd480RKwrtu3uZ2iU1bh593fkgBcOBrcb9cLjA,1702
27
+ esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=PjLs_xhpd3rTViaLbWUEvJ1LWDPTKEZFj57iPk8wswo,1462
28
+ esd_services_api_client/nexus/algorithms/recursive.py,sha256=MAyfj3gQZ0zQB5hU3RHIOfH889v_Wtp4GHBhB-trr2w,1862
29
+ esd_services_api_client/nexus/configurations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
30
+ esd_services_api_client/nexus/configurations/algorithm_configuration.py,sha256=eE7diX2PATCGkmqhvFOcZwXrr6vns4fqnJGmgNvhhZM,1091
31
+ esd_services_api_client/nexus/core/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
32
+ esd_services_api_client/nexus/core/app_core.py,sha256=FzotOWZ8Yc3njI1wCnSZAi2MhENdHHXfm3v9V0WUAPQ,9425
33
+ esd_services_api_client/nexus/core/app_dependencies.py,sha256=GChNBRE-9CGqZ41xQOlTvt_XAniyoXnO3GrxTSBih9w,6145
34
+ esd_services_api_client/nexus/exceptions/__init__.py,sha256=feN33VdqB5-2bD9aJesJl_OlsKrNNo3hZCnQgKuaU9k,696
35
+ esd_services_api_client/nexus/exceptions/_nexus_error.py,sha256=QvtY38mNoIA6t26dUN6UIsaPfljhtVNsbQVS7ksMb-Q,895
36
+ esd_services_api_client/nexus/exceptions/input_reader_error.py,sha256=Chy8XW6Ien4-bkZZ1CmP8CWU49mi2hobS6L_R59ONs8,1765
37
+ esd_services_api_client/nexus/exceptions/startup_error.py,sha256=HPTlrg2voRbGXZsakWcVT7lcGSaccSR-GsBIo08SJ4A,1556
38
+ esd_services_api_client/nexus/input/__init__.py,sha256=ahJ7yUuRTCNIYB5GitBRGAPZZX82ghISzF6lm_CkyEA,819
39
+ esd_services_api_client/nexus/input/_functions.py,sha256=9bySbwcXGV7JIsQbSqg3YuW7avJUErsvsS2ZDHRWfT0,2918
40
+ esd_services_api_client/nexus/input/input_processor.py,sha256=ZUdpz2M6PfsF-99-DpCMDtD9Id0SQYhC-HbjosMz8O8,4609
41
+ esd_services_api_client/nexus/input/input_reader.py,sha256=MtZNdxjWLjoSqlSlwa6f7BIqsu-_GoQav5tCz2g6WrA,3214
42
+ esd_services_api_client/nexus/input/payload_reader.py,sha256=Kq0xN1Shyqv71v6YkcrqVTDbmsEjZc8ithsXYpyu87M,2516
43
+ esd_services_api_client-2.1.2.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
44
+ esd_services_api_client-2.1.2.dist-info/METADATA,sha256=xr1Dw96DI1t-qKxCb1aq48Dyxt-co-BzwRePv-Jgoz0,1292
45
+ esd_services_api_client-2.1.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
46
+ esd_services_api_client-2.1.2.dist-info/RECORD,,
@@ -1,46 +0,0 @@
1
- esd_services_api_client/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
2
- esd_services_api_client/_version.py,sha256=LbU43-7hsLmdXWI0wTAl3y6D3Tr7CbJiDOLXwl7clj8,22
3
- esd_services_api_client/beast/__init__.py,sha256=NTaz_7YoLPK8MCLwbwqH7rW1zDWLxXu2T7fGmMmRxyg,718
4
- esd_services_api_client/beast/v3/__init__.py,sha256=TRjB4-T6eIORpMvdylb32_GinrIpYNFmAdshSC1HqHg,749
5
- esd_services_api_client/beast/v3/_connector.py,sha256=oPizDQ1KOKOfiyh-jAofKodlpRzrRiELv-rmP_o_oio,11473
6
- esd_services_api_client/beast/v3/_models.py,sha256=hvh8QlwYLCfdBJxUKPutOq0xFn9rUU1q6UgHEpoZaFM,6761
7
- esd_services_api_client/boxer/README.md,sha256=-MAhYUPvmEMcgx_lo_2PlH_gZI1lndGv8fnDQYIpurU,3618
8
- esd_services_api_client/boxer/__init__.py,sha256=OYsWvdnLan0kmjUcH4I2-m1rbPeARKp5iqhp8uyudPk,780
9
- esd_services_api_client/boxer/_auth.py,sha256=vA7T9y0oZV2f17UWQ2or9CK8vAsNnHB10G5HNQe1l1I,7440
10
- esd_services_api_client/boxer/_base.py,sha256=PSU08QzTKFkXfzx7fF5FIHBOZXshxNEd1J_qKGo0Rd0,976
11
- esd_services_api_client/boxer/_connector.py,sha256=kATr7IyQKtvsGlvgVInO_DsGGC4yCC1aW3RxoXt-QIM,8672
12
- esd_services_api_client/boxer/_models.py,sha256=q_xRHOXlRroKrDrNUxQJUFGSotxYvUbRY1rYmx8UfJg,1697
13
- esd_services_api_client/common/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
14
- esd_services_api_client/crystal/__init__.py,sha256=afSGQRkDic0ECsJfgu3b291kX8CyU57sjreqxj5cx-s,787
15
- esd_services_api_client/crystal/_api_versions.py,sha256=2BMiQRS0D8IEpWCCys3dge5alVBRCZrOuCR1QAn8UIM,832
16
- esd_services_api_client/crystal/_connector.py,sha256=WjfMezWXia41Z8aiNupaT577fk9Sx6uy6V23O6y9hfI,12870
17
- esd_services_api_client/crystal/_models.py,sha256=eRhGAl8LjglCyIFwf1bcFBhjbpSuRYucuF2LO388L2E,4025
18
- esd_services_api_client/nexus/README.md,sha256=Q3laWkuCxqVBFsfP58mGAys-jVZeesroIRRMiaQMntc,8688
19
- esd_services_api_client/nexus/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
20
- esd_services_api_client/nexus/abstractions/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
21
- esd_services_api_client/nexus/abstractions/logger_factory.py,sha256=jw1xnrU2dEWSUhqNut4RzjRPZFmeNx6czOU_8-q4kuo,2014
22
- esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=Y2jwpSa0cLyMtGxOSEpg3F8evvSMxYHx5t5KD92KotE,2759
23
- esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=Mv9BWdxw8VY4Gi4EOrdxWK1zsR3-fqIbpyF1xHWchbE,1495
24
- esd_services_api_client/nexus/algorithms/__init__.py,sha256=ySnLfuYuxi28EaqT18oW2Ka50ZR7WOvJlLhmIts-Xgw,898
25
- esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=_3QHWmBDnfByEJTftf4jKzje5GNqftb0_aCfp-8ksCU,2436
26
- esd_services_api_client/nexus/algorithms/distributed.py,sha256=kZS4InM4GoT4T6UrXRnfoDn4zxE2iGuqLvxBrzBetZc,1697
27
- esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=DOU52Me_W0luUPWmIRBFkJdfapyEkPqVbbvy4V5yQZ8,1457
28
- esd_services_api_client/nexus/algorithms/recursive.py,sha256=r4_LIlC_hhIBSrKtL99xE9md3SahctF3R3GT1tIwlg4,1857
29
- esd_services_api_client/nexus/configurations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
30
- esd_services_api_client/nexus/configurations/algorithm_configuration.py,sha256=sg3ul-Vqd39li42cfMsdWtFJIBNfQ52XLmqKO2sryDg,1086
31
- esd_services_api_client/nexus/core/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
32
- esd_services_api_client/nexus/core/app_core.py,sha256=XvPWnr6wJckjNoM4vWg3CYBnrJOaWZFxiIil1sK-qNc,9420
33
- esd_services_api_client/nexus/core/app_dependencies.py,sha256=rIk6R-InJeF1P0DC4bhj4kJTNNxYg39Vhuz8AXfV6pk,6140
34
- esd_services_api_client/nexus/exceptions/__init__.py,sha256=JgPXhrvBIi0U1QOF90TYHS8jv_SBQEoRLsEg6rrtRoc,691
35
- esd_services_api_client/nexus/exceptions/_nexus_error.py,sha256=b3L8JnNvV2jdxNfuFWh9-j4kVb_VX7gNH5WHKcC-R78,890
36
- esd_services_api_client/nexus/exceptions/input_reader_error.py,sha256=D-xYTKRNREQ2-NGhc88GHOmXCvLNsIVQsH8wf0LLC_0,1760
37
- esd_services_api_client/nexus/exceptions/startup_error.py,sha256=f2PIOSdLgT-42eKD6ec8p7nROADshMawCsDGDUbxO_w,1546
38
- esd_services_api_client/nexus/input/__init__.py,sha256=DEdzkK43xjNl3XPQy3Q8xHGzXYsMwxCMs1QdEAsS4FI,814
39
- esd_services_api_client/nexus/input/_functions.py,sha256=TLb3yDW3ci4_viS52nOUK-efoG6Ok_c6cf3J895phZ0,2480
40
- esd_services_api_client/nexus/input/input_processor.py,sha256=FEYvlgVwEHU58wRolNX6z6svgJl-gcfNk6RAkL4Yb_s,1803
41
- esd_services_api_client/nexus/input/input_reader.py,sha256=aVhVGDrZCH4UPr6JlhvxgPZIplzbhEqd_njKZfpNFAI,3172
42
- esd_services_api_client/nexus/input/payload_reader.py,sha256=__r_QjIFRAWwx56X5WUK1qensJUae0vZEb422dzOgSY,2511
43
- esd_services_api_client-2.1.0.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
44
- esd_services_api_client-2.1.0.dist-info/METADATA,sha256=iDTAP-IzN03N2AtLsm6tlueG6cw2sVvjApJ8eh71fv4,1292
45
- esd_services_api_client-2.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
46
- esd_services_api_client-2.1.0.dist-info/RECORD,,