esd-services-api-client 2.1.0__py3-none-any.whl → 2.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- esd_services_api_client/__init__.py +1 -1
- esd_services_api_client/_version.py +1 -1
- esd_services_api_client/beast/__init__.py +1 -1
- esd_services_api_client/beast/v3/__init__.py +1 -1
- esd_services_api_client/beast/v3/_connector.py +1 -1
- esd_services_api_client/beast/v3/_models.py +1 -1
- esd_services_api_client/boxer/__init__.py +1 -1
- esd_services_api_client/boxer/_auth.py +1 -1
- esd_services_api_client/boxer/_base.py +1 -1
- esd_services_api_client/boxer/_connector.py +1 -1
- esd_services_api_client/boxer/_models.py +1 -1
- esd_services_api_client/common/__init__.py +1 -1
- esd_services_api_client/crystal/__init__.py +1 -1
- esd_services_api_client/crystal/_api_versions.py +1 -1
- esd_services_api_client/crystal/_connector.py +1 -1
- esd_services_api_client/crystal/_models.py +1 -1
- esd_services_api_client/nexus/__init__.py +1 -1
- esd_services_api_client/nexus/abstractions/__init__.py +1 -1
- esd_services_api_client/nexus/abstractions/logger_factory.py +1 -1
- esd_services_api_client/nexus/abstractions/nexus_object.py +5 -2
- esd_services_api_client/nexus/abstractions/socket_provider.py +14 -5
- esd_services_api_client/nexus/algorithms/__init__.py +1 -1
- esd_services_api_client/nexus/algorithms/_baseline_algorithm.py +28 -19
- esd_services_api_client/nexus/algorithms/distributed.py +1 -1
- esd_services_api_client/nexus/algorithms/minimalistic.py +1 -1
- esd_services_api_client/nexus/algorithms/recursive.py +1 -1
- esd_services_api_client/nexus/configurations/algorithm_configuration.py +1 -1
- esd_services_api_client/nexus/core/__init__.py +1 -1
- esd_services_api_client/nexus/core/app_core.py +1 -1
- esd_services_api_client/nexus/core/app_dependencies.py +1 -1
- esd_services_api_client/nexus/exceptions/__init__.py +1 -1
- esd_services_api_client/nexus/exceptions/_nexus_error.py +1 -1
- esd_services_api_client/nexus/exceptions/input_reader_error.py +1 -1
- esd_services_api_client/nexus/exceptions/startup_error.py +2 -2
- esd_services_api_client/nexus/input/__init__.py +1 -1
- esd_services_api_client/nexus/input/_functions.py +21 -4
- esd_services_api_client/nexus/input/input_processor.py +90 -3
- esd_services_api_client/nexus/input/input_reader.py +14 -15
- esd_services_api_client/nexus/input/payload_reader.py +1 -1
- {esd_services_api_client-2.1.0.dist-info → esd_services_api_client-2.1.2.dist-info}/METADATA +1 -1
- esd_services_api_client-2.1.2.dist-info/RECORD +46 -0
- esd_services_api_client-2.1.0.dist-info/RECORD +0 -46
- {esd_services_api_client-2.1.0.dist-info → esd_services_api_client-2.1.2.dist-info}/LICENSE +0 -0
- {esd_services_api_client-2.1.0.dist-info → esd_services_api_client-2.1.2.dist-info}/WHEEL +0 -0
@@ -1 +1 @@
|
|
1
|
-
__version__ = '2.1.
|
1
|
+
__version__ = '2.1.2'
|
@@ -1,7 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
Connector for Beast Workload Manager (Spark AKS)
|
3
3
|
"""
|
4
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
4
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
5
5
|
#
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
7
|
# you may not use this file except in compliance with the License.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
Models for Beast connector
|
3
3
|
"""
|
4
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
4
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
5
5
|
#
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
7
|
# you may not use this file except in compliance with the License.
|
@@ -2,7 +2,7 @@
|
|
2
2
|
Boxer Auth classes
|
3
3
|
Based on https://docs.python-requests.org/en/master/user/advanced/#custom-authentication
|
4
4
|
"""
|
5
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
6
6
|
#
|
7
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
8
|
# you may not use this file except in compliance with the License.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
Interfaces for boxer API
|
3
3
|
"""
|
4
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
4
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
5
5
|
#
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
7
|
# you may not use this file except in compliance with the License.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
Connector for Boxer Auth API.
|
3
3
|
"""
|
4
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
4
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
5
5
|
#
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
7
|
# you may not use this file except in compliance with the License.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
API versioning for arcane connector
|
3
3
|
"""
|
4
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
4
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
5
5
|
#
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
7
|
# you may not use this file except in compliance with the License.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
Connector for Crystal Job Runtime (AKS)
|
3
3
|
"""
|
4
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
4
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
5
5
|
#
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
7
|
# you may not use this file except in compliance with the License.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
Models for Crystal connector
|
3
3
|
"""
|
4
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
4
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
5
5
|
#
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
7
|
# you may not use this file except in compliance with the License.
|
@@ -2,7 +2,7 @@
|
|
2
2
|
Logger factory for async loggers.
|
3
3
|
"""
|
4
4
|
|
5
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
6
6
|
#
|
7
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
8
|
# you may not use this file except in compliance with the License.
|
@@ -3,7 +3,7 @@
|
|
3
3
|
"""
|
4
4
|
import re
|
5
5
|
|
6
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
6
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
7
7
|
#
|
8
8
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
9
|
# you may not use this file except in compliance with the License.
|
@@ -95,5 +95,8 @@ class NexusObject(Generic[TPayload, TResult], ABC):
|
|
95
95
|
return re.sub(
|
96
96
|
r"(?<!^)(?=[A-Z])",
|
97
97
|
"_",
|
98
|
-
cls.__name__.lower()
|
98
|
+
cls.__name__.lower()
|
99
|
+
.replace("reader", "")
|
100
|
+
.replace("processor", "")
|
101
|
+
.replace("algorithm", ""),
|
99
102
|
)
|
@@ -1,9 +1,8 @@
|
|
1
1
|
"""
|
2
2
|
Socket provider for all data sockets used by algorithms.
|
3
3
|
"""
|
4
|
-
import json
|
5
4
|
|
6
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
7
6
|
#
|
8
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
8
|
# you may not use this file except in compliance with the License.
|
@@ -18,10 +17,15 @@ import json
|
|
18
17
|
# limitations under the License.
|
19
18
|
#
|
20
19
|
|
21
|
-
|
20
|
+
import json
|
21
|
+
from typing import final
|
22
22
|
|
23
23
|
from adapta.process_communication import DataSocket
|
24
24
|
|
25
|
+
from esd_services_api_client.nexus.exceptions.startup_error import (
|
26
|
+
FatalStartupConfigurationError,
|
27
|
+
)
|
28
|
+
|
25
29
|
|
26
30
|
@final
|
27
31
|
class ExternalSocketProvider:
|
@@ -32,11 +36,16 @@ class ExternalSocketProvider:
|
|
32
36
|
def __init__(self, *sockets: DataSocket):
|
33
37
|
self._sockets = {socket.alias: socket for socket in sockets}
|
34
38
|
|
35
|
-
def socket(self, name: str) ->
|
39
|
+
def socket(self, name: str) -> DataSocket:
|
36
40
|
"""
|
37
41
|
Retrieve a socket if it exists.
|
38
42
|
"""
|
39
|
-
|
43
|
+
if name in self._sockets:
|
44
|
+
return self._sockets[name]
|
45
|
+
|
46
|
+
raise FatalStartupConfigurationError(
|
47
|
+
missing_entry=f"socket with alias `{name}`"
|
48
|
+
)
|
40
49
|
|
41
50
|
@classmethod
|
42
51
|
def from_serialized(cls, socket_list_ser: str) -> "ExternalSocketProvider":
|
@@ -1,9 +1,8 @@
|
|
1
1
|
"""
|
2
2
|
Base algorithm
|
3
3
|
"""
|
4
|
-
import asyncio
|
5
4
|
|
6
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
7
6
|
#
|
8
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
8
|
# you may not use this file except in compliance with the License.
|
@@ -20,18 +19,21 @@ import asyncio
|
|
20
19
|
|
21
20
|
|
22
21
|
from abc import abstractmethod
|
23
|
-
from functools import reduce
|
22
|
+
from functools import reduce, partial
|
24
23
|
|
25
24
|
from adapta.metrics import MetricsProvider
|
25
|
+
from adapta.utils.decorators import run_time_metrics_async
|
26
26
|
|
27
27
|
from esd_services_api_client.nexus.abstractions.nexus_object import (
|
28
28
|
NexusObject,
|
29
29
|
TPayload,
|
30
|
-
TResult,
|
31
30
|
AlgorithmResult,
|
32
31
|
)
|
33
32
|
from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
|
34
|
-
from esd_services_api_client.nexus.input.input_processor import
|
33
|
+
from esd_services_api_client.nexus.input.input_processor import (
|
34
|
+
InputProcessor,
|
35
|
+
resolve_processors,
|
36
|
+
)
|
35
37
|
|
36
38
|
|
37
39
|
class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
|
@@ -54,24 +56,31 @@ class BaselineAlgorithm(NexusObject[TPayload, AlgorithmResult]):
|
|
54
56
|
Core logic for this algorithm. Implementing this method is mandatory.
|
55
57
|
"""
|
56
58
|
|
59
|
+
@property
|
60
|
+
def _metric_tags(self) -> dict[str, str]:
|
61
|
+
return {"algorithm": self.__class__.alias()}
|
62
|
+
|
57
63
|
async def run(self, **kwargs) -> AlgorithmResult:
|
58
64
|
"""
|
59
65
|
Coroutine that executes the algorithm logic.
|
60
66
|
"""
|
61
67
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
68
|
+
@run_time_metrics_async(
|
69
|
+
metric_name="algorthm_run",
|
70
|
+
on_finish_message_template="Finished running {algorithm} in {elapsed:.2f}s seconds",
|
71
|
+
template_args={
|
72
|
+
"algorithm": self.__class__.alias().upper(),
|
73
|
+
},
|
74
|
+
)
|
75
|
+
async def _measured_run(**run_args):
|
76
|
+
return await self._run(**run_args)
|
67
77
|
|
68
|
-
|
69
|
-
input_processor.__class__.__name__.lower(): asyncio.create_task(
|
70
|
-
_process(input_processor)
|
71
|
-
)
|
72
|
-
for input_processor in self._input_processors
|
73
|
-
}
|
74
|
-
await asyncio.wait(fs=process_tasks.values())
|
75
|
-
results = [task.result() for task in process_tasks.values()]
|
78
|
+
results = await resolve_processors(*self._input_processors, **kwargs)
|
76
79
|
|
77
|
-
return await
|
80
|
+
return await partial(
|
81
|
+
_measured_run,
|
82
|
+
**reduce(lambda a, b: a | b, [result for _, result in results.items()]),
|
83
|
+
metric_tags=self._metric_tags,
|
84
|
+
metrics_provider=self._metrics_provider,
|
85
|
+
logger=self._logger,
|
86
|
+
)()
|
@@ -3,7 +3,7 @@
|
|
3
3
|
Sub-problems can be Distributed, Minimalistic or Recursive as well.
|
4
4
|
"""
|
5
5
|
|
6
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
6
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
7
7
|
#
|
8
8
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
9
|
# you may not use this file except in compliance with the License.
|
@@ -2,7 +2,7 @@
|
|
2
2
|
Simple algorithm with a single train/predict/solve iteration.
|
3
3
|
"""
|
4
4
|
|
5
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
6
6
|
#
|
7
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
8
|
# you may not use this file except in compliance with the License.
|
@@ -2,7 +2,7 @@
|
|
2
2
|
Algorithm that uses its output to alter the input for the next iteration, until a certain condition is met.
|
3
3
|
"""
|
4
4
|
|
5
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
6
6
|
#
|
7
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
8
|
# you may not use this file except in compliance with the License.
|
@@ -7,7 +7,7 @@ from dataclasses import dataclass
|
|
7
7
|
from dataclasses_json import DataClassJsonMixin
|
8
8
|
|
9
9
|
|
10
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
10
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
11
11
|
#
|
12
12
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
13
13
|
# you may not use this file except in compliance with the License.
|
@@ -2,7 +2,7 @@
|
|
2
2
|
Base classes for custom exceptions.
|
3
3
|
"""
|
4
4
|
|
5
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
6
6
|
#
|
7
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
8
|
# you may not use this file except in compliance with the License.
|
@@ -1,8 +1,8 @@
|
|
1
1
|
"""
|
2
|
-
|
2
|
+
App startup exceptions.
|
3
3
|
"""
|
4
4
|
|
5
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
6
6
|
#
|
7
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
8
|
# you may not use this file except in compliance with the License.
|
@@ -2,7 +2,7 @@
|
|
2
2
|
Utility functions to handle input processing.
|
3
3
|
"""
|
4
4
|
|
5
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
6
6
|
#
|
7
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
8
|
# you may not use this file except in compliance with the License.
|
@@ -30,6 +30,9 @@ from esd_services_api_client.nexus.exceptions.input_reader_error import (
|
|
30
30
|
from esd_services_api_client.nexus.input.input_reader import InputReader
|
31
31
|
|
32
32
|
|
33
|
+
_reader_cache = {}
|
34
|
+
|
35
|
+
|
33
36
|
def resolve_reader_exc_type(
|
34
37
|
ex: BaseException,
|
35
38
|
) -> Union[Type[FatalInputReaderError], Type[TransientInputReaderError]]:
|
@@ -61,12 +64,26 @@ async def resolve_readers(
|
|
61
64
|
|
62
65
|
async def _read(input_reader: InputReader):
|
63
66
|
async with input_reader as instance:
|
64
|
-
|
67
|
+
result = await instance.read()
|
68
|
+
_reader_cache[input_reader.__class__.alias()] = result
|
69
|
+
return result
|
70
|
+
|
71
|
+
cached = {
|
72
|
+
reader.__class__.alias(): reader.data
|
73
|
+
for reader in readers
|
74
|
+
if reader.__class__.alias() in _reader_cache
|
75
|
+
}
|
76
|
+
if len(cached) == len(readers):
|
77
|
+
return cached
|
65
78
|
|
66
79
|
read_tasks: dict[str, asyncio.Task] = {
|
67
80
|
reader.__class__.alias(): asyncio.create_task(_read(reader))
|
68
81
|
for reader in readers
|
82
|
+
if reader.__class__.alias() not in _reader_cache
|
69
83
|
}
|
70
|
-
|
84
|
+
if len(read_tasks) > 0:
|
85
|
+
await asyncio.wait(fs=read_tasks.values())
|
71
86
|
|
72
|
-
return {
|
87
|
+
return {
|
88
|
+
alias: get_result(alias, task) for alias, task in read_tasks.items()
|
89
|
+
} | cached
|
@@ -1,8 +1,9 @@
|
|
1
1
|
"""
|
2
2
|
Input processing.
|
3
3
|
"""
|
4
|
+
import asyncio
|
4
5
|
|
5
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
6
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
6
7
|
#
|
7
8
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
9
|
# you may not use this file except in compliance with the License.
|
@@ -18,8 +19,11 @@
|
|
18
19
|
#
|
19
20
|
|
20
21
|
from abc import abstractmethod
|
22
|
+
from functools import partial
|
23
|
+
from typing import Optional
|
21
24
|
|
22
25
|
from adapta.metrics import MetricsProvider
|
26
|
+
from adapta.utils.decorators import run_time_metrics_async
|
23
27
|
|
24
28
|
from esd_services_api_client.nexus.abstractions.nexus_object import (
|
25
29
|
NexusObject,
|
@@ -27,9 +31,14 @@ from esd_services_api_client.nexus.abstractions.nexus_object import (
|
|
27
31
|
TResult,
|
28
32
|
)
|
29
33
|
from esd_services_api_client.nexus.abstractions.logger_factory import LoggerFactory
|
30
|
-
from esd_services_api_client.nexus.input._functions import
|
34
|
+
from esd_services_api_client.nexus.input._functions import (
|
35
|
+
resolve_readers,
|
36
|
+
resolve_reader_exc_type,
|
37
|
+
)
|
31
38
|
from esd_services_api_client.nexus.input.input_reader import InputReader
|
32
39
|
|
40
|
+
_processor_cache = {}
|
41
|
+
|
33
42
|
|
34
43
|
class InputProcessor(NexusObject[TPayload, TResult]):
|
35
44
|
"""
|
@@ -46,12 +55,90 @@ class InputProcessor(NexusObject[TPayload, TResult]):
|
|
46
55
|
super().__init__(metrics_provider, logger_factory)
|
47
56
|
self._readers = readers
|
48
57
|
self._payload = payload
|
58
|
+
self._result: Optional[TResult] = None
|
49
59
|
|
50
60
|
async def _read_input(self) -> dict[str, TResult]:
|
51
61
|
return await resolve_readers(*self._readers)
|
52
62
|
|
63
|
+
@property
|
64
|
+
def result(self) -> dict[str, TResult]:
|
65
|
+
"""
|
66
|
+
Data returned by this processor
|
67
|
+
"""
|
68
|
+
return self._result
|
69
|
+
|
53
70
|
@abstractmethod
|
54
|
-
async def
|
71
|
+
async def _process_input(self, **kwargs) -> dict[str, TResult]:
|
55
72
|
"""
|
56
73
|
Input processing logic. Implement this method to prepare data for your algorithm code.
|
57
74
|
"""
|
75
|
+
|
76
|
+
@property
|
77
|
+
def _metric_tags(self) -> dict[str, str]:
|
78
|
+
return {"processor": self.__class__.alias()}
|
79
|
+
|
80
|
+
async def process_input(self, **kwargs) -> dict[str, TResult]:
|
81
|
+
"""
|
82
|
+
Input processing coroutine. Do not override this method.
|
83
|
+
"""
|
84
|
+
|
85
|
+
@run_time_metrics_async(
|
86
|
+
metric_name="input_process",
|
87
|
+
on_finish_message_template="Finished processing {processor} in {elapsed:.2f}s seconds",
|
88
|
+
template_args={
|
89
|
+
"processor": self.__class__.alias().upper(),
|
90
|
+
},
|
91
|
+
)
|
92
|
+
async def _process(**_) -> dict[str, TResult]:
|
93
|
+
return await self._process_input(**kwargs)
|
94
|
+
|
95
|
+
if self._result is None:
|
96
|
+
self._result = await partial(
|
97
|
+
_process,
|
98
|
+
metric_tags=self._metric_tags,
|
99
|
+
metrics_provider=self._metrics_provider,
|
100
|
+
logger=self._logger,
|
101
|
+
)()
|
102
|
+
|
103
|
+
return self._result
|
104
|
+
|
105
|
+
|
106
|
+
async def resolve_processors(
|
107
|
+
*processors: InputProcessor[TPayload, TResult], **kwargs
|
108
|
+
) -> dict[str, dict[str, TResult]]:
|
109
|
+
"""
|
110
|
+
Concurrently resolve `result` property of all processors by invoking their `process_input` method.
|
111
|
+
"""
|
112
|
+
|
113
|
+
def get_result(alias: str, completed_task: asyncio.Task) -> dict[str, TResult]:
|
114
|
+
reader_exc = completed_task.exception()
|
115
|
+
if reader_exc:
|
116
|
+
raise resolve_reader_exc_type(reader_exc)(alias, reader_exc) from reader_exc
|
117
|
+
|
118
|
+
return completed_task.result()
|
119
|
+
|
120
|
+
async def _process(input_processor: InputProcessor):
|
121
|
+
async with input_processor as instance:
|
122
|
+
result = await instance.process_input(**kwargs)
|
123
|
+
_processor_cache[input_processor.__class__.alias()] = result
|
124
|
+
return result
|
125
|
+
|
126
|
+
cached = {
|
127
|
+
processor.__class__.alias(): processor.result
|
128
|
+
for processor in processors
|
129
|
+
if processor.__class__.alias() in _processor_cache
|
130
|
+
}
|
131
|
+
if len(cached) == len(processors):
|
132
|
+
return cached
|
133
|
+
|
134
|
+
process_tasks: dict[str, asyncio.Task] = {
|
135
|
+
processor.__class__.alias(): asyncio.create_task(_process(processor))
|
136
|
+
for processor in processors
|
137
|
+
if processor.__class__.alias() not in _processor_cache
|
138
|
+
}
|
139
|
+
if len(process_tasks) > 0:
|
140
|
+
await asyncio.wait(fs=process_tasks.values())
|
141
|
+
|
142
|
+
return {
|
143
|
+
alias: get_result(alias, task) for alias, task in process_tasks.items()
|
144
|
+
} | cached
|
@@ -1,9 +1,8 @@
|
|
1
1
|
"""
|
2
2
|
Input reader.
|
3
3
|
"""
|
4
|
-
import functools
|
5
4
|
|
6
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
7
6
|
#
|
8
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
9
8
|
# you may not use this file except in compliance with the License.
|
@@ -56,10 +55,10 @@ class InputReader(NexusObject[TPayload, TResult]):
|
|
56
55
|
self._readers = readers
|
57
56
|
self._payload = payload
|
58
57
|
|
59
|
-
@
|
58
|
+
@property
|
60
59
|
def data(self) -> Optional[TResult]:
|
61
60
|
"""
|
62
|
-
Data
|
61
|
+
Data returned by this reader
|
63
62
|
"""
|
64
63
|
return self._data
|
65
64
|
|
@@ -75,11 +74,11 @@ class InputReader(NexusObject[TPayload, TResult]):
|
|
75
74
|
|
76
75
|
async def read(self) -> TResult:
|
77
76
|
"""
|
78
|
-
Coroutine that reads the data from external store and converts it to a dataframe.
|
77
|
+
Coroutine that reads the data from external store and converts it to a dataframe, or generates data locally. Do not override this method.
|
79
78
|
"""
|
80
79
|
|
81
80
|
@run_time_metrics_async(
|
82
|
-
metric_name="
|
81
|
+
metric_name="input_read",
|
83
82
|
on_finish_message_template="Finished reading {entity} from path {data_path} in {elapsed:.2f}s seconds"
|
84
83
|
if self.socket
|
85
84
|
else "Finished reading {entity} in {elapsed:.2f}s seconds",
|
@@ -89,14 +88,14 @@ class InputReader(NexusObject[TPayload, TResult]):
|
|
89
88
|
| ({"data_path": self.socket.data_path} if self.socket else {}),
|
90
89
|
)
|
91
90
|
async def _read(**_) -> TResult:
|
92
|
-
|
93
|
-
self._data = await self._read_input()
|
91
|
+
return await self._read_input()
|
94
92
|
|
95
|
-
|
93
|
+
if self._data is None:
|
94
|
+
self._data = await partial(
|
95
|
+
_read,
|
96
|
+
metric_tags=self._metric_tags,
|
97
|
+
metrics_provider=self._metrics_provider,
|
98
|
+
logger=self._logger,
|
99
|
+
)()
|
96
100
|
|
97
|
-
return
|
98
|
-
_read,
|
99
|
-
metric_tags=self._metric_tags,
|
100
|
-
metrics_provider=self._metrics_provider,
|
101
|
-
logger=self._logger,
|
102
|
-
)()
|
101
|
+
return self._data
|
@@ -2,7 +2,7 @@
|
|
2
2
|
Code infrastructure for manipulating payload received from Crystal SAS URI
|
3
3
|
"""
|
4
4
|
|
5
|
-
# Copyright (c) 2023. ECCO Sneaks & Data
|
5
|
+
# Copyright (c) 2023-2024. ECCO Sneaks & Data
|
6
6
|
#
|
7
7
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
8
|
# you may not use this file except in compliance with the License.
|
@@ -0,0 +1,46 @@
|
|
1
|
+
esd_services_api_client/__init__.py,sha256=L-cEW1mVbnTJLCLG5V6Ucw7zBgx1zf0t1bYcQC1heyw,603
|
2
|
+
esd_services_api_client/_version.py,sha256=m5qImnzcnIhayvILFVqEnXPYsN-vE0vxokygykKhRfw,22
|
3
|
+
esd_services_api_client/beast/__init__.py,sha256=zNhXcHSP5w4P9quM1XP4oXVJEccvC_VScG41TZ0GzZ8,723
|
4
|
+
esd_services_api_client/beast/v3/__init__.py,sha256=FtumtInoDyCCRE424Llqv8QZLRuwXzj-smyfu1od1nc,754
|
5
|
+
esd_services_api_client/beast/v3/_connector.py,sha256=WNmCiTXFRb3q56mrr7ZbqBHWDUxbfyWhiWlBFLUIOnc,11478
|
6
|
+
esd_services_api_client/beast/v3/_models.py,sha256=UyBAEtJhDtF_qPE6TB9QWoffvNk1nY-AXm9bMTVBT2U,6766
|
7
|
+
esd_services_api_client/boxer/README.md,sha256=-MAhYUPvmEMcgx_lo_2PlH_gZI1lndGv8fnDQYIpurU,3618
|
8
|
+
esd_services_api_client/boxer/__init__.py,sha256=yDEcfL-9q0H91UWr-al-1iMJtVLv-oIXky6hUmFZs0g,785
|
9
|
+
esd_services_api_client/boxer/_auth.py,sha256=q9vU4agH-s6HPpVTEdkXFA_RvO92hogOfjurfjHBmBg,7445
|
10
|
+
esd_services_api_client/boxer/_base.py,sha256=LMKDArELZlWVKJH10coUFSSN8MFH6KfHCBncEDjsiIU,981
|
11
|
+
esd_services_api_client/boxer/_connector.py,sha256=gUplRNAxcOoZfMPee0ZUJmhbzYmreys9cMeIMHmIyKM,8677
|
12
|
+
esd_services_api_client/boxer/_models.py,sha256=ursQIR_c9jcVfRKc0LH1OuVL2KFD6kqojyGBadRfRPI,1702
|
13
|
+
esd_services_api_client/common/__init__.py,sha256=L-cEW1mVbnTJLCLG5V6Ucw7zBgx1zf0t1bYcQC1heyw,603
|
14
|
+
esd_services_api_client/crystal/__init__.py,sha256=oeyJjdQ9EpTnIq6XnjPq5v0DWPdHqi4uEfRIcD1mNZA,792
|
15
|
+
esd_services_api_client/crystal/_api_versions.py,sha256=GHbmV_5lP9fP72TZE0j_ZeQSeJjMRcRaBRxNJbz-MWQ,837
|
16
|
+
esd_services_api_client/crystal/_connector.py,sha256=wT8SahCkkRWPoHcUSLz0I-sjeK_9OSYtt07zJKie0CU,12875
|
17
|
+
esd_services_api_client/crystal/_models.py,sha256=OCaidMqipl-TA8VLs7v12BppyBR3QpRA5jU2KjZxU_Q,4030
|
18
|
+
esd_services_api_client/nexus/README.md,sha256=Q3laWkuCxqVBFsfP58mGAys-jVZeesroIRRMiaQMntc,8688
|
19
|
+
esd_services_api_client/nexus/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
|
20
|
+
esd_services_api_client/nexus/abstractions/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
|
21
|
+
esd_services_api_client/nexus/abstractions/logger_factory.py,sha256=9biONvCqNrP__yrmeRkoDL05TMA5v-LyrcKwgiKG59U,2019
|
22
|
+
esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=Be90h8iVDp2o1recnx8-Ousm3OjaVjIjthPqE5u5vik,2828
|
23
|
+
esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=Rwa_aPErI4Es5AdyCd3EoGze7mg2D70u8kuc2UGEBaI,1729
|
24
|
+
esd_services_api_client/nexus/algorithms/__init__.py,sha256=yMvLFSqg5eUKOXI0zMFX69Ni0ibKQHOqAnrZsxQqhOo,903
|
25
|
+
esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=UxqGFzpl-8w4SLJX0GMxLzRinRnFVoBT2zBKzHOapBM,2694
|
26
|
+
esd_services_api_client/nexus/algorithms/distributed.py,sha256=vkKSCsd480RKwrtu3uZ2iU1bh593fkgBcOBrcb9cLjA,1702
|
27
|
+
esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=PjLs_xhpd3rTViaLbWUEvJ1LWDPTKEZFj57iPk8wswo,1462
|
28
|
+
esd_services_api_client/nexus/algorithms/recursive.py,sha256=MAyfj3gQZ0zQB5hU3RHIOfH889v_Wtp4GHBhB-trr2w,1862
|
29
|
+
esd_services_api_client/nexus/configurations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
30
|
+
esd_services_api_client/nexus/configurations/algorithm_configuration.py,sha256=eE7diX2PATCGkmqhvFOcZwXrr6vns4fqnJGmgNvhhZM,1091
|
31
|
+
esd_services_api_client/nexus/core/__init__.py,sha256=sOgKKq3_LZGbLmQMtMS7lDw2hv027qownTmNIRV0BB8,627
|
32
|
+
esd_services_api_client/nexus/core/app_core.py,sha256=FzotOWZ8Yc3njI1wCnSZAi2MhENdHHXfm3v9V0WUAPQ,9425
|
33
|
+
esd_services_api_client/nexus/core/app_dependencies.py,sha256=GChNBRE-9CGqZ41xQOlTvt_XAniyoXnO3GrxTSBih9w,6145
|
34
|
+
esd_services_api_client/nexus/exceptions/__init__.py,sha256=feN33VdqB5-2bD9aJesJl_OlsKrNNo3hZCnQgKuaU9k,696
|
35
|
+
esd_services_api_client/nexus/exceptions/_nexus_error.py,sha256=QvtY38mNoIA6t26dUN6UIsaPfljhtVNsbQVS7ksMb-Q,895
|
36
|
+
esd_services_api_client/nexus/exceptions/input_reader_error.py,sha256=Chy8XW6Ien4-bkZZ1CmP8CWU49mi2hobS6L_R59ONs8,1765
|
37
|
+
esd_services_api_client/nexus/exceptions/startup_error.py,sha256=HPTlrg2voRbGXZsakWcVT7lcGSaccSR-GsBIo08SJ4A,1556
|
38
|
+
esd_services_api_client/nexus/input/__init__.py,sha256=ahJ7yUuRTCNIYB5GitBRGAPZZX82ghISzF6lm_CkyEA,819
|
39
|
+
esd_services_api_client/nexus/input/_functions.py,sha256=9bySbwcXGV7JIsQbSqg3YuW7avJUErsvsS2ZDHRWfT0,2918
|
40
|
+
esd_services_api_client/nexus/input/input_processor.py,sha256=ZUdpz2M6PfsF-99-DpCMDtD9Id0SQYhC-HbjosMz8O8,4609
|
41
|
+
esd_services_api_client/nexus/input/input_reader.py,sha256=MtZNdxjWLjoSqlSlwa6f7BIqsu-_GoQav5tCz2g6WrA,3214
|
42
|
+
esd_services_api_client/nexus/input/payload_reader.py,sha256=Kq0xN1Shyqv71v6YkcrqVTDbmsEjZc8ithsXYpyu87M,2516
|
43
|
+
esd_services_api_client-2.1.2.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
|
44
|
+
esd_services_api_client-2.1.2.dist-info/METADATA,sha256=xr1Dw96DI1t-qKxCb1aq48Dyxt-co-BzwRePv-Jgoz0,1292
|
45
|
+
esd_services_api_client-2.1.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
46
|
+
esd_services_api_client-2.1.2.dist-info/RECORD,,
|
@@ -1,46 +0,0 @@
|
|
1
|
-
esd_services_api_client/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
|
2
|
-
esd_services_api_client/_version.py,sha256=LbU43-7hsLmdXWI0wTAl3y6D3Tr7CbJiDOLXwl7clj8,22
|
3
|
-
esd_services_api_client/beast/__init__.py,sha256=NTaz_7YoLPK8MCLwbwqH7rW1zDWLxXu2T7fGmMmRxyg,718
|
4
|
-
esd_services_api_client/beast/v3/__init__.py,sha256=TRjB4-T6eIORpMvdylb32_GinrIpYNFmAdshSC1HqHg,749
|
5
|
-
esd_services_api_client/beast/v3/_connector.py,sha256=oPizDQ1KOKOfiyh-jAofKodlpRzrRiELv-rmP_o_oio,11473
|
6
|
-
esd_services_api_client/beast/v3/_models.py,sha256=hvh8QlwYLCfdBJxUKPutOq0xFn9rUU1q6UgHEpoZaFM,6761
|
7
|
-
esd_services_api_client/boxer/README.md,sha256=-MAhYUPvmEMcgx_lo_2PlH_gZI1lndGv8fnDQYIpurU,3618
|
8
|
-
esd_services_api_client/boxer/__init__.py,sha256=OYsWvdnLan0kmjUcH4I2-m1rbPeARKp5iqhp8uyudPk,780
|
9
|
-
esd_services_api_client/boxer/_auth.py,sha256=vA7T9y0oZV2f17UWQ2or9CK8vAsNnHB10G5HNQe1l1I,7440
|
10
|
-
esd_services_api_client/boxer/_base.py,sha256=PSU08QzTKFkXfzx7fF5FIHBOZXshxNEd1J_qKGo0Rd0,976
|
11
|
-
esd_services_api_client/boxer/_connector.py,sha256=kATr7IyQKtvsGlvgVInO_DsGGC4yCC1aW3RxoXt-QIM,8672
|
12
|
-
esd_services_api_client/boxer/_models.py,sha256=q_xRHOXlRroKrDrNUxQJUFGSotxYvUbRY1rYmx8UfJg,1697
|
13
|
-
esd_services_api_client/common/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
|
14
|
-
esd_services_api_client/crystal/__init__.py,sha256=afSGQRkDic0ECsJfgu3b291kX8CyU57sjreqxj5cx-s,787
|
15
|
-
esd_services_api_client/crystal/_api_versions.py,sha256=2BMiQRS0D8IEpWCCys3dge5alVBRCZrOuCR1QAn8UIM,832
|
16
|
-
esd_services_api_client/crystal/_connector.py,sha256=WjfMezWXia41Z8aiNupaT577fk9Sx6uy6V23O6y9hfI,12870
|
17
|
-
esd_services_api_client/crystal/_models.py,sha256=eRhGAl8LjglCyIFwf1bcFBhjbpSuRYucuF2LO388L2E,4025
|
18
|
-
esd_services_api_client/nexus/README.md,sha256=Q3laWkuCxqVBFsfP58mGAys-jVZeesroIRRMiaQMntc,8688
|
19
|
-
esd_services_api_client/nexus/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
|
20
|
-
esd_services_api_client/nexus/abstractions/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
|
21
|
-
esd_services_api_client/nexus/abstractions/logger_factory.py,sha256=jw1xnrU2dEWSUhqNut4RzjRPZFmeNx6czOU_8-q4kuo,2014
|
22
|
-
esd_services_api_client/nexus/abstractions/nexus_object.py,sha256=Y2jwpSa0cLyMtGxOSEpg3F8evvSMxYHx5t5KD92KotE,2759
|
23
|
-
esd_services_api_client/nexus/abstractions/socket_provider.py,sha256=Mv9BWdxw8VY4Gi4EOrdxWK1zsR3-fqIbpyF1xHWchbE,1495
|
24
|
-
esd_services_api_client/nexus/algorithms/__init__.py,sha256=ySnLfuYuxi28EaqT18oW2Ka50ZR7WOvJlLhmIts-Xgw,898
|
25
|
-
esd_services_api_client/nexus/algorithms/_baseline_algorithm.py,sha256=_3QHWmBDnfByEJTftf4jKzje5GNqftb0_aCfp-8ksCU,2436
|
26
|
-
esd_services_api_client/nexus/algorithms/distributed.py,sha256=kZS4InM4GoT4T6UrXRnfoDn4zxE2iGuqLvxBrzBetZc,1697
|
27
|
-
esd_services_api_client/nexus/algorithms/minimalistic.py,sha256=DOU52Me_W0luUPWmIRBFkJdfapyEkPqVbbvy4V5yQZ8,1457
|
28
|
-
esd_services_api_client/nexus/algorithms/recursive.py,sha256=r4_LIlC_hhIBSrKtL99xE9md3SahctF3R3GT1tIwlg4,1857
|
29
|
-
esd_services_api_client/nexus/configurations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
30
|
-
esd_services_api_client/nexus/configurations/algorithm_configuration.py,sha256=sg3ul-Vqd39li42cfMsdWtFJIBNfQ52XLmqKO2sryDg,1086
|
31
|
-
esd_services_api_client/nexus/core/__init__.py,sha256=e7RPs-qJNQqDHj121TeYx-_YadZSOIyJuAPyhSSXRsE,622
|
32
|
-
esd_services_api_client/nexus/core/app_core.py,sha256=XvPWnr6wJckjNoM4vWg3CYBnrJOaWZFxiIil1sK-qNc,9420
|
33
|
-
esd_services_api_client/nexus/core/app_dependencies.py,sha256=rIk6R-InJeF1P0DC4bhj4kJTNNxYg39Vhuz8AXfV6pk,6140
|
34
|
-
esd_services_api_client/nexus/exceptions/__init__.py,sha256=JgPXhrvBIi0U1QOF90TYHS8jv_SBQEoRLsEg6rrtRoc,691
|
35
|
-
esd_services_api_client/nexus/exceptions/_nexus_error.py,sha256=b3L8JnNvV2jdxNfuFWh9-j4kVb_VX7gNH5WHKcC-R78,890
|
36
|
-
esd_services_api_client/nexus/exceptions/input_reader_error.py,sha256=D-xYTKRNREQ2-NGhc88GHOmXCvLNsIVQsH8wf0LLC_0,1760
|
37
|
-
esd_services_api_client/nexus/exceptions/startup_error.py,sha256=f2PIOSdLgT-42eKD6ec8p7nROADshMawCsDGDUbxO_w,1546
|
38
|
-
esd_services_api_client/nexus/input/__init__.py,sha256=DEdzkK43xjNl3XPQy3Q8xHGzXYsMwxCMs1QdEAsS4FI,814
|
39
|
-
esd_services_api_client/nexus/input/_functions.py,sha256=TLb3yDW3ci4_viS52nOUK-efoG6Ok_c6cf3J895phZ0,2480
|
40
|
-
esd_services_api_client/nexus/input/input_processor.py,sha256=FEYvlgVwEHU58wRolNX6z6svgJl-gcfNk6RAkL4Yb_s,1803
|
41
|
-
esd_services_api_client/nexus/input/input_reader.py,sha256=aVhVGDrZCH4UPr6JlhvxgPZIplzbhEqd_njKZfpNFAI,3172
|
42
|
-
esd_services_api_client/nexus/input/payload_reader.py,sha256=__r_QjIFRAWwx56X5WUK1qensJUae0vZEb422dzOgSY,2511
|
43
|
-
esd_services_api_client-2.1.0.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
|
44
|
-
esd_services_api_client-2.1.0.dist-info/METADATA,sha256=iDTAP-IzN03N2AtLsm6tlueG6cw2sVvjApJ8eh71fv4,1292
|
45
|
-
esd_services_api_client-2.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
46
|
-
esd_services_api_client-2.1.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|