deisa-dask 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deisa_dask-0.3.0/LICENSE +21 -0
- deisa_dask-0.3.0/PKG-INFO +38 -0
- deisa_dask-0.3.0/README.md +0 -0
- deisa_dask-0.3.0/setup.cfg +4 -0
- deisa_dask-0.3.0/setup.py +88 -0
- deisa_dask-0.3.0/src/deisa/dask/__init__.py +39 -0
- deisa_dask-0.3.0/src/deisa/dask/__version__.py +8 -0
- deisa_dask-0.3.0/src/deisa/dask/bridge.py +135 -0
- deisa_dask-0.3.0/src/deisa/dask/deisa.py +373 -0
- deisa_dask-0.3.0/src/deisa/dask/handshake.py +146 -0
- deisa_dask-0.3.0/src/deisa/dask/py.typed +0 -0
- deisa_dask-0.3.0/src/deisa_dask.egg-info/PKG-INFO +38 -0
- deisa_dask-0.3.0/src/deisa_dask.egg-info/SOURCES.txt +16 -0
- deisa_dask-0.3.0/src/deisa_dask.egg-info/dependency_links.txt +1 -0
- deisa_dask-0.3.0/src/deisa_dask.egg-info/requires.txt +8 -0
- deisa_dask-0.3.0/src/deisa_dask.egg-info/top_level.txt +1 -0
- deisa_dask-0.3.0/test/test_deisa.py +750 -0
- deisa_dask-0.3.0/test/test_handshake.py +79 -0
deisa_dask-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 deisa-project
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deisa-dask
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Deisa: Dask-Enabled In Situ Analytics
|
|
5
|
+
Home-page: https://github.com/deisa-project/deisa-dask
|
|
6
|
+
Author: Benoît Martin
|
|
7
|
+
Author-email: bmartin@cea.fr
|
|
8
|
+
License: MIT
|
|
9
|
+
Project-URL: Bug Reports, https://github.com/deisa-project/deisa-dask/issues
|
|
10
|
+
Project-URL: Source, https://github.com/deisa-project/deisa-dask
|
|
11
|
+
Keywords: deisa in-situ
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Development Status :: 3 - Alpha
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: deisa-core==0.1.0
|
|
20
|
+
Requires-Dist: dask
|
|
21
|
+
Requires-Dist: distributed
|
|
22
|
+
Provides-Extra: test
|
|
23
|
+
Requires-Dist: pytest; extra == "test"
|
|
24
|
+
Requires-Dist: pytest-xdist; extra == "test"
|
|
25
|
+
Requires-Dist: numpy; extra == "test"
|
|
26
|
+
Dynamic: author
|
|
27
|
+
Dynamic: author-email
|
|
28
|
+
Dynamic: classifier
|
|
29
|
+
Dynamic: description-content-type
|
|
30
|
+
Dynamic: home-page
|
|
31
|
+
Dynamic: keywords
|
|
32
|
+
Dynamic: license
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
Dynamic: project-url
|
|
35
|
+
Dynamic: provides-extra
|
|
36
|
+
Dynamic: requires-dist
|
|
37
|
+
Dynamic: requires-python
|
|
38
|
+
Dynamic: summary
|
|
File without changes
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# Copyright (C) 2015-2023 Commissariat a l'energie atomique et aux energies alternatives (CEA)
|
|
3
|
+
#
|
|
4
|
+
# All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# Redistribution and use in source and binary forms, with or without
|
|
7
|
+
# modification, are permitted provided that the following conditions are met:
|
|
8
|
+
# * Redistributions of source code must retain the above copyright notice,
|
|
9
|
+
# this list of conditions and the following disclaimer.
|
|
10
|
+
# * Redistributions in binary form must reproduce the above copyright notice,
|
|
11
|
+
# this list of conditions and the following disclaimer in the documentation
|
|
12
|
+
# and/or other materials provided with the distribution.
|
|
13
|
+
# * Neither the names of CEA, nor the names of the contributors may be used to
|
|
14
|
+
# endorse or promote products derived from this software without specific
|
|
15
|
+
# prior written permission.
|
|
16
|
+
#
|
|
17
|
+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
18
|
+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
19
|
+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
20
|
+
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
21
|
+
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
22
|
+
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
23
|
+
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
24
|
+
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
25
|
+
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
26
|
+
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
27
|
+
# POSSIBILITY OF SUCH DAMAGE.
|
|
28
|
+
# =============================================================================
|
|
29
|
+
|
|
30
|
+
from setuptools import setup, find_namespace_packages
|
|
31
|
+
|
|
32
|
+
def read_version():
|
|
33
|
+
ns = {}
|
|
34
|
+
with open("src/deisa/dask/__version__.py") as f:
|
|
35
|
+
exec(f.read(), ns)
|
|
36
|
+
return ns["__version__"]
|
|
37
|
+
|
|
38
|
+
def readme():
|
|
39
|
+
with open('README.md', 'r') as f:
|
|
40
|
+
return f.read()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
setup(name='deisa-dask',
|
|
44
|
+
version=read_version(),
|
|
45
|
+
|
|
46
|
+
description='Deisa: Dask-Enabled In Situ Analytics',
|
|
47
|
+
long_description=readme(),
|
|
48
|
+
long_description_content_type='text/markdown',
|
|
49
|
+
license='MIT',
|
|
50
|
+
|
|
51
|
+
url='https://github.com/deisa-project/deisa-dask',
|
|
52
|
+
project_urls={
|
|
53
|
+
'Bug Reports': 'https://github.com/deisa-project/deisa-dask/issues',
|
|
54
|
+
'Source': 'https://github.com/deisa-project/deisa-dask',
|
|
55
|
+
},
|
|
56
|
+
|
|
57
|
+
author='Benoît Martin',
|
|
58
|
+
author_email='bmartin@cea.fr',
|
|
59
|
+
|
|
60
|
+
python_requires='>=3.10',
|
|
61
|
+
|
|
62
|
+
keywords='deisa in-situ',
|
|
63
|
+
|
|
64
|
+
package_dir={'': 'src'},
|
|
65
|
+
packages=find_namespace_packages(where='src', include=['deisa.dask']),
|
|
66
|
+
|
|
67
|
+
install_requires=[
|
|
68
|
+
"deisa-core==0.1.0",
|
|
69
|
+
'dask',
|
|
70
|
+
'distributed'
|
|
71
|
+
],
|
|
72
|
+
|
|
73
|
+
extras_require={
|
|
74
|
+
"test": [
|
|
75
|
+
"pytest",
|
|
76
|
+
"pytest-xdist",
|
|
77
|
+
"numpy",
|
|
78
|
+
]
|
|
79
|
+
},
|
|
80
|
+
test_suite='test',
|
|
81
|
+
|
|
82
|
+
classifiers=[
|
|
83
|
+
"Programming Language :: Python :: 3.10",
|
|
84
|
+
"License :: OSI Approved :: MIT License",
|
|
85
|
+
"Operating System :: OS Independent",
|
|
86
|
+
"Development Status :: 3 - Alpha"
|
|
87
|
+
]
|
|
88
|
+
)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
###################################################################################################
|
|
2
|
+
# Copyright (c) 2025 Commissariat a l'énergie atomique et aux énergies alternatives (CEA)
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
###################################################################################################
|
|
5
|
+
from importlib.metadata import version, PackageNotFoundError
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
__version__ = version("deisa-dask") # installed version
|
|
9
|
+
except PackageNotFoundError:
|
|
10
|
+
from .__version__ import __version__ # fallback
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
|
|
14
|
+
from distributed import Client
|
|
15
|
+
|
|
16
|
+
from .bridge import Bridge
|
|
17
|
+
from .deisa import Deisa
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def get_connection_info(dask_scheduler_address: str | Client) -> Client:
|
|
21
|
+
if isinstance(dask_scheduler_address, Client):
|
|
22
|
+
client = dask_scheduler_address
|
|
23
|
+
elif isinstance(dask_scheduler_address, str):
|
|
24
|
+
try:
|
|
25
|
+
client = Client(address=dask_scheduler_address)
|
|
26
|
+
except ValueError:
|
|
27
|
+
# try scheduler_file
|
|
28
|
+
if os.path.isfile(dask_scheduler_address):
|
|
29
|
+
client = Client(scheduler_file=dask_scheduler_address)
|
|
30
|
+
else:
|
|
31
|
+
raise ValueError(
|
|
32
|
+
"dask_scheduler_address must be a string containing the address of the scheduler, "
|
|
33
|
+
"or a string containing a file name to a dask scheduler file, or a Dask Client object.")
|
|
34
|
+
else:
|
|
35
|
+
raise ValueError(
|
|
36
|
+
"dask_scheduler_address must be a string containing the address of the scheduler, "
|
|
37
|
+
"or a string containing a file name to a dask scheduler file, or a Dask Client object.")
|
|
38
|
+
|
|
39
|
+
return client
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
###################################################################################################
|
|
2
|
+
# Copyright (c) 2025 Commissariat a l'énergie atomique et aux énergies alternatives (CEA)
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: MIT
|
|
5
|
+
#
|
|
6
|
+
###################################################################################################
|
|
7
|
+
|
|
8
|
+
__version__ = "0.3.0"
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# Copyright (C) 2025 Commissariat a l'energie atomique et aux energies alternatives (CEA)
|
|
3
|
+
#
|
|
4
|
+
# All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# Redistribution and use in source and binary forms, with or without
|
|
7
|
+
# modification, are permitted provided that the following conditions are met:
|
|
8
|
+
# * Redistributions of source code must retain the above copyright notice,
|
|
9
|
+
# this list of conditions and the following disclaimer.
|
|
10
|
+
# * Redistributions in binary form must reproduce the above copyright notice,
|
|
11
|
+
# this list of conditions and the following disclaimer in the documentation
|
|
12
|
+
# and/or other materials provided with the distribution.
|
|
13
|
+
# * Neither the names of CEA, nor the names of the contributors may be used to
|
|
14
|
+
# endorse or promote products derived from this software without specific
|
|
15
|
+
# prior written permission.
|
|
16
|
+
#
|
|
17
|
+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
18
|
+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
19
|
+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
20
|
+
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
21
|
+
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
22
|
+
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
23
|
+
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
24
|
+
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
25
|
+
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
26
|
+
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
27
|
+
# POSSIBILITY OF SUCH DAMAGE.
|
|
28
|
+
# =============================================================================
|
|
29
|
+
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
import numpy as np
|
|
33
|
+
from deisa.core import validate_system_metadata, validate_arrays_metadata, IBridge
|
|
34
|
+
from distributed import Client, Queue, Variable, Lock
|
|
35
|
+
from distributed.utils import TimeoutError
|
|
36
|
+
|
|
37
|
+
from deisa.dask.deisa import LOCK_PREFIX, VARIABLE_PREFIX
|
|
38
|
+
from deisa.dask.handshake import Handshake
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Bridge(IBridge):
|
|
42
|
+
|
|
43
|
+
def __init__(self, id: int, arrays_metadata: dict[str, dict], system_metadata: dict[str, Any], *args, **kwargs):
|
|
44
|
+
"""
|
|
45
|
+
Initializes an object to manage communication between an MPI-based distributed
|
|
46
|
+
system and a Dask-based framework. The class ensures proper allocation of workers
|
|
47
|
+
among processes and instantiates the required communication objects like queues.
|
|
48
|
+
|
|
49
|
+
:param id: Unique identifier in the computation. This may be the rank of this MPI process.
|
|
50
|
+
:type id: int
|
|
51
|
+
|
|
52
|
+
:param mpi_comm_size: Total number of MPI processes involved in the computation.
|
|
53
|
+
:type mpi_comm_size: int
|
|
54
|
+
|
|
55
|
+
:param arrays_metadata: A dictionary containing metadata about the Dask arrays
|
|
56
|
+
eg: arrays_metadata = {
|
|
57
|
+
'global_t': {
|
|
58
|
+
'size': [20, 20]
|
|
59
|
+
'subsize': [10, 10]
|
|
60
|
+
}
|
|
61
|
+
'global_p': {
|
|
62
|
+
'size': [100, 100]
|
|
63
|
+
'subsize': [50, 50]
|
|
64
|
+
}
|
|
65
|
+
:type arrays_metadata: dict[str, dict]
|
|
66
|
+
|
|
67
|
+
:param connection: A function that returns a connected Dask Client.
|
|
68
|
+
:type connection: Callable
|
|
69
|
+
|
|
70
|
+
:param kwargs: Passed to Handshake
|
|
71
|
+
:type kwargs: dict
|
|
72
|
+
"""
|
|
73
|
+
super().__init__(id, arrays_metadata, system_metadata, *args, **kwargs)
|
|
74
|
+
self.system_metadata = validate_system_metadata(system_metadata)
|
|
75
|
+
self.client: Client = self.system_metadata['connection']
|
|
76
|
+
self.arrays_metadata = validate_arrays_metadata(arrays_metadata)
|
|
77
|
+
self.mpi_rank = id
|
|
78
|
+
self.futures = []
|
|
79
|
+
|
|
80
|
+
# blocking until analytics is ready
|
|
81
|
+
Handshake('bridge', self.client, id=id, max=self.system_metadata['nb_bridges'],
|
|
82
|
+
arrays_metadata=self.arrays_metadata, **kwargs)
|
|
83
|
+
|
|
84
|
+
def send(self, array_name: str, data: np.ndarray, iteration: int, chunked: bool = True):
|
|
85
|
+
"""
|
|
86
|
+
Publishes data to the distributed workers and communicates metadata and data future via a queue. This method is used
|
|
87
|
+
to send data to workers in a distributed computing setup and ensures that both the metadata about the data and the
|
|
88
|
+
data itself (in the form of a future) are made available to the relevant processes. Metadata includes information
|
|
89
|
+
such as iteration number, MPI rank, data shape, and data type.
|
|
90
|
+
|
|
91
|
+
:param array_name: Name of the array associated with the data
|
|
92
|
+
:type array_name: str
|
|
93
|
+
:param data: The data to be distributed among the workers
|
|
94
|
+
:type data: numpy.ndarray
|
|
95
|
+
:param iteration: The iteration number associated with the data
|
|
96
|
+
:type iteration: int
|
|
97
|
+
:param chunked: Defines if the data is a chunk.
|
|
98
|
+
:type chunked: bool
|
|
99
|
+
:return: None
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
assert self.client.status == 'running', "Client is not connected to a scheduler. Please check your connection."
|
|
103
|
+
|
|
104
|
+
# TODO: select workers to send data to. self.client.scatter(data, direct=True, workers=self.workers)
|
|
105
|
+
f = self.client.scatter(data, direct=True) # send data to workers
|
|
106
|
+
|
|
107
|
+
# TODO: this is a memory leak. Find a way to release the futures once they are used to build a dask array in the client code.
|
|
108
|
+
self.futures.append(f)
|
|
109
|
+
|
|
110
|
+
to_send = {
|
|
111
|
+
'rank': self.mpi_rank,
|
|
112
|
+
'shape': data.shape,
|
|
113
|
+
'dtype': data.dtype,
|
|
114
|
+
'iteration': iteration,
|
|
115
|
+
'future': f
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
q = Queue(array_name, client=self.client)
|
|
119
|
+
q.put(to_send)
|
|
120
|
+
|
|
121
|
+
# TODO: what to do if error ?
|
|
122
|
+
|
|
123
|
+
def get(self, key: str, default: Any = None, chunked: bool = False, delete: bool = True):
|
|
124
|
+
if chunked:
|
|
125
|
+
raise NotImplementedError() # TODO
|
|
126
|
+
else:
|
|
127
|
+
try:
|
|
128
|
+
with Lock(f'{LOCK_PREFIX}{key}'):
|
|
129
|
+
return Variable(f'{VARIABLE_PREFIX}{key}', client=self.client).get(timeout=0)
|
|
130
|
+
except TimeoutError:
|
|
131
|
+
return default
|
|
132
|
+
finally:
|
|
133
|
+
if delete:
|
|
134
|
+
with Lock(f'{LOCK_PREFIX}{key}'):
|
|
135
|
+
Variable(f'{VARIABLE_PREFIX}{key}', client=self.client).delete()
|