deisa-dask 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 deisa-project
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,38 @@
1
+ Metadata-Version: 2.4
2
+ Name: deisa-dask
3
+ Version: 0.3.0
4
+ Summary: Deisa: Dask-Enabled In Situ Analytics
5
+ Home-page: https://github.com/deisa-project/deisa-dask
6
+ Author: Benoît Martin
7
+ Author-email: bmartin@cea.fr
8
+ License: MIT
9
+ Project-URL: Bug Reports, https://github.com/deisa-project/deisa-dask/issues
10
+ Project-URL: Source, https://github.com/deisa-project/deisa-dask
11
+ Keywords: deisa in-situ
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Development Status :: 3 - Alpha
16
+ Requires-Python: >=3.10
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: deisa-core==0.1.0
20
+ Requires-Dist: dask
21
+ Requires-Dist: distributed
22
+ Provides-Extra: test
23
+ Requires-Dist: pytest; extra == "test"
24
+ Requires-Dist: pytest-xdist; extra == "test"
25
+ Requires-Dist: numpy; extra == "test"
26
+ Dynamic: author
27
+ Dynamic: author-email
28
+ Dynamic: classifier
29
+ Dynamic: description-content-type
30
+ Dynamic: home-page
31
+ Dynamic: keywords
32
+ Dynamic: license
33
+ Dynamic: license-file
34
+ Dynamic: project-url
35
+ Dynamic: provides-extra
36
+ Dynamic: requires-dist
37
+ Dynamic: requires-python
38
+ Dynamic: summary
File without changes
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,88 @@
1
+ # =============================================================================
2
+ # Copyright (C) 2015-2023 Commissariat a l'energie atomique et aux energies alternatives (CEA)
3
+ #
4
+ # All rights reserved.
5
+ #
6
+ # Redistribution and use in source and binary forms, with or without
7
+ # modification, are permitted provided that the following conditions are met:
8
+ # * Redistributions of source code must retain the above copyright notice,
9
+ # this list of conditions and the following disclaimer.
10
+ # * Redistributions in binary form must reproduce the above copyright notice,
11
+ # this list of conditions and the following disclaimer in the documentation
12
+ # and/or other materials provided with the distribution.
13
+ # * Neither the names of CEA, nor the names of the contributors may be used to
14
+ # endorse or promote products derived from this software without specific
15
+ # prior written permission.
16
+ #
17
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18
+ # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
+ # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21
+ # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22
+ # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23
+ # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25
+ # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26
+ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
+ # POSSIBILITY OF SUCH DAMAGE.
28
+ # =============================================================================
29
+
30
+ from setuptools import setup, find_namespace_packages
31
+
32
+ def read_version():
33
+ ns = {}
34
+ with open("src/deisa/dask/__version__.py") as f:
35
+ exec(f.read(), ns)
36
+ return ns["__version__"]
37
+
38
+ def readme():
39
+ with open('README.md', 'r') as f:
40
+ return f.read()
41
+
42
+
43
+ setup(name='deisa-dask',
44
+ version=read_version(),
45
+
46
+ description='Deisa: Dask-Enabled In Situ Analytics',
47
+ long_description=readme(),
48
+ long_description_content_type='text/markdown',
49
+ license='MIT',
50
+
51
+ url='https://github.com/deisa-project/deisa-dask',
52
+ project_urls={
53
+ 'Bug Reports': 'https://github.com/deisa-project/deisa-dask/issues',
54
+ 'Source': 'https://github.com/deisa-project/deisa-dask',
55
+ },
56
+
57
+ author='Benoît Martin',
58
+ author_email='bmartin@cea.fr',
59
+
60
+ python_requires='>=3.10',
61
+
62
+ keywords='deisa in-situ',
63
+
64
+ package_dir={'': 'src'},
65
+ packages=find_namespace_packages(where='src', include=['deisa.dask']),
66
+
67
+ install_requires=[
68
+ "deisa-core==0.1.0",
69
+ 'dask',
70
+ 'distributed'
71
+ ],
72
+
73
+ extras_require={
74
+ "test": [
75
+ "pytest",
76
+ "pytest-xdist",
77
+ "numpy",
78
+ ]
79
+ },
80
+ test_suite='test',
81
+
82
+ classifiers=[
83
+ "Programming Language :: Python :: 3.10",
84
+ "License :: OSI Approved :: MIT License",
85
+ "Operating System :: OS Independent",
86
+ "Development Status :: 3 - Alpha"
87
+ ]
88
+ )
@@ -0,0 +1,39 @@
1
+ ###################################################################################################
2
+ # Copyright (c) 2025 Commissariat a l'énergie atomique et aux énergies alternatives (CEA)
3
+ # SPDX-License-Identifier: MIT
4
+ ###################################################################################################
5
+ from importlib.metadata import version, PackageNotFoundError
6
+
7
+ try:
8
+ __version__ = version("deisa-dask") # installed version
9
+ except PackageNotFoundError:
10
+ from .__version__ import __version__ # fallback
11
+
12
+ import os
13
+
14
+ from distributed import Client
15
+
16
+ from .bridge import Bridge
17
+ from .deisa import Deisa
18
+
19
+
20
+ def get_connection_info(dask_scheduler_address: str | Client) -> Client:
21
+ if isinstance(dask_scheduler_address, Client):
22
+ client = dask_scheduler_address
23
+ elif isinstance(dask_scheduler_address, str):
24
+ try:
25
+ client = Client(address=dask_scheduler_address)
26
+ except ValueError:
27
+ # try scheduler_file
28
+ if os.path.isfile(dask_scheduler_address):
29
+ client = Client(scheduler_file=dask_scheduler_address)
30
+ else:
31
+ raise ValueError(
32
+ "dask_scheduler_address must be a string containing the address of the scheduler, "
33
+ "or a string containing a file name to a dask scheduler file, or a Dask Client object.")
34
+ else:
35
+ raise ValueError(
36
+ "dask_scheduler_address must be a string containing the address of the scheduler, "
37
+ "or a string containing a file name to a dask scheduler file, or a Dask Client object.")
38
+
39
+ return client
@@ -0,0 +1,8 @@
1
+ ###################################################################################################
2
+ # Copyright (c) 2025 Commissariat a l'énergie atomique et aux énergies alternatives (CEA)
3
+ #
4
+ # SPDX-License-Identifier: MIT
5
+ #
6
+ ###################################################################################################
7
+
8
+ __version__ = "0.3.0"
@@ -0,0 +1,135 @@
1
+ # =============================================================================
2
+ # Copyright (C) 2025 Commissariat a l'energie atomique et aux energies alternatives (CEA)
3
+ #
4
+ # All rights reserved.
5
+ #
6
+ # Redistribution and use in source and binary forms, with or without
7
+ # modification, are permitted provided that the following conditions are met:
8
+ # * Redistributions of source code must retain the above copyright notice,
9
+ # this list of conditions and the following disclaimer.
10
+ # * Redistributions in binary form must reproduce the above copyright notice,
11
+ # this list of conditions and the following disclaimer in the documentation
12
+ # and/or other materials provided with the distribution.
13
+ # * Neither the names of CEA, nor the names of the contributors may be used to
14
+ # endorse or promote products derived from this software without specific
15
+ # prior written permission.
16
+ #
17
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18
+ # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
+ # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21
+ # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22
+ # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23
+ # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25
+ # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26
+ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
+ # POSSIBILITY OF SUCH DAMAGE.
28
+ # =============================================================================
29
+
30
+ from typing import Any
31
+
32
+ import numpy as np
33
+ from deisa.core import validate_system_metadata, validate_arrays_metadata, IBridge
34
+ from distributed import Client, Queue, Variable, Lock
35
+ from distributed.utils import TimeoutError
36
+
37
+ from deisa.dask.deisa import LOCK_PREFIX, VARIABLE_PREFIX
38
+ from deisa.dask.handshake import Handshake
39
+
40
+
41
+ class Bridge(IBridge):
42
+
43
+ def __init__(self, id: int, arrays_metadata: dict[str, dict], system_metadata: dict[str, Any], *args, **kwargs):
44
+ """
45
+ Initializes an object to manage communication between an MPI-based distributed
46
+ system and a Dask-based framework. The class ensures proper allocation of workers
47
+ among processes and instantiates the required communication objects like queues.
48
+
49
+ :param id: Unique identifier in the computation. This may be the rank of this MPI process.
50
+ :type id: int
51
+
52
+ :param mpi_comm_size: Total number of MPI processes involved in the computation.
53
+ :type mpi_comm_size: int
54
+
55
+ :param arrays_metadata: A dictionary containing metadata about the Dask arrays
56
+ eg: arrays_metadata = {
57
+ 'global_t': {
58
+ 'size': [20, 20]
59
+ 'subsize': [10, 10]
60
+ }
61
+ 'global_p': {
62
+ 'size': [100, 100]
63
+ 'subsize': [50, 50]
64
+ }
65
+ :type arrays_metadata: dict[str, dict]
66
+
67
+ :param connection: A function that returns a connected Dask Client.
68
+ :type connection: Callable
69
+
70
+ :param kwargs: Passed to Handshake
71
+ :type kwargs: dict
72
+ """
73
+ super().__init__(id, arrays_metadata, system_metadata, *args, **kwargs)
74
+ self.system_metadata = validate_system_metadata(system_metadata)
75
+ self.client: Client = self.system_metadata['connection']
76
+ self.arrays_metadata = validate_arrays_metadata(arrays_metadata)
77
+ self.mpi_rank = id
78
+ self.futures = []
79
+
80
+ # blocking until analytics is ready
81
+ Handshake('bridge', self.client, id=id, max=self.system_metadata['nb_bridges'],
82
+ arrays_metadata=self.arrays_metadata, **kwargs)
83
+
84
+ def send(self, array_name: str, data: np.ndarray, iteration: int, chunked: bool = True):
85
+ """
86
+ Publishes data to the distributed workers and communicates metadata and data future via a queue. This method is used
87
+ to send data to workers in a distributed computing setup and ensures that both the metadata about the data and the
88
+ data itself (in the form of a future) are made available to the relevant processes. Metadata includes information
89
+ such as iteration number, MPI rank, data shape, and data type.
90
+
91
+ :param array_name: Name of the array associated with the data
92
+ :type array_name: str
93
+ :param data: The data to be distributed among the workers
94
+ :type data: numpy.ndarray
95
+ :param iteration: The iteration number associated with the data
96
+ :type iteration: int
97
+ :param chunked: Defines if the data is a chunk.
98
+ :type chunked: bool
99
+ :return: None
100
+ """
101
+
102
+ assert self.client.status == 'running', "Client is not connected to a scheduler. Please check your connection."
103
+
104
+ # TODO: select workers to send data to. self.client.scatter(data, direct=True, workers=self.workers)
105
+ f = self.client.scatter(data, direct=True) # send data to workers
106
+
107
+ # TODO: this is a memory leak. Find a way to release the futures once they are used to build a dask array in the client code.
108
+ self.futures.append(f)
109
+
110
+ to_send = {
111
+ 'rank': self.mpi_rank,
112
+ 'shape': data.shape,
113
+ 'dtype': data.dtype,
114
+ 'iteration': iteration,
115
+ 'future': f
116
+ }
117
+
118
+ q = Queue(array_name, client=self.client)
119
+ q.put(to_send)
120
+
121
+ # TODO: what to do if error ?
122
+
123
+ def get(self, key: str, default: Any = None, chunked: bool = False, delete: bool = True):
124
+ if chunked:
125
+ raise NotImplementedError() # TODO
126
+ else:
127
+ try:
128
+ with Lock(f'{LOCK_PREFIX}{key}'):
129
+ return Variable(f'{VARIABLE_PREFIX}{key}', client=self.client).get(timeout=0)
130
+ except TimeoutError:
131
+ return default
132
+ finally:
133
+ if delete:
134
+ with Lock(f'{LOCK_PREFIX}{key}'):
135
+ Variable(f'{VARIABLE_PREFIX}{key}', client=self.client).delete()