deisa-dask 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
deisa/dask/__init__.py ADDED
@@ -0,0 +1,39 @@
1
+ ###################################################################################################
2
+ # Copyright (c) 2025 Commissariat a l'énergie atomique et aux énergies alternatives (CEA)
3
+ # SPDX-License-Identifier: MIT
4
+ ###################################################################################################
5
+ from importlib.metadata import version, PackageNotFoundError
6
+
7
+ try:
8
+ __version__ = version("deisa-dask") # installed version
9
+ except PackageNotFoundError:
10
+ from .__version__ import __version__ # fallback
11
+
12
+ import os
13
+
14
+ from distributed import Client
15
+
16
+ from .bridge import Bridge
17
+ from .deisa import Deisa
18
+
19
+
20
+ def get_connection_info(dask_scheduler_address: str | Client) -> Client:
21
+ if isinstance(dask_scheduler_address, Client):
22
+ client = dask_scheduler_address
23
+ elif isinstance(dask_scheduler_address, str):
24
+ try:
25
+ client = Client(address=dask_scheduler_address)
26
+ except ValueError:
27
+ # try scheduler_file
28
+ if os.path.isfile(dask_scheduler_address):
29
+ client = Client(scheduler_file=dask_scheduler_address)
30
+ else:
31
+ raise ValueError(
32
+ "dask_scheduler_address must be a string containing the address of the scheduler, "
33
+ "or a string containing a file name to a dask scheduler file, or a Dask Client object.")
34
+ else:
35
+ raise ValueError(
36
+ "dask_scheduler_address must be a string containing the address of the scheduler, "
37
+ "or a string containing a file name to a dask scheduler file, or a Dask Client object.")
38
+
39
+ return client
@@ -0,0 +1,8 @@
1
+ ###################################################################################################
2
+ # Copyright (c) 2025 Commissariat a l'énergie atomique et aux énergies alternatives (CEA)
3
+ #
4
+ # SPDX-License-Identifier: MIT
5
+ #
6
+ ###################################################################################################
7
+
8
+ __version__ = "0.3.0"
deisa/dask/bridge.py ADDED
@@ -0,0 +1,135 @@
1
+ # =============================================================================
2
+ # Copyright (C) 2025 Commissariat a l'energie atomique et aux energies alternatives (CEA)
3
+ #
4
+ # All rights reserved.
5
+ #
6
+ # Redistribution and use in source and binary forms, with or without
7
+ # modification, are permitted provided that the following conditions are met:
8
+ # * Redistributions of source code must retain the above copyright notice,
9
+ # this list of conditions and the following disclaimer.
10
+ # * Redistributions in binary form must reproduce the above copyright notice,
11
+ # this list of conditions and the following disclaimer in the documentation
12
+ # and/or other materials provided with the distribution.
13
+ # * Neither the names of CEA, nor the names of the contributors may be used to
14
+ # endorse or promote products derived from this software without specific
15
+ # prior written permission.
16
+ #
17
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18
+ # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
+ # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21
+ # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22
+ # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23
+ # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25
+ # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26
+ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
+ # POSSIBILITY OF SUCH DAMAGE.
28
+ # =============================================================================
29
+
30
+ from typing import Any
31
+
32
+ import numpy as np
33
+ from deisa.core import validate_system_metadata, validate_arrays_metadata, IBridge
34
+ from distributed import Client, Queue, Variable, Lock
35
+ from distributed.utils import TimeoutError
36
+
37
+ from deisa.dask.deisa import LOCK_PREFIX, VARIABLE_PREFIX
38
+ from deisa.dask.handshake import Handshake
39
+
40
+
41
+ class Bridge(IBridge):
42
+
43
+ def __init__(self, id: int, arrays_metadata: dict[str, dict], system_metadata: dict[str, Any], *args, **kwargs):
44
+ """
45
+ Initializes an object to manage communication between an MPI-based distributed
46
+ system and a Dask-based framework. The class ensures proper allocation of workers
47
+ among processes and instantiates the required communication objects like queues.
48
+
49
+ :param id: Unique identifier in the computation. This may be the rank of this MPI process.
50
+ :type id: int
51
+
52
+ :param mpi_comm_size: Total number of MPI processes involved in the computation.
53
+ :type mpi_comm_size: int
54
+
55
+ :param arrays_metadata: A dictionary containing metadata about the Dask arrays
56
+ eg: arrays_metadata = {
57
+ 'global_t': {
58
+ 'size': [20, 20]
59
+ 'subsize': [10, 10]
60
+ }
61
+ 'global_p': {
62
+ 'size': [100, 100]
63
+ 'subsize': [50, 50]
64
+ }
65
+ :type arrays_metadata: dict[str, dict]
66
+
67
+ :param connection: A function that returns a connected Dask Client.
68
+ :type connection: Callable
69
+
70
+ :param kwargs: Passed to Handshake
71
+ :type kwargs: dict
72
+ """
73
+ super().__init__(id, arrays_metadata, system_metadata, *args, **kwargs)
74
+ self.system_metadata = validate_system_metadata(system_metadata)
75
+ self.client: Client = self.system_metadata['connection']
76
+ self.arrays_metadata = validate_arrays_metadata(arrays_metadata)
77
+ self.mpi_rank = id
78
+ self.futures = []
79
+
80
+ # blocking until analytics is ready
81
+ Handshake('bridge', self.client, id=id, max=self.system_metadata['nb_bridges'],
82
+ arrays_metadata=self.arrays_metadata, **kwargs)
83
+
84
+ def send(self, array_name: str, data: np.ndarray, iteration: int, chunked: bool = True):
85
+ """
86
+ Publishes data to the distributed workers and communicates metadata and data future via a queue. This method is used
87
+ to send data to workers in a distributed computing setup and ensures that both the metadata about the data and the
88
+ data itself (in the form of a future) are made available to the relevant processes. Metadata includes information
89
+ such as iteration number, MPI rank, data shape, and data type.
90
+
91
+ :param array_name: Name of the array associated with the data
92
+ :type array_name: str
93
+ :param data: The data to be distributed among the workers
94
+ :type data: numpy.ndarray
95
+ :param iteration: The iteration number associated with the data
96
+ :type iteration: int
97
+ :param chunked: Defines if the data is a chunk.
98
+ :type chunked: bool
99
+ :return: None
100
+ """
101
+
102
+ assert self.client.status == 'running', "Client is not connected to a scheduler. Please check your connection."
103
+
104
+ # TODO: select workers to send data to. self.client.scatter(data, direct=True, workers=self.workers)
105
+ f = self.client.scatter(data, direct=True) # send data to workers
106
+
107
+ # TODO: this is a memory leak. Find a way to release the futures once they are used to build a dask array in the client code.
108
+ self.futures.append(f)
109
+
110
+ to_send = {
111
+ 'rank': self.mpi_rank,
112
+ 'shape': data.shape,
113
+ 'dtype': data.dtype,
114
+ 'iteration': iteration,
115
+ 'future': f
116
+ }
117
+
118
+ q = Queue(array_name, client=self.client)
119
+ q.put(to_send)
120
+
121
+ # TODO: what to do if error ?
122
+
123
+ def get(self, key: str, default: Any = None, chunked: bool = False, delete: bool = True):
124
+ if chunked:
125
+ raise NotImplementedError() # TODO
126
+ else:
127
+ try:
128
+ with Lock(f'{LOCK_PREFIX}{key}'):
129
+ return Variable(f'{VARIABLE_PREFIX}{key}', client=self.client).get(timeout=0)
130
+ except TimeoutError:
131
+ return default
132
+ finally:
133
+ if delete:
134
+ with Lock(f'{LOCK_PREFIX}{key}'):
135
+ Variable(f'{VARIABLE_PREFIX}{key}', client=self.client).delete()
deisa/dask/deisa.py ADDED
@@ -0,0 +1,373 @@
1
+ # =============================================================================
2
+ # Copyright (C) 2025 Commissariat a l'energie atomique et aux energies alternatives (CEA)
3
+ #
4
+ # All rights reserved.
5
+ #
6
+ # Redistribution and use in source and binary forms, with or without
7
+ # modification, are permitted provided that the following conditions are met:
8
+ # * Redistributions of source code must retain the above copyright notice,
9
+ # this list of conditions and the following disclaimer.
10
+ # * Redistributions in binary form must reproduce the above copyright notice,
11
+ # this list of conditions and the following disclaimer in the documentation
12
+ # and/or other materials provided with the distribution.
13
+ # * Neither the names of CEA, nor the names of the contributors may be used to
14
+ # endorse or promote products derived from this software without specific
15
+ # prior written permission.
16
+ #
17
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18
+ # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
+ # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21
+ # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22
+ # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23
+ # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25
+ # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26
+ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
+ # POSSIBILITY OF SUCH DAMAGE.
28
+ # =============================================================================
29
+
30
+ import asyncio
31
+ import collections
32
+ import gc
33
+ import sys
34
+ import threading
35
+ import traceback
36
+ from typing import Callable, Union, Tuple, List, Final, Literal
37
+
38
+ import dask
39
+ import dask.array as da
40
+ import numpy as np
41
+ from dask.array import Array
42
+ from deisa.core.interface import IDeisa, SupportsSlidingWindow
43
+ from distributed import Client, Future, Queue, Variable, Lock
44
+
45
+ from deisa.dask.handshake import Handshake
46
+
47
+ SLIDING_WINDOW_THREAD_PREFIX: Final[str] = "deisa_sliding_window_callback_"
48
+ LOCK_PREFIX: Final[str] = "deisa_lock_"
49
+ VARIABLE_PREFIX: Final[str] = "deisa_variable_"
50
+ DEFAULT_SLIDING_WINDOW_SIZE: int = 1
51
+
52
+
53
+ class Deisa(IDeisa):
54
+ Callback_args = Union[str, Tuple[str], Tuple[str, int]] # array_name, window_size
55
+
56
+ def __init__(self, get_connection_info: Callable[[], Client], *args, **kwargs):
57
+ """
58
+ Initializes the distributed processing environment and configures workers using
59
+ a Dask scheduler. This class handles setting up a Dask client and ensures the
60
+ specified number of workers are available for distributed computation tasks.
61
+
62
+ :param get_connection_info: A function that returns a connected Dask Client.
63
+ :type get_connection_info: Callable
64
+ """
65
+ # dask.config.set({"distributed.deploy.lost-worker-timeout": 60, "distributed.workers.memory.spill":0.97, "distributed.workers.memory.target":0.95, "distributed.workers.memory.terminate":0.99 })
66
+
67
+ super().__init__(get_connection_info, *args, **kwargs)
68
+ self.client: Client = get_connection_info()
69
+
70
+ # blocking until all bridges are ready
71
+ handshake = Handshake('deisa', self.client, **kwargs)
72
+
73
+ self.mpi_comm_size = handshake.get_nb_bridges()
74
+ self.arrays_metadata = handshake.get_arrays_metadata()
75
+ self.sliding_window_callback_threads: dict[str, threading.Thread] = {}
76
+ self.sliding_window_callback_thread_lock = threading.Lock()
77
+
78
+ def __del__(self):
79
+ if hasattr(self, 'sliding_window_callback_threads'): # may not be the case if an exception is thrown in ctor
80
+ for thread in self.sliding_window_callback_threads.values():
81
+ self.__stop_join_thread(thread)
82
+ gc.collect()
83
+
84
+ @staticmethod
85
+ def __stop_join_thread(thread: threading.Thread):
86
+ thread.stop = True
87
+ thread.join()
88
+
89
+ # exc = getattr(thread, "exception", None)
90
+ # if exc:
91
+ # # print(f"Exception encountered: {exc['traceback']}", file=sys.stderr, flush=True)
92
+ # # raise exc['exception']
93
+ # pass
94
+
95
+ def close(self):
96
+ self.__del__()
97
+
98
+ def get_array(self, name: str, timeout=None) -> tuple[Array, int]:
99
+ """Retrieve a Dask array for a given array name."""
100
+
101
+ # if self.arrays_metadata is None:
102
+ # self.arrays_metadata = Queue("Arrays", client=self.client).get(timeout=timeout)
103
+ # arrays_metadata will look something like this:
104
+ # arrays_metadata = {
105
+ # 'global_t': {
106
+ # 'size': [20, 20]
107
+ # 'subsize': [10, 10]
108
+ # }
109
+ # 'global_p': {
110
+ # 'size': [100, 100]
111
+ # 'subsize': [50, 50]
112
+ # }
113
+
114
+ if self.arrays_metadata.get(name) is None:
115
+ raise ValueError(f"Array '{name}' is not known.")
116
+
117
+ res = []
118
+ iteration = 0
119
+ l = self.client.sync(self.__get_all_chunks, Queue(name, client=self.client),
120
+ self.mpi_comm_size, timeout=timeout)
121
+ for m in l:
122
+ assert type(m) is dict, "Metadata must be a dictionary."
123
+ assert type(m['future']) is Future, "Data future must be a Dask future."
124
+ m['da'] = da.from_delayed(dask.delayed(m['future']), m['shape'], dtype=m['dtype'])
125
+ res.append(m)
126
+ iteration = m['iteration']
127
+
128
+ # create dask array from blocks
129
+ res.sort(key=lambda x: x['rank']) # sort by mpi rank
130
+ chunks = [item['da'] for item in res] # extract ordered dask arrays
131
+ darr = self.__tile_dask_blocks(chunks, self.arrays_metadata[name]['size'])
132
+ return darr, iteration
133
+
134
+ @staticmethod
135
+ def __default_exception_handler(array_name, e):
136
+ print(f"Exception from {array_name} thread: {e}", file=sys.stderr, flush=True)
137
+
138
+ def register_sliding_window_callback(self,
139
+ callback: SupportsSlidingWindow.Callback,
140
+ array_name: str, window_size: int = DEFAULT_SLIDING_WINDOW_SIZE,
141
+ exception_handler: SupportsSlidingWindow.ExceptionHandler = __default_exception_handler) -> str:
142
+ """
143
+ Register a sliding-window callback for a single array.
144
+ """
145
+ parsed = [(array_name, window_size)]
146
+ return self._register_sliding_window_callbacks_impl(
147
+ callback,
148
+ parsed,
149
+ exception_handler=exception_handler,
150
+ when='AND')
151
+
152
+ def register_sliding_window_callbacks(self,
153
+ callback: SupportsSlidingWindow.Callback,
154
+ *callback_args: Callback_args,
155
+ exception_handler: SupportsSlidingWindow.ExceptionHandler = __default_exception_handler,
156
+ when: Literal['AND', 'OR'] = 'AND') -> str:
157
+ """
158
+ Register a sliding-window callback for one or more arrays.
159
+
160
+ Supports:
161
+ - "array"
162
+ - ("array", window_size)
163
+ - mixed forms
164
+ """
165
+ if not callback_args:
166
+ raise TypeError(
167
+ "register_sliding_window_callbacks requires at least one array name "
168
+ "or (name, window_size) tuple"
169
+ )
170
+
171
+ parsed: List[Tuple[str, int]] = []
172
+
173
+ for arg in callback_args:
174
+ if isinstance(arg, str):
175
+ parsed.append((arg, DEFAULT_SLIDING_WINDOW_SIZE))
176
+ elif isinstance(arg, tuple):
177
+ if len(arg) == 1:
178
+ parsed.append((arg[0], DEFAULT_SLIDING_WINDOW_SIZE))
179
+ elif len(arg) == 2:
180
+ name, ws = arg
181
+ if not isinstance(name, str) or not isinstance(ws, int):
182
+ raise TypeError("tuple must be (str, int)")
183
+ parsed.append((name, ws))
184
+ else:
185
+ raise TypeError("tuple must be (str,) or (str, int)")
186
+ else:
187
+ raise TypeError("callback_args must be str or tuple")
188
+
189
+ return self._register_sliding_window_callbacks_impl(
190
+ callback,
191
+ parsed,
192
+ exception_handler=exception_handler,
193
+ when=when)
194
+
195
+ def _register_sliding_window_callbacks_impl(self,
196
+ callback: SupportsSlidingWindow.Callback,
197
+ parsed: List[Tuple[str, int]],
198
+ *,
199
+ exception_handler: SupportsSlidingWindow.ExceptionHandler,
200
+ when: str) -> str:
201
+ """
202
+ Supports:
203
+ - (callback, "array_name", window_size=K)
204
+ - (callback, ("name1", k1), ("name2", k2), ..., when='AND')
205
+ - mixed: (callback, "a", ("b", 3)) -> "a" gets default window_size
206
+ """
207
+ if when not in ('AND', 'OR'):
208
+ raise ValueError("when must be 'AND' or 'OR'")
209
+
210
+ for array_name, _ in parsed:
211
+ if array_name not in self.arrays_metadata:
212
+ raise ValueError(f'unknown array name: {array_name}')
213
+
214
+ def queue_watcher(arrays: List[Tuple[str, int]]):
215
+ current_windows = {}
216
+ for arr_name, window_size in arrays:
217
+ current_windows[arr_name] = {
218
+ 'window': collections.deque(maxlen=window_size),
219
+ 'changed': False,
220
+ }
221
+
222
+ t = threading.current_thread()
223
+ while not getattr(t, "stop", False):
224
+ for arr_name, d in current_windows.items():
225
+ try:
226
+ darr, iteration = self.get_array(arr_name, timeout='1s')
227
+ d['window'].append(darr)
228
+ d['changed'] = True
229
+ windows = [list(dd['window']) for dd in current_windows.values()]
230
+
231
+ if when == 'OR':
232
+ callback(*windows, timestep=iteration)
233
+ d['changed'] = False
234
+ else: # AND
235
+ if all(dd['changed'] for dd in current_windows.values()):
236
+ callback(*windows, timestep=iteration)
237
+ for dd in current_windows.values():
238
+ dd['changed'] = False
239
+
240
+ except TimeoutError:
241
+ pass
242
+ except BaseException as e:
243
+ setattr(t, "exception", (e, traceback.format_exc()))
244
+ try:
245
+ exception_handler(arr_name, e)
246
+ except BaseException:
247
+ with self.sliding_window_callback_thread_lock:
248
+ print(
249
+ f"Exception thrown in exception handler for {arr_name}. "
250
+ f"Unregistering callback.",
251
+ file=sys.stderr,
252
+ )
253
+ self.unregister_sliding_window_callback(arr_name)
254
+
255
+ callback_id = self.__get_callback_id(*parsed)
256
+ if callback_id not in self.sliding_window_callback_threads:
257
+ thread = threading.Thread(
258
+ target=queue_watcher,
259
+ name=f"{SLIDING_WINDOW_THREAD_PREFIX}{callback_id}",
260
+ args=(parsed,),
261
+ )
262
+ self.sliding_window_callback_threads[callback_id] = thread
263
+ thread.start()
264
+
265
+ return callback_id
266
+
267
+ def unregister_sliding_window_callback(self, *array_names: Callback_args) -> None:
268
+ """
269
+ Unregisters a sliding window callback for the specified array name. This method removes the
270
+ callback thread associated with the array name. If the thread exists, it stops the thread and waits
271
+ for it to finish execution.
272
+
273
+ :param array_name: The name of the array for which the sliding window callback is to be unregistered.
274
+ Must be a string.
275
+ :return: None
276
+ """
277
+
278
+ callback_id = self.__get_callback_id(*array_names)
279
+ thread = self.sliding_window_callback_threads.pop(callback_id, None)
280
+ if thread:
281
+ self.__stop_join_thread(thread)
282
+
283
+ def set(self, key: str, data: Union[Future, object], chunked=False):
284
+ if chunked:
285
+ raise NotImplementedError() # TODO
286
+ else:
287
+ with Lock(f'{LOCK_PREFIX}{key}'):
288
+ Variable(f'{VARIABLE_PREFIX}{key}', client=self.client).set(data)
289
+
290
+ def delete(self, key: str) -> None:
291
+ with Lock(f'{LOCK_PREFIX}{key}'):
292
+ Variable(f'{VARIABLE_PREFIX}{key}', client=self.client).delete()
293
+
294
+ @staticmethod
295
+ async def __get_all_chunks(q: Queue, mpi_comm_size: int, timeout=None) -> list[tuple[dict, Future]]:
296
+ """This will return a list of tuples (metadata, data_future) for all chunks in the queue."""
297
+ try:
298
+ res = []
299
+ for _ in range(mpi_comm_size):
300
+ res.append(q.get(timeout=timeout))
301
+ return await asyncio.gather(*res)
302
+ except asyncio.TimeoutError:
303
+ raise TimeoutError(f"Timeout reached while waiting for chunks in queue '{q.name}'.")
304
+
305
+ @staticmethod
306
+ def __tile_dask_blocks(blocks: list[da.Array], global_shape: tuple[int, ...]) -> da.Array:
307
+ """
308
+ Given a flat list of N-dimensional Dask arrays, tile them into a single Dask array.
309
+ The tiling layout is inferred from the provided global shape.
310
+
311
+ Parameters:
312
+ blocks (list of dask.array): Flat list of Dask arrays. All must have the same shape.
313
+ global_shape (tuple of int): Shape of the full array to reconstruct.
314
+
315
+ Returns:
316
+ dask.array.Array: Combined tiled Dask array.
317
+ """
318
+ if not blocks:
319
+ raise ValueError("No blocks provided.")
320
+
321
+ block_shape = blocks[0].shape
322
+ ndim = len(block_shape)
323
+
324
+ if len(global_shape) != ndim:
325
+ raise ValueError("global_shape must have the same number of dimensions as blocks.")
326
+
327
+ # Check that all blocks have the same shape
328
+ for b in blocks:
329
+ if b.shape != block_shape:
330
+ raise ValueError("All blocks must have the same shape.")
331
+
332
+ # Compute how many blocks are needed per dimension
333
+ tile_counts = tuple(g // b for g, b in zip(global_shape, block_shape))
334
+
335
+ if np.prod(tile_counts) != len(blocks):
336
+ raise ValueError(
337
+ f"Mismatch between number of blocks ({len(blocks)}) and expected number from global_shape {global_shape} "
338
+ f"with block shape {block_shape} (expected {np.prod(tile_counts)} blocks)."
339
+ )
340
+
341
+ # Reshape the flat list into an N-dimensional grid of blocks
342
+ def nest_blocks(flat_blocks, shape):
343
+ """Nest a flat list of blocks into a nested list matching the grid shape."""
344
+ if len(shape) == 1:
345
+ return flat_blocks
346
+ else:
347
+ size = shape[0]
348
+ stride = int(len(flat_blocks) / size)
349
+ return [nest_blocks(flat_blocks[i * stride:(i + 1) * stride], shape[1:]) for i in range(size)]
350
+
351
+ nested = nest_blocks(blocks, tile_counts)
352
+
353
+ # Use da.block to combine blocks
354
+ return da.block(nested)
355
+
356
+ @staticmethod
357
+ def __get_callback_id(*callback_args: Callback_args) -> str:
358
+ """Flatten callback_args to a tuple of array names."""
359
+ array_names = []
360
+ for arg in callback_args:
361
+ if isinstance(arg, str):
362
+ array_names.append(arg)
363
+ elif isinstance(arg, tuple):
364
+ if len(arg) == 1 and isinstance(arg[0], str):
365
+ array_names.append(arg[0])
366
+ elif len(arg) == 2 and isinstance(arg[0], str) and isinstance(arg[1], int):
367
+ array_names.append(arg[0])
368
+ else:
369
+ raise TypeError(
370
+ "Tuple callback_args must be either (array_name,) or (array_name, window_size: int)")
371
+ else:
372
+ raise TypeError("callback_args must be str or a tuple")
373
+ return str(array_names)
@@ -0,0 +1,146 @@
1
+ # =============================================================================
2
+ # Copyright (C) 2025 Commissariat a l'energie atomique et aux energies alternatives (CEA)
3
+ #
4
+ # All rights reserved.
5
+ #
6
+ # Redistribution and use in source and binary forms, with or without
7
+ # modification, are permitted provided that the following conditions are met:
8
+ # * Redistributions of source code must retain the above copyright notice,
9
+ # this list of conditions and the following disclaimer.
10
+ # * Redistributions in binary form must reproduce the above copyright notice,
11
+ # this list of conditions and the following disclaimer in the documentation
12
+ # and/or other materials provided with the distribution.
13
+ # * Neither the names of CEA, nor the names of the contributors may be used to
14
+ # endorse or promote products derived from this software without specific
15
+ # prior written permission.
16
+ #
17
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18
+ # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
+ # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21
+ # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22
+ # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23
+ # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25
+ # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26
+ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
+ # POSSIBILITY OF SUCH DAMAGE.
28
+ # =============================================================================
29
+
30
+ import asyncio
31
+
32
+ from dask.distributed import Variable
33
+ from distributed import Client, Future, get_client, Lock
34
+
35
+
36
+ class Handshake:
37
+ DEISA_HANDSHAKE_ACTOR_FUTURE_VARIABLE = 'deisa_handshake_actor_future'
38
+ DEISA_WAIT_FOR_GO_VARIABLE = 'deisa_handshake_wait_for_go'
39
+
40
+ class HandshakeActor:
41
+ bridges = []
42
+ max_bridges = 0
43
+ arrays_metadata = {}
44
+ analytics_ready = False
45
+
46
+ def __init__(self):
47
+ self.bridges = []
48
+ self.max_bridges = 0
49
+ self.arrays_metadata = {}
50
+ self.analytics_ready = False
51
+ self.client = get_client()
52
+
53
+ def add_bridge(self, id: int, max: int) -> None:
54
+ if max == 0:
55
+ raise ValueError('max cannot be 0.')
56
+ elif self.max_bridges == 0:
57
+ self.max_bridges = max
58
+ elif self.max_bridges != max:
59
+ raise ValueError(f'Value {max} for bridge {id} is unexpected. Expecting max={self.max_bridges}.')
60
+ elif len(self.bridges) >= max:
61
+ raise RuntimeError(f'add_bridge cannot be called more than {max} times.')
62
+
63
+ self.bridges.append(id)
64
+ if self.__is_everyone_ready():
65
+ self.__go()
66
+
67
+ def set_analytics_ready(self) -> None:
68
+ self.analytics_ready = True
69
+ if self.__are_bridges_ready():
70
+ self.__go()
71
+
72
+ def set_arrays_metadata(self, arrays_metadata: dict) -> None | Future:
73
+ self.arrays_metadata = arrays_metadata
74
+
75
+ def get_arrays_metadata(self) -> dict | Future:
76
+ return self.arrays_metadata
77
+
78
+ def get_max_bridges(self) -> int | Future:
79
+ return self.max_bridges
80
+
81
+ def __are_bridges_ready(self) -> bool | Future:
82
+ return self.max_bridges != 0 and len(self.bridges) == self.max_bridges
83
+
84
+ def __is_everyone_ready(self) -> bool | Future:
85
+ return self.__are_bridges_ready() and self.analytics_ready
86
+
87
+ def __go(self):
88
+ Variable(Handshake.DEISA_WAIT_FOR_GO_VARIABLE, client=self.client).set(None)
89
+
90
+ def __init__(self, who: str, client: Client, **kwargs):
91
+ self.client = client
92
+ # self.client.direct_to_workers() # TODO
93
+ self.handshake_actor = self.__get_handshake_actor()
94
+ assert self.handshake_actor is not None
95
+
96
+ if who == 'bridge':
97
+ self.start_bridge(**kwargs)
98
+ elif who == 'deisa':
99
+ self.start_deisa(**kwargs)
100
+ else:
101
+ raise ValueError("Expecting 'bridge' or 'deisa'.")
102
+
103
+ def start_bridge(self, id: int, max: int, arrays_metadata: dict, wait_for_go=True) -> None:
104
+ """
105
+ Bridge must wait for analytics to be ready.
106
+ """
107
+ assert self.handshake_actor is not None
108
+ self.handshake_actor.add_bridge(id, max)
109
+
110
+ if id == 0:
111
+ self.handshake_actor.set_arrays_metadata(arrays_metadata).result()
112
+
113
+ # wait for go
114
+ if wait_for_go:
115
+ self.__wait_for_go()
116
+
117
+ def start_deisa(self, wait_for_go=True) -> None:
118
+ """
119
+ When analytics is ready, notify all Bridges
120
+ """
121
+ assert self.handshake_actor is not None
122
+ self.handshake_actor.set_analytics_ready()
123
+
124
+ # wait for go
125
+ if wait_for_go:
126
+ self.__wait_for_go()
127
+
128
+ def get_arrays_metadata(self) -> dict:
129
+ assert self.handshake_actor is not None
130
+ return self.handshake_actor.get_arrays_metadata().result()
131
+
132
+ def get_nb_bridges(self) -> int:
133
+ assert self.handshake_actor is not None
134
+ return self.handshake_actor.get_max_bridges().result()
135
+
136
+ def __get_handshake_actor(self) -> HandshakeActor:
137
+ with Lock(Handshake.DEISA_HANDSHAKE_ACTOR_FUTURE_VARIABLE):
138
+ try:
139
+ return Variable(Handshake.DEISA_HANDSHAKE_ACTOR_FUTURE_VARIABLE, client=self.client).get(timeout=0).result()
140
+ except asyncio.exceptions.TimeoutError:
141
+ actor_future = self.client.submit(Handshake.HandshakeActor, actor=True)
142
+ Variable(Handshake.DEISA_HANDSHAKE_ACTOR_FUTURE_VARIABLE, client=self.client).set(actor_future)
143
+ return actor_future.result()
144
+
145
+ def __wait_for_go(self) -> None:
146
+ Variable(Handshake.DEISA_WAIT_FOR_GO_VARIABLE, client=self.client).get()
deisa/dask/py.typed ADDED
File without changes
@@ -0,0 +1,38 @@
1
+ Metadata-Version: 2.4
2
+ Name: deisa-dask
3
+ Version: 0.3.0
4
+ Summary: Deisa: Dask-Enabled In Situ Analytics
5
+ Home-page: https://github.com/deisa-project/deisa-dask
6
+ Author: Benoît Martin
7
+ Author-email: bmartin@cea.fr
8
+ License: MIT
9
+ Project-URL: Bug Reports, https://github.com/deisa-project/deisa-dask/issues
10
+ Project-URL: Source, https://github.com/deisa-project/deisa-dask
11
+ Keywords: deisa in-situ
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Development Status :: 3 - Alpha
16
+ Requires-Python: >=3.10
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: deisa-core==0.1.0
20
+ Requires-Dist: dask
21
+ Requires-Dist: distributed
22
+ Provides-Extra: test
23
+ Requires-Dist: pytest; extra == "test"
24
+ Requires-Dist: pytest-xdist; extra == "test"
25
+ Requires-Dist: numpy; extra == "test"
26
+ Dynamic: author
27
+ Dynamic: author-email
28
+ Dynamic: classifier
29
+ Dynamic: description-content-type
30
+ Dynamic: home-page
31
+ Dynamic: keywords
32
+ Dynamic: license
33
+ Dynamic: license-file
34
+ Dynamic: project-url
35
+ Dynamic: provides-extra
36
+ Dynamic: requires-dist
37
+ Dynamic: requires-python
38
+ Dynamic: summary
@@ -0,0 +1,11 @@
1
+ deisa/dask/__init__.py,sha256=jY2FOi4BxYRZFG_m21msp5Y4D9VXKQNrg125TeOiUIg,1621
2
+ deisa/dask/__version__.py,sha256=pVM9pFBko7_pioepIaFDX4DzdcE3AFjCcXqZBtQYJK0,349
3
+ deisa/dask/bridge.py,sha256=TV2atJngLjR0-9FZaDAWcIQOk2ybnufzRQKGg9hak3Y,6268
4
+ deisa/dask/deisa.py,sha256=9FClbHqnVswbe1ZjTa45FeL9mfPn9Je6fJpwWT8sHWg,16524
5
+ deisa/dask/handshake.py,sha256=apdD-VogFTf7VNGtDunqbsSc6eLQLBcZnay73rYrYBY,6029
6
+ deisa/dask/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
+ deisa_dask-0.3.0.dist-info/licenses/LICENSE,sha256=z14wBfeyikSU-KT1OSq2Z7Dnl5vho48YmyjX-kWgXRM,1070
8
+ deisa_dask-0.3.0.dist-info/METADATA,sha256=NnWe69j0h0UxF1jdNmIbSNcN_M4ARKM9hlwkr_L7QzI,1167
9
+ deisa_dask-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
10
+ deisa_dask-0.3.0.dist-info/top_level.txt,sha256=I_RhfpzuA-XBujEw_fY6pzeVVty-gER2hkLZ6WgM7Uw,6
11
+ deisa_dask-0.3.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (80.9.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 deisa-project
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ deisa