askap-patchwork 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
patchwork/write.py ADDED
@@ -0,0 +1,502 @@
1
+ """Writing a mosaic: making its tiles with an executor, writing them to FITS or zarr, and writing its noise.
2
+
3
+ The work is handed to a ``concurrent.futures.Executor``: threads, or dask
4
+ workers through ``patchwork.pipeline.flow``.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import logging
10
+ import sys
11
+ import time
12
+ from concurrent.futures import (
13
+ FIRST_COMPLETED,
14
+ Executor,
15
+ Future,
16
+ ThreadPoolExecutor,
17
+ wait,
18
+ )
19
+ from contextlib import ExitStack, closing
20
+ from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
21
+
22
+ import numpy as np
23
+ from astropy.io import fits
24
+
25
+ from patchwork.io import (
26
+ beams_header,
27
+ create_output_fits,
28
+ data_index,
29
+ data_location,
30
+ read_rows,
31
+ spectral_axis,
32
+ write_block,
33
+ write_rows,
34
+ )
35
+ from patchwork.logging import logger
36
+ from patchwork.resample import resolve_backend, tile_threads
37
+ from patchwork.tile import mosaic_tile
38
+
39
+ if TYPE_CHECKING:
40
+ from collections.abc import Callable, Generator, Iterable
41
+
42
+ from numpy.typing import NDArray
43
+
44
+ from patchwork.io import DataLocation, Slices
45
+ from patchwork.options import MosaicOptions, RunOptions
46
+ from patchwork.plan import MosaicNames, MosaicPlan
47
+ from patchwork.tile import TileResult
48
+
49
+
50
+ def create_outputs(plan: MosaicPlan, run_options: RunOptions) -> None:
51
+ """Create the empty outputs of a planned mosaic: its FITS images and weights, which tiles are written into, or all its zarr stores."""
52
+ if run_options.output_format == "zarr":
53
+ create_zarr_outputs(plan, run_options.zarr_chunk)
54
+ return
55
+ files = (*plan.names.images, *plan.names.weights)
56
+ if run_options.fits_writer == "workers":
57
+ # The tasks write the noise too, so it is created with the rest
58
+ files = (*files, *plan.names.noises)
59
+ for path, header in zip(files, plan.headers[: len(files)], strict=True):
60
+ create_output_fits(path, header, plan.beams)
61
+
62
+
63
+ def _output_channels(plan: MosaicPlan, channels: slice) -> slice:
64
+ """The channels of the outputs of a mosaic that hold some channels of its inputs."""
65
+ start = channels.start - (plan.channels.start or 0)
66
+ return slice(start, start + channels.stop - channels.start)
67
+
68
+
69
+ def write_tiles(plan: MosaicPlan, results: Iterable[TileResult]) -> None:
70
+ """Write finished tiles into the output FITS files of a mosaic."""
71
+ with ExitStack() as stack:
72
+ image_hdus = [
73
+ stack.enter_context(fits.open(path, mode="update", memmap=True))[0]
74
+ for path in plan.names.images
75
+ ]
76
+ weight_hdus = [
77
+ stack.enter_context(fits.open(path, mode="update", memmap=True))[0]
78
+ for path in plan.names.weights
79
+ ]
80
+ for result in results:
81
+ channels = _output_channels(plan, result.channels)
82
+ for hdu, block in zip(
83
+ (*image_hdus, *weight_hdus),
84
+ (*result.images, *result.weights),
85
+ strict=True,
86
+ ):
87
+ write_block(hdu, result.slices, block, channels)
88
+ logger.debug(f"Wrote tile {result.slices} of channels {result.channels}")
89
+
90
+
91
+ def write_noise(plan: MosaicPlan, strip_rows: int = 1024) -> None:
92
+ """Write the noise of each weight of a written mosaic, the inverse of its square root, channel by channel.
93
+
94
+ Each noise image has the header, and any BEAMS table, of its weight, in the
95
+ units of the images of its group. Pixels with no weight are blank.
96
+
97
+ Args:
98
+ plan (MosaicPlan): The plan of the mosaic, whose files are written
99
+ strip_rows (int, optional): Rows of the files worked on at a time. Defaults to 1024.
100
+ """
101
+ if not plan.names.noises:
102
+ return
103
+ for group, (weight_path, noise_path) in enumerate(
104
+ zip(plan.names.weights, plan.names.noises, strict=True)
105
+ ):
106
+ image_path = plan.names.images[plan.groups.index(group)]
107
+ with fits.open(weight_path, memmap=True) as weight_hdul:
108
+ header = weight_hdul[0].header.copy()
109
+ header["BUNIT"] = fits.getheader(image_path).get("BUNIT", "")
110
+ create_output_fits(noise_path, header)
111
+ if "BEAMS" in weight_hdul:
112
+ fits.append(
113
+ noise_path, weight_hdul["BEAMS"].data, weight_hdul["BEAMS"].header
114
+ )
115
+ weight = weight_hdul[0].data
116
+ with fits.open(noise_path, mode="update", memmap=True) as noise_hdul:
117
+ noise = noise_hdul[0].data
118
+ axis = spectral_axis(header)
119
+ n_channels = 1 if axis is None else int(header[f"NAXIS{axis}"])
120
+ for channel in range(n_channels):
121
+ for row in range(0, int(header["NAXIS2"]), strip_rows):
122
+ index = data_index(
123
+ header,
124
+ (slice(row, row + strip_rows), slice(None)),
125
+ slice(channel, channel + 1),
126
+ )
127
+ noise[index] = noise_of(np.asarray(weight[index]))
128
+ logger.info(f"Written {noise_path}")
129
+
130
+
131
+ def noise_of(weight: NDArray[np.floating]) -> NDArray[np.float32]:
132
+ """The noise of a weight, the inverse of its square root, which is blank where there is no weight."""
133
+ with np.errstate(divide="ignore", invalid="ignore"):
134
+ return np.where(weight > 0, 1 / np.sqrt(weight), np.nan).astype(np.float32)
135
+
136
+
137
+ def create_zarr_outputs(plan: MosaicPlan, chunk: int) -> None:
138
+ """Create the zarr stores of a mosaic, laid out as rm-lite reads them.
139
+
140
+ Each is one (channel, y, x) float32 array, chunked over every channel, with
141
+ the FITS header of the cube and the restoring beam of each channel in its
142
+ attributes.
143
+ """
144
+ import zarr # noqa: PLC0415
145
+ from zarr.codecs import BloscCodec # noqa: PLC0415
146
+
147
+ n_channels = plan.channels.stop - plan.channels.start
148
+ for path, header in zip(
149
+ (*plan.names.images, *plan.names.weights, *plan.names.noises),
150
+ plan.headers,
151
+ strict=True,
152
+ ):
153
+ array = zarr.create_array(
154
+ store=str(path),
155
+ shape=(n_channels, *plan.frame.shape),
156
+ chunks=(
157
+ n_channels,
158
+ min(chunk, plan.frame.shape[0]),
159
+ min(chunk, plan.frame.shape[1]),
160
+ ),
161
+ dtype="float32",
162
+ fill_value=np.nan,
163
+ compressors=(BloscCodec(cname="lz4", clevel=5, shuffle="shuffle"),),
164
+ overwrite=True,
165
+ )
166
+ # The header holds one beam only if every channel shares it, as a FITS cube's does
167
+ stored = header.copy()
168
+ beams_header(stored, plan.beams)
169
+ array.attrs["fits_header"] = stored.tostring()
170
+ array.attrs["beams"] = [
171
+ list(beam) if beam.is_usable() else None for beam in plan.beams
172
+ ]
173
+
174
+
175
+ def mosaic_tile_to_zarr(
176
+ tile: Slices,
177
+ plan: MosaicPlan,
178
+ options: MosaicOptions,
179
+ channel_chunk: int,
180
+ ) -> None:
181
+ """Make a tile over every channel, a few channels at a time, and write it into the zarr stores of the mosaic.
182
+
183
+ One task owns each chunk of the stores, so tasks never write the same chunk.
184
+ """
185
+ import zarr # noqa: PLC0415
186
+
187
+ start, stop = plan.channels.start, plan.channels.stop
188
+ results = [
189
+ mosaic_tile(
190
+ tile,
191
+ plan.frame,
192
+ plan.inputs,
193
+ options,
194
+ slice(first, min(first + channel_chunk, stop)),
195
+ )
196
+ for first in range(start, stop, channel_chunk)
197
+ ]
198
+ for index, path in enumerate(plan.names.images):
199
+ zarr.open_array(str(path), mode="r+")[(slice(None), *tile)] = np.concatenate(
200
+ [result.images[index] for result in results]
201
+ )
202
+ for index, path in enumerate(plan.names.weights):
203
+ zarr.open_array(str(path), mode="r+")[(slice(None), *tile)] = np.concatenate(
204
+ [result.weights[index] for result in results]
205
+ )
206
+
207
+
208
+ def noise_zarr_tile(tile: Slices, plan: MosaicPlan) -> None:
209
+ """Write a tile of the noise stores of a mosaic from its weights."""
210
+ import zarr # noqa: PLC0415
211
+
212
+ index = (slice(None), *tile)
213
+ for weight_path, noise_path in zip(
214
+ plan.names.weights, plan.names.noises, strict=True
215
+ ):
216
+ weight = np.asarray(zarr.open_array(str(weight_path), mode="r")[index])
217
+ zarr.open_array(str(noise_path), mode="r+")[index] = noise_of(weight)
218
+
219
+
220
+ class FitsLocations(NamedTuple):
221
+ """Where the data of each FITS output of a mosaic lies, for tasks to write their strips into"""
222
+
223
+ images: tuple[DataLocation, ...]
224
+ """The data of each image"""
225
+ weights: tuple[DataLocation, ...]
226
+ """The data of each weight"""
227
+ noises: tuple[DataLocation, ...]
228
+ """The data of each noise image"""
229
+
230
+
231
+ def fits_locations(plan: MosaicPlan) -> FitsLocations:
232
+ """Find where the data of each created FITS output of a mosaic lies."""
233
+ names = plan.names
234
+ return FitsLocations(
235
+ images=tuple(map(data_location, names.images)),
236
+ weights=tuple(map(data_location, names.weights)),
237
+ noises=tuple(map(data_location, names.noises)),
238
+ )
239
+
240
+
241
+ def mosaic_strip_to_fits(
242
+ plan: MosaicPlan,
243
+ strip: Slices,
244
+ options: MosaicOptions,
245
+ channels: slice,
246
+ locations: FitsLocations,
247
+ ) -> None:
248
+ """Make a strip of whole rows over some channels, and write it into the FITS outputs of the mosaic.
249
+
250
+ Each task writes only its own rows, so no two tasks write the same bytes.
251
+ """
252
+ result = mosaic_tile(strip, plan.frame, plan.inputs, options, channels)
253
+ written = _output_channels(plan, result.channels)
254
+ files = (*locations.images, *locations.weights)
255
+ blocks = (*result.images, *result.weights)
256
+
257
+ def write(index: int) -> None:
258
+ write_rows(files[index], strip[0], written, blocks[index])
259
+
260
+ threads = 1
261
+ if resolve_backend(options.resample.backend) == "fused":
262
+ threads = min(tile_threads(options.resample.tile_threads), len(files))
263
+ started = time.perf_counter()
264
+ if threads == 1:
265
+ for index in range(len(files)):
266
+ write(index)
267
+ else:
268
+ # A tile made on several threads would otherwise wait for one to write it
269
+ with ThreadPoolExecutor(threads, thread_name_prefix="patchwork-write") as pool:
270
+ list(pool.map(write, range(len(files))))
271
+ if logger.isEnabledFor(logging.DEBUG):
272
+ written_bytes = sum(block.nbytes for block in blocks)
273
+ logger.debug(
274
+ f"Wrote rows {strip[0].start}-{strip[0].stop}, channels {written.start}-{written.stop} "
275
+ f"({written_bytes / 2**20:.0f} MB) in {time.perf_counter() - started:.1f} s on {threads} threads"
276
+ )
277
+
278
+
279
+ def noise_fits_strip(strip: Slices, locations: FitsLocations) -> None:
280
+ """Write a strip of the FITS noise outputs of a mosaic from its weights."""
281
+ rows = strip[0]
282
+ for weight_location, noise_location in zip(
283
+ locations.weights, locations.noises, strict=True
284
+ ):
285
+ for channel in range(weight_location.n_channels):
286
+ noise = noise_of(read_rows(weight_location, rows, channel))
287
+ write_rows(noise_location, rows, slice(channel, channel + 1), noise[None])
288
+
289
+
290
+ Call = tuple["Callable[..., Any]", tuple[Any, ...]]
291
+ """A function and the arguments to call it with"""
292
+
293
+ Result = TypeVar("Result")
294
+
295
+
296
+ def _duration(seconds: float) -> str:
297
+ """A duration such as ``42s``, ``3m 05s`` or ``1h 02m``."""
298
+ minutes, whole_seconds = divmod(round(seconds), 60)
299
+ hours, minutes = divmod(minutes, 60)
300
+ if hours:
301
+ return f"{hours}h {minutes:02d}m"
302
+ if minutes:
303
+ return f"{minutes}m {whole_seconds:02d}s"
304
+ return f"{whole_seconds}s"
305
+
306
+
307
+ def logged_progress(
308
+ results: Iterable[Result], total: int, what: str, interval_s: float = 30.0
309
+ ) -> Generator[Result, None, None]:
310
+ """Pass results on as they arrive, logging how many of ``total`` are done every ``interval_s`` seconds, and once all are.
311
+
312
+ A result is counted once it has been taken, so for tiles written as they
313
+ arrive the count is of tiles made and written.
314
+
315
+ Args:
316
+ results (Iterable[Result]): The results, such as those of ``bounded_results``
317
+ total (int): How many results there are
318
+ what (str): What the results are, such as ``tiles``
319
+ interval_s (float, optional): The least time in seconds between messages. Defaults to 30.0.
320
+
321
+ Yields:
322
+ Result: Each result, unchanged
323
+ """
324
+ start = logged = time.monotonic()
325
+ done = 0
326
+ for result in results:
327
+ yield result
328
+ done += 1
329
+ now = time.monotonic()
330
+ if done < total and now - logged < interval_s:
331
+ continue
332
+ logged, elapsed = now, now - start
333
+ message = f"Made {done} of {total} {what} ({done / total:.0%}) in {_duration(elapsed)}"
334
+ if done < total:
335
+ message += f", about {_duration(elapsed / done * (total - done))} left"
336
+ logger.info(message)
337
+
338
+
339
+ def bounded_results(
340
+ executor: Executor,
341
+ calls: Iterable[Call],
342
+ limit: Callable[[], int],
343
+ refresh_s: float = 5.0,
344
+ ) -> Generator[Any, None, None]:
345
+ """The results of calls made by an executor, as they finish, with no more than ``limit()`` submitted and not yet taken.
346
+
347
+ The next call is submitted only once a result is taken. An executor that
348
+ hands results back as soon as they finish, as dask's does, therefore never
349
+ holds more than ``limit()`` of them in this process, however slowly they are
350
+ taken. Should a call fail, or the results stop being taken, the calls still
351
+ waiting are cancelled.
352
+
353
+ Args:
354
+ executor (Executor): Makes the calls
355
+ calls (Iterable[Call]): The functions to call, and their arguments
356
+ limit (Callable[[], int]): The most calls submitted and not yet taken. It is asked again every ``refresh_s`` seconds, so it can follow a cluster as it scales.
357
+ refresh_s (float, optional): Seconds between asking ``limit`` again. Defaults to 5.0.
358
+
359
+ Yields:
360
+ Any: The result of each call, in the order they finish
361
+ """
362
+ remaining = iter(calls)
363
+ pending: set[Future[Any]] = set()
364
+ size, asked = max(limit(), 1), time.monotonic()
365
+ try:
366
+ while True:
367
+ if time.monotonic() - asked > refresh_s:
368
+ size, asked = max(limit(), 1), time.monotonic()
369
+ while len(pending) < size and (call := next(remaining, None)) is not None:
370
+ function, args = call
371
+ pending.add(executor.submit(function, *args))
372
+ if not pending:
373
+ return
374
+ done, pending = wait(pending, return_when=FIRST_COMPLETED)
375
+ for future in done:
376
+ yield future.result()
377
+ finally:
378
+ for future in pending:
379
+ future.cancel()
380
+
381
+
382
+ def run_tiles(
383
+ plan: MosaicPlan,
384
+ options: MosaicOptions,
385
+ run_options: RunOptions,
386
+ executor: Executor,
387
+ share: Callable[[Any], Any] | None = None,
388
+ capacity: Callable[[], int] | None = None,
389
+ ) -> MosaicNames:
390
+ """Make the tiles of a planned mosaic with an executor, write them, and finish the mosaic.
391
+
392
+ FITS files are written by this process as tiles arrive. Zarr stores are
393
+ written by the tasks, one chunk each. Each tile is finished once written,
394
+ so only the noise of the mosaic is written after them. No more than
395
+ ``tasks_in_flight`` tiles are made at once, so this process holds no more
396
+ than that many.
397
+
398
+ Args:
399
+ plan (MosaicPlan): The plan, from ``patchwork.plan.plan_mosaic``, whose outputs are created
400
+ options (MosaicOptions): The mosaicking options
401
+ run_options (RunOptions): How the mosaic is made and written
402
+ executor (Executor): Runs the tasks, such as a thread pool, or a dask ``Client.get_executor()``
403
+ share (Callable[[Any], Any] | None, optional): Turns the plan into what is sent to each task, such as dask's ``scatter``. Defaults to None, sending it as it is.
404
+ capacity (Callable[[], int] | None, optional): How many tasks the executor runs at once, asked again as tiles are made, such as the threads of a dask cluster as it scales. Without ``tasks_in_flight`` twice this many tiles are made at once. Defaults to None, one.
405
+
406
+ Returns:
407
+ MosaicNames: The files the mosaic was written to
408
+ """
409
+ shared = plan if share is None else share(plan)
410
+ # Only a FITS mosaic written by this process holds the finished tiles, so
411
+ # only it needs a bound; the others queue every task, so an adaptive dask
412
+ # cluster sees the whole mosaic waiting and grows to its maximum at once
413
+ held_here = (
414
+ run_options.output_format == "fits" and run_options.fits_writer != "workers"
415
+ )
416
+
417
+ def in_flight() -> int:
418
+ if run_options.tasks_in_flight is not None:
419
+ return run_options.tasks_in_flight
420
+ if not held_here:
421
+ return sys.maxsize
422
+ return 2 * max(1 if capacity is None else capacity(), 1)
423
+
424
+ def results(calls: Iterable[Call]) -> closing[Generator[Any, None, None]]:
425
+ return closing(bounded_results(executor, calls, in_flight))
426
+
427
+ chunks = [
428
+ slice(first, min(first + run_options.channel_chunk, plan.channels.stop))
429
+ for first in range(
430
+ plan.channels.start, plan.channels.stop, run_options.channel_chunk
431
+ )
432
+ ]
433
+ if run_options.output_format == "fits" and run_options.fits_writer == "workers":
434
+ _run_strips(plan, options, shared, chunks, results)
435
+ return plan.names
436
+ if run_options.output_format == "fits":
437
+ logger.info(
438
+ f"Making {len(plan.tiles)} tiles in each of {len(chunks)} chunks of channels, "
439
+ f"{in_flight()} at once"
440
+ )
441
+ with results(
442
+ (_plan_tile, (shared, tile, options, chunk))
443
+ for chunk in chunks
444
+ for tile in plan.tiles
445
+ ) as tiles_made:
446
+ write_tiles(
447
+ plan,
448
+ logged_progress(tiles_made, len(plan.tiles) * len(chunks), "tiles"),
449
+ )
450
+ write_noise(plan)
451
+ return plan.names
452
+
453
+ logger.info(
454
+ f"Making {len(plan.tiles)} zarr chunks"
455
+ + (
456
+ f", {in_flight()} at once"
457
+ if run_options.tasks_in_flight
458
+ else ", all queued"
459
+ )
460
+ )
461
+ with results(
462
+ (mosaic_tile_to_zarr, (tile, shared, options, run_options.channel_chunk))
463
+ for tile in plan.tiles
464
+ ) as chunks_made:
465
+ list(logged_progress(chunks_made, len(plan.tiles), "zarr chunks"))
466
+ if plan.names.noises:
467
+ with results((noise_zarr_tile, (tile, shared)) for tile in plan.tiles) as noise:
468
+ list(noise)
469
+ return plan.names
470
+
471
+
472
+ def _run_strips(
473
+ plan: MosaicPlan,
474
+ options: MosaicOptions,
475
+ shared: Any,
476
+ chunks: list[slice],
477
+ results: Callable[[Iterable[Call]], closing[Generator[Any, None, None]]],
478
+ ) -> None:
479
+ """Make the strips of a planned FITS mosaic, each written by its task, then write their noise."""
480
+ locations = fits_locations(plan)
481
+ logger.info(
482
+ f"Making {len(plan.tiles)} strips in each of {len(chunks)} chunks of channels, "
483
+ "each written by its task"
484
+ )
485
+ with results(
486
+ (mosaic_strip_to_fits, (shared, strip, options, chunk, locations))
487
+ for chunk in chunks
488
+ for strip in plan.tiles
489
+ ) as made:
490
+ list(logged_progress(made, len(plan.tiles) * len(chunks), "strips"))
491
+ if plan.names.noises:
492
+ with results(
493
+ (noise_fits_strip, (strip, locations)) for strip in plan.tiles
494
+ ) as noise:
495
+ list(noise)
496
+
497
+
498
+ def _plan_tile(
499
+ plan: MosaicPlan, tile: Slices, options: MosaicOptions, channels: slice
500
+ ) -> TileResult:
501
+ """Make one tile of a planned mosaic, over some channels."""
502
+ return mosaic_tile(tile, plan.frame, plan.inputs, options, channels)