askap-patchwork 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askap_patchwork-0.1.0.dist-info/METADATA +179 -0
- askap_patchwork-0.1.0.dist-info/RECORD +33 -0
- askap_patchwork-0.1.0.dist-info/WHEEL +4 -0
- askap_patchwork-0.1.0.dist-info/entry_points.txt +4 -0
- askap_patchwork-0.1.0.dist-info/licenses/LICENSE +28 -0
- patchwork/__init__.py +11 -0
- patchwork/__main__.py +8 -0
- patchwork/_version.py +24 -0
- patchwork/_version.pyi +2 -0
- patchwork/cli.py +60 -0
- patchwork/combine.py +146 -0
- patchwork/convolve.py +534 -0
- patchwork/coordinates.py +222 -0
- patchwork/frame.py +439 -0
- patchwork/fused.py +618 -0
- patchwork/fused_kernels.py +695 -0
- patchwork/io.py +668 -0
- patchwork/logging.py +62 -0
- patchwork/mosaic.py +546 -0
- patchwork/numba_kernels.py +203 -0
- patchwork/options.py +211 -0
- patchwork/pipeline/__init__.py +3 -0
- patchwork/pipeline/cluster.py +73 -0
- patchwork/pipeline/flow.py +143 -0
- patchwork/plan.py +818 -0
- patchwork/polarisation.py +178 -0
- patchwork/primary_beam.py +636 -0
- patchwork/py.typed +0 -0
- patchwork/resample.py +267 -0
- patchwork/rotate.py +319 -0
- patchwork/tile.py +344 -0
- patchwork/weights.py +175 -0
- patchwork/write.py +502 -0
patchwork/write.py
ADDED
|
@@ -0,0 +1,502 @@
|
|
|
1
|
+
"""Writing a mosaic: making its tiles with an executor, writing them to FITS or zarr, and writing its noise.
|
|
2
|
+
|
|
3
|
+
The work is handed to a ``concurrent.futures.Executor``: threads, or dask
|
|
4
|
+
workers through ``patchwork.pipeline.flow``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
from concurrent.futures import (
|
|
13
|
+
FIRST_COMPLETED,
|
|
14
|
+
Executor,
|
|
15
|
+
Future,
|
|
16
|
+
ThreadPoolExecutor,
|
|
17
|
+
wait,
|
|
18
|
+
)
|
|
19
|
+
from contextlib import ExitStack, closing
|
|
20
|
+
from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
from astropy.io import fits
|
|
24
|
+
|
|
25
|
+
from patchwork.io import (
|
|
26
|
+
beams_header,
|
|
27
|
+
create_output_fits,
|
|
28
|
+
data_index,
|
|
29
|
+
data_location,
|
|
30
|
+
read_rows,
|
|
31
|
+
spectral_axis,
|
|
32
|
+
write_block,
|
|
33
|
+
write_rows,
|
|
34
|
+
)
|
|
35
|
+
from patchwork.logging import logger
|
|
36
|
+
from patchwork.resample import resolve_backend, tile_threads
|
|
37
|
+
from patchwork.tile import mosaic_tile
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
from collections.abc import Callable, Generator, Iterable
|
|
41
|
+
|
|
42
|
+
from numpy.typing import NDArray
|
|
43
|
+
|
|
44
|
+
from patchwork.io import DataLocation, Slices
|
|
45
|
+
from patchwork.options import MosaicOptions, RunOptions
|
|
46
|
+
from patchwork.plan import MosaicNames, MosaicPlan
|
|
47
|
+
from patchwork.tile import TileResult
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def create_outputs(plan: MosaicPlan, run_options: RunOptions) -> None:
|
|
51
|
+
"""Create the empty outputs of a planned mosaic: its FITS images and weights, which tiles are written into, or all its zarr stores."""
|
|
52
|
+
if run_options.output_format == "zarr":
|
|
53
|
+
create_zarr_outputs(plan, run_options.zarr_chunk)
|
|
54
|
+
return
|
|
55
|
+
files = (*plan.names.images, *plan.names.weights)
|
|
56
|
+
if run_options.fits_writer == "workers":
|
|
57
|
+
# The tasks write the noise too, so it is created with the rest
|
|
58
|
+
files = (*files, *plan.names.noises)
|
|
59
|
+
for path, header in zip(files, plan.headers[: len(files)], strict=True):
|
|
60
|
+
create_output_fits(path, header, plan.beams)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _output_channels(plan: MosaicPlan, channels: slice) -> slice:
|
|
64
|
+
"""The channels of the outputs of a mosaic that hold some channels of its inputs."""
|
|
65
|
+
start = channels.start - (plan.channels.start or 0)
|
|
66
|
+
return slice(start, start + channels.stop - channels.start)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def write_tiles(plan: MosaicPlan, results: Iterable[TileResult]) -> None:
|
|
70
|
+
"""Write finished tiles into the output FITS files of a mosaic."""
|
|
71
|
+
with ExitStack() as stack:
|
|
72
|
+
image_hdus = [
|
|
73
|
+
stack.enter_context(fits.open(path, mode="update", memmap=True))[0]
|
|
74
|
+
for path in plan.names.images
|
|
75
|
+
]
|
|
76
|
+
weight_hdus = [
|
|
77
|
+
stack.enter_context(fits.open(path, mode="update", memmap=True))[0]
|
|
78
|
+
for path in plan.names.weights
|
|
79
|
+
]
|
|
80
|
+
for result in results:
|
|
81
|
+
channels = _output_channels(plan, result.channels)
|
|
82
|
+
for hdu, block in zip(
|
|
83
|
+
(*image_hdus, *weight_hdus),
|
|
84
|
+
(*result.images, *result.weights),
|
|
85
|
+
strict=True,
|
|
86
|
+
):
|
|
87
|
+
write_block(hdu, result.slices, block, channels)
|
|
88
|
+
logger.debug(f"Wrote tile {result.slices} of channels {result.channels}")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def write_noise(plan: MosaicPlan, strip_rows: int = 1024) -> None:
|
|
92
|
+
"""Write the noise of each weight of a written mosaic, the inverse of its square root, channel by channel.
|
|
93
|
+
|
|
94
|
+
Each noise image has the header, and any BEAMS table, of its weight, in the
|
|
95
|
+
units of the images of its group. Pixels with no weight are blank.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
plan (MosaicPlan): The plan of the mosaic, whose files are written
|
|
99
|
+
strip_rows (int, optional): Rows of the files worked on at a time. Defaults to 1024.
|
|
100
|
+
"""
|
|
101
|
+
if not plan.names.noises:
|
|
102
|
+
return
|
|
103
|
+
for group, (weight_path, noise_path) in enumerate(
|
|
104
|
+
zip(plan.names.weights, plan.names.noises, strict=True)
|
|
105
|
+
):
|
|
106
|
+
image_path = plan.names.images[plan.groups.index(group)]
|
|
107
|
+
with fits.open(weight_path, memmap=True) as weight_hdul:
|
|
108
|
+
header = weight_hdul[0].header.copy()
|
|
109
|
+
header["BUNIT"] = fits.getheader(image_path).get("BUNIT", "")
|
|
110
|
+
create_output_fits(noise_path, header)
|
|
111
|
+
if "BEAMS" in weight_hdul:
|
|
112
|
+
fits.append(
|
|
113
|
+
noise_path, weight_hdul["BEAMS"].data, weight_hdul["BEAMS"].header
|
|
114
|
+
)
|
|
115
|
+
weight = weight_hdul[0].data
|
|
116
|
+
with fits.open(noise_path, mode="update", memmap=True) as noise_hdul:
|
|
117
|
+
noise = noise_hdul[0].data
|
|
118
|
+
axis = spectral_axis(header)
|
|
119
|
+
n_channels = 1 if axis is None else int(header[f"NAXIS{axis}"])
|
|
120
|
+
for channel in range(n_channels):
|
|
121
|
+
for row in range(0, int(header["NAXIS2"]), strip_rows):
|
|
122
|
+
index = data_index(
|
|
123
|
+
header,
|
|
124
|
+
(slice(row, row + strip_rows), slice(None)),
|
|
125
|
+
slice(channel, channel + 1),
|
|
126
|
+
)
|
|
127
|
+
noise[index] = noise_of(np.asarray(weight[index]))
|
|
128
|
+
logger.info(f"Written {noise_path}")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def noise_of(weight: NDArray[np.floating]) -> NDArray[np.float32]:
|
|
132
|
+
"""The noise of a weight, the inverse of its square root, which is blank where there is no weight."""
|
|
133
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
134
|
+
return np.where(weight > 0, 1 / np.sqrt(weight), np.nan).astype(np.float32)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def create_zarr_outputs(plan: MosaicPlan, chunk: int) -> None:
|
|
138
|
+
"""Create the zarr stores of a mosaic, laid out as rm-lite reads them.
|
|
139
|
+
|
|
140
|
+
Each is one (channel, y, x) float32 array, chunked over every channel, with
|
|
141
|
+
the FITS header of the cube and the restoring beam of each channel in its
|
|
142
|
+
attributes.
|
|
143
|
+
"""
|
|
144
|
+
import zarr # noqa: PLC0415
|
|
145
|
+
from zarr.codecs import BloscCodec # noqa: PLC0415
|
|
146
|
+
|
|
147
|
+
n_channels = plan.channels.stop - plan.channels.start
|
|
148
|
+
for path, header in zip(
|
|
149
|
+
(*plan.names.images, *plan.names.weights, *plan.names.noises),
|
|
150
|
+
plan.headers,
|
|
151
|
+
strict=True,
|
|
152
|
+
):
|
|
153
|
+
array = zarr.create_array(
|
|
154
|
+
store=str(path),
|
|
155
|
+
shape=(n_channels, *plan.frame.shape),
|
|
156
|
+
chunks=(
|
|
157
|
+
n_channels,
|
|
158
|
+
min(chunk, plan.frame.shape[0]),
|
|
159
|
+
min(chunk, plan.frame.shape[1]),
|
|
160
|
+
),
|
|
161
|
+
dtype="float32",
|
|
162
|
+
fill_value=np.nan,
|
|
163
|
+
compressors=(BloscCodec(cname="lz4", clevel=5, shuffle="shuffle"),),
|
|
164
|
+
overwrite=True,
|
|
165
|
+
)
|
|
166
|
+
# The header holds one beam only if every channel shares it, as a FITS cube's does
|
|
167
|
+
stored = header.copy()
|
|
168
|
+
beams_header(stored, plan.beams)
|
|
169
|
+
array.attrs["fits_header"] = stored.tostring()
|
|
170
|
+
array.attrs["beams"] = [
|
|
171
|
+
list(beam) if beam.is_usable() else None for beam in plan.beams
|
|
172
|
+
]
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def mosaic_tile_to_zarr(
|
|
176
|
+
tile: Slices,
|
|
177
|
+
plan: MosaicPlan,
|
|
178
|
+
options: MosaicOptions,
|
|
179
|
+
channel_chunk: int,
|
|
180
|
+
) -> None:
|
|
181
|
+
"""Make a tile over every channel, a few channels at a time, and write it into the zarr stores of the mosaic.
|
|
182
|
+
|
|
183
|
+
One task owns each chunk of the stores, so tasks never write the same chunk.
|
|
184
|
+
"""
|
|
185
|
+
import zarr # noqa: PLC0415
|
|
186
|
+
|
|
187
|
+
start, stop = plan.channels.start, plan.channels.stop
|
|
188
|
+
results = [
|
|
189
|
+
mosaic_tile(
|
|
190
|
+
tile,
|
|
191
|
+
plan.frame,
|
|
192
|
+
plan.inputs,
|
|
193
|
+
options,
|
|
194
|
+
slice(first, min(first + channel_chunk, stop)),
|
|
195
|
+
)
|
|
196
|
+
for first in range(start, stop, channel_chunk)
|
|
197
|
+
]
|
|
198
|
+
for index, path in enumerate(plan.names.images):
|
|
199
|
+
zarr.open_array(str(path), mode="r+")[(slice(None), *tile)] = np.concatenate(
|
|
200
|
+
[result.images[index] for result in results]
|
|
201
|
+
)
|
|
202
|
+
for index, path in enumerate(plan.names.weights):
|
|
203
|
+
zarr.open_array(str(path), mode="r+")[(slice(None), *tile)] = np.concatenate(
|
|
204
|
+
[result.weights[index] for result in results]
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def noise_zarr_tile(tile: Slices, plan: MosaicPlan) -> None:
|
|
209
|
+
"""Write a tile of the noise stores of a mosaic from its weights."""
|
|
210
|
+
import zarr # noqa: PLC0415
|
|
211
|
+
|
|
212
|
+
index = (slice(None), *tile)
|
|
213
|
+
for weight_path, noise_path in zip(
|
|
214
|
+
plan.names.weights, plan.names.noises, strict=True
|
|
215
|
+
):
|
|
216
|
+
weight = np.asarray(zarr.open_array(str(weight_path), mode="r")[index])
|
|
217
|
+
zarr.open_array(str(noise_path), mode="r+")[index] = noise_of(weight)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class FitsLocations(NamedTuple):
|
|
221
|
+
"""Where the data of each FITS output of a mosaic lies, for tasks to write their strips into"""
|
|
222
|
+
|
|
223
|
+
images: tuple[DataLocation, ...]
|
|
224
|
+
"""The data of each image"""
|
|
225
|
+
weights: tuple[DataLocation, ...]
|
|
226
|
+
"""The data of each weight"""
|
|
227
|
+
noises: tuple[DataLocation, ...]
|
|
228
|
+
"""The data of each noise image"""
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def fits_locations(plan: MosaicPlan) -> FitsLocations:
|
|
232
|
+
"""Find where the data of each created FITS output of a mosaic lies."""
|
|
233
|
+
names = plan.names
|
|
234
|
+
return FitsLocations(
|
|
235
|
+
images=tuple(map(data_location, names.images)),
|
|
236
|
+
weights=tuple(map(data_location, names.weights)),
|
|
237
|
+
noises=tuple(map(data_location, names.noises)),
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def mosaic_strip_to_fits(
|
|
242
|
+
plan: MosaicPlan,
|
|
243
|
+
strip: Slices,
|
|
244
|
+
options: MosaicOptions,
|
|
245
|
+
channels: slice,
|
|
246
|
+
locations: FitsLocations,
|
|
247
|
+
) -> None:
|
|
248
|
+
"""Make a strip of whole rows over some channels, and write it into the FITS outputs of the mosaic.
|
|
249
|
+
|
|
250
|
+
Each task writes only its own rows, so no two tasks write the same bytes.
|
|
251
|
+
"""
|
|
252
|
+
result = mosaic_tile(strip, plan.frame, plan.inputs, options, channels)
|
|
253
|
+
written = _output_channels(plan, result.channels)
|
|
254
|
+
files = (*locations.images, *locations.weights)
|
|
255
|
+
blocks = (*result.images, *result.weights)
|
|
256
|
+
|
|
257
|
+
def write(index: int) -> None:
|
|
258
|
+
write_rows(files[index], strip[0], written, blocks[index])
|
|
259
|
+
|
|
260
|
+
threads = 1
|
|
261
|
+
if resolve_backend(options.resample.backend) == "fused":
|
|
262
|
+
threads = min(tile_threads(options.resample.tile_threads), len(files))
|
|
263
|
+
started = time.perf_counter()
|
|
264
|
+
if threads == 1:
|
|
265
|
+
for index in range(len(files)):
|
|
266
|
+
write(index)
|
|
267
|
+
else:
|
|
268
|
+
# A tile made on several threads would otherwise wait for one to write it
|
|
269
|
+
with ThreadPoolExecutor(threads, thread_name_prefix="patchwork-write") as pool:
|
|
270
|
+
list(pool.map(write, range(len(files))))
|
|
271
|
+
if logger.isEnabledFor(logging.DEBUG):
|
|
272
|
+
written_bytes = sum(block.nbytes for block in blocks)
|
|
273
|
+
logger.debug(
|
|
274
|
+
f"Wrote rows {strip[0].start}-{strip[0].stop}, channels {written.start}-{written.stop} "
|
|
275
|
+
f"({written_bytes / 2**20:.0f} MB) in {time.perf_counter() - started:.1f} s on {threads} threads"
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def noise_fits_strip(strip: Slices, locations: FitsLocations) -> None:
|
|
280
|
+
"""Write a strip of the FITS noise outputs of a mosaic from its weights."""
|
|
281
|
+
rows = strip[0]
|
|
282
|
+
for weight_location, noise_location in zip(
|
|
283
|
+
locations.weights, locations.noises, strict=True
|
|
284
|
+
):
|
|
285
|
+
for channel in range(weight_location.n_channels):
|
|
286
|
+
noise = noise_of(read_rows(weight_location, rows, channel))
|
|
287
|
+
write_rows(noise_location, rows, slice(channel, channel + 1), noise[None])
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
Call = tuple["Callable[..., Any]", tuple[Any, ...]]
|
|
291
|
+
"""A function and the arguments to call it with"""
|
|
292
|
+
|
|
293
|
+
Result = TypeVar("Result")
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _duration(seconds: float) -> str:
|
|
297
|
+
"""A duration such as ``42s``, ``3m 05s`` or ``1h 02m``."""
|
|
298
|
+
minutes, whole_seconds = divmod(round(seconds), 60)
|
|
299
|
+
hours, minutes = divmod(minutes, 60)
|
|
300
|
+
if hours:
|
|
301
|
+
return f"{hours}h {minutes:02d}m"
|
|
302
|
+
if minutes:
|
|
303
|
+
return f"{minutes}m {whole_seconds:02d}s"
|
|
304
|
+
return f"{whole_seconds}s"
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def logged_progress(
|
|
308
|
+
results: Iterable[Result], total: int, what: str, interval_s: float = 30.0
|
|
309
|
+
) -> Generator[Result, None, None]:
|
|
310
|
+
"""Pass results on as they arrive, logging how many of ``total`` are done every ``interval_s`` seconds, and once all are.
|
|
311
|
+
|
|
312
|
+
A result is counted once it has been taken, so for tiles written as they
|
|
313
|
+
arrive the count is of tiles made and written.
|
|
314
|
+
|
|
315
|
+
Args:
|
|
316
|
+
results (Iterable[Result]): The results, such as those of ``bounded_results``
|
|
317
|
+
total (int): How many results there are
|
|
318
|
+
what (str): What the results are, such as ``tiles``
|
|
319
|
+
interval_s (float, optional): The least time in seconds between messages. Defaults to 30.0.
|
|
320
|
+
|
|
321
|
+
Yields:
|
|
322
|
+
Result: Each result, unchanged
|
|
323
|
+
"""
|
|
324
|
+
start = logged = time.monotonic()
|
|
325
|
+
done = 0
|
|
326
|
+
for result in results:
|
|
327
|
+
yield result
|
|
328
|
+
done += 1
|
|
329
|
+
now = time.monotonic()
|
|
330
|
+
if done < total and now - logged < interval_s:
|
|
331
|
+
continue
|
|
332
|
+
logged, elapsed = now, now - start
|
|
333
|
+
message = f"Made {done} of {total} {what} ({done / total:.0%}) in {_duration(elapsed)}"
|
|
334
|
+
if done < total:
|
|
335
|
+
message += f", about {_duration(elapsed / done * (total - done))} left"
|
|
336
|
+
logger.info(message)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def bounded_results(
|
|
340
|
+
executor: Executor,
|
|
341
|
+
calls: Iterable[Call],
|
|
342
|
+
limit: Callable[[], int],
|
|
343
|
+
refresh_s: float = 5.0,
|
|
344
|
+
) -> Generator[Any, None, None]:
|
|
345
|
+
"""The results of calls made by an executor, as they finish, with no more than ``limit()`` submitted and not yet taken.
|
|
346
|
+
|
|
347
|
+
The next call is submitted only once a result is taken. An executor that
|
|
348
|
+
hands results back as soon as they finish, as dask's does, therefore never
|
|
349
|
+
holds more than ``limit()`` of them in this process, however slowly they are
|
|
350
|
+
taken. Should a call fail, or the results stop being taken, the calls still
|
|
351
|
+
waiting are cancelled.
|
|
352
|
+
|
|
353
|
+
Args:
|
|
354
|
+
executor (Executor): Makes the calls
|
|
355
|
+
calls (Iterable[Call]): The functions to call, and their arguments
|
|
356
|
+
limit (Callable[[], int]): The most calls submitted and not yet taken. It is asked again every ``refresh_s`` seconds, so it can follow a cluster as it scales.
|
|
357
|
+
refresh_s (float, optional): Seconds between asking ``limit`` again. Defaults to 5.0.
|
|
358
|
+
|
|
359
|
+
Yields:
|
|
360
|
+
Any: The result of each call, in the order they finish
|
|
361
|
+
"""
|
|
362
|
+
remaining = iter(calls)
|
|
363
|
+
pending: set[Future[Any]] = set()
|
|
364
|
+
size, asked = max(limit(), 1), time.monotonic()
|
|
365
|
+
try:
|
|
366
|
+
while True:
|
|
367
|
+
if time.monotonic() - asked > refresh_s:
|
|
368
|
+
size, asked = max(limit(), 1), time.monotonic()
|
|
369
|
+
while len(pending) < size and (call := next(remaining, None)) is not None:
|
|
370
|
+
function, args = call
|
|
371
|
+
pending.add(executor.submit(function, *args))
|
|
372
|
+
if not pending:
|
|
373
|
+
return
|
|
374
|
+
done, pending = wait(pending, return_when=FIRST_COMPLETED)
|
|
375
|
+
for future in done:
|
|
376
|
+
yield future.result()
|
|
377
|
+
finally:
|
|
378
|
+
for future in pending:
|
|
379
|
+
future.cancel()
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def run_tiles(
|
|
383
|
+
plan: MosaicPlan,
|
|
384
|
+
options: MosaicOptions,
|
|
385
|
+
run_options: RunOptions,
|
|
386
|
+
executor: Executor,
|
|
387
|
+
share: Callable[[Any], Any] | None = None,
|
|
388
|
+
capacity: Callable[[], int] | None = None,
|
|
389
|
+
) -> MosaicNames:
|
|
390
|
+
"""Make the tiles of a planned mosaic with an executor, write them, and finish the mosaic.
|
|
391
|
+
|
|
392
|
+
FITS files are written by this process as tiles arrive. Zarr stores are
|
|
393
|
+
written by the tasks, one chunk each. Each tile is finished once written,
|
|
394
|
+
so only the noise of the mosaic is written after them. No more than
|
|
395
|
+
``tasks_in_flight`` tiles are made at once, so this process holds no more
|
|
396
|
+
than that many.
|
|
397
|
+
|
|
398
|
+
Args:
|
|
399
|
+
plan (MosaicPlan): The plan, from ``patchwork.plan.plan_mosaic``, whose outputs are created
|
|
400
|
+
options (MosaicOptions): The mosaicking options
|
|
401
|
+
run_options (RunOptions): How the mosaic is made and written
|
|
402
|
+
executor (Executor): Runs the tasks, such as a thread pool, or a dask ``Client.get_executor()``
|
|
403
|
+
share (Callable[[Any], Any] | None, optional): Turns the plan into what is sent to each task, such as dask's ``scatter``. Defaults to None, sending it as it is.
|
|
404
|
+
capacity (Callable[[], int] | None, optional): How many tasks the executor runs at once, asked again as tiles are made, such as the threads of a dask cluster as it scales. Without ``tasks_in_flight`` twice this many tiles are made at once. Defaults to None, one.
|
|
405
|
+
|
|
406
|
+
Returns:
|
|
407
|
+
MosaicNames: The files the mosaic was written to
|
|
408
|
+
"""
|
|
409
|
+
shared = plan if share is None else share(plan)
|
|
410
|
+
# Only a FITS mosaic written by this process holds the finished tiles, so
|
|
411
|
+
# only it needs a bound; the others queue every task, so an adaptive dask
|
|
412
|
+
# cluster sees the whole mosaic waiting and grows to its maximum at once
|
|
413
|
+
held_here = (
|
|
414
|
+
run_options.output_format == "fits" and run_options.fits_writer != "workers"
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
def in_flight() -> int:
|
|
418
|
+
if run_options.tasks_in_flight is not None:
|
|
419
|
+
return run_options.tasks_in_flight
|
|
420
|
+
if not held_here:
|
|
421
|
+
return sys.maxsize
|
|
422
|
+
return 2 * max(1 if capacity is None else capacity(), 1)
|
|
423
|
+
|
|
424
|
+
def results(calls: Iterable[Call]) -> closing[Generator[Any, None, None]]:
|
|
425
|
+
return closing(bounded_results(executor, calls, in_flight))
|
|
426
|
+
|
|
427
|
+
chunks = [
|
|
428
|
+
slice(first, min(first + run_options.channel_chunk, plan.channels.stop))
|
|
429
|
+
for first in range(
|
|
430
|
+
plan.channels.start, plan.channels.stop, run_options.channel_chunk
|
|
431
|
+
)
|
|
432
|
+
]
|
|
433
|
+
if run_options.output_format == "fits" and run_options.fits_writer == "workers":
|
|
434
|
+
_run_strips(plan, options, shared, chunks, results)
|
|
435
|
+
return plan.names
|
|
436
|
+
if run_options.output_format == "fits":
|
|
437
|
+
logger.info(
|
|
438
|
+
f"Making {len(plan.tiles)} tiles in each of {len(chunks)} chunks of channels, "
|
|
439
|
+
f"{in_flight()} at once"
|
|
440
|
+
)
|
|
441
|
+
with results(
|
|
442
|
+
(_plan_tile, (shared, tile, options, chunk))
|
|
443
|
+
for chunk in chunks
|
|
444
|
+
for tile in plan.tiles
|
|
445
|
+
) as tiles_made:
|
|
446
|
+
write_tiles(
|
|
447
|
+
plan,
|
|
448
|
+
logged_progress(tiles_made, len(plan.tiles) * len(chunks), "tiles"),
|
|
449
|
+
)
|
|
450
|
+
write_noise(plan)
|
|
451
|
+
return plan.names
|
|
452
|
+
|
|
453
|
+
logger.info(
|
|
454
|
+
f"Making {len(plan.tiles)} zarr chunks"
|
|
455
|
+
+ (
|
|
456
|
+
f", {in_flight()} at once"
|
|
457
|
+
if run_options.tasks_in_flight
|
|
458
|
+
else ", all queued"
|
|
459
|
+
)
|
|
460
|
+
)
|
|
461
|
+
with results(
|
|
462
|
+
(mosaic_tile_to_zarr, (tile, shared, options, run_options.channel_chunk))
|
|
463
|
+
for tile in plan.tiles
|
|
464
|
+
) as chunks_made:
|
|
465
|
+
list(logged_progress(chunks_made, len(plan.tiles), "zarr chunks"))
|
|
466
|
+
if plan.names.noises:
|
|
467
|
+
with results((noise_zarr_tile, (tile, shared)) for tile in plan.tiles) as noise:
|
|
468
|
+
list(noise)
|
|
469
|
+
return plan.names
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _run_strips(
|
|
473
|
+
plan: MosaicPlan,
|
|
474
|
+
options: MosaicOptions,
|
|
475
|
+
shared: Any,
|
|
476
|
+
chunks: list[slice],
|
|
477
|
+
results: Callable[[Iterable[Call]], closing[Generator[Any, None, None]]],
|
|
478
|
+
) -> None:
|
|
479
|
+
"""Make the strips of a planned FITS mosaic, each written by its task, then write their noise."""
|
|
480
|
+
locations = fits_locations(plan)
|
|
481
|
+
logger.info(
|
|
482
|
+
f"Making {len(plan.tiles)} strips in each of {len(chunks)} chunks of channels, "
|
|
483
|
+
"each written by its task"
|
|
484
|
+
)
|
|
485
|
+
with results(
|
|
486
|
+
(mosaic_strip_to_fits, (shared, strip, options, chunk, locations))
|
|
487
|
+
for chunk in chunks
|
|
488
|
+
for strip in plan.tiles
|
|
489
|
+
) as made:
|
|
490
|
+
list(logged_progress(made, len(plan.tiles) * len(chunks), "strips"))
|
|
491
|
+
if plan.names.noises:
|
|
492
|
+
with results(
|
|
493
|
+
(noise_fits_strip, (strip, locations)) for strip in plan.tiles
|
|
494
|
+
) as noise:
|
|
495
|
+
list(noise)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _plan_tile(
|
|
499
|
+
plan: MosaicPlan, tile: Slices, options: MosaicOptions, channels: slice
|
|
500
|
+
) -> TileResult:
|
|
501
|
+
"""Make one tile of a planned mosaic, over some channels."""
|
|
502
|
+
return mosaic_tile(tile, plan.frame, plan.inputs, options, channels)
|