askap-patchwork 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askap_patchwork-0.1.0.dist-info/METADATA +179 -0
- askap_patchwork-0.1.0.dist-info/RECORD +33 -0
- askap_patchwork-0.1.0.dist-info/WHEEL +4 -0
- askap_patchwork-0.1.0.dist-info/entry_points.txt +4 -0
- askap_patchwork-0.1.0.dist-info/licenses/LICENSE +28 -0
- patchwork/__init__.py +11 -0
- patchwork/__main__.py +8 -0
- patchwork/_version.py +24 -0
- patchwork/_version.pyi +2 -0
- patchwork/cli.py +60 -0
- patchwork/combine.py +146 -0
- patchwork/convolve.py +534 -0
- patchwork/coordinates.py +222 -0
- patchwork/frame.py +439 -0
- patchwork/fused.py +618 -0
- patchwork/fused_kernels.py +695 -0
- patchwork/io.py +668 -0
- patchwork/logging.py +62 -0
- patchwork/mosaic.py +546 -0
- patchwork/numba_kernels.py +203 -0
- patchwork/options.py +211 -0
- patchwork/pipeline/__init__.py +3 -0
- patchwork/pipeline/cluster.py +73 -0
- patchwork/pipeline/flow.py +143 -0
- patchwork/plan.py +818 -0
- patchwork/polarisation.py +178 -0
- patchwork/primary_beam.py +636 -0
- patchwork/py.typed +0 -0
- patchwork/resample.py +267 -0
- patchwork/rotate.py +319 -0
- patchwork/tile.py +344 -0
- patchwork/weights.py +175 -0
- patchwork/write.py +502 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Numba versions of the resampling loops, used when numba is installed.
|
|
2
|
+
|
|
3
|
+
They follow ``patchwork.resample``, so both give the same values to round-off.
|
|
4
|
+
A stack of images on one grid, such as the channels of a cube, is interpolated
|
|
5
|
+
with the taps of each position found once. Each runs on one thread and releases the GIL, so tiles can be made
|
|
6
|
+
in parallel by threads or dask workers without oversubscribing the cores.
|
|
7
|
+
|
|
8
|
+
The compiled kernels are cached in ``cache_directory()``.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import functools
|
|
14
|
+
import os
|
|
15
|
+
import stat
|
|
16
|
+
import tempfile
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import TYPE_CHECKING, Any
|
|
19
|
+
|
|
20
|
+
import numba
|
|
21
|
+
import numpy as np
|
|
22
|
+
|
|
23
|
+
from patchwork.logging import logger
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from collections.abc import Callable
|
|
27
|
+
|
|
28
|
+
from numpy.typing import NDArray
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@functools.cache
|
|
32
|
+
def cache_directory() -> str:
|
|
33
|
+
"""The directory numba caches these kernels in: ``NUMBA_CACHE_DIR`` if it is set, otherwise one of this user's own in the temporary directory.
|
|
34
|
+
|
|
35
|
+
numba's default is a ``__pycache__`` directory beside this module. When
|
|
36
|
+
several machines share one installation, as the workers of a cluster
|
|
37
|
+
often do, they all compile the kernels at once into that directory. One
|
|
38
|
+
can then read the cache's index while another replaces it, which on NFS
|
|
39
|
+
fails with "Stale file handle" and takes the task with it. The temporary
|
|
40
|
+
directory (``TMPDIR``, usually ``/tmp``) is on each machine instead.
|
|
41
|
+
|
|
42
|
+
numba loads whatever it finds in its cache, so the directory must belong
|
|
43
|
+
to this user and be closed to everyone else. If the usual one does not,
|
|
44
|
+
perhaps because someone else made it first, a new one is made with
|
|
45
|
+
``tempfile.mkdtemp`` and used instead, and the kernels compile again in
|
|
46
|
+
each process. On Windows, which has no user ids, the temporary directory
|
|
47
|
+
already belongs to each user, so the directory is made there and used.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
str: The directory
|
|
51
|
+
"""
|
|
52
|
+
if numba.config.CACHE_DIR:
|
|
53
|
+
return str(numba.config.CACHE_DIR)
|
|
54
|
+
getuid = getattr(os, "getuid", None)
|
|
55
|
+
user = None if getuid is None else getuid()
|
|
56
|
+
name = "patchwork-numba" if user is None else f"patchwork-numba-{user}"
|
|
57
|
+
path = Path(tempfile.gettempdir()) / name
|
|
58
|
+
try:
|
|
59
|
+
path.mkdir(mode=0o700, exist_ok=True)
|
|
60
|
+
# Not followed, so a link someone else left in its place is not used
|
|
61
|
+
found = path.lstat()
|
|
62
|
+
private = stat.S_ISDIR(found.st_mode) and (
|
|
63
|
+
user is None or (found.st_uid == user and not found.st_mode & 0o077)
|
|
64
|
+
)
|
|
65
|
+
except OSError:
|
|
66
|
+
private = False
|
|
67
|
+
if not private:
|
|
68
|
+
path = Path(tempfile.mkdtemp(prefix="patchwork-numba-"))
|
|
69
|
+
logger.debug(f"Caching the numba kernels in {path}")
|
|
70
|
+
return str(path)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _kernel(function: Callable[..., Any], **options: Any) -> Any:
|
|
74
|
+
"""Compile a function with numba, releasing the GIL, caching it in ``cache_directory()``.
|
|
75
|
+
|
|
76
|
+
numba chooses where a function is cached when it is decorated, so its
|
|
77
|
+
setting is changed only while each kernel is, and anything else in the
|
|
78
|
+
process is cached where numba would cache it. ``options`` are passed to
|
|
79
|
+
``numba.njit``.
|
|
80
|
+
"""
|
|
81
|
+
setting = numba.config.CACHE_DIR
|
|
82
|
+
numba.config.CACHE_DIR = cache_directory()
|
|
83
|
+
try:
|
|
84
|
+
return numba.njit(cache=True, nogil=True, **options)(function)
|
|
85
|
+
finally:
|
|
86
|
+
numba.config.CACHE_DIR = setting
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@_kernel
|
|
90
|
+
def _taps(
|
|
91
|
+
position: float,
|
|
92
|
+
code: int,
|
|
93
|
+
width: int,
|
|
94
|
+
weights: NDArray[np.float64],
|
|
95
|
+
shift_trig: NDArray[np.float64],
|
|
96
|
+
) -> int:
|
|
97
|
+
"""Fill ``weights`` with the taps that interpolate a position, returning the index of the first.
|
|
98
|
+
|
|
99
|
+
``shift_trig`` holds the cosine and sine of ``pi k / a`` for each tap ``k`` of
|
|
100
|
+
a Lanczos kernel of half-width ``a``.
|
|
101
|
+
"""
|
|
102
|
+
if code == 0:
|
|
103
|
+
weights[0] = 1.0
|
|
104
|
+
return int(np.floor(position + 0.5))
|
|
105
|
+
|
|
106
|
+
base = np.floor(position)
|
|
107
|
+
fraction = position - base
|
|
108
|
+
first = width // 2 - 1
|
|
109
|
+
if code == 3:
|
|
110
|
+
# sin(pi (f - k)) is +-sin(pi f), and sin(pi (f - k) / a) follows by angle
|
|
111
|
+
# addition, so a Lanczos kernel needs three trigonometric calls, not 2 per tap
|
|
112
|
+
a = width // 2
|
|
113
|
+
sin_f = np.sin(np.pi * fraction)
|
|
114
|
+
sin_fa, cos_fa = np.sin(np.pi * fraction / a), np.cos(np.pi * fraction / a)
|
|
115
|
+
total = 0.0
|
|
116
|
+
for k in range(width):
|
|
117
|
+
shift = k - first
|
|
118
|
+
distance = fraction - shift
|
|
119
|
+
if abs(distance) < 1e-12:
|
|
120
|
+
weight = 1.0
|
|
121
|
+
else:
|
|
122
|
+
sin_d = -sin_f if shift & 1 else sin_f
|
|
123
|
+
sin_da = sin_fa * shift_trig[0, k] - cos_fa * shift_trig[1, k]
|
|
124
|
+
weight = a * sin_d * sin_da / (np.pi * distance) ** 2
|
|
125
|
+
weights[k] = weight
|
|
126
|
+
total += weight
|
|
127
|
+
for k in range(width):
|
|
128
|
+
weights[k] /= total
|
|
129
|
+
return int(base) - first
|
|
130
|
+
|
|
131
|
+
for k in range(width):
|
|
132
|
+
distance = fraction - (k - first)
|
|
133
|
+
if code == 1:
|
|
134
|
+
weights[k] = max(1.0 - abs(distance), 0.0)
|
|
135
|
+
else:
|
|
136
|
+
t = abs(distance)
|
|
137
|
+
if t <= 1:
|
|
138
|
+
weights[k] = (1.5 * t - 2.5) * t**2 + 1
|
|
139
|
+
elif t < 2:
|
|
140
|
+
weights[k] = ((-0.5 * t + 2.5) * t - 4) * t + 2
|
|
141
|
+
else:
|
|
142
|
+
weights[k] = 0.0
|
|
143
|
+
return int(base) - first
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@_kernel
|
|
147
|
+
def interpolate_stack(
|
|
148
|
+
windows: NDArray[np.float64],
|
|
149
|
+
x: NDArray[np.float64],
|
|
150
|
+
y: NDArray[np.float64],
|
|
151
|
+
code: int,
|
|
152
|
+
width: int,
|
|
153
|
+
) -> tuple[NDArray[np.float64], NDArray[np.bool]]:
|
|
154
|
+
"""Interpolate a (y, x, image) stack at pixel positions, as ``patchwork.resample.interpolate_stack`` does.
|
|
155
|
+
|
|
156
|
+
The taps of each position are found once and used for every image. Values
|
|
157
|
+
and validity are returned shaped ``(*x.shape, image)``.
|
|
158
|
+
"""
|
|
159
|
+
ny, nx, n_images = windows.shape
|
|
160
|
+
x_flat = x.ravel()
|
|
161
|
+
y_flat = y.ravel()
|
|
162
|
+
values = np.zeros((x_flat.size, n_images))
|
|
163
|
+
valid = np.zeros((x_flat.size, n_images), dtype=np.bool_)
|
|
164
|
+
x_weights = np.empty(width)
|
|
165
|
+
y_weights = np.empty(width)
|
|
166
|
+
shifts = np.pi * (np.arange(width) - (width // 2 - 1)) / max(width // 2, 1)
|
|
167
|
+
shift_trig = np.vstack((np.cos(shifts), np.sin(shifts)))
|
|
168
|
+
total = np.empty(n_images)
|
|
169
|
+
is_valid = np.empty(n_images, dtype=np.bool_)
|
|
170
|
+
|
|
171
|
+
for pixel in range(x_flat.size):
|
|
172
|
+
x_position, y_position = x_flat[pixel], y_flat[pixel]
|
|
173
|
+
if not (np.isfinite(x_position) and np.isfinite(y_position)):
|
|
174
|
+
continue
|
|
175
|
+
x_first = _taps(x_position, code, width, x_weights, shift_trig)
|
|
176
|
+
y_first = _taps(y_position, code, width, y_weights, shift_trig)
|
|
177
|
+
|
|
178
|
+
total[:] = 0.0
|
|
179
|
+
is_valid[:] = True
|
|
180
|
+
for j in range(width):
|
|
181
|
+
row = y_first + j
|
|
182
|
+
row_outside = row < 0 or row >= ny
|
|
183
|
+
row = min(max(row, 0), ny - 1)
|
|
184
|
+
for i in range(width):
|
|
185
|
+
column = x_first + i
|
|
186
|
+
outside = row_outside or column < 0 or column >= nx
|
|
187
|
+
column = min(max(column, 0), nx - 1)
|
|
188
|
+
|
|
189
|
+
tap_weight = y_weights[j] * x_weights[i]
|
|
190
|
+
carries_weight = abs(tap_weight) > 1e-9
|
|
191
|
+
for image in range(n_images):
|
|
192
|
+
value = windows[row, column, image]
|
|
193
|
+
blank = not np.isfinite(value)
|
|
194
|
+
if carries_weight and (outside or blank):
|
|
195
|
+
is_valid[image] = False
|
|
196
|
+
total[image] += tap_weight * (0.0 if blank else value)
|
|
197
|
+
|
|
198
|
+
for image in range(n_images):
|
|
199
|
+
if is_valid[image]:
|
|
200
|
+
values[pixel, image] = total[image]
|
|
201
|
+
valid[pixel, image] = True
|
|
202
|
+
|
|
203
|
+
return values.reshape((*x.shape, n_images)), valid.reshape((*x.shape, n_images))
|
patchwork/options.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""Options that control each stage of the mosaicking"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import TYPE_CHECKING, Literal, NamedTuple
|
|
7
|
+
|
|
8
|
+
from capn_crunch import BaseOptions, add_options_to_parser, create_options_from_parser
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from argparse import ArgumentParser, Namespace
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class FrameOptions(BaseOptions):
|
|
15
|
+
"""How the output frame of the mosaic is constructed. The ``reference`` frame
|
|
16
|
+
reuses the pixel grid of one input, while the ``fit`` frame builds a new grid
|
|
17
|
+
from all inputs. Either way the frame is sized to cover every input."""
|
|
18
|
+
|
|
19
|
+
frame_type: Literal["reference", "fit"] = "reference"
|
|
20
|
+
"""Reuse the grid of a reference input (``reference``), or fit a new grid to all inputs (``fit``)"""
|
|
21
|
+
reference_index: int | None = None
|
|
22
|
+
"""Index of the input whose grid a ``reference`` frame reuses. If None the central input is used"""
|
|
23
|
+
centre: tuple[float, float] | None = None
|
|
24
|
+
"""The (RA, Dec) in degrees of the frame centre. If None it is derived from the inputs"""
|
|
25
|
+
centre_type: Literal["all", "most"] = "all"
|
|
26
|
+
"""How a ``fit`` frame finds its centre: the middle of all inputs (``all``), or of the region most inputs cover (``most``)"""
|
|
27
|
+
pixel_scale_type: Literal["median", "min", "max"] = "median"
|
|
28
|
+
"""Which of the input pixel scales a ``fit`` frame takes"""
|
|
29
|
+
pixel_scale_arcsec: float | None = None
|
|
30
|
+
"""The pixel scale of a ``fit`` frame in arcseconds, overriding ``pixel_scale_type``"""
|
|
31
|
+
projection: str = "SIN"
|
|
32
|
+
"""The projection code of a ``fit`` frame, or of a ``reference`` frame given a new ``centre``"""
|
|
33
|
+
image_size: tuple[int, int] | None = None
|
|
34
|
+
"""The (nx, ny) size of a ``fit`` frame in pixels. If None it is sized to cover all inputs"""
|
|
35
|
+
full_frame: bool = False
|
|
36
|
+
"""Cover the whole images of the inputs. Otherwise the frame is trimmed to where they contribute, within ``pb_cutoff`` of their primary beams, as no pixel beyond that has a value. A frame of a given ``image_size`` is never trimmed"""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class PrimaryBeamOptions(BaseOptions):
|
|
40
|
+
"""The primary beam model that weights each input"""
|
|
41
|
+
|
|
42
|
+
pb_model: Literal["gaussian", "holography", "none"] = "gaussian"
|
|
43
|
+
"""The primary beam model. With ``none`` every input has a flat response"""
|
|
44
|
+
aperture_m: float = 12.0
|
|
45
|
+
"""Dish diameter in metres of the Gaussian beam"""
|
|
46
|
+
fwhm_scaling: float = 1.09
|
|
47
|
+
"""The Gaussian beam FWHM, in units of wavelength over the dish diameter"""
|
|
48
|
+
holography_cube: Path | None = None
|
|
49
|
+
"""Holography cube of beam responses, with axes (beam, stokes, freq, dec, ra). Needed by the ``holography`` model"""
|
|
50
|
+
holography_alpha_rad: float = 0.0
|
|
51
|
+
"""Rotation in radians of the holography beams from the frame the cube was measured in"""
|
|
52
|
+
holography_scale: Literal["centre", "stored"] = "centre"
|
|
53
|
+
"""Scale each holography beam to 1 at its centre, where calibration sets the gain (``centre``), or use it as the cube stores it, with ``pb_cutoff`` measured from the beam's peak (``stored``)"""
|
|
54
|
+
pb_cutoff: float = 0.01
|
|
55
|
+
"""Each input only contributes where its primary beam is at least this response, whatever its noise. The mosaic is not blanked by its weight. With 0 or less an input contributes wherever its beam responds at all"""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class WeightOptions(BaseOptions):
|
|
59
|
+
"""How each input is weighted. ``weight_type`` sets where the
|
|
60
|
+
weights come from and ``weight_state`` whether the inputs are already corrected
|
|
61
|
+
for the primary beam."""
|
|
62
|
+
|
|
63
|
+
weight_type: Literal["primary_beam", "weight_image", "combined"] = "combined"
|
|
64
|
+
"""Weight by the primary beam squared, by a weight (an image or a single value), or by both"""
|
|
65
|
+
weight_state: Literal["corrected", "inherent", "weighted"] = "inherent"
|
|
66
|
+
"""Whether inputs are corrected for the primary beam (``corrected``), still carry it (``inherent``), or are already weighted (``weighted``)"""
|
|
67
|
+
image_weight: Literal["mad", "std", "none"] = "mad"
|
|
68
|
+
"""Without weight images, weight each input by its inverse variance from this noise statistic. With ``none`` every input weighs 1"""
|
|
69
|
+
weight_stride: int = 8
|
|
70
|
+
"""Use every n'th row when measuring the noise of an input, which reads only those rows"""
|
|
71
|
+
weight_map_type: Literal["weight", "variance", "rms"] = "weight"
|
|
72
|
+
"""What the values of weight images are"""
|
|
73
|
+
noise_image: bool = False
|
|
74
|
+
"""Also write the noise of the mosaic, the inverse square root of its weight. This is the standard deviation of each pixel when every input is weighted by the inverse of its variance, as ``image_weight`` and ``variance`` or ``rms`` weight maps weight them, and otherwise only relative"""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class ResampleOptions(BaseOptions):
|
|
78
|
+
"""How inputs are resampled onto the output frame"""
|
|
79
|
+
|
|
80
|
+
kernel: Literal[
|
|
81
|
+
"nearest", "linear", "cubic", "lanczos2", "lanczos3", "lanczos4"
|
|
82
|
+
] = "cubic"
|
|
83
|
+
"""The interpolation kernel"""
|
|
84
|
+
tile_size: int = 512
|
|
85
|
+
"""Size in pixels of the square tiles the output frame is worked in. Each is a task, whose memory grows with its area"""
|
|
86
|
+
coordinate_step: int = 16
|
|
87
|
+
"""Spacing in pixels of the grid on which coordinates are transformed exactly, and between which they are interpolated. Interpolated positions are checked to be within 0.001 pixels, and every pixel is transformed where they are not. 1 transforms every pixel"""
|
|
88
|
+
backend: Literal["auto", "numpy", "numba", "fused"] = "auto"
|
|
89
|
+
"""Interpolate with numpy or numba, or make each tile in one compiled pass with numba (``fused``), which can use several threads (``tile_threads``). ``auto`` uses numba when it is installed"""
|
|
90
|
+
kernel_threads: int = 0
|
|
91
|
+
"""Threads of ``tile_threads`` that the ``fused`` kernels run on; the rest read the inputs. 0 gives the kernels 5 of every 8 threads when there are at least 4, which balanced the two on a 16-core node, and has every thread do both with fewer. Setting it to ``tile_threads`` or more has every thread do both, so the readers compete with the kernels for CPUs, and a parallel kernel waits for its slowest thread"""
|
|
92
|
+
tile_threads: int = 1
|
|
93
|
+
"""Threads each task uses: the ``fused`` backend makes each tile with them, and the noise of each image is measured with them by every backend. 0 uses every CPU the process may run on, shared between the tasks it runs at once: between ``max_workers``, or the threads of its dask worker. Leave it at 1 where each core has a worker of its own, and give a number where several worker processes share the CPUs of one job"""
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class PolarisationOptions(BaseOptions):
|
|
97
|
+
"""How linear polarisation is mosaicked. Stokes Q and U are rotated from the
|
|
98
|
+
frame of each input into the frame of the mosaic, which matters most towards
|
|
99
|
+
the celestial poles. With a holography primary beam the leakage of Stokes I
|
|
100
|
+
into Q, U and V can be removed."""
|
|
101
|
+
|
|
102
|
+
input_frame: Literal["reference", "crval", "north"] = "reference"
|
|
103
|
+
"""What the Q and U of each input are referenced to. ``reference``: north at ``frame_centre``, or at the position the input's POLFRAME keywords give; an input of Q and U with neither is refused. ``crval``: north at each input's own reference position, as an imager references a single pointing. ``north``: north at every pixel. An input whose POLFRAME keywords say otherwise is refused"""
|
|
104
|
+
frame_centre: tuple[float, float] | None = None
|
|
105
|
+
"""The (RA, Dec) in degrees that the Q and U of every input are referenced to north at, such as the boresight of an ASKAP observation, with ``input_frame="reference"``. It also orients holography beams"""
|
|
106
|
+
output_frame: Literal["north", "centre"] = "north"
|
|
107
|
+
"""Reference the Q and U of the mosaic to north at every pixel (``north``, the IAU convention), or to north at the centre of the frame (``centre``), which stays smooth across a pole"""
|
|
108
|
+
remove_leakage: bool = False
|
|
109
|
+
"""Subtract the leakage of Stokes I into Q, U and V given by the holography cube, which needs a Stokes I image of each input"""
|
|
110
|
+
keep_centre_leakage: bool = False
|
|
111
|
+
"""Keep the leakage at the beam centre. Otherwise it is removed before the leakage is subtracted, as calibration has already removed it there"""
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class CombineOptions(BaseOptions):
|
|
115
|
+
"""How resampled inputs are combined"""
|
|
116
|
+
|
|
117
|
+
combine_type: Literal["weighted", "average", "sum"] = "weighted"
|
|
118
|
+
"""Combine as a weighted mean, a plain mean or a sum"""
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class RunOptions(BaseOptions):
|
|
122
|
+
"""How a mosaic is made: which channels of cubes, how many each task makes,
|
|
123
|
+
whether the inputs are first convolved to a common beam, and what the
|
|
124
|
+
mosaic is written as"""
|
|
125
|
+
|
|
126
|
+
channels: tuple[int, int] | None = None
|
|
127
|
+
"""The (start, stop) channels to mosaic, counted from 0 and stopping before ``stop``. If None every channel is"""
|
|
128
|
+
channel_chunk: int = 16
|
|
129
|
+
"""Number of channels each task makes. More channels share more of the work (the positions and primary beams of a tile), and take more memory"""
|
|
130
|
+
tasks_in_flight: int | None = None
|
|
131
|
+
"""Most tiles made at once. The process writing a FITS mosaic holds each finished tile until it is written, so this bounds its memory: if None, twice the number of tasks the executor runs at once. Where the workers write the mosaic (``fits_writer`` ``workers``, or zarr) nothing is held, and None queues every tile, so an adaptive dask cluster sees all the work and grows to its maximum"""
|
|
132
|
+
convolve: Literal["none", "natural", "total"] = "none"
|
|
133
|
+
"""Convolve the inputs to a common beam first: each channel to the smallest beam that encloses that channel of every input (``natural``), or every channel to one beam (``total``). For images, which have one channel, the two are the same"""
|
|
134
|
+
output_format: Literal["fits", "zarr"] = "fits"
|
|
135
|
+
"""Write the mosaic as FITS, or as zarr stores (one (channel, y, x) array per image, as rm-lite reads them)"""
|
|
136
|
+
fits_writer: Literal["driver", "workers"] = "driver"
|
|
137
|
+
"""Which processes write FITS outputs: the one driving the mosaic, as tiles arrive (``driver``), or the tasks themselves, each into its own strip of whole rows (``workers``). Workers need a file system that keeps one file consistent while several machines write to it, such as Lustre or GPFS, and not NFS"""
|
|
138
|
+
zarr_chunk: int = 256
|
|
139
|
+
"""Size in pixels of the square chunks of zarr output. Each is made over every channel by one task"""
|
|
140
|
+
keep_convolved: bool = False
|
|
141
|
+
"""Keep the convolved inputs, which are otherwise removed once the mosaic is made"""
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class ConvolveOptions(BaseOptions):
|
|
145
|
+
"""How images are convolved to a common restoring beam"""
|
|
146
|
+
|
|
147
|
+
target_beam: tuple[float, float, float] | None = None
|
|
148
|
+
"""The (major, minor, position angle) of the beam to convolve to, in arcseconds and degrees. If None the smallest beam that encloses all inputs is used"""
|
|
149
|
+
beam_cutoff_arcsec: float | None = None
|
|
150
|
+
"""Images whose beam major axis is larger than this are blanked and left out of the common beam"""
|
|
151
|
+
circularise: bool = False
|
|
152
|
+
"""Make the common beam circular, with its minor axis set to its major axis"""
|
|
153
|
+
tolerance: float = 1e-4
|
|
154
|
+
"""Tolerance of the common beam search"""
|
|
155
|
+
nsamps: int = 200
|
|
156
|
+
"""Number of samples of the common beam search"""
|
|
157
|
+
epsilon: float = 5e-4
|
|
158
|
+
"""Epsilon of the common beam search"""
|
|
159
|
+
suffix: str = "conv"
|
|
160
|
+
"""Suffix inserted before ``.fits`` in the names of convolved images"""
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class RotateOptions(BaseOptions):
|
|
164
|
+
"""How the Q and U of images or cubes are rotated from the frame they were
|
|
165
|
+
imaged in to north at each pixel, the IAU convention, as ``patchwork mosaic``
|
|
166
|
+
rotates them in flight"""
|
|
167
|
+
|
|
168
|
+
input_frame: Literal["reference", "crval"] = "reference"
|
|
169
|
+
"""What the Q and U of each input are referenced to, as for the mosaic. ``reference``: north at ``frame_centre``, or at the position the input's POLFRAME keywords give; an input with neither is refused. ``crval``: north at each input's own reference position, as an imager references a single pointing. An input whose keywords say otherwise is refused, as rotating it from another frame would turn it twice"""
|
|
170
|
+
frame_centre: tuple[float, float] | None = None
|
|
171
|
+
"""The (RA, Dec) in degrees that the Q and U of every input are referenced to north at, such as the boresight of an ASKAP observation, with ``input_frame="reference"``"""
|
|
172
|
+
suffix: str = "rot"
|
|
173
|
+
"""Suffix inserted before ``.fits`` in the names of rotated images"""
|
|
174
|
+
reverse_rotation: bool = False
|
|
175
|
+
"""Turn Q and U the opposite way, by minus the angle between the frame of each input and north at each pixel. This is for testing the direction of the rotation, such as where the angles of overlapping fields disagree. The outputs still record that they are referenced to north at each pixel, and their HISTORY that the rotation was reversed"""
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class MosaicOptions(NamedTuple):
|
|
179
|
+
"""The options of every stage of the mosaicking"""
|
|
180
|
+
|
|
181
|
+
frame: FrameOptions = FrameOptions()
|
|
182
|
+
"""How the output frame is constructed"""
|
|
183
|
+
primary_beam: PrimaryBeamOptions = PrimaryBeamOptions()
|
|
184
|
+
"""The primary beam model"""
|
|
185
|
+
weight: WeightOptions = WeightOptions()
|
|
186
|
+
"""How each input is weighted"""
|
|
187
|
+
resample: ResampleOptions = ResampleOptions()
|
|
188
|
+
"""How inputs are resampled"""
|
|
189
|
+
combine: CombineOptions = CombineOptions()
|
|
190
|
+
"""How resampled inputs are combined"""
|
|
191
|
+
polarisation: PolarisationOptions = PolarisationOptions()
|
|
192
|
+
"""How linear polarisation is mosaicked"""
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def add_mosaic_options_to_parser(parser: ArgumentParser) -> ArgumentParser:
|
|
196
|
+
"""Add the options of every mosaicking stage to a parser."""
|
|
197
|
+
for options_class in MosaicOptions._field_defaults.values():
|
|
198
|
+
add_options_to_parser(parser=parser, options_class=type(options_class))
|
|
199
|
+
return parser
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def create_mosaic_options_from_parser(args: Namespace) -> MosaicOptions:
|
|
203
|
+
"""Create the options of every mosaicking stage from parsed arguments."""
|
|
204
|
+
return MosaicOptions(
|
|
205
|
+
**{
|
|
206
|
+
name: create_options_from_parser(
|
|
207
|
+
parser_namespace=args, options_class=type(default)
|
|
208
|
+
)
|
|
209
|
+
for name, default in MosaicOptions._field_defaults.items()
|
|
210
|
+
}
|
|
211
|
+
)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Dask clusters that the pipeline runs on"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
import logging
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import TYPE_CHECKING, Any
|
|
9
|
+
|
|
10
|
+
import yaml
|
|
11
|
+
from distributed import LocalCluster
|
|
12
|
+
from distributed.diagnostics.plugin import WorkerPlugin
|
|
13
|
+
|
|
14
|
+
from patchwork.logging import logger, setup_logging
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from distributed import Worker
|
|
18
|
+
from distributed.deploy.cluster import Cluster
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# Without distributed's type hints, as in pre-commit's mypy, WorkerPlugin is Any
|
|
22
|
+
class LogOnWorkers(WorkerPlugin): # type: ignore[misc,unused-ignore]
|
|
23
|
+
"""Prints patchwork's messages on every worker of a cluster, including those that join later"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, level: int | str = logging.INFO) -> None:
|
|
26
|
+
self.level = level
|
|
27
|
+
|
|
28
|
+
def setup(self, worker: Worker) -> None: # noqa: ARG002
|
|
29
|
+
"""Print patchwork's messages on the worker."""
|
|
30
|
+
setup_logging(self.level)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def get_cluster_spec(cluster_config: str | Path) -> dict[str, Any]:
|
|
34
|
+
"""Read a cluster specification, with keys ``cluster_class``, ``cluster_kwargs`` and ``adapt_kwargs``."""
|
|
35
|
+
with Path(cluster_config).open() as config_file:
|
|
36
|
+
spec: dict[str, Any] = yaml.safe_load(config_file)
|
|
37
|
+
if "cluster_class" not in spec:
|
|
38
|
+
msg = f"{cluster_config} does not name a cluster_class"
|
|
39
|
+
raise ValueError(msg)
|
|
40
|
+
return spec
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def get_cluster(
|
|
44
|
+
cluster_config: str | Path | None = None, n_workers: int | None = None
|
|
45
|
+
) -> Cluster:
|
|
46
|
+
"""Start a dask cluster, either on this machine or from a cluster specification.
|
|
47
|
+
|
|
48
|
+
A specification names the class of the cluster, such as
|
|
49
|
+
``dask_jobqueue.SLURMCluster``, the arguments it is created with, and
|
|
50
|
+
optionally how it adapts its number of workers. Without one a local cluster
|
|
51
|
+
of single threaded workers is started.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
cluster_config (str | Path | None, optional): YAML cluster specification. Defaults to None, a local cluster.
|
|
55
|
+
n_workers (int | None, optional): Number of workers, when the cluster does not adapt. Defaults to None.
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
Cluster: The running cluster
|
|
59
|
+
"""
|
|
60
|
+
if cluster_config is None:
|
|
61
|
+
return LocalCluster(n_workers=n_workers, threads_per_worker=1)
|
|
62
|
+
|
|
63
|
+
spec = get_cluster_spec(cluster_config)
|
|
64
|
+
module_name, class_name = spec["cluster_class"].rsplit(".", 1)
|
|
65
|
+
cluster_class = getattr(importlib.import_module(module_name), class_name)
|
|
66
|
+
cluster: Cluster = cluster_class(**spec.get("cluster_kwargs", {}))
|
|
67
|
+
logger.info(f"Started a {class_name} from {cluster_config}")
|
|
68
|
+
|
|
69
|
+
if spec.get("adapt_kwargs"):
|
|
70
|
+
cluster.adapt(**spec["adapt_kwargs"])
|
|
71
|
+
elif n_workers is not None:
|
|
72
|
+
cluster.scale(n_workers)
|
|
73
|
+
return cluster
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""The patchwork pipeline: mosaic images or cubes of Stokes I, Q, U and V on a dask cluster.
|
|
2
|
+
|
|
3
|
+
The images are given as one list, in any order, with a weights log or weight
|
|
4
|
+
image for each. Each one's Stokes parameter comes from its header, and those
|
|
5
|
+
on one pixel grid are taken to be one beam. With --convolve the inputs are
|
|
6
|
+
first convolved to a common beam, one to each task. Each task then makes one
|
|
7
|
+
tile of the mosaic over some channels, for every Stokes parameter at once.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import time
|
|
13
|
+
from argparse import ArgumentParser, Namespace
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
|
+
|
|
17
|
+
from distributed import Client
|
|
18
|
+
|
|
19
|
+
from patchwork.logging import logger, setup_logging
|
|
20
|
+
from patchwork.mosaic import (
|
|
21
|
+
add_log_level_argument,
|
|
22
|
+
add_mosaic_arguments,
|
|
23
|
+
mosaic,
|
|
24
|
+
options_from_args,
|
|
25
|
+
)
|
|
26
|
+
from patchwork.options import ConvolveOptions, MosaicOptions, RunOptions
|
|
27
|
+
from patchwork.pipeline.cluster import LogOnWorkers, get_cluster
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from patchwork.plan import MosaicNames
|
|
31
|
+
|
|
32
|
+
SCATTER_ATTEMPTS = 5
|
|
33
|
+
"""Times the plan is offered to the cluster before giving up, as workers may leave while it is sent"""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def run_pipeline(
|
|
37
|
+
images: list[Path],
|
|
38
|
+
output_base: str | Path,
|
|
39
|
+
client: Client,
|
|
40
|
+
options: MosaicOptions = MosaicOptions(), # noqa: B008
|
|
41
|
+
run_options: RunOptions = RunOptions(), # noqa: B008
|
|
42
|
+
convolve_options: ConvolveOptions = ConvolveOptions(), # noqa: B008
|
|
43
|
+
weights: list[Path] | None = None,
|
|
44
|
+
stokes_i_images: list[Path] | None = None,
|
|
45
|
+
overwrite: bool = False,
|
|
46
|
+
) -> MosaicNames:
|
|
47
|
+
"""Mosaic images or cubes of Stokes I, Q, U and V on a dask cluster.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
images (list[Path]): The images or cubes, in any order
|
|
51
|
+
output_base (str | Path): Base name of the output files
|
|
52
|
+
client (Client): Client of the cluster to run on
|
|
53
|
+
options (MosaicOptions, optional): The mosaicking options. Defaults to MosaicOptions().
|
|
54
|
+
run_options (RunOptions, optional): Which channels are mosaicked, whether the inputs are convolved first, and how the mosaic is written. Defaults to RunOptions().
|
|
55
|
+
convolve_options (ConvolveOptions, optional): How the common beam is found and convolved to. Defaults to ConvolveOptions().
|
|
56
|
+
weights (list[Path] | None, optional): A weights log or weight image for each image, in the same order. Defaults to None, weighting by noise.
|
|
57
|
+
stokes_i_images (list[Path] | None, optional): Stokes I images to remove the leakage with, but not mosaic. Defaults to None.
|
|
58
|
+
overwrite (bool, optional): Replace existing outputs. Defaults to False.
|
|
59
|
+
|
|
60
|
+
Returns:
|
|
61
|
+
MosaicNames: The files the mosaic was written to
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def share(plan: Any) -> Any:
|
|
65
|
+
# A plan is a tuple, which scatter would otherwise split into its fields.
|
|
66
|
+
# It is not broadcast: the workers fetch it from each other as their
|
|
67
|
+
# first tasks need it, whereas broadcasting replicates it to a list of
|
|
68
|
+
# workers that adaptive scaling may retire meanwhile, which the
|
|
69
|
+
# scheduler raises as a KeyError. The same can befall the one worker
|
|
70
|
+
# it is sent to, so the scatter is retried
|
|
71
|
+
for attempt in range(1, SCATTER_ATTEMPTS):
|
|
72
|
+
try:
|
|
73
|
+
return client.scatter([plan])[0]
|
|
74
|
+
except (KeyError, OSError) as error:
|
|
75
|
+
logger.warning(
|
|
76
|
+
f"Sending the plan to the workers failed ({error!r}), "
|
|
77
|
+
f"retrying ({attempt} of {SCATTER_ATTEMPTS - 1})"
|
|
78
|
+
)
|
|
79
|
+
time.sleep(attempt)
|
|
80
|
+
return client.scatter([plan])[0]
|
|
81
|
+
|
|
82
|
+
return mosaic(
|
|
83
|
+
images=images,
|
|
84
|
+
output_base=output_base,
|
|
85
|
+
options=options,
|
|
86
|
+
run_options=run_options,
|
|
87
|
+
convolve_options=convolve_options,
|
|
88
|
+
weights=weights,
|
|
89
|
+
stokes_i_images=stokes_i_images,
|
|
90
|
+
overwrite=overwrite,
|
|
91
|
+
# The tasks write files, so are not pure: each call is a task of its own
|
|
92
|
+
executor=client.get_executor(pure=False),
|
|
93
|
+
share=share,
|
|
94
|
+
capacity=lambda: sum(client.nthreads().values()),
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def get_parser() -> ArgumentParser:
|
|
99
|
+
"""The parser of the ``patchwork-pipeline`` command."""
|
|
100
|
+
parser = add_log_level_argument(
|
|
101
|
+
add_mosaic_arguments(ArgumentParser(description=__doc__))
|
|
102
|
+
)
|
|
103
|
+
parser.add_argument(
|
|
104
|
+
"--cluster-config",
|
|
105
|
+
type=Path,
|
|
106
|
+
default=None,
|
|
107
|
+
help="YAML specification of the dask cluster to run on. If not given a local cluster is started",
|
|
108
|
+
)
|
|
109
|
+
parser.add_argument(
|
|
110
|
+
"--n-workers",
|
|
111
|
+
type=int,
|
|
112
|
+
default=None,
|
|
113
|
+
help="Number of dask workers, when the cluster does not adapt",
|
|
114
|
+
)
|
|
115
|
+
return parser
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def cli(args: Namespace | None = None) -> None:
|
|
119
|
+
"""Run the ``patchwork-pipeline`` command."""
|
|
120
|
+
setup_logging()
|
|
121
|
+
args = get_parser().parse_args() if args is None else args
|
|
122
|
+
setup_logging(args.log_level)
|
|
123
|
+
options, run_options, convolve_options = options_from_args(args)
|
|
124
|
+
with (
|
|
125
|
+
get_cluster(args.cluster_config, n_workers=args.n_workers) as cluster,
|
|
126
|
+
Client(cluster) as client,
|
|
127
|
+
):
|
|
128
|
+
client.register_plugin(LogOnWorkers(args.log_level))
|
|
129
|
+
run_pipeline(
|
|
130
|
+
images=args.images,
|
|
131
|
+
output_base=args.output_base,
|
|
132
|
+
client=client,
|
|
133
|
+
options=options,
|
|
134
|
+
run_options=run_options,
|
|
135
|
+
convolve_options=convolve_options,
|
|
136
|
+
weights=args.weights,
|
|
137
|
+
stokes_i_images=args.stokes_i_images,
|
|
138
|
+
overwrite=args.overwrite,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
if __name__ == "__main__":
|
|
143
|
+
cli()
|