askap-patchwork 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askap_patchwork-0.1.0.dist-info/METADATA +179 -0
- askap_patchwork-0.1.0.dist-info/RECORD +33 -0
- askap_patchwork-0.1.0.dist-info/WHEEL +4 -0
- askap_patchwork-0.1.0.dist-info/entry_points.txt +4 -0
- askap_patchwork-0.1.0.dist-info/licenses/LICENSE +28 -0
- patchwork/__init__.py +11 -0
- patchwork/__main__.py +8 -0
- patchwork/_version.py +24 -0
- patchwork/_version.pyi +2 -0
- patchwork/cli.py +60 -0
- patchwork/combine.py +146 -0
- patchwork/convolve.py +534 -0
- patchwork/coordinates.py +222 -0
- patchwork/frame.py +439 -0
- patchwork/fused.py +618 -0
- patchwork/fused_kernels.py +695 -0
- patchwork/io.py +668 -0
- patchwork/logging.py +62 -0
- patchwork/mosaic.py +546 -0
- patchwork/numba_kernels.py +203 -0
- patchwork/options.py +211 -0
- patchwork/pipeline/__init__.py +3 -0
- patchwork/pipeline/cluster.py +73 -0
- patchwork/pipeline/flow.py +143 -0
- patchwork/plan.py +818 -0
- patchwork/polarisation.py +178 -0
- patchwork/primary_beam.py +636 -0
- patchwork/py.typed +0 -0
- patchwork/resample.py +267 -0
- patchwork/rotate.py +319 -0
- patchwork/tile.py +344 -0
- patchwork/weights.py +175 -0
- patchwork/write.py +502 -0
patchwork/io.py
ADDED
|
@@ -0,0 +1,668 @@
|
|
|
1
|
+
"""Reading input images and cubes, and writing mosaics to FITS"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import functools
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import warnings
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import TYPE_CHECKING, NamedTuple
|
|
11
|
+
|
|
12
|
+
import astropy.units as u
|
|
13
|
+
import numpy as np
|
|
14
|
+
from astropy.io import fits
|
|
15
|
+
from astropy.table import Table
|
|
16
|
+
from astropy.wcs import WCS, FITSFixedWarning
|
|
17
|
+
from radio_beam import Beam, Beams
|
|
18
|
+
|
|
19
|
+
from patchwork.logging import logger
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from collections.abc import Iterable
|
|
23
|
+
|
|
24
|
+
from numpy.typing import NDArray
|
|
25
|
+
|
|
26
|
+
Slices = tuple[slice, slice]
|
|
27
|
+
|
|
28
|
+
BLANK_BEAM = float(np.finfo(np.float32).tiny)
|
|
29
|
+
"""The value fitscube writes into a BEAMS table for a channel with no restoring beam, as CASA rejects NaN"""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class BeamShape(NamedTuple):
|
|
33
|
+
"""A restoring beam held as plain floats, as astropy quantities do not travel
|
|
34
|
+
well to dask workers"""
|
|
35
|
+
|
|
36
|
+
bmaj_arcsec: float
|
|
37
|
+
"""Major axis FWHM in arcseconds"""
|
|
38
|
+
bmin_arcsec: float
|
|
39
|
+
"""Minor axis FWHM in arcseconds"""
|
|
40
|
+
bpa_deg: float
|
|
41
|
+
"""Position angle of the major axis, east of north, in degrees"""
|
|
42
|
+
|
|
43
|
+
def __str__(self) -> str:
|
|
44
|
+
return f'{self.bmaj_arcsec:g}" x {self.bmin_arcsec:g}", PA {self.bpa_deg:g} deg'
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
def from_header(cls, header: fits.Header) -> BeamShape | None:
|
|
48
|
+
"""The beam described by the BMAJ, BMIN and BPA of a header, if it has one."""
|
|
49
|
+
if "BMAJ" not in header or "BMIN" not in header:
|
|
50
|
+
return None
|
|
51
|
+
return cls(
|
|
52
|
+
bmaj_arcsec=float(header["BMAJ"]) * 3600,
|
|
53
|
+
bmin_arcsec=float(header["BMIN"]) * 3600,
|
|
54
|
+
bpa_deg=float(header.get("BPA", 0.0)),
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
@classmethod
|
|
58
|
+
def from_radio_beam(cls, beam: Beam) -> BeamShape:
|
|
59
|
+
"""Convert a ``radio_beam.Beam``."""
|
|
60
|
+
return cls(
|
|
61
|
+
bmaj_arcsec=float(beam.major.to(u.arcsec).value),
|
|
62
|
+
bmin_arcsec=float(beam.minor.to(u.arcsec).value),
|
|
63
|
+
bpa_deg=float(beam.pa.to(u.deg).value),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def to_radio_beam(self) -> Beam:
|
|
67
|
+
"""Convert to a ``radio_beam.Beam``."""
|
|
68
|
+
return Beam(
|
|
69
|
+
major=self.bmaj_arcsec * u.arcsec,
|
|
70
|
+
minor=self.bmin_arcsec * u.arcsec,
|
|
71
|
+
pa=self.bpa_deg * u.deg,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
def is_usable(self) -> bool:
|
|
75
|
+
"""Whether the beam is a real one: finite, and larger than zero or the blank placeholder."""
|
|
76
|
+
return bool(
|
|
77
|
+
np.all(np.isfinite(self))
|
|
78
|
+
and self.bmaj_arcsec > BLANK_BEAM
|
|
79
|
+
and self.bmin_arcsec > BLANK_BEAM
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def to_header(self, header: fits.Header) -> None:
|
|
83
|
+
"""Write the beam into a header."""
|
|
84
|
+
header["BMAJ"] = self.bmaj_arcsec / 3600
|
|
85
|
+
header["BMIN"] = self.bmin_arcsec / 3600
|
|
86
|
+
header["BPA"] = self.bpa_deg
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
NO_BEAM = BeamShape(np.nan, np.nan, np.nan)
|
|
90
|
+
"""The beam of a channel that holds no image"""
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class ImageInfo(NamedTuple):
|
|
94
|
+
"""What the mosaicking needs to know about an input image or cube, read from its header"""
|
|
95
|
+
|
|
96
|
+
path: Path
|
|
97
|
+
"""Path to the image"""
|
|
98
|
+
shape: tuple[int, int]
|
|
99
|
+
"""The (ny, nx) shape of the image plane"""
|
|
100
|
+
wcs: WCS
|
|
101
|
+
"""Celestial WCS of the image plane"""
|
|
102
|
+
header: fits.Header
|
|
103
|
+
"""The full header of the image"""
|
|
104
|
+
frequency_hz: float | None
|
|
105
|
+
"""Frequency of the image (its first channel), if known"""
|
|
106
|
+
beam: BeamShape | None
|
|
107
|
+
"""Restoring beam of the image (its first channel), if known"""
|
|
108
|
+
beam_number: int | None
|
|
109
|
+
"""The beam number, used to pick the beam from a holography cube"""
|
|
110
|
+
beam_centre: tuple[float, float]
|
|
111
|
+
"""The (RA, Dec) in degrees of the primary beam centre"""
|
|
112
|
+
stokes: int | None = None
|
|
113
|
+
"""The FITS code of the Stokes parameter of the image (1 to 4 for I, Q, U and V), if it has a STOKES axis"""
|
|
114
|
+
n_channels: int = 1
|
|
115
|
+
"""The number of channels along the FREQ axis"""
|
|
116
|
+
frequencies_hz: tuple[float, ...] = ()
|
|
117
|
+
"""The frequency of each channel, if known"""
|
|
118
|
+
channel_beams: tuple[BeamShape | None, ...] = ()
|
|
119
|
+
"""The restoring beam of each channel, from a BEAMS table or the header, where None is unknown"""
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def beam_number_from_name(path: str | Path) -> int | None:
|
|
123
|
+
"""The beam number in a name such as ``SB1234.field.beam07.fits``, if there is one."""
|
|
124
|
+
matches = re.findall(r"beam(\d+)", Path(path).name, flags=re.IGNORECASE)
|
|
125
|
+
return int(matches[-1]) if matches else None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _find_axis(header: fits.Header, prefix: str) -> int | None:
|
|
129
|
+
"""The FITS number of the first axis whose CTYPE starts with ``prefix``."""
|
|
130
|
+
for axis in range(1, header.get("NAXIS", 0) + 1):
|
|
131
|
+
if str(header.get(f"CTYPE{axis}", "")).upper().startswith(prefix):
|
|
132
|
+
return axis
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _axis_values(header: fits.Header, axis: int) -> NDArray[np.float64]:
|
|
137
|
+
"""The world value of each pixel along an axis."""
|
|
138
|
+
pixels = np.arange(header[f"NAXIS{axis}"]) + 1
|
|
139
|
+
crpix = header.get(f"CRPIX{axis}", 1.0)
|
|
140
|
+
return np.asarray(
|
|
141
|
+
header[f"CRVAL{axis}"] + (pixels - crpix) * header.get(f"CDELT{axis}", 1.0)
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def spectral_axis(header: fits.Header) -> int | None:
|
|
146
|
+
"""The FITS number of the FREQ axis, if there is one."""
|
|
147
|
+
return _find_axis(header, "FREQ")
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def stokes_from_header(header: fits.Header) -> int | None:
|
|
151
|
+
"""The FITS code of the Stokes parameter of the first pixel of a STOKES axis, if there is one."""
|
|
152
|
+
axis = _find_axis(header, "STOKES")
|
|
153
|
+
if axis is None:
|
|
154
|
+
return None
|
|
155
|
+
return round(float(_axis_values(header, axis)[0]))
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def frequencies_from_header(header: fits.Header) -> tuple[float, ...]:
|
|
159
|
+
"""The frequency of each channel of a FREQ axis, else the rest frequency, if either is known."""
|
|
160
|
+
axis = spectral_axis(header)
|
|
161
|
+
if axis is not None:
|
|
162
|
+
return tuple(float(value) for value in _axis_values(header, axis))
|
|
163
|
+
for key in ("RESTFRQ", "RESTFREQ"):
|
|
164
|
+
if key in header:
|
|
165
|
+
return (float(header[key]),)
|
|
166
|
+
return ()
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def frequency_from_header(header: fits.Header) -> float | None:
|
|
170
|
+
"""The frequency of the first pixel of a FREQ axis, else the rest frequency, if either is known."""
|
|
171
|
+
frequencies = frequencies_from_header(header)
|
|
172
|
+
return frequencies[0] if frequencies else None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def full_wcs(header: fits.Header) -> WCS:
|
|
176
|
+
"""The WCS of a header, without the warnings astropy raises as it tidies real headers."""
|
|
177
|
+
with warnings.catch_warnings():
|
|
178
|
+
warnings.simplefilter("ignore", FITSFixedWarning)
|
|
179
|
+
return WCS(header)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def celestial_wcs(header: fits.Header) -> WCS:
|
|
183
|
+
"""The celestial WCS of a header."""
|
|
184
|
+
return full_wcs(header).celestial
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _check_axes(header: fits.Header, path: Path) -> None:
|
|
188
|
+
"""Raise unless an image has its celestial axes first, and no axis beyond them but FREQ is longer than one."""
|
|
189
|
+
full = full_wcs(header)
|
|
190
|
+
if (full.wcs.lng, full.wcs.lat) != (0, 1):
|
|
191
|
+
msg = f"The first two axes of {path} must be celestial longitude and latitude"
|
|
192
|
+
raise ValueError(msg)
|
|
193
|
+
for axis in range(3, header["NAXIS"] + 1):
|
|
194
|
+
if header[f"NAXIS{axis}"] > 1 and axis != spectral_axis(header):
|
|
195
|
+
msg = (
|
|
196
|
+
f"Axis {axis} ({header.get(f'CTYPE{axis}')}) of {path} has more than one "
|
|
197
|
+
"pixel. Only a FREQ axis may, so each cube must hold one Stokes parameter"
|
|
198
|
+
)
|
|
199
|
+
raise ValueError(msg)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def channel_beams_from_hdul(
|
|
203
|
+
hdul: fits.HDUList, n_channels: int
|
|
204
|
+
) -> tuple[BeamShape | None, ...]:
|
|
205
|
+
"""The restoring beam of each channel: from a BEAMS table if there is one, otherwise the header beam."""
|
|
206
|
+
if "BEAMS" not in hdul:
|
|
207
|
+
return (BeamShape.from_header(hdul[0].header),) * n_channels
|
|
208
|
+
|
|
209
|
+
table = hdul["BEAMS"]
|
|
210
|
+
# radio_beam reads the axes in the units of their columns, which are not always arcsec
|
|
211
|
+
found = Beams.from_fits_bintable(table)
|
|
212
|
+
major, minor = found.major.to_value(u.arcsec), found.minor.to_value(u.arcsec)
|
|
213
|
+
pa = found.pa.to_value(u.deg)
|
|
214
|
+
channels, pols = table.data["CHAN"], table.data["POL"]
|
|
215
|
+
beams: list[BeamShape | None] = [NO_BEAM] * n_channels
|
|
216
|
+
for row in np.flatnonzero(pols == np.min(pols)):
|
|
217
|
+
if 0 <= channels[row] < n_channels:
|
|
218
|
+
beams[channels[row]] = BeamShape(
|
|
219
|
+
float(major[row]), float(minor[row]), float(pa[row])
|
|
220
|
+
)
|
|
221
|
+
return tuple(beams)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def read_image_info(
|
|
225
|
+
path: str | Path,
|
|
226
|
+
beam_number: int | None = None,
|
|
227
|
+
frequency_hz: float | None = None,
|
|
228
|
+
beam_centre: tuple[float, float] | None = None,
|
|
229
|
+
) -> ImageInfo:
|
|
230
|
+
"""Read what the mosaicking needs to know from the header of an image or cube.
|
|
231
|
+
|
|
232
|
+
Unless given, the beam number is taken from the file name, the frequency
|
|
233
|
+
from the header, and the beam centre from the reference value of the
|
|
234
|
+
celestial axes.
|
|
235
|
+
|
|
236
|
+
Args:
|
|
237
|
+
path (str | Path): The image to read
|
|
238
|
+
beam_number (int | None, optional): Beam number of the image. Defaults to None.
|
|
239
|
+
frequency_hz (float | None, optional): Frequency of an image with no frequency in its header. Defaults to None.
|
|
240
|
+
beam_centre (tuple[float, float] | None, optional): (RA, Dec) in degrees of the primary beam centre. Defaults to None.
|
|
241
|
+
|
|
242
|
+
Returns:
|
|
243
|
+
ImageInfo: The description of the image
|
|
244
|
+
"""
|
|
245
|
+
path = Path(path)
|
|
246
|
+
with fits.open(path, memmap=True) as hdul:
|
|
247
|
+
header = hdul[0].header.copy()
|
|
248
|
+
_check_axes(header=header, path=path)
|
|
249
|
+
axis = spectral_axis(header)
|
|
250
|
+
n_channels = 1 if axis is None else int(header[f"NAXIS{axis}"])
|
|
251
|
+
channel_beams = channel_beams_from_hdul(hdul, n_channels)
|
|
252
|
+
|
|
253
|
+
frequencies = frequencies_from_header(header) or (
|
|
254
|
+
(frequency_hz,) if frequency_hz else ()
|
|
255
|
+
)
|
|
256
|
+
wcs = celestial_wcs(header)
|
|
257
|
+
return ImageInfo(
|
|
258
|
+
path=path,
|
|
259
|
+
shape=(int(header["NAXIS2"]), int(header["NAXIS1"])),
|
|
260
|
+
wcs=wcs,
|
|
261
|
+
header=header,
|
|
262
|
+
frequency_hz=frequencies[0] if frequencies else None,
|
|
263
|
+
beam=channel_beams[0],
|
|
264
|
+
beam_number=beam_number
|
|
265
|
+
if beam_number is not None
|
|
266
|
+
else beam_number_from_name(path),
|
|
267
|
+
beam_centre=beam_centre or (float(wcs.wcs.crval[0]), float(wcs.wcs.crval[1])),
|
|
268
|
+
stokes=stokes_from_header(header),
|
|
269
|
+
n_channels=n_channels,
|
|
270
|
+
frequencies_hz=frequencies,
|
|
271
|
+
channel_beams=channel_beams,
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def data_index(
|
|
276
|
+
header: fits.Header, region: Slices | None = None, channels: slice | None = None
|
|
277
|
+
) -> tuple[slice, ...]:
|
|
278
|
+
"""The index into the data of a FITS image of a region of some of its channels, keeping every axis."""
|
|
279
|
+
naxis = int(header["NAXIS"])
|
|
280
|
+
ys, xs = region or (slice(None), slice(None))
|
|
281
|
+
spectral = spectral_axis(header)
|
|
282
|
+
index = [
|
|
283
|
+
(channels or slice(None)) if naxis - numpy_axis == spectral else slice(0, 1)
|
|
284
|
+
for numpy_axis in range(naxis - 2)
|
|
285
|
+
]
|
|
286
|
+
return (*index, ys, xs)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def read_block(
|
|
290
|
+
path: str | Path,
|
|
291
|
+
region: Slices | None = None,
|
|
292
|
+
channels: slice | None = None,
|
|
293
|
+
dtype: type[np.floating] = np.float64,
|
|
294
|
+
out: NDArray[np.floating] | None = None,
|
|
295
|
+
) -> NDArray[np.floating]:
|
|
296
|
+
"""Read a region of some channels of a FITS image or cube, as (channel, y, x).
|
|
297
|
+
|
|
298
|
+
The file is memory mapped, so only the region is read from disk.
|
|
299
|
+
|
|
300
|
+
Args:
|
|
301
|
+
path (str | Path): The image or cube to read
|
|
302
|
+
region (Slices | None, optional): The (y, x) slices to read. Defaults to None, reading all of each plane.
|
|
303
|
+
channels (slice | None, optional): The channels to read. Defaults to None, reading every channel.
|
|
304
|
+
dtype (type[np.floating], optional): The type of the block. Defaults to np.float64.
|
|
305
|
+
out (NDArray[np.floating] | None, optional): A (channel, y, x) array to read the block into, in its own type. Defaults to None, making a new one.
|
|
306
|
+
|
|
307
|
+
Returns:
|
|
308
|
+
NDArray[np.floating]: The block, with NaN wherever there is no data
|
|
309
|
+
"""
|
|
310
|
+
with fits.open(path, memmap=True) as hdul:
|
|
311
|
+
hdu = hdul[0]
|
|
312
|
+
data = hdu.data[data_index(hdu.header, region, channels)]
|
|
313
|
+
if out is not None:
|
|
314
|
+
out[...] = data.reshape(-1, *data.shape[-2:])
|
|
315
|
+
return out
|
|
316
|
+
# One copy, in the machine's byte order, which no longer needs the file
|
|
317
|
+
block = np.array(data, dtype=dtype)
|
|
318
|
+
return block.reshape(-1, *block.shape[-2:])
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def read_plane(path: str | Path, region: Slices | None = None) -> NDArray[np.float64]:
|
|
322
|
+
"""Read the image plane of a FITS file, or a region of it.
|
|
323
|
+
|
|
324
|
+
Args:
|
|
325
|
+
path (str | Path): The image to read
|
|
326
|
+
region (Slices | None, optional): The (y, x) slices to read. Defaults to None, reading all of it.
|
|
327
|
+
|
|
328
|
+
Raises:
|
|
329
|
+
ValueError: Raised if the image has more than one channel
|
|
330
|
+
|
|
331
|
+
Returns:
|
|
332
|
+
NDArray[np.float64]: The plane, with NaN wherever there is no data
|
|
333
|
+
"""
|
|
334
|
+
block = read_block(path, region)
|
|
335
|
+
if block.shape[0] > 1:
|
|
336
|
+
msg = f"{path} has {block.shape[0]} channels, but one plane is read from it"
|
|
337
|
+
raise ValueError(msg)
|
|
338
|
+
return np.asarray(block[0])
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def write_block(
|
|
342
|
+
hdu: fits.PrimaryHDU,
|
|
343
|
+
region: Slices,
|
|
344
|
+
block: NDArray[np.floating],
|
|
345
|
+
channels: slice | None = None,
|
|
346
|
+
) -> None:
|
|
347
|
+
"""Write a (channel, y, x) block into the data of a FITS image opened for update."""
|
|
348
|
+
index = data_index(hdu.header, region, channels)
|
|
349
|
+
target = hdu.data[index]
|
|
350
|
+
target[...] = np.reshape(block, target.shape)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
class DataLocation(NamedTuple):
|
|
354
|
+
"""Where the data of a FITS image lies in its file, to read and write whole rows of it in place"""
|
|
355
|
+
|
|
356
|
+
path: Path
|
|
357
|
+
"""The file"""
|
|
358
|
+
offset: int
|
|
359
|
+
"""The byte the data starts at"""
|
|
360
|
+
shape: tuple[int, ...]
|
|
361
|
+
"""The shape of the data, in numpy order"""
|
|
362
|
+
spectral: int | None
|
|
363
|
+
"""The numpy axis of the channels, if there is one"""
|
|
364
|
+
|
|
365
|
+
@property
|
|
366
|
+
def n_channels(self) -> int:
|
|
367
|
+
"""The number of channels of the data."""
|
|
368
|
+
return 1 if self.spectral is None else self.shape[self.spectral]
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def data_location(path: str | Path) -> DataLocation:
|
|
372
|
+
"""Find where the data of a float32 FITS image lies in its file.
|
|
373
|
+
|
|
374
|
+
Raises:
|
|
375
|
+
ValueError: Raised if its data are not plain float32, which rows are read and written as
|
|
376
|
+
"""
|
|
377
|
+
with fits.open(path) as hdul:
|
|
378
|
+
header = hdul[0].header
|
|
379
|
+
offset = int(hdul.fileinfo(0)["datLoc"])
|
|
380
|
+
if (
|
|
381
|
+
header["BITPIX"] != -32
|
|
382
|
+
or header.get("BSCALE", 1) != 1
|
|
383
|
+
or header.get("BZERO", 0)
|
|
384
|
+
):
|
|
385
|
+
msg = f"The data of {path} are not plain float32, so its rows cannot be written in place"
|
|
386
|
+
raise ValueError(msg)
|
|
387
|
+
naxis = int(header["NAXIS"])
|
|
388
|
+
axis = spectral_axis(header)
|
|
389
|
+
return DataLocation(
|
|
390
|
+
path=Path(path),
|
|
391
|
+
offset=offset,
|
|
392
|
+
shape=tuple(int(header[f"NAXIS{number}"]) for number in range(naxis, 0, -1)),
|
|
393
|
+
spectral=None if axis is None else naxis - axis,
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@functools.lru_cache(maxsize=4096)
|
|
398
|
+
def _mapped_location(path: str, modified_ns: int, size: int) -> DataLocation | None: # noqa: ARG001
|
|
399
|
+
"""Where the data of a FITS file lies, if it can be read straight from the file: plain float32 that is not compressed and is all there.
|
|
400
|
+
|
|
401
|
+
It is kept for the file as it was when asked, by its time of modification
|
|
402
|
+
and size, so a file that is rewritten is looked at again.
|
|
403
|
+
"""
|
|
404
|
+
with Path(path).open("rb") as file:
|
|
405
|
+
# A compressed file opens as FITS too, but its bytes are not the data
|
|
406
|
+
if file.read(9) != b"SIMPLE =":
|
|
407
|
+
return None
|
|
408
|
+
try:
|
|
409
|
+
location = data_location(path)
|
|
410
|
+
except ValueError:
|
|
411
|
+
return None
|
|
412
|
+
if location.offset + 4 * int(np.prod(location.shape)) > size:
|
|
413
|
+
return None
|
|
414
|
+
return location
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def read_window(
|
|
418
|
+
path: str | Path,
|
|
419
|
+
region: Slices,
|
|
420
|
+
channels: slice | None,
|
|
421
|
+
out: NDArray[np.floating],
|
|
422
|
+
) -> NDArray[np.floating]:
|
|
423
|
+
"""Read a region of some channels of a FITS image or cube into ``out``, as ``read_block`` does.
|
|
424
|
+
|
|
425
|
+
``read_block`` opens the file with astropy each time, which parses its
|
|
426
|
+
header while holding the GIL. Many windows of one file are read from
|
|
427
|
+
threads here, so the header is parsed once, and each window is then copied
|
|
428
|
+
from a memory map of the file, which does not hold the GIL. A file whose
|
|
429
|
+
data are not plain float32 is read by ``read_block``.
|
|
430
|
+
|
|
431
|
+
Args:
|
|
432
|
+
path (str | Path): The image or cube to read
|
|
433
|
+
region (Slices): The (y, x) slices to read
|
|
434
|
+
channels (slice | None): The channels to read, or None for every channel
|
|
435
|
+
out (NDArray[np.floating]): The (channel, y, x) array the window is read into
|
|
436
|
+
|
|
437
|
+
Returns:
|
|
438
|
+
NDArray[np.floating]: ``out``, with NaN wherever there is no data
|
|
439
|
+
"""
|
|
440
|
+
status = Path(path).stat()
|
|
441
|
+
location = _mapped_location(str(path), status.st_mtime_ns, status.st_size)
|
|
442
|
+
if location is None:
|
|
443
|
+
return read_block(path, region, channels, out=out)
|
|
444
|
+
data = np.memmap(
|
|
445
|
+
path, dtype=">f4", mode="r", offset=location.offset, shape=location.shape
|
|
446
|
+
)
|
|
447
|
+
index = tuple(
|
|
448
|
+
(channels or slice(None)) if axis == location.spectral else slice(0, 1)
|
|
449
|
+
for axis in range(len(location.shape) - 2)
|
|
450
|
+
)
|
|
451
|
+
window = data[(*index, *region)]
|
|
452
|
+
out[...] = window.reshape(-1, *window.shape[-2:])
|
|
453
|
+
return out
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _rows_offset(location: DataLocation, channel: int, row: int) -> int:
|
|
457
|
+
"""The byte at which a row of a channel of a float32 FITS image starts."""
|
|
458
|
+
index = [0] * len(location.shape)
|
|
459
|
+
index[-2] = row
|
|
460
|
+
if location.spectral is not None:
|
|
461
|
+
index[location.spectral] = channel
|
|
462
|
+
return location.offset + 4 * int(np.ravel_multi_index(index, location.shape))
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def write_rows(
|
|
466
|
+
location: DataLocation, rows: slice, channels: slice, block: NDArray[np.floating]
|
|
467
|
+
) -> None:
|
|
468
|
+
"""Write whole rows of some channels, a (channel, y, x) block, into a float32 FITS image in place.
|
|
469
|
+
|
|
470
|
+
Each channel's rows are one run of bytes, written without reading or
|
|
471
|
+
rewriting anything else in the file, so processes on different machines
|
|
472
|
+
may write different rows of one file at once on a file system that keeps it
|
|
473
|
+
consistent, such as Lustre.
|
|
474
|
+
"""
|
|
475
|
+
n_rows, n_channels = rows.stop - rows.start, channels.stop - channels.start
|
|
476
|
+
expected = (n_channels, n_rows, location.shape[-1])
|
|
477
|
+
# A block of the wrong shape would run on into the bytes of other rows or channels
|
|
478
|
+
if (
|
|
479
|
+
block.shape != expected
|
|
480
|
+
or rows.start < 0
|
|
481
|
+
or rows.stop > location.shape[-2]
|
|
482
|
+
or channels.start < 0
|
|
483
|
+
or channels.stop > location.n_channels
|
|
484
|
+
):
|
|
485
|
+
msg = (
|
|
486
|
+
f"A block of shape {block.shape} does not fill rows {rows.start} to {rows.stop} "
|
|
487
|
+
f"of channels {channels.start} to {channels.stop} of {location.path}, "
|
|
488
|
+
f"whose data are {location.shape}"
|
|
489
|
+
)
|
|
490
|
+
raise ValueError(msg)
|
|
491
|
+
with location.path.open("r+b") as file:
|
|
492
|
+
for index, channel in enumerate(range(channels.start, channels.stop)):
|
|
493
|
+
# The bytes of the rows as FITS holds them, without a second copy
|
|
494
|
+
data = memoryview(np.ascontiguousarray(block[index], dtype=">f4")).cast("B")
|
|
495
|
+
offset = _rows_offset(location, channel, rows.start)
|
|
496
|
+
written = 0
|
|
497
|
+
while written < len(data):
|
|
498
|
+
count = os.pwrite(file.fileno(), data[written:], offset + written)
|
|
499
|
+
if count <= 0:
|
|
500
|
+
msg = f"Could not write rows of {location.path}"
|
|
501
|
+
raise OSError(msg)
|
|
502
|
+
written += count
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def read_rows(location: DataLocation, rows: slice, channel: int) -> NDArray[np.float32]:
|
|
506
|
+
"""Read whole rows of one channel of a float32 FITS image, as (y, x)."""
|
|
507
|
+
n_rows = rows.stop - rows.start
|
|
508
|
+
size = 4 * n_rows * location.shape[-1]
|
|
509
|
+
offset = _rows_offset(location, channel, rows.start)
|
|
510
|
+
data = b""
|
|
511
|
+
with location.path.open("rb") as file:
|
|
512
|
+
while len(data) < size:
|
|
513
|
+
chunk = os.pread(file.fileno(), size - len(data), offset + len(data))
|
|
514
|
+
if not chunk:
|
|
515
|
+
msg = (
|
|
516
|
+
f"{location.path} ends before row {rows.stop} of channel {channel}"
|
|
517
|
+
)
|
|
518
|
+
raise ValueError(msg)
|
|
519
|
+
data += chunk
|
|
520
|
+
return np.frombuffer(data, dtype=">f4").astype(np.float32).reshape(n_rows, -1)
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def suffixed_name(path: str | Path, suffix: str) -> Path:
|
|
524
|
+
"""The name of an output made from an input: ``suffix`` inserted before a final ``.fits``, or added with ``.fits`` after any other name, so it never names the input itself."""
|
|
525
|
+
path = Path(path)
|
|
526
|
+
name = path.name
|
|
527
|
+
if name.lower().endswith(".fits"):
|
|
528
|
+
return path.with_name(f"{name[:-5]}.{suffix}{name[-5:]}")
|
|
529
|
+
return path.with_name(f"{name}.{suffix}.fits")
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def check_output_fits(
|
|
533
|
+
output_fits: str | Path,
|
|
534
|
+
overwrite: bool,
|
|
535
|
+
input_fits: Iterable[str | Path | None] = (),
|
|
536
|
+
) -> None:
|
|
537
|
+
"""Check that an output FITS image may be written, before any work is done to
|
|
538
|
+
make it. An existing output is an error unless ``overwrite`` is set, in which
|
|
539
|
+
case it is replaced with a warning. An output that is also one of the inputs,
|
|
540
|
+
even when reached through another path such as a symbolic link, is always an
|
|
541
|
+
error: outputs are created empty before the inputs are read, so writing one
|
|
542
|
+
over an input would destroy it.
|
|
543
|
+
|
|
544
|
+
Args:
|
|
545
|
+
output_fits (str | Path): The output that will be written
|
|
546
|
+
overwrite (bool): Whether an existing output may be replaced
|
|
547
|
+
input_fits (Iterable[str | Path | None], optional): The inputs of the operation, where None is skipped. Defaults to ().
|
|
548
|
+
|
|
549
|
+
Raises:
|
|
550
|
+
FileExistsError: Raised when the output is one of the inputs, or already exists and ``overwrite`` is False
|
|
551
|
+
"""
|
|
552
|
+
output_fits = Path(output_fits)
|
|
553
|
+
if not output_fits.exists():
|
|
554
|
+
return
|
|
555
|
+
|
|
556
|
+
if any(
|
|
557
|
+
Path(path).exists() and output_fits.samefile(path)
|
|
558
|
+
for path in input_fits
|
|
559
|
+
if path is not None
|
|
560
|
+
):
|
|
561
|
+
msg = f"{output_fits} is also an input, which writing it would destroy. Choose another output name"
|
|
562
|
+
raise FileExistsError(msg)
|
|
563
|
+
if not overwrite:
|
|
564
|
+
msg = f"{output_fits} already exists. Set overwrite=True (--overwrite on the CLI) to replace it"
|
|
565
|
+
raise FileExistsError(msg)
|
|
566
|
+
logger.warning(f"Overwriting the existing {output_fits}")
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def beams_header(header: fits.Header, beams: tuple[BeamShape, ...]) -> bool:
|
|
570
|
+
"""Describe the restoring beams of the channels in a header, returning whether a BEAMS table is needed.
|
|
571
|
+
|
|
572
|
+
One beam for every channel is written as BMAJ, BMIN and BPA. Beams that
|
|
573
|
+
differ need a BEAMS table, which the header then announces as fitscube does.
|
|
574
|
+
"""
|
|
575
|
+
for key in ("BMAJ", "BMIN", "BPA", "CASAMBM"):
|
|
576
|
+
header.remove(key, ignore_missing=True, remove_all=True)
|
|
577
|
+
if not beams:
|
|
578
|
+
return False
|
|
579
|
+
if all(np.allclose(beam, beams[0], equal_nan=True) for beam in beams):
|
|
580
|
+
if beams[0].is_usable():
|
|
581
|
+
beams[0].to_header(header)
|
|
582
|
+
return False
|
|
583
|
+
|
|
584
|
+
header["CASAMBM"] = True
|
|
585
|
+
header["COMMENT"] = "The PSF in each image plane varies."
|
|
586
|
+
header["COMMENT"] = "Full beam information is stored in the second FITS extension."
|
|
587
|
+
header["COMMENT"] = (
|
|
588
|
+
f"The value '{BLANK_BEAM}' represents a NaN PSF in the beamtable."
|
|
589
|
+
)
|
|
590
|
+
return True
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def beam_table(beams: tuple[BeamShape, ...], stokes_index: int = 0) -> fits.BinTableHDU:
|
|
594
|
+
"""A BEAMS table of the restoring beam of each channel, as CASA and fitscube write it."""
|
|
595
|
+
values = np.array(
|
|
596
|
+
[beam if beam.is_usable() else (BLANK_BEAM,) * 3 for beam in beams],
|
|
597
|
+
dtype=np.float64,
|
|
598
|
+
).reshape(-1, 3)
|
|
599
|
+
table = Table(
|
|
600
|
+
data=[
|
|
601
|
+
values[:, 0],
|
|
602
|
+
values[:, 1],
|
|
603
|
+
values[:, 2],
|
|
604
|
+
np.arange(len(beams)),
|
|
605
|
+
np.full(len(beams), stokes_index),
|
|
606
|
+
],
|
|
607
|
+
names=["BMAJ", "BMIN", "BPA", "CHAN", "POL"],
|
|
608
|
+
dtype=["f4", "f4", "f4", "i4", "i4"],
|
|
609
|
+
units=["arcsec", "arcsec", "deg", None, None],
|
|
610
|
+
)
|
|
611
|
+
hdu = fits.table_to_hdu(table)
|
|
612
|
+
hdu.header["EXTNAME"] = "BEAMS"
|
|
613
|
+
hdu.header["NCHAN"] = len(beams)
|
|
614
|
+
hdu.header["NPOL"] = 1
|
|
615
|
+
return hdu
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def create_output_fits(
|
|
619
|
+
path: str | Path, header: fits.Header, beams: tuple[BeamShape, ...] = ()
|
|
620
|
+
) -> Path:
|
|
621
|
+
"""Create a float32 FITS file shaped by ``header`` without holding its data in memory.
|
|
622
|
+
|
|
623
|
+
The data starts as zeros, and is filled by writing blocks into it. The
|
|
624
|
+
restoring beams of the channels go into the header, or into a BEAMS table
|
|
625
|
+
after the data when they differ.
|
|
626
|
+
|
|
627
|
+
Args:
|
|
628
|
+
path (str | Path): The file to create, replacing any that exists
|
|
629
|
+
header (fits.Header): Header giving the shape and coordinates of the data
|
|
630
|
+
beams (tuple[BeamShape, ...], optional): The restoring beam of each channel. Defaults to (), leaving the header beam as it is.
|
|
631
|
+
|
|
632
|
+
Returns:
|
|
633
|
+
Path: The created file
|
|
634
|
+
"""
|
|
635
|
+
path = Path(path)
|
|
636
|
+
naxis = int(header["NAXIS"])
|
|
637
|
+
shape = [int(header[f"NAXIS{axis}"]) for axis in range(1, naxis + 1)]
|
|
638
|
+
|
|
639
|
+
structural = {"SIMPLE", "BITPIX", "NAXIS", "EXTEND", "BSCALE", "BZERO", "BLANK"}
|
|
640
|
+
# Checksums of the reference's data do not hold for the new file
|
|
641
|
+
structural |= {"CHECKSUM", "DATASUM"}
|
|
642
|
+
structural |= {f"NAXIS{axis}" for axis in range(1, naxis + 1)}
|
|
643
|
+
out_header = fits.Header(
|
|
644
|
+
[
|
|
645
|
+
("SIMPLE", True),
|
|
646
|
+
("BITPIX", -32),
|
|
647
|
+
("NAXIS", naxis),
|
|
648
|
+
*[(f"NAXIS{axis}", size) for axis, size in enumerate(shape, start=1)],
|
|
649
|
+
("EXTEND", True),
|
|
650
|
+
]
|
|
651
|
+
)
|
|
652
|
+
for card in header.cards:
|
|
653
|
+
if card.keyword not in structural:
|
|
654
|
+
out_header.append(card)
|
|
655
|
+
needs_table = beams_header(out_header, beams) if beams else False
|
|
656
|
+
|
|
657
|
+
header_bytes = out_header.tostring().encode("ascii")
|
|
658
|
+
data_bytes = 4 * int(np.prod(shape))
|
|
659
|
+
padded_bytes = -(-data_bytes // 2880) * 2880
|
|
660
|
+
with path.open("wb") as out_file:
|
|
661
|
+
out_file.write(header_bytes)
|
|
662
|
+
out_file.seek(len(header_bytes) + padded_bytes - 1)
|
|
663
|
+
out_file.write(b"\0")
|
|
664
|
+
if needs_table:
|
|
665
|
+
table = beam_table(beams)
|
|
666
|
+
fits.append(path, table.data, table.header)
|
|
667
|
+
|
|
668
|
+
return path
|