fitscube 2.8.0__tar.gz → 2.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fitscube-2.8.0 → fitscube-2.9.0}/PKG-INFO +3 -3
- {fitscube-2.8.0 → fitscube-2.9.0}/README.md +2 -2
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/_version.py +2 -2
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/bounding_box.py +2 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/combine_fits.py +142 -26
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/extract.py +95 -25
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_combine_guards.py +170 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_extract.py +164 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_frequencies.py +26 -1
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_times.py +39 -1
- {fitscube-2.8.0 → fitscube-2.9.0}/.github/CONTRIBUTING.md +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/.github/dependabot.yml +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/.github/release.yml +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/.github/workflows/cd.yml +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/.github/workflows/ci.yml +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/.gitignore +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/.pre-commit-config.yaml +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/CHANGELOG.md +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/LICENSE +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/__init__.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/asyncio.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/cli.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/exceptions.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/logging.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/fitscube/version.pyi +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/noxfile.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/pyproject.toml +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/__init__.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/conftest.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/data/cube.zip +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/data/images.zip +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/data/time_images.zip +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/data/timecube.zip +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_bb.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_combine.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_combine_layouts.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_compress.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_package.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/tests/test_zero_beams.py +0 -0
- {fitscube-2.8.0 → fitscube-2.9.0}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: fitscube
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.9.0
|
|
4
4
|
Summary: A package to produce produce FITS cubes.
|
|
5
5
|
Project-URL: Homepage, https://github.com/AlecThomson/fitscube
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/AlecThomson/fitscube/issues
|
|
@@ -169,7 +169,7 @@ options:
|
|
|
169
169
|
usage: fitscube combine [-h] [-o] [--create-blanks] [--time-domain]
|
|
170
170
|
[--spec-file SPEC_FILE | --specs SPECS [SPECS ...] | --ignore-spec] [-v]
|
|
171
171
|
[--max-workers MAX_WORKERS] [--bounding-box] [--invalidate-zeros]
|
|
172
|
-
[--no-blank-zero-beams] [--floating {
|
|
172
|
+
[--no-blank-zero-beams] [--floating {32,64}] [--compress]
|
|
173
173
|
[--compress-method {gzip,pgzip}]
|
|
174
174
|
file_list [file_list ...] out_cube
|
|
175
175
|
|
|
@@ -199,7 +199,7 @@ options:
|
|
|
199
199
|
Keep images whose restoring beam is exactly zero. By default such images
|
|
200
200
|
are blanked with NaNs, as a zero beam means no PSF was fitted for that
|
|
201
201
|
plane (e.g. a wsclean model image planted by -fit-spectral-pol)
|
|
202
|
-
--floating {
|
|
202
|
+
--floating {32,64}
|
|
203
203
|
The number of floating point bits to use in the out cube. If None the
|
|
204
204
|
input data precision is used.
|
|
205
205
|
--compress Gzip-compress the output cube
|
|
@@ -96,7 +96,7 @@ options:
|
|
|
96
96
|
usage: fitscube combine [-h] [-o] [--create-blanks] [--time-domain]
|
|
97
97
|
[--spec-file SPEC_FILE | --specs SPECS [SPECS ...] | --ignore-spec] [-v]
|
|
98
98
|
[--max-workers MAX_WORKERS] [--bounding-box] [--invalidate-zeros]
|
|
99
|
-
[--no-blank-zero-beams] [--floating {
|
|
99
|
+
[--no-blank-zero-beams] [--floating {32,64}] [--compress]
|
|
100
100
|
[--compress-method {gzip,pgzip}]
|
|
101
101
|
file_list [file_list ...] out_cube
|
|
102
102
|
|
|
@@ -126,7 +126,7 @@ options:
|
|
|
126
126
|
Keep images whose restoring beam is exactly zero. By default such images
|
|
127
127
|
are blanked with NaNs, as a zero beam means no PSF was fitted for that
|
|
128
128
|
plane (e.g. a wsclean model image planted by -fit-spectral-pol)
|
|
129
|
-
--floating {
|
|
129
|
+
--floating {32,64}
|
|
130
130
|
The number of floating point bits to use in the out cube. If None the
|
|
131
131
|
input data precision is used.
|
|
132
132
|
--compress Gzip-compress the output cube
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '2.
|
|
22
|
-
__version_tuple__ = version_tuple = (2,
|
|
21
|
+
__version__ = version = '2.9.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 9, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -163,6 +163,8 @@ async def get_bounding_box_for_fits_coro(
|
|
|
163
163
|
data = np.squeeze(data)
|
|
164
164
|
|
|
165
165
|
if invalidate_zeros:
|
|
166
|
+
# Integer data cannot hold NaN
|
|
167
|
+
data = data.astype(np.result_type(data.dtype, np.float32))
|
|
166
168
|
data[data == 0.0] = np.nan
|
|
167
169
|
|
|
168
170
|
return await asyncio.to_thread(create_bound_box_plane, image_data=data)
|
|
@@ -22,7 +22,7 @@ from collections.abc import Callable
|
|
|
22
22
|
from functools import partial
|
|
23
23
|
from io import BufferedRandom
|
|
24
24
|
from pathlib import Path
|
|
25
|
-
from typing import IO, Any, Literal, NamedTuple, TypeVar, cast
|
|
25
|
+
from typing import IO, Any, Literal, NamedTuple, TypeVar, cast, get_args
|
|
26
26
|
|
|
27
27
|
import astropy.units as u
|
|
28
28
|
import numpy as np
|
|
@@ -65,7 +65,7 @@ BIT_DICT = {
|
|
|
65
65
|
16: 2,
|
|
66
66
|
8: 1,
|
|
67
67
|
}
|
|
68
|
-
FLOAT_LENGTH = Literal[
|
|
68
|
+
FLOAT_LENGTH = Literal[32, 64]
|
|
69
69
|
|
|
70
70
|
|
|
71
71
|
def fits_dtype(bit_pix: int) -> str:
|
|
@@ -86,6 +86,20 @@ warnings.filterwarnings("ignore", category=UserWarning, module="astropy.io.fits"
|
|
|
86
86
|
warnings.filterwarnings("ignore", category=VerifyWarning)
|
|
87
87
|
|
|
88
88
|
|
|
89
|
+
def check_float_length(float_length: int | None) -> None:
|
|
90
|
+
"""Check that float_length is a FITS float width.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
float_length (int | None): Requested float width in bits
|
|
94
|
+
|
|
95
|
+
Raises:
|
|
96
|
+
ValueError: If float_length is not None, 32 or 64
|
|
97
|
+
"""
|
|
98
|
+
if float_length not in (None, *get_args(FLOAT_LENGTH)):
|
|
99
|
+
msg = f"{float_length=} is not a FITS float width. Use 32 or 64."
|
|
100
|
+
raise ValueError(msg)
|
|
101
|
+
|
|
102
|
+
|
|
89
103
|
class InitResult(NamedTuple):
|
|
90
104
|
"""Initialization result."""
|
|
91
105
|
|
|
@@ -274,6 +288,49 @@ async def check_matching_axes_coro(
|
|
|
274
288
|
check_matching_axes = sync_wrapper(check_matching_axes_coro)
|
|
275
289
|
|
|
276
290
|
|
|
291
|
+
async def check_matching_scaling_coro(
|
|
292
|
+
file_list: list[Path],
|
|
293
|
+
max_workers: int | None = None,
|
|
294
|
+
) -> None:
|
|
295
|
+
"""Confirm every input of an integer cube stores its data the same way.
|
|
296
|
+
|
|
297
|
+
An integer cube holds the stored values of every input under the first
|
|
298
|
+
input's BITPIX, BSCALE, BZERO and BLANK, so an input that differs would be
|
|
299
|
+
silently mis-scaled, truncated or wrapped. Float inputs are not checked.
|
|
300
|
+
|
|
301
|
+
Args:
|
|
302
|
+
file_list (list[Path]): The FITS images to check
|
|
303
|
+
max_workers (int | None, optional): Maximum number of concurrent header reads. Defaults to None.
|
|
304
|
+
|
|
305
|
+
Raises:
|
|
306
|
+
ValueError: If any integer image differs in BITPIX, BSCALE, BZERO or BLANK from the first
|
|
307
|
+
"""
|
|
308
|
+
headers = await read_headers_coro(file_list, max_workers=max_workers)
|
|
309
|
+
if headers[0]["BITPIX"] < 0:
|
|
310
|
+
return
|
|
311
|
+
check_values_match(
|
|
312
|
+
file_list,
|
|
313
|
+
[
|
|
314
|
+
(
|
|
315
|
+
header["BITPIX"],
|
|
316
|
+
header.get("BSCALE", 1.0),
|
|
317
|
+
header.get("BZERO", 0.0),
|
|
318
|
+
header.get("BLANK"),
|
|
319
|
+
)
|
|
320
|
+
for header in headers
|
|
321
|
+
],
|
|
322
|
+
describe=lambda scaling: f"(BITPIX, BSCALE, BZERO, BLANK)={scaling}",
|
|
323
|
+
requirement=(
|
|
324
|
+
"the same BITPIX, BSCALE, BZERO and BLANK to make an integer cube "
|
|
325
|
+
"(pass float_length=32 or 64 to make a float cube instead)"
|
|
326
|
+
),
|
|
327
|
+
exception=ValueError,
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
check_matching_scaling = sync_wrapper(check_matching_scaling_coro)
|
|
332
|
+
|
|
333
|
+
|
|
277
334
|
# https://stackoverflow.com/a/66082278
|
|
278
335
|
def np_arange_fix(start: float, stop: float, step: float) -> NDArray[np.float64]:
|
|
279
336
|
n = (stop - start) / step + 1
|
|
@@ -297,14 +354,15 @@ def isin_close(
|
|
|
297
354
|
NDArray[np.bool_]: Boolean array
|
|
298
355
|
"""
|
|
299
356
|
if time_domain_mode:
|
|
300
|
-
#
|
|
357
|
+
# ~0.5 s for MJD seconds, so integrations of a second or more are told apart
|
|
301
358
|
rtol = 1e-10
|
|
302
359
|
atol = 1e-9
|
|
303
360
|
else:
|
|
304
|
-
#
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
361
|
+
# ~1 Hz at 1 GHz. The np.isclose default of rtol=1e-5 (10 kHz) would
|
|
362
|
+
# match several kHz-wide zoom channels and snap off-grid inputs.
|
|
363
|
+
rtol = 1e-9
|
|
364
|
+
atol = 1e-5
|
|
365
|
+
return np.isclose(element[:, None], test_element, rtol=rtol, atol=atol).any(1)
|
|
308
366
|
|
|
309
367
|
|
|
310
368
|
def grid_step(diffs: ArrayLike) -> float:
|
|
@@ -316,7 +374,9 @@ def grid_step(diffs: ArrayLike) -> float:
|
|
|
316
374
|
overestimates the step (e.g. present channels 0, 3, 5 give ``min == 2`` but
|
|
317
375
|
the true step is 1). Falls back to ``min(diffs)`` if the gcd is numerically
|
|
318
376
|
unstable or does not divide every diff, so the common case is unchanged (any
|
|
319
|
-
surviving adjacent pair makes the gcd equal the minimum diff exactly).
|
|
377
|
+
surviving adjacent pair makes the gcd equal the minimum diff exactly). Also
|
|
378
|
+
falls back when the gcd is implausibly small: diffs of 1e6 and 1e6 + 2 have a
|
|
379
|
+
float gcd of 2.
|
|
320
380
|
|
|
321
381
|
Args:
|
|
322
382
|
diffs (ArrayLike): Differences between consecutive spec values.
|
|
@@ -337,7 +397,8 @@ def grid_step(diffs: ArrayLike) -> float:
|
|
|
337
397
|
g = a
|
|
338
398
|
|
|
339
399
|
divides_all = np.all(np.abs(np.round(diffs / g) - diffs / g) <= 1e-3)
|
|
340
|
-
|
|
400
|
+
# A gcd far below every diff is float noise, not a real channel width.
|
|
401
|
+
if g <= tol or not divides_all or g * 16 < min_diff:
|
|
341
402
|
return float(min_diff)
|
|
342
403
|
return float(g)
|
|
343
404
|
|
|
@@ -354,7 +415,7 @@ def even_spacing(specs: u.Quantity, time_domain_mode: bool = False) -> Spequency
|
|
|
354
415
|
specs (u.Quantity): Original frequencies/times
|
|
355
416
|
|
|
356
417
|
Raises:
|
|
357
|
-
IrregularSpacingException: If a grid through the inputs would drop any of them
|
|
418
|
+
IrregularSpacingException: If a grid through the inputs would drop any of them, or is implausibly large
|
|
358
419
|
|
|
359
420
|
Returns:
|
|
360
421
|
SpequencyInfo: specs, missing_chan_idx
|
|
@@ -369,6 +430,15 @@ def even_spacing(specs: u.Quantity, time_domain_mode: bool = False) -> Spequency
|
|
|
369
430
|
|
|
370
431
|
diffs = np.diff(specs_arr)
|
|
371
432
|
step = grid_step(diffs)
|
|
433
|
+
span = specs_arr[-1] - specs_arr[0]
|
|
434
|
+
max_channels = 1000 * len(specs_arr)
|
|
435
|
+
if span / step > max_channels:
|
|
436
|
+
msg = (
|
|
437
|
+
f"A regular grid of step {step} {specs.unit} over {len(specs_arr)} values "
|
|
438
|
+
f"spanning {span} {specs.unit} would need {span / step + 1:.0f} channels. "
|
|
439
|
+
"Combine without blank channels."
|
|
440
|
+
)
|
|
441
|
+
raise IrregularSpacingException(msg)
|
|
372
442
|
# Create a new array with the fundamental grid step
|
|
373
443
|
new_specs = np_arange_fix(specs_arr[0], specs_arr[-1], step)
|
|
374
444
|
missing_chan_idx = np.logical_not(
|
|
@@ -446,7 +516,9 @@ async def create_cube_from_scratch_coro(
|
|
|
446
516
|
fobj.seek(file_length)
|
|
447
517
|
fobj.write(b"\0")
|
|
448
518
|
|
|
449
|
-
with fits.open(
|
|
519
|
+
with fits.open(
|
|
520
|
+
output_file, mode="denywrite", memmap=True, do_not_scale_image_data=True
|
|
521
|
+
) as hdu_list:
|
|
450
522
|
hdu = hdu_list[0]
|
|
451
523
|
data = hdu.data
|
|
452
524
|
on_disk_shape = data.shape
|
|
@@ -483,7 +555,7 @@ async def create_output_cube_coro(
|
|
|
483
555
|
overwrite (bool, optional): If True the out the output cube will overwrite any existing file. Defaults to False.
|
|
484
556
|
time_domain_mode (bool, optional): Whether to join images via the DATE-OBS (e.g. time) axis. If False cube joined along frequency axis. Defaults to False.
|
|
485
557
|
bounding_box (BoundingBox | None, optional): Whether a trimming operation should be applied to input images. If a BoundingBox is supplied this will be used to update the reference pixel position and data shape indicators. Defaults to None.
|
|
486
|
-
float_length (Literal[
|
|
558
|
+
float_length (Literal[32, 64] | None, optional): The precision of the output data. If None drawn from input data. Otherwise values accepted are 32 and 64. Defaults to None.
|
|
487
559
|
|
|
488
560
|
Returns:
|
|
489
561
|
InitResult: Details of the output cube, including the output header
|
|
@@ -493,7 +565,10 @@ async def create_output_cube_coro(
|
|
|
493
565
|
unit = u.s if time_domain_mode else u.Hz
|
|
494
566
|
ctype = "TIME" if time_domain_mode else "FREQ"
|
|
495
567
|
|
|
496
|
-
|
|
568
|
+
# Only the shape is used, and a scaled image cannot be memory-mapped
|
|
569
|
+
old_data, old_header = fits.getdata(
|
|
570
|
+
old_name, header=True, memmap=True, do_not_scale_image_data=True
|
|
571
|
+
)
|
|
497
572
|
sorted_specs = np.sort(specs)
|
|
498
573
|
if time_domain_mode:
|
|
499
574
|
logger.info("Computing time-differences")
|
|
@@ -559,6 +634,13 @@ async def create_output_cube_coro(
|
|
|
559
634
|
new_header[f"CDELT{fits_idx}"] = np.median(np.diff(specs)).value
|
|
560
635
|
new_header[f"CUNIT{fits_idx}"] = f"{unit:fits}"
|
|
561
636
|
new_header[f"CTYPE{fits_idx}"] = ctype
|
|
637
|
+
if time_domain_mode:
|
|
638
|
+
# Times are UTC MJD seconds (see utc_to_mjdsec), so replace any
|
|
639
|
+
# reference time copied from the input
|
|
640
|
+
for k in ("JDREF", "JDREFI", "JDREFF", "MJDREFI", "MJDREFF", "DATEREF"):
|
|
641
|
+
new_header.pop(k, None)
|
|
642
|
+
new_header["MJDREF"] = 0.0
|
|
643
|
+
new_header["TIMESYS"] = "UTC"
|
|
562
644
|
|
|
563
645
|
# Figure out the correct number of dimensions to use
|
|
564
646
|
_no_of_naxis = [k for k in new_header if k.startswith("NAXIS") and k != "NAXIS"]
|
|
@@ -568,13 +650,6 @@ async def create_output_cube_coro(
|
|
|
568
650
|
key = f"{k}{fits_idx}"
|
|
569
651
|
logger.debug(f"{key}={new_header[key]}")
|
|
570
652
|
|
|
571
|
-
# Add extra transform fields for consistency
|
|
572
|
-
if ("CD1_1" in new_header or "PC1_1" in new_header) and fits_idx != 1:
|
|
573
|
-
transform_type = "CD" if "CD1_1" in new_header else "PC"
|
|
574
|
-
pv1 = f"{transform_type}{fits_idx}_{fits_idx}"
|
|
575
|
-
logger.info(f"Adding {pv1} to header")
|
|
576
|
-
new_header[pv1] = 1.0
|
|
577
|
-
|
|
578
653
|
if ignore_spec or not even_spec:
|
|
579
654
|
logger.info(
|
|
580
655
|
f"Ignore the spequency information, {ignore_spec=} or {not even_spec=}"
|
|
@@ -584,6 +659,16 @@ async def create_output_cube_coro(
|
|
|
584
659
|
new_header[f"CTYPE{fits_idx}"] = "CHAN"
|
|
585
660
|
new_header[f"CRVAL{fits_idx}"] = 1
|
|
586
661
|
|
|
662
|
+
# Add extra transform fields for consistency. A CD matrix replaces CDELT,
|
|
663
|
+
# so it must carry the step itself.
|
|
664
|
+
if ("CD1_1" in new_header or "PC1_1" in new_header) and fits_idx != 1:
|
|
665
|
+
transform_type = "CD" if "CD1_1" in new_header else "PC"
|
|
666
|
+
pv1 = f"{transform_type}{fits_idx}_{fits_idx}"
|
|
667
|
+
logger.info(f"Adding {pv1} to header")
|
|
668
|
+
new_header[pv1] = (
|
|
669
|
+
new_header[f"CDELT{fits_idx}"] if transform_type == "CD" else 1.0
|
|
670
|
+
)
|
|
671
|
+
|
|
587
672
|
if has_beams and not single_beam:
|
|
588
673
|
tiny = np.finfo(np.float32).tiny
|
|
589
674
|
new_header["CASAMBM"] = True
|
|
@@ -623,13 +708,16 @@ async def create_output_cube_coro(
|
|
|
623
708
|
# 64 numpy.int64
|
|
624
709
|
# -32 numpy.float32
|
|
625
710
|
# -64 numpy.float64
|
|
626
|
-
|
|
627
|
-
f"{float_length=} not in {BIT_DICT=}"
|
|
628
|
-
)
|
|
711
|
+
check_float_length(float_length)
|
|
629
712
|
bit_pix = int(float_length)
|
|
630
713
|
logger.info(f"Specified {float_length=}, corresponding to {bit_pix=}")
|
|
631
714
|
new_header["BITPIX"] = -abs(bit_pix)
|
|
632
715
|
|
|
716
|
+
if new_header["BITPIX"] < 0:
|
|
717
|
+
# Float planes are written as physical values
|
|
718
|
+
for k in ("BSCALE", "BZERO", "BLANK"):
|
|
719
|
+
new_header.pop(k, None)
|
|
720
|
+
|
|
633
721
|
# Planes are written at an offset of (plane bytes * channel), which only
|
|
634
722
|
# lands on the right plane when the spectral axis is the slowest-varying
|
|
635
723
|
# one - i.e. every axis above it in the cube is degenerate.
|
|
@@ -727,6 +815,7 @@ async def parse_specs_coro(
|
|
|
727
815
|
|
|
728
816
|
Raises:
|
|
729
817
|
ValueError: If both spec_file and spec_list are specified
|
|
818
|
+
ValueError: If any frequency/time is NaN or infinite
|
|
730
819
|
KeyError: If 2D and (REFFREQ or DATE-OBS) is not in header
|
|
731
820
|
ValueError: If not 2D and FREQ is not in header
|
|
732
821
|
|
|
@@ -786,6 +875,14 @@ async def parse_specs_coro(
|
|
|
786
875
|
|
|
787
876
|
specs = file_specs.copy()
|
|
788
877
|
|
|
878
|
+
not_finite = np.flatnonzero(~np.isfinite(file_specs.value))
|
|
879
|
+
if len(not_finite):
|
|
880
|
+
msg = (
|
|
881
|
+
f"{spequencies.capitalize()} must be finite, but value {not_finite[0]} "
|
|
882
|
+
f"(for {file_list[not_finite[0]]}) is {file_specs[not_finite[0]]}"
|
|
883
|
+
)
|
|
884
|
+
raise ValueError(msg)
|
|
885
|
+
|
|
789
886
|
if create_blanks:
|
|
790
887
|
msg = f"Trying to create a blank cube with evenly spaced {spequencies}"
|
|
791
888
|
logger.info(msg)
|
|
@@ -1035,9 +1132,13 @@ def load_and_preprocess_fits_data(
|
|
|
1035
1132
|
bounding_box: BoundingBox | None = None,
|
|
1036
1133
|
invalidate_zeros: bool = False,
|
|
1037
1134
|
wipe_with_nan: bool = False,
|
|
1135
|
+
stored_values: bool = False,
|
|
1038
1136
|
) -> NDArray[np.floating[Any]]:
|
|
1039
1137
|
# Use memmap=False to force the data to be read into memory - gives a speedup
|
|
1040
|
-
|
|
1138
|
+
# stored_values skips BSCALE/BZERO, for integer cubes sharing the input scaling
|
|
1139
|
+
plane: NDArray[np.floating[Any]] = fits.getdata(
|
|
1140
|
+
filename=file_path, memmap=False, do_not_scale_image_data=stored_values
|
|
1141
|
+
)
|
|
1041
1142
|
|
|
1042
1143
|
if bounding_box is not None:
|
|
1043
1144
|
plane = plane[
|
|
@@ -1051,6 +1152,8 @@ def load_and_preprocess_fits_data(
|
|
|
1051
1152
|
return np.full(plane.shape, np.nan)
|
|
1052
1153
|
|
|
1053
1154
|
if invalidate_zeros:
|
|
1155
|
+
# Integer data cannot hold NaN
|
|
1156
|
+
plane = plane.astype(np.result_type(plane.dtype, np.float32))
|
|
1054
1157
|
plane[plane == 0.0] = np.nan
|
|
1055
1158
|
|
|
1056
1159
|
return plane
|
|
@@ -1076,6 +1179,7 @@ async def process_channel(
|
|
|
1076
1179
|
bounding_box=bounding_box,
|
|
1077
1180
|
invalidate_zeros=invalidate_zeros,
|
|
1078
1181
|
wipe_with_nan=is_missing or blank_channel,
|
|
1182
|
+
stored_values=new_header["BITPIX"] > 0,
|
|
1079
1183
|
)
|
|
1080
1184
|
|
|
1081
1185
|
if "BITPIX" in new_header:
|
|
@@ -1140,19 +1244,31 @@ async def combine_fits_coro(
|
|
|
1140
1244
|
time_domain_mode (bool, optional): Work in time domain mode - make a time-cube. Default = False.
|
|
1141
1245
|
bounding_box (bool | BoundingBox, optional): Clip invalid/padded pixels when crafting the fits cube. When True an extra read of the input data is needed, but output cube is smaller. A BoundingBox may be supplied directly (see `get_common_bounding_box`) to force several cubes onto an identical pixel grid. Defaults to False.
|
|
1142
1246
|
invalidate_zeros (bool, optionals): Set pixels whose values are exactly zero to NaNs. Defaults to False.
|
|
1143
|
-
float_length (Literal[
|
|
1247
|
+
float_length (Literal[32, 64] | None, optional): The floating point precision in bits to use when creating the output cube. If None the size of the input data are used. Defaults to None.
|
|
1144
1248
|
blank_zero_beams (bool, optional): Blank (NaN) any input image whose restoring beam is exactly zero. wsclean writes such a beam when a plane holds no fitted PSF - e.g. the model image `-fit-spectral-pol` plants into a channel - and those planes are not comparable to the rest of the cube. Defaults to True.
|
|
1145
1249
|
|
|
1146
1250
|
Raises:
|
|
1147
1251
|
ShapeMismatchException: If the input images do not share a pixel grid
|
|
1252
|
+
ValueError: If integer inputs differ in BITPIX, BSCALE, BZERO or BLANK and float_length is None
|
|
1253
|
+
ValueError: If invalidate_zeros is set for integer output
|
|
1254
|
+
ValueError: If float_length is not None, 32 or 64
|
|
1148
1255
|
|
|
1149
1256
|
Returns:
|
|
1150
1257
|
tuple[fits.HDUList, u.Quantity]: The combined FITS cube and frequencies
|
|
1151
1258
|
"""
|
|
1152
1259
|
# TODO: Check that all files have the same WCS
|
|
1153
1260
|
|
|
1261
|
+
check_float_length(float_length)
|
|
1154
1262
|
await check_matching_shapes_coro(file_list=file_list, max_workers=max_workers)
|
|
1155
1263
|
await check_matching_axes_coro(file_list=file_list, max_workers=max_workers)
|
|
1264
|
+
if float_length is None:
|
|
1265
|
+
await check_matching_scaling_coro(file_list=file_list, max_workers=max_workers)
|
|
1266
|
+
if invalidate_zeros and fits.getheader(file_list[0])["BITPIX"] > 0:
|
|
1267
|
+
msg = (
|
|
1268
|
+
"invalidate_zeros writes NaNs, which integer output data cannot "
|
|
1269
|
+
"hold. Pass float_length=32 or 64."
|
|
1270
|
+
)
|
|
1271
|
+
raise ValueError(msg)
|
|
1156
1272
|
|
|
1157
1273
|
file_specs, specs, missing_chan_idx = await parse_specs_coro(
|
|
1158
1274
|
spec_file=spec_file,
|
|
@@ -1432,7 +1548,7 @@ def get_parser(
|
|
|
1432
1548
|
parser.add_argument(
|
|
1433
1549
|
"--floating",
|
|
1434
1550
|
type=int,
|
|
1435
|
-
choices=(
|
|
1551
|
+
choices=get_args(FLOAT_LENGTH),
|
|
1436
1552
|
default=None,
|
|
1437
1553
|
help="The number of floating point bits to use in the out cube. If None the input data precision is used.",
|
|
1438
1554
|
)
|
|
@@ -60,12 +60,12 @@ class TargetWCS:
|
|
|
60
60
|
"""The axis count that the frequency corresponds to"""
|
|
61
61
|
ctype: str
|
|
62
62
|
"""The FITS WCS key name"""
|
|
63
|
-
crpix:
|
|
64
|
-
"""The reference index position"""
|
|
63
|
+
crpix: float
|
|
64
|
+
"""The reference index position (FITS 1-based)"""
|
|
65
65
|
crval: float
|
|
66
66
|
"""The reference value stored in the header"""
|
|
67
67
|
cdelt: float
|
|
68
|
-
"""The step between planes"""
|
|
68
|
+
"""The step between planes, with any CDi_i or PCi_i applied"""
|
|
69
69
|
cunit: str
|
|
70
70
|
"""the unit of the target axis"""
|
|
71
71
|
|
|
@@ -158,6 +158,21 @@ def get_output_path(input_path: Path, target_index: TargetIndex) -> Path:
|
|
|
158
158
|
return output_path
|
|
159
159
|
|
|
160
160
|
|
|
161
|
+
def check_output_is_not_input(fits_cube: Path, output_path: Path) -> None:
|
|
162
|
+
"""Refuse to write a plane over the cube it is read from.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
fits_cube (Path): The cube planes are read from
|
|
166
|
+
output_path (Path): Where a plane is to be written
|
|
167
|
+
|
|
168
|
+
Raises:
|
|
169
|
+
ValueError: If ``output_path`` is the cube
|
|
170
|
+
"""
|
|
171
|
+
if output_path.exists() and output_path.samefile(fits_cube):
|
|
172
|
+
msg = f"Output path {output_path} is the input cube {fits_cube}"
|
|
173
|
+
raise ValueError(msg)
|
|
174
|
+
|
|
175
|
+
|
|
161
176
|
def fits_file_contains_beam_table(header: fits.header.Header | Path) -> bool:
|
|
162
177
|
"""Consider whether a fits file contains a beam table
|
|
163
178
|
|
|
@@ -236,21 +251,62 @@ def find_target_axis(
|
|
|
236
251
|
# Remember that range upper limit is exclusive, and
|
|
237
252
|
# we start counting from 1
|
|
238
253
|
for axis in range(1, naxis + 1):
|
|
239
|
-
if axis_name in header
|
|
254
|
+
if axis_name in header.get(f"CTYPE{axis}", ""):
|
|
240
255
|
logger.info(f"Found {axis_name} at {axis=}")
|
|
256
|
+
# CDi_i replaces CDELTi, PCi_i scales it. Absent keys take FITS defaults.
|
|
257
|
+
cdelt = header.get(
|
|
258
|
+
f"CD{axis}_{axis}",
|
|
259
|
+
header.get(f"CDELT{axis}", 1.0) * header.get(f"PC{axis}_{axis}", 1.0),
|
|
260
|
+
)
|
|
241
261
|
return TargetWCS(
|
|
242
262
|
axis=axis,
|
|
243
263
|
ctype=header[f"CTYPE{axis}"],
|
|
244
|
-
crpix=header
|
|
245
|
-
crval=header
|
|
246
|
-
cdelt=
|
|
247
|
-
cunit=header
|
|
264
|
+
crpix=header.get(f"CRPIX{axis}", 0.0),
|
|
265
|
+
crval=header.get(f"CRVAL{axis}", 0.0),
|
|
266
|
+
cdelt=cdelt,
|
|
267
|
+
cunit=header.get(f"CUNIT{axis}", ""),
|
|
248
268
|
)
|
|
249
269
|
|
|
250
270
|
msg = "Did not find the frequency axis"
|
|
251
271
|
raise TargetAxisMissingException(msg)
|
|
252
272
|
|
|
253
273
|
|
|
274
|
+
def find_plane_axis(header: fits.header.Header) -> tuple[TargetWCS, str]:
|
|
275
|
+
"""Find the FREQ or TIME axis to split a cube along.
|
|
276
|
+
|
|
277
|
+
Args:
|
|
278
|
+
header (fits.header.Header): The header from the fits cube
|
|
279
|
+
|
|
280
|
+
Raises:
|
|
281
|
+
TargetAxisMissingException: If there is neither a FREQ nor a TIME axis
|
|
282
|
+
ValueError: If both the FREQ and TIME axes are longer than one
|
|
283
|
+
|
|
284
|
+
Returns:
|
|
285
|
+
tuple[TargetWCS, str]: The axis, and its name (FREQ or TIME)
|
|
286
|
+
"""
|
|
287
|
+
found: dict[str, TargetWCS] = {}
|
|
288
|
+
for axis_name in ("FREQ", "TIME"):
|
|
289
|
+
try:
|
|
290
|
+
found[axis_name] = find_target_axis(header=header, target_index=axis_name)
|
|
291
|
+
except TargetAxisMissingException:
|
|
292
|
+
continue
|
|
293
|
+
if not found:
|
|
294
|
+
msg = "Did not find a FREQ or TIME axis"
|
|
295
|
+
raise TargetAxisMissingException(msg)
|
|
296
|
+
|
|
297
|
+
# A time cube made from images keeps their degenerate FREQ axis
|
|
298
|
+
long_axes = [
|
|
299
|
+
axis_name
|
|
300
|
+
for axis_name, target_wcs in found.items()
|
|
301
|
+
if header[f"NAXIS{target_wcs.axis}"] > 1
|
|
302
|
+
]
|
|
303
|
+
if len(long_axes) > 1:
|
|
304
|
+
msg = f"Both {long_axes} axes have more than one plane, so cannot pick one"
|
|
305
|
+
raise ValueError(msg)
|
|
306
|
+
axis_name = long_axes[0] if long_axes else next(iter(found))
|
|
307
|
+
return found[axis_name], axis_name
|
|
308
|
+
|
|
309
|
+
|
|
254
310
|
def create_plane_target_wcs(
|
|
255
311
|
original_freq_wcs: TargetWCS, target_index: int | TargetIndex
|
|
256
312
|
) -> TargetWCS:
|
|
@@ -264,7 +320,10 @@ def create_plane_target_wcs(
|
|
|
264
320
|
TargetWCS: The frequency information for a channel
|
|
265
321
|
"""
|
|
266
322
|
index = target_index if isinstance(target_index, int) else target_index.axis_index
|
|
267
|
-
|
|
323
|
+
# FITS pixels are 1-based, so the 0-based index is pixel index + 1
|
|
324
|
+
update_index = original_freq_wcs.crval + (
|
|
325
|
+
(index + 1 - original_freq_wcs.crpix) * original_freq_wcs.cdelt
|
|
326
|
+
)
|
|
268
327
|
return TargetWCS(
|
|
269
328
|
axis=original_freq_wcs.axis,
|
|
270
329
|
ctype=original_freq_wcs.ctype,
|
|
@@ -303,7 +362,13 @@ def update_header_for_target_axis(
|
|
|
303
362
|
out_header[f"CRPIX{_idx}"] = plane_freq_wcs.crpix
|
|
304
363
|
out_header[f"CRVAL{_idx}"] = plane_freq_wcs.crval
|
|
305
364
|
out_header[f"CDELT{_idx}"] = plane_freq_wcs.cdelt
|
|
306
|
-
|
|
365
|
+
# cdelt is the full step, so it goes into CDi_i, and PCi_i no longer scales it
|
|
366
|
+
if f"CD{_idx}_{_idx}" in out_header:
|
|
367
|
+
out_header[f"CD{_idx}_{_idx}"] = plane_freq_wcs.cdelt
|
|
368
|
+
if f"PC{_idx}_{_idx}" in out_header:
|
|
369
|
+
out_header[f"PC{_idx}_{_idx}"] = 1.0
|
|
370
|
+
if plane_freq_wcs.cunit:
|
|
371
|
+
out_header[f"CUNIT{_idx}"] = plane_freq_wcs.cunit
|
|
307
372
|
|
|
308
373
|
if extract_beam_from_file and fits_file_contains_beam_table(
|
|
309
374
|
header=extract_beam_from_file
|
|
@@ -354,6 +419,7 @@ def extract_plane_from_cube(fits_cube: Path, extract_options: ExtractOptions) ->
|
|
|
354
419
|
target_index=target_index,
|
|
355
420
|
)
|
|
356
421
|
)
|
|
422
|
+
check_output_is_not_input(fits_cube=fits_cube, output_path=output_path)
|
|
357
423
|
|
|
358
424
|
logger.info(f"Opening {fits_cube=}")
|
|
359
425
|
with fits.open(
|
|
@@ -371,8 +437,8 @@ def extract_plane_from_cube(fits_cube: Path, extract_options: ExtractOptions) ->
|
|
|
371
437
|
target_axis_wcs = find_target_axis(header=header, target_index=target_index)
|
|
372
438
|
target_cube_index = len(data.shape) - target_axis_wcs.axis
|
|
373
439
|
|
|
374
|
-
if target_index.axis_index
|
|
375
|
-
msg = f"{
|
|
440
|
+
if not 0 <= target_index.axis_index < data.shape[target_cube_index]:
|
|
441
|
+
msg = f"{target_index.axis_index=} outside of {target_index.axis_name} axis of {data.shape=}, axis shape of {data.shape[target_cube_index]}"
|
|
376
442
|
raise ChannelMissingException(msg)
|
|
377
443
|
|
|
378
444
|
# Get the channel index requested
|
|
@@ -427,17 +493,17 @@ def read_beam_table(open_fits: fits.HDUList, hdu_index: int = 0) -> Beams | None
|
|
|
427
493
|
def _plane_header(
|
|
428
494
|
header: fits.header.Header,
|
|
429
495
|
target_wcs: TargetWCS,
|
|
430
|
-
|
|
496
|
+
target_index: TargetIndex,
|
|
431
497
|
beams: Beams | None,
|
|
432
498
|
) -> fits.header.Header:
|
|
433
|
-
"""The header of a single extracted
|
|
499
|
+
"""The header of a single extracted plane, beam included"""
|
|
434
500
|
out_header = update_header_for_target_axis(
|
|
435
501
|
header=header,
|
|
436
502
|
target_wcs=target_wcs,
|
|
437
|
-
target_index=
|
|
503
|
+
target_index=target_index,
|
|
438
504
|
)
|
|
439
505
|
if beams is not None:
|
|
440
|
-
beam = beams[
|
|
506
|
+
beam = beams[target_index.axis_index]
|
|
441
507
|
out_header["BMAJ"] = beam.major.to(u.deg).value
|
|
442
508
|
out_header["BMIN"] = beam.minor.to(u.deg).value
|
|
443
509
|
out_header["BPA"] = beam.pa.to(u.deg).value
|
|
@@ -451,7 +517,7 @@ def _plane_header(
|
|
|
451
517
|
async def extract_planes_from_cube_coro(
|
|
452
518
|
fits_cube: Path, extract_planes_options: ExtractPlanesOptions | None = None
|
|
453
519
|
) -> list[Path]:
|
|
454
|
-
"""Extract every channel of a cube into its own file.
|
|
520
|
+
"""Extract every channel or timestep of a cube into its own file.
|
|
455
521
|
|
|
456
522
|
Calling ``extract_plane_from_cube`` once per channel reopens the cube and
|
|
457
523
|
re-reads its whole beam table each time, which grows with the square of the
|
|
@@ -463,6 +529,7 @@ async def extract_planes_from_cube_coro(
|
|
|
463
529
|
|
|
464
530
|
Raises:
|
|
465
531
|
ShapeMismatchException: If the supplied output paths do not match the channel count
|
|
532
|
+
ValueError: If both the FREQ and TIME axes are longer than one
|
|
466
533
|
|
|
467
534
|
Returns:
|
|
468
535
|
list[Path]: The output file of each channel, in channel order
|
|
@@ -475,19 +542,20 @@ async def extract_planes_from_cube_coro(
|
|
|
475
542
|
) as open_fits:
|
|
476
543
|
header = open_fits[options.hdu_index].header
|
|
477
544
|
data = open_fits[options.hdu_index].data
|
|
478
|
-
target_wcs =
|
|
479
|
-
channels =
|
|
545
|
+
target_wcs, axis_name = find_plane_axis(header=header)
|
|
546
|
+
channels = int(header[f"NAXIS{target_wcs.axis}"])
|
|
547
|
+
index_key = "time_index" if axis_name == "TIME" else "channel_index"
|
|
548
|
+
target_indices = [
|
|
549
|
+
create_target_index(**{index_key: channel}) for channel in range(channels)
|
|
550
|
+
]
|
|
480
551
|
# The FITS axis order is reversed against the numpy one
|
|
481
552
|
cube_axis = data.ndim - target_wcs.axis
|
|
482
553
|
beams = read_beam_table(open_fits=open_fits, hdu_index=options.hdu_index)
|
|
483
554
|
|
|
484
555
|
if options.output_paths is None:
|
|
485
556
|
output_paths = [
|
|
486
|
-
get_output_path(
|
|
487
|
-
|
|
488
|
-
target_index=create_target_index(channel_index=channel),
|
|
489
|
-
)
|
|
490
|
-
for channel in range(channels)
|
|
557
|
+
get_output_path(input_path=fits_cube, target_index=target_index)
|
|
558
|
+
for target_index in target_indices
|
|
491
559
|
]
|
|
492
560
|
elif len(options.output_paths) != channels:
|
|
493
561
|
msg = (
|
|
@@ -497,6 +565,8 @@ async def extract_planes_from_cube_coro(
|
|
|
497
565
|
raise ShapeMismatchException(msg)
|
|
498
566
|
else:
|
|
499
567
|
output_paths = options.output_paths
|
|
568
|
+
for output_path in output_paths:
|
|
569
|
+
check_output_is_not_input(fits_cube=fits_cube, output_path=output_path)
|
|
500
570
|
|
|
501
571
|
def _write_plane(channel: int) -> None:
|
|
502
572
|
plane = np.expand_dims(
|
|
@@ -508,7 +578,7 @@ async def extract_planes_from_cube_coro(
|
|
|
508
578
|
header=_plane_header(
|
|
509
579
|
header=header,
|
|
510
580
|
target_wcs=target_wcs,
|
|
511
|
-
|
|
581
|
+
target_index=target_indices[channel],
|
|
512
582
|
beams=beams,
|
|
513
583
|
),
|
|
514
584
|
overwrite=options.overwrite,
|
|
@@ -8,11 +8,14 @@ import astropy.units as u
|
|
|
8
8
|
import numpy as np
|
|
9
9
|
import pytest
|
|
10
10
|
from astropy.io import fits
|
|
11
|
+
from astropy.wcs import WCS
|
|
11
12
|
from fitscube.bounding_box import get_common_bounding_box
|
|
12
13
|
from fitscube.combine_fits import (
|
|
14
|
+
FLOAT_LENGTH,
|
|
13
15
|
check_matching_axes,
|
|
14
16
|
check_matching_shapes,
|
|
15
17
|
combine_fits,
|
|
18
|
+
get_parser,
|
|
16
19
|
)
|
|
17
20
|
from fitscube.exceptions import (
|
|
18
21
|
AxisOrderException,
|
|
@@ -182,6 +185,7 @@ def test_blank_channels_are_created(
|
|
|
182
185
|
"memmap",
|
|
183
186
|
"lazy_load_hdus",
|
|
184
187
|
"ext",
|
|
188
|
+
"do_not_scale_image_data",
|
|
185
189
|
}
|
|
186
190
|
if unexpected:
|
|
187
191
|
msg = f"getdata() got unexpected keyword arguments {unexpected}"
|
|
@@ -409,3 +413,169 @@ def test_bounding_box_can_be_supplied(tmp_path: Path, specs: u.Quantity) -> None
|
|
|
409
413
|
|
|
410
414
|
# The weights alone would have given the full, untrimmed grid
|
|
411
415
|
assert get_common_bounding_box(file_list=weights).x_span == 8
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
@pytest.mark.parametrize("bad", [np.nan, np.inf])
|
|
419
|
+
@pytest.mark.parametrize("create_blanks", [False, True])
|
|
420
|
+
def test_non_finite_specs_raise(
|
|
421
|
+
tmp_path: Path, file_list: list[Path], bad: float, create_blanks: bool
|
|
422
|
+
) -> None:
|
|
423
|
+
"""NaN failed deep in the header write, and inf was written as a CHAN axis."""
|
|
424
|
+
with pytest.raises(ValueError, match="must be finite"):
|
|
425
|
+
combine_fits(
|
|
426
|
+
file_list=file_list,
|
|
427
|
+
out_cube=tmp_path / "cube.fits",
|
|
428
|
+
spec_list=[1e9, bad, 1.002e9, 1.003e9],
|
|
429
|
+
create_blanks=create_blanks,
|
|
430
|
+
overwrite=True,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def make_scaled_plane(
|
|
435
|
+
path: Path,
|
|
436
|
+
spec: u.Quantity,
|
|
437
|
+
stored: int,
|
|
438
|
+
bscale: float = 1.0,
|
|
439
|
+
bzero: float = 0.0,
|
|
440
|
+
blank: int | None = None,
|
|
441
|
+
dtype: Any = np.int16,
|
|
442
|
+
) -> Path:
|
|
443
|
+
"""Write an integer plane whose stored values are ``stored``."""
|
|
444
|
+
header = fits.getheader(make_plane(path, spec, dtype=dtype))
|
|
445
|
+
hdu = fits.PrimaryHDU(np.full((1, 1, 8, 8), stored, dtype=dtype), header=header)
|
|
446
|
+
hdu.header["BSCALE"] = bscale
|
|
447
|
+
hdu.header["BZERO"] = bzero
|
|
448
|
+
if blank is not None:
|
|
449
|
+
hdu.header["BLANK"] = blank
|
|
450
|
+
hdu.writeto(path, overwrite=True)
|
|
451
|
+
return path
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
@pytest.mark.parametrize(
|
|
455
|
+
"odd_one_out",
|
|
456
|
+
[
|
|
457
|
+
{"bscale": 0.5},
|
|
458
|
+
{"bzero": 100.0},
|
|
459
|
+
{"blank": -32768, "stored": -32768},
|
|
460
|
+
{"dtype": np.int32, "stored": 100_000},
|
|
461
|
+
],
|
|
462
|
+
)
|
|
463
|
+
def test_integer_inputs_must_share_scaling(
|
|
464
|
+
tmp_path: Path, specs: u.Quantity, odd_one_out: dict[str, Any]
|
|
465
|
+
) -> None:
|
|
466
|
+
"""An odd input was silently truncated, wrapped or had BLANK written as 0."""
|
|
467
|
+
file_list = [
|
|
468
|
+
make_scaled_plane(tmp_path / f"int_{i}.fits", spec, stored=i)
|
|
469
|
+
for i, spec in enumerate(specs[:3])
|
|
470
|
+
]
|
|
471
|
+
file_list[1] = make_scaled_plane(
|
|
472
|
+
tmp_path / "odd.fits", specs[1], **{"stored": 5, **odd_one_out}
|
|
473
|
+
)
|
|
474
|
+
expected = [fits.getdata(path)[0, 0, 0, 0] for path in file_list]
|
|
475
|
+
|
|
476
|
+
with pytest.raises(ValueError, match="float_length"):
|
|
477
|
+
combine_fits(
|
|
478
|
+
file_list=file_list, out_cube=tmp_path / "cube.fits", overwrite=True
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
out_cube = tmp_path / "cube.fits"
|
|
482
|
+
combine_fits(
|
|
483
|
+
file_list=file_list, out_cube=out_cube, overwrite=True, float_length=64
|
|
484
|
+
)
|
|
485
|
+
assert np.allclose(fits.getdata(out_cube)[:, 0, 0, 0], expected, equal_nan=True)
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
@pytest.mark.parametrize("float_length", [None, 32])
|
|
489
|
+
def test_shared_integer_scaling_is_kept(
|
|
490
|
+
tmp_path: Path, specs: u.Quantity, float_length: FLOAT_LENGTH | None
|
|
491
|
+
) -> None:
|
|
492
|
+
"""A scaled first input could not be memory-mapped, so nothing was written."""
|
|
493
|
+
file_list = [
|
|
494
|
+
make_scaled_plane(
|
|
495
|
+
tmp_path / f"int_{i}.fits", spec, stored=i, bscale=2.0, bzero=32768.0
|
|
496
|
+
)
|
|
497
|
+
for i, spec in enumerate(specs)
|
|
498
|
+
]
|
|
499
|
+
expected = [fits.getdata(path)[0, 0, 0, 0] for path in file_list]
|
|
500
|
+
|
|
501
|
+
out_cube = tmp_path / "cube.fits"
|
|
502
|
+
combine_fits(
|
|
503
|
+
file_list=file_list,
|
|
504
|
+
out_cube=out_cube,
|
|
505
|
+
overwrite=True,
|
|
506
|
+
float_length=float_length,
|
|
507
|
+
)
|
|
508
|
+
assert np.allclose(fits.getdata(out_cube)[:, 0, 0, 0], expected)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def test_cd_matrix_carries_the_step(tmp_path: Path, specs: u.Quantity) -> None:
|
|
512
|
+
"""CDi_i overrides CDELTi, and was set to 1 for the spectral axis."""
|
|
513
|
+
file_list = []
|
|
514
|
+
for i, spec in enumerate(specs):
|
|
515
|
+
path = make_plane(tmp_path / f"cd_{i}.fits", spec, value=float(i))
|
|
516
|
+
with fits.open(path, mode="update") as hdu_list:
|
|
517
|
+
header = hdu_list[0].header
|
|
518
|
+
for axis in range(1, 5):
|
|
519
|
+
header[f"CD{axis}_{axis}"] = header.pop(f"CDELT{axis}")
|
|
520
|
+
file_list.append(path)
|
|
521
|
+
|
|
522
|
+
out_cube = tmp_path / "cube.fits"
|
|
523
|
+
combine_fits(file_list=file_list, out_cube=out_cube, overwrite=True)
|
|
524
|
+
|
|
525
|
+
wcs = WCS(fits.getheader(out_cube))
|
|
526
|
+
out_specs = wcs.spectral.pixel_to_world(np.arange(len(specs)))
|
|
527
|
+
assert np.allclose(out_specs.to(u.Hz).value, specs.to(u.Hz).value)
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
@pytest.mark.parametrize("bounding_box", [False, True])
|
|
531
|
+
def test_invalidate_zeros_on_integer_input(
|
|
532
|
+
tmp_path: Path, specs: u.Quantity, bounding_box: bool
|
|
533
|
+
) -> None:
|
|
534
|
+
"""NaN was written into the integer plane, which numpy refuses."""
|
|
535
|
+
file_list = [
|
|
536
|
+
make_plane(tmp_path / f"int_{i}.fits", spec, value=float(i), dtype=np.int16)
|
|
537
|
+
for i, spec in enumerate(specs)
|
|
538
|
+
]
|
|
539
|
+
out_cube = tmp_path / "cube.fits"
|
|
540
|
+
|
|
541
|
+
with pytest.raises(ValueError, match="float_length"):
|
|
542
|
+
combine_fits(
|
|
543
|
+
file_list=file_list,
|
|
544
|
+
out_cube=out_cube,
|
|
545
|
+
overwrite=True,
|
|
546
|
+
invalidate_zeros=True,
|
|
547
|
+
bounding_box=bounding_box,
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
combine_fits(
|
|
551
|
+
file_list=file_list,
|
|
552
|
+
out_cube=out_cube,
|
|
553
|
+
overwrite=True,
|
|
554
|
+
invalidate_zeros=True,
|
|
555
|
+
bounding_box=bounding_box,
|
|
556
|
+
float_length=32,
|
|
557
|
+
)
|
|
558
|
+
cube = fits.getdata(out_cube)
|
|
559
|
+
assert np.isnan(cube[0]).all()
|
|
560
|
+
assert np.allclose(cube[1:, 0, 0, 0], [1, 2, 3])
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
@pytest.mark.parametrize("float_length", [8, 16])
|
|
564
|
+
def test_float_length_must_be_fits_float(
|
|
565
|
+
tmp_path: Path, file_list: list[Path], float_length: int
|
|
566
|
+
) -> None:
|
|
567
|
+
"""BITPIX -8 and -16 are not FITS floats, and raised a KeyError."""
|
|
568
|
+
out_cube = tmp_path / "cube.fits"
|
|
569
|
+
with pytest.raises(ValueError, match="32 or 64"):
|
|
570
|
+
combine_fits(
|
|
571
|
+
file_list=file_list,
|
|
572
|
+
out_cube=out_cube,
|
|
573
|
+
overwrite=True,
|
|
574
|
+
float_length=float_length, # type: ignore[arg-type]
|
|
575
|
+
)
|
|
576
|
+
assert not out_cube.exists()
|
|
577
|
+
|
|
578
|
+
with pytest.raises(SystemExit):
|
|
579
|
+
get_parser().parse_args(
|
|
580
|
+
[*map(str, file_list), str(out_cube), "--floating", str(float_length)]
|
|
581
|
+
)
|
|
@@ -7,6 +7,7 @@ from pathlib import Path
|
|
|
7
7
|
import numpy as np
|
|
8
8
|
import pytest
|
|
9
9
|
from astropy.io import fits
|
|
10
|
+
from astropy.wcs import WCS
|
|
10
11
|
from fitscube.exceptions import (
|
|
11
12
|
ChannelMissingException,
|
|
12
13
|
ShapeMismatchException,
|
|
@@ -339,3 +340,166 @@ def test_extract_planes_output_path_count(cube_path, tmpdir) -> None:
|
|
|
339
340
|
cube_path,
|
|
340
341
|
ExtractPlanesOptions(output_paths=[Path(tmpdir) / "only_one.fits"]),
|
|
341
342
|
)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _write_crpix_cube(path: Path, crpix: float | None) -> Path:
|
|
346
|
+
"""A 4 channel (FREQ, DEC, RA) cube whose reference pixel is not 1"""
|
|
347
|
+
header = fits.Header()
|
|
348
|
+
header["CTYPE1"] = "RA---SIN"
|
|
349
|
+
header["CTYPE2"] = "DEC--SIN"
|
|
350
|
+
header["CTYPE3"] = "FREQ"
|
|
351
|
+
header["CRVAL3"] = 1e9
|
|
352
|
+
header["CDELT3"] = 1e6
|
|
353
|
+
header["CUNIT3"] = "Hz"
|
|
354
|
+
if crpix is not None:
|
|
355
|
+
header["CRPIX3"] = crpix
|
|
356
|
+
data = np.arange(4, dtype=np.float32)[:, None, None] * np.ones((4, 2, 2))
|
|
357
|
+
fits.writeto(path, data=data, header=header)
|
|
358
|
+
return path
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
@pytest.mark.parametrize(("crpix", "expected"), [(3.0, 0.998e9), (None, 1.001e9)])
|
|
362
|
+
def test_extract_respects_crpix(tmpdir, crpix, expected) -> None:
|
|
363
|
+
"""The plane value used to ignore CRPIX, i.e. assume CRPIX = 1"""
|
|
364
|
+
cube = _write_crpix_cube(Path(tmpdir) / "cube.fits", crpix)
|
|
365
|
+
|
|
366
|
+
output_file = Path(tmpdir) / "plane.fits"
|
|
367
|
+
extract_plane_from_cube(
|
|
368
|
+
fits_cube=cube,
|
|
369
|
+
extract_options=ExtractOptions(channel_index=0, output_path=output_file),
|
|
370
|
+
)
|
|
371
|
+
header = fits.getheader(output_file)
|
|
372
|
+
assert header["CRPIX3"] == 1
|
|
373
|
+
assert np.isclose(header["CRVAL3"], expected)
|
|
374
|
+
|
|
375
|
+
plane_paths = extract_planes_from_cube(
|
|
376
|
+
fits_cube=cube,
|
|
377
|
+
extract_planes_options=ExtractPlanesOptions(
|
|
378
|
+
output_paths=[Path(tmpdir) / f"p{i}.fits" for i in range(4)]
|
|
379
|
+
),
|
|
380
|
+
)
|
|
381
|
+
for channel, plane_path in enumerate(plane_paths):
|
|
382
|
+
assert np.isclose(
|
|
383
|
+
fits.getheader(plane_path)["CRVAL3"], expected + channel * 1e6
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
@pytest.mark.parametrize("channel", [4, -1])
|
|
388
|
+
def test_extract_index_outside_axis(tmpdir, channel) -> None:
|
|
389
|
+
"""One past the end hit an IndexError, and -1 got the wrong CRVAL"""
|
|
390
|
+
cube = _write_crpix_cube(Path(tmpdir) / "cube.fits", 1.0)
|
|
391
|
+
with pytest.raises(ChannelMissingException):
|
|
392
|
+
extract_plane_from_cube(
|
|
393
|
+
fits_cube=cube,
|
|
394
|
+
extract_options=ExtractOptions(
|
|
395
|
+
channel_index=channel, output_path=Path(tmpdir) / "plane.fits"
|
|
396
|
+
),
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def test_extract_refuses_to_overwrite_cube(tmpdir) -> None:
|
|
401
|
+
"""With overwrite set, an output path naming the cube replaced it with one plane"""
|
|
402
|
+
cube = _write_crpix_cube(Path(tmpdir) / "cube.fits", 1.0)
|
|
403
|
+
# Same file under another spelling
|
|
404
|
+
same_cube = Path(tmpdir) / "." / "cube.fits"
|
|
405
|
+
|
|
406
|
+
with pytest.raises(ValueError, match="is the input cube"):
|
|
407
|
+
extract_plane_from_cube(
|
|
408
|
+
fits_cube=cube,
|
|
409
|
+
extract_options=ExtractOptions(
|
|
410
|
+
channel_index=2, output_path=same_cube, overwrite=True
|
|
411
|
+
),
|
|
412
|
+
)
|
|
413
|
+
with pytest.raises(ValueError, match="is the input cube"):
|
|
414
|
+
extract_planes_from_cube(
|
|
415
|
+
fits_cube=cube,
|
|
416
|
+
extract_planes_options=ExtractPlanesOptions(
|
|
417
|
+
output_paths=[Path(tmpdir) / f"p{i}.fits" for i in range(3)]
|
|
418
|
+
+ [same_cube],
|
|
419
|
+
overwrite=True,
|
|
420
|
+
),
|
|
421
|
+
)
|
|
422
|
+
assert fits.getdata(cube).shape == (4, 2, 2)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
@pytest.mark.filterwarnings("ignore:'cdfix' made the change")
|
|
426
|
+
@pytest.mark.parametrize(
|
|
427
|
+
"keywords",
|
|
428
|
+
[
|
|
429
|
+
{"CDELT3": None, "CUNIT3": None, "CD3_3": 1e6},
|
|
430
|
+
{"CDELT3": 5e5, "PC3_3": 2.0},
|
|
431
|
+
{"CDELT3": None, "CRVAL3": 1e6, "CRPIX3": 1000.0},
|
|
432
|
+
],
|
|
433
|
+
)
|
|
434
|
+
def test_extract_follows_fits_wcs(tmpdir, keywords) -> None:
|
|
435
|
+
"""Missing CDELT or CUNIT raised a KeyError, and CDi_i and PCi_i were ignored"""
|
|
436
|
+
cube = _write_crpix_cube(Path(tmpdir) / "cube.fits", 1.0)
|
|
437
|
+
with fits.open(cube, mode="update") as hdu_list:
|
|
438
|
+
for key, value in keywords.items():
|
|
439
|
+
if value is None:
|
|
440
|
+
del hdu_list[0].header[key]
|
|
441
|
+
else:
|
|
442
|
+
hdu_list[0].header[key] = value
|
|
443
|
+
cube_wcs = WCS(fits.getheader(cube))
|
|
444
|
+
|
|
445
|
+
for channel in (0, 3):
|
|
446
|
+
output_file = Path(tmpdir) / f"plane_{channel}.fits"
|
|
447
|
+
extract_plane_from_cube(
|
|
448
|
+
fits_cube=cube,
|
|
449
|
+
extract_options=ExtractOptions(
|
|
450
|
+
channel_index=channel, output_path=output_file
|
|
451
|
+
),
|
|
452
|
+
)
|
|
453
|
+
plane_wcs = WCS(fits.getheader(output_file))
|
|
454
|
+
assert np.isclose(
|
|
455
|
+
plane_wcs.spectral.pixel_to_world_values(0),
|
|
456
|
+
cube_wcs.spectral.pixel_to_world_values(channel),
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def test_extract_planes_time_cube(timecube_path, tmpdir) -> None:
|
|
461
|
+
"""Only a FREQ axis was looked for, so time cubes could not be split"""
|
|
462
|
+
output_paths = [Path(tmpdir) / f"t{i}.fits" for i in range(724)]
|
|
463
|
+
extract_planes_from_cube(
|
|
464
|
+
fits_cube=timecube_path,
|
|
465
|
+
extract_planes_options=ExtractPlanesOptions(output_paths=output_paths),
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
single = Path(tmpdir) / "single.fits"
|
|
469
|
+
extract_plane_from_cube(
|
|
470
|
+
fits_cube=timecube_path,
|
|
471
|
+
extract_options=ExtractOptions(time_index=200, output_path=single),
|
|
472
|
+
)
|
|
473
|
+
assert np.array_equal(
|
|
474
|
+
fits.getdata(output_paths[200]), fits.getdata(single), equal_nan=True
|
|
475
|
+
)
|
|
476
|
+
assert fits.getheader(output_paths[200]) == fits.getheader(single)
|
|
477
|
+
assert get_output_path(
|
|
478
|
+
timecube_path, create_target_index(time_index=3)
|
|
479
|
+
).name.endswith("timestep-3.fits")
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def test_extract_planes_skips_degenerate_freq(tmpdir) -> None:
|
|
483
|
+
"""A time cube built from images keeps their one-plane FREQ axis"""
|
|
484
|
+
cube = Path(tmpdir) / "cube.fits"
|
|
485
|
+
header = fits.Header()
|
|
486
|
+
header["CTYPE1"] = "RA---SIN"
|
|
487
|
+
header["CTYPE2"] = "DEC--SIN"
|
|
488
|
+
header["CTYPE3"] = "FREQ"
|
|
489
|
+
header["CTYPE4"] = "TIME"
|
|
490
|
+
header["CDELT4"] = 10.0
|
|
491
|
+
fits.writeto(
|
|
492
|
+
cube,
|
|
493
|
+
np.arange(3, dtype=np.float32)[:, None, None, None] * np.ones((3, 1, 2, 2)),
|
|
494
|
+
header,
|
|
495
|
+
)
|
|
496
|
+
|
|
497
|
+
output_paths = extract_planes_from_cube(
|
|
498
|
+
fits_cube=cube,
|
|
499
|
+
extract_planes_options=ExtractPlanesOptions(
|
|
500
|
+
output_paths=[Path(tmpdir) / f"t{i}.fits" for i in range(3)]
|
|
501
|
+
),
|
|
502
|
+
)
|
|
503
|
+
for timestep, path in enumerate(output_paths):
|
|
504
|
+
assert fits.getdata(path).shape == (1, 1, 2, 2)
|
|
505
|
+
assert np.all(fits.getdata(path) == timestep)
|
|
@@ -6,7 +6,8 @@ import astropy.units as u
|
|
|
6
6
|
import numpy as np
|
|
7
7
|
import pytest
|
|
8
8
|
from astropy.io import fits
|
|
9
|
-
from fitscube.combine_fits import combine_fits, even_spacing, parse_specs
|
|
9
|
+
from fitscube.combine_fits import combine_fits, even_spacing, grid_step, parse_specs
|
|
10
|
+
from fitscube.exceptions import IrregularSpacingException
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
@pytest.fixture
|
|
@@ -121,3 +122,27 @@ def test_even_spacing_non_adjacent_gaps():
|
|
|
121
122
|
assert missing_chan_idx.sum() == 3
|
|
122
123
|
expected_missing = np.array([False, True, True, False, True, False])
|
|
123
124
|
assert np.array_equal(missing_chan_idx, expected_missing)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_float_gcd_does_not_explode_the_grid():
|
|
128
|
+
# The float gcd of 1e6 and 1e6 + 2 Hz is 2 Hz, a grid of a million channels
|
|
129
|
+
assert grid_step([1e6, 1e6 + 2]) == 1e6
|
|
130
|
+
with pytest.raises(IrregularSpacingException, match="would drop inputs"):
|
|
131
|
+
even_spacing(np.array([0.0, 1e6, 2e6 + 2]) * u.Hz)
|
|
132
|
+
# Missing channels 1, 2 and 4 still give the unit step
|
|
133
|
+
assert grid_step([3.0, 2.0]) == 1.0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_huge_grids_are_refused():
|
|
137
|
+
with pytest.raises(IrregularSpacingException, match="would need"):
|
|
138
|
+
# A unit step over a 1e6 span would be a million mostly blank channels
|
|
139
|
+
even_spacing(np.array([0.0, 1.0, 2.0, 1e6]) * u.Hz)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_even_spacing_narrow_channels():
|
|
143
|
+
# 1.16 kHz zoom channels at 1.4 GHz, one missing. A 10 kHz match tolerance
|
|
144
|
+
# would put every input on several grid points.
|
|
145
|
+
specs = (1.4e9 + np.delete(np.arange(64), 10) * 1157.4) * u.Hz
|
|
146
|
+
new_specs, missing_chan_idx = even_spacing(specs)
|
|
147
|
+
assert len(new_specs) == 64
|
|
148
|
+
assert np.flatnonzero(missing_chan_idx).tolist() == [10]
|
|
@@ -6,7 +6,9 @@ import astropy.units as u
|
|
|
6
6
|
import numpy as np
|
|
7
7
|
import pytest
|
|
8
8
|
from astropy.io import fits
|
|
9
|
-
from
|
|
9
|
+
from astropy.time import Time
|
|
10
|
+
from astropy.wcs import WCS
|
|
11
|
+
from fitscube.combine_fits import combine_fits, even_spacing, parse_specs
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
def test_parse_specs(file_list: list[Path], even_specs: u.Quantity):
|
|
@@ -228,3 +230,39 @@ def test_wsclean_images_create_axis_withbb_invalidatezeros(
|
|
|
228
230
|
# The TIME axis will be appended as a new dimension
|
|
229
231
|
cube_image_data = cube_data[i]
|
|
230
232
|
assert np.allclose(image_data.squeeze()[5:-5, 5:-5], cube_image_data.squeeze())
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
@pytest.mark.filterwarnings("ignore:'datfix' made the change")
|
|
236
|
+
@pytest.mark.filterwarnings("ignore:ERFA function")
|
|
237
|
+
def test_time_axis_reference_is_set(time_image_paths, tmpdir) -> None:
|
|
238
|
+
"""MJDREF and TIMESYS were copied from the first input, shifting the axis"""
|
|
239
|
+
for path in time_image_paths:
|
|
240
|
+
with fits.open(path, mode="update") as hdu_list:
|
|
241
|
+
hdu_list[0].header["MJDREF"] = 51544.0
|
|
242
|
+
hdu_list[0].header["TIMESYS"] = "TT"
|
|
243
|
+
|
|
244
|
+
out_cube = Path(tmpdir) / "time_cube_ref.fits"
|
|
245
|
+
combine_fits(
|
|
246
|
+
file_list=time_image_paths,
|
|
247
|
+
out_cube=out_cube,
|
|
248
|
+
overwrite=True,
|
|
249
|
+
time_domain_mode=True,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
header = fits.getheader(out_cube)
|
|
253
|
+
assert header["MJDREF"] == 0.0
|
|
254
|
+
assert header["TIMESYS"] == "UTC"
|
|
255
|
+
wcs = WCS(header)
|
|
256
|
+
first_time = wcs.pixel_to_world(*([0] * wcs.pixel_n_dim))[-1]
|
|
257
|
+
date_obs = Time(fits.getheader(time_image_paths[0])["DATE-OBS"], scale="utc")
|
|
258
|
+
# UTC MJD seconds skip leap seconds, which a FITS reader counts (37 s since 1972)
|
|
259
|
+
assert abs(first_time - date_obs) < 60 * u.s
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def test_even_spacing_one_second_steps():
|
|
263
|
+
"""rtol and atol were passed to np.isclose swapped, giving a ~5 s tolerance
|
|
264
|
+
that matched every grid point of a 1 s grid"""
|
|
265
|
+
times = (5.18e9 + np.array([0.0, 1.0, 3.0, 4.0])) * u.s
|
|
266
|
+
new_times, missing_chan_idx = even_spacing(times, time_domain_mode=True)
|
|
267
|
+
assert len(new_times) == 5
|
|
268
|
+
assert np.array_equal(missing_chan_idx, [False, False, True, False, False])
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|