fitscube 2.10.0__tar.gz → 2.12.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fitscube-2.10.0 → fitscube-2.12.0}/.github/workflows/cd.yml +1 -1
- {fitscube-2.10.0 → fitscube-2.12.0}/.github/workflows/ci.yml +2 -2
- {fitscube-2.10.0 → fitscube-2.12.0}/PKG-INFO +16 -4
- {fitscube-2.10.0 → fitscube-2.12.0}/README.md +13 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/_version.py +2 -2
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/combine_fits.py +89 -86
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/compress.py +22 -15
- {fitscube-2.10.0 → fitscube-2.12.0}/pyproject.toml +4 -5
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_combine_guards.py +30 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_compress.py +70 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_frequencies.py +20 -9
- fitscube-2.12.0/tests/test_racs_all.py +112 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/uv.lock +71 -445
- {fitscube-2.10.0 → fitscube-2.12.0}/.github/CONTRIBUTING.md +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/.github/dependabot.yml +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/.github/release.yml +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/.gitignore +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/.pre-commit-config.yaml +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/CHANGELOG.md +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/LICENSE +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/__init__.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/asyncio.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/bounding_box.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/cli.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/exceptions.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/extract.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/logging.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/version.pyi +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/noxfile.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/__init__.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/conftest.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/data/cube.zip +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/data/images.zip +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/data/time_images.zip +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/data/timecube.zip +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_bb.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_combine.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_combine_layouts.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_extract.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_package.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_times.py +0 -0
- {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_zero_beams.py +0 -0
|
@@ -34,7 +34,7 @@ jobs:
|
|
|
34
34
|
strategy:
|
|
35
35
|
fail-fast: false
|
|
36
36
|
matrix:
|
|
37
|
-
python-version: ["3.
|
|
37
|
+
python-version: ["3.11", "3.12", "3.13", "3.14"]
|
|
38
38
|
runs-on: [ubuntu-latest]
|
|
39
39
|
|
|
40
40
|
steps:
|
|
@@ -55,6 +55,6 @@ jobs:
|
|
|
55
55
|
--durations=20
|
|
56
56
|
|
|
57
57
|
- name: Upload coverage report
|
|
58
|
-
uses: codecov/codecov-action@v7.
|
|
58
|
+
uses: codecov/codecov-action@v7.1.1
|
|
59
59
|
with:
|
|
60
60
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: fitscube
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.12.0
|
|
4
4
|
Summary: A package to produce produce FITS cubes.
|
|
5
5
|
Project-URL: Homepage, https://github.com/AlecThomson/fitscube
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/AlecThomson/fitscube/issues
|
|
@@ -37,15 +37,14 @@ Classifier: Operating System :: OS Independent
|
|
|
37
37
|
Classifier: Programming Language :: Python
|
|
38
38
|
Classifier: Programming Language :: Python :: 3
|
|
39
39
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
40
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
41
40
|
Classifier: Programming Language :: Python :: 3.11
|
|
42
41
|
Classifier: Programming Language :: Python :: 3.12
|
|
43
42
|
Classifier: Programming Language :: Python :: 3.13
|
|
44
43
|
Classifier: Programming Language :: Python :: 3.14
|
|
45
44
|
Classifier: Topic :: Scientific/Engineering
|
|
46
45
|
Classifier: Typing :: Typed
|
|
47
|
-
Requires-Python: >=3.
|
|
48
|
-
Requires-Dist: astropy>=
|
|
46
|
+
Requires-Python: >=3.11
|
|
47
|
+
Requires-Dist: astropy>=7.2
|
|
49
48
|
Requires-Dist: numpy>=2
|
|
50
49
|
Requires-Dist: radio-beam
|
|
51
50
|
Requires-Dist: tqdm
|
|
@@ -329,6 +328,19 @@ Quantise once. Every decompress and recompress with `tile-rice` adds a new error
|
|
|
329
328
|
unless given `allow_requantize=True`. Smooth images such as weights do not shrink under
|
|
330
329
|
RICE; use `tile-gzip` for them.
|
|
331
330
|
|
|
331
|
+
## Blank channels
|
|
332
|
+
|
|
333
|
+
`--create-blanks` puts the inputs on one evenly spaced axis, with NaN planes
|
|
334
|
+
where there is no input. It picks the widest step that holds every input, so
|
|
335
|
+
images of uneven width, or images either side of a gap between bands (as in
|
|
336
|
+
RACS-low3, -mid and -high together), still make a single cube.
|
|
337
|
+
|
|
338
|
+
An input counts as on the grid if it is within ~1 Hz at 1 GHz of a plane (or
|
|
339
|
+
~0.5 s for times). If any input is off by more than float rounding, `fitscube`
|
|
340
|
+
logs a warning, and the cube's `CSYERi` (the FITS systematic error of the axis)
|
|
341
|
+
records the largest offset. A grid of more than 1000 planes per input, or one
|
|
342
|
+
that cannot hold every input, is refused.
|
|
343
|
+
|
|
332
344
|
## Zero-sized beams
|
|
333
345
|
|
|
334
346
|
wsclean writes `BMAJ = BMIN = 0` into an image when no PSF was fitted for that
|
|
@@ -256,6 +256,19 @@ Quantise once. Every decompress and recompress with `tile-rice` adds a new error
|
|
|
256
256
|
unless given `allow_requantize=True`. Smooth images such as weights do not shrink under
|
|
257
257
|
RICE; use `tile-gzip` for them.
|
|
258
258
|
|
|
259
|
+
## Blank channels
|
|
260
|
+
|
|
261
|
+
`--create-blanks` puts the inputs on one evenly spaced axis, with NaN planes
|
|
262
|
+
where there is no input. It picks the widest step that holds every input, so
|
|
263
|
+
images of uneven width, or images either side of a gap between bands (as in
|
|
264
|
+
RACS-low3, -mid and -high together), still make a single cube.
|
|
265
|
+
|
|
266
|
+
An input counts as on the grid if it is within ~1 Hz at 1 GHz of a plane (or
|
|
267
|
+
~0.5 s for times). If any input is off by more than float rounding, `fitscube`
|
|
268
|
+
logs a warning, and the cube's `CSYERi` (the FITS systematic error of the axis)
|
|
269
|
+
records the largest offset. A grid of more than 1000 planes per input, or one
|
|
270
|
+
that cannot hold every input, is refused.
|
|
271
|
+
|
|
259
272
|
## Zero-sized beams
|
|
260
273
|
|
|
261
274
|
wsclean writes `BMAJ = BMIN = 0` into an image when no PSF was fitted for that
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '2.
|
|
22
|
-
__version_tuple__ = version_tuple = (2,
|
|
21
|
+
__version__ = version = '2.12.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 12, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -29,7 +29,7 @@ from astropy.io.fits.verify import VerifyWarning
|
|
|
29
29
|
from astropy.table import Table
|
|
30
30
|
from astropy.time import Time
|
|
31
31
|
from astropy.wcs import WCS, WCSSUB_STOKES
|
|
32
|
-
from numpy.typing import
|
|
32
|
+
from numpy.typing import NDArray
|
|
33
33
|
from radio_beam import Beam, Beams
|
|
34
34
|
from radio_beam.beam import NoBeamException
|
|
35
35
|
from tqdm.asyncio import tqdm
|
|
@@ -324,130 +324,120 @@ async def check_matching_scaling_coro(
|
|
|
324
324
|
check_matching_scaling = sync_wrapper(check_matching_scaling_coro)
|
|
325
325
|
|
|
326
326
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
stop += step * max(0.1, x) if x < 0.5 else 0
|
|
332
|
-
return np.arange(start, stop, step)
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
def isin_close(
|
|
336
|
-
element: NDArray[np.floating[Any]],
|
|
337
|
-
test_element: ArrayLike,
|
|
338
|
-
time_domain_mode: bool = False,
|
|
339
|
-
) -> NDArray[np.bool_]:
|
|
340
|
-
"""Check if element is in test_element, within a tolerance.
|
|
341
|
-
|
|
342
|
-
Args:
|
|
343
|
-
element (NDArray[np.floating[Any]]): Element to check
|
|
344
|
-
test_element (ArrayLike): Element to check against
|
|
345
|
-
|
|
346
|
-
Returns:
|
|
347
|
-
NDArray[np.bool_]: Boolean array
|
|
348
|
-
"""
|
|
327
|
+
def match_tolerance(
|
|
328
|
+
values: NDArray[np.floating[Any]], time_domain_mode: bool = False
|
|
329
|
+
) -> NDArray[np.floating[Any]]:
|
|
330
|
+
"""How far a value may sit from a grid point and still be on it."""
|
|
349
331
|
if time_domain_mode:
|
|
350
332
|
# ~0.5 s for MJD seconds, so integrations of a second or more are told apart
|
|
351
|
-
rtol = 1e-10
|
|
352
|
-
atol = 1e-9
|
|
333
|
+
rtol, atol = 1e-10, 1e-9
|
|
353
334
|
else:
|
|
354
335
|
# ~1 Hz at 1 GHz. The np.isclose default of rtol=1e-5 (10 kHz) would
|
|
355
336
|
# match several kHz-wide zoom channels and snap off-grid inputs.
|
|
356
|
-
rtol = 1e-9
|
|
357
|
-
|
|
358
|
-
|
|
337
|
+
rtol, atol = 1e-9, 1e-5
|
|
338
|
+
return atol + rtol * np.abs(values)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
class GridFit(NamedTuple):
|
|
342
|
+
"""A regular grid through a set of sorted values."""
|
|
359
343
|
|
|
344
|
+
start: float
|
|
345
|
+
"""Value of the first grid point"""
|
|
346
|
+
step: float
|
|
347
|
+
"""Grid step"""
|
|
348
|
+
index: NDArray[np.int64]
|
|
349
|
+
"""Grid point of each value"""
|
|
350
|
+
residuals: NDArray[np.float64]
|
|
351
|
+
"""Distance of each value from its grid point"""
|
|
360
352
|
|
|
361
|
-
def grid_step(diffs: ArrayLike) -> float:
|
|
362
|
-
"""Recover the fundamental channel step from the observed diffs.
|
|
363
353
|
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
float gcd of 2.
|
|
354
|
+
def fit_grid(
|
|
355
|
+
specs: NDArray[np.floating[Any]], time_domain_mode: bool = False
|
|
356
|
+
) -> GridFit | None:
|
|
357
|
+
"""Find the widest regular grid that holds every sorted value.
|
|
358
|
+
|
|
359
|
+
The step divides the smallest gap, so steps of that gap over 1, 2, 3...
|
|
360
|
+
are tried in turn. Each is fitted to all the values, and the first that
|
|
361
|
+
puts every value on a grid point within the match tolerance wins.
|
|
373
362
|
|
|
374
363
|
Args:
|
|
375
|
-
|
|
364
|
+
specs (NDArray[np.floating[Any]]): Sorted, distinct values
|
|
365
|
+
time_domain_mode (bool, optional): Use the tolerance for times. Defaults to False.
|
|
376
366
|
|
|
377
367
|
Returns:
|
|
378
|
-
|
|
368
|
+
GridFit | None: The grid, or None if none of at most 1000 points per value fits
|
|
379
369
|
"""
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
370
|
+
offsets = specs - specs[0]
|
|
371
|
+
span = offsets[-1]
|
|
372
|
+
min_gap = np.min(np.diff(specs))
|
|
373
|
+
tol = match_tolerance(specs, time_domain_mode)
|
|
374
|
+
max_channels = 1000 * len(specs)
|
|
375
|
+
|
|
376
|
+
divisor = 1
|
|
377
|
+
while span * divisor / min_gap < max_channels:
|
|
378
|
+
index = np.round(offsets * divisor / min_gap)
|
|
379
|
+
step, start = np.polyfit(index, offsets, 1)
|
|
380
|
+
residuals = np.abs(offsets - start - index * step)
|
|
381
|
+
if np.all(residuals <= tol):
|
|
382
|
+
return GridFit(
|
|
383
|
+
start=float(specs[0] + start),
|
|
384
|
+
step=float(step),
|
|
385
|
+
index=index.astype(np.int64),
|
|
386
|
+
residuals=residuals,
|
|
387
|
+
)
|
|
388
|
+
divisor += 1
|
|
391
389
|
|
|
392
|
-
|
|
393
|
-
# A gcd far below every diff is float noise, not a real channel width.
|
|
394
|
-
if g <= tol or not divides_all or g * 16 < min_diff:
|
|
395
|
-
return float(min_diff)
|
|
396
|
-
return float(g)
|
|
390
|
+
return None
|
|
397
391
|
|
|
398
392
|
|
|
399
393
|
def even_spacing(specs: u.Quantity, time_domain_mode: bool = False) -> SpequencyInfo:
|
|
400
394
|
"""Make the frequencies or times evenly spaced.
|
|
401
395
|
|
|
402
|
-
A regular grid cannot always hold every input - an irregular spacing
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
396
|
+
A regular grid cannot always hold every input - an irregular spacing leaves
|
|
397
|
+
inputs sitting between grid points, and equal values collapse onto one.
|
|
398
|
+
Those inputs would be dropped from the cube without trace, so this raises
|
|
399
|
+
rather than writing a cube that is missing data. Inputs that sit on the grid
|
|
400
|
+
only within the match tolerance are kept, with a warning.
|
|
406
401
|
|
|
407
402
|
Args:
|
|
408
403
|
specs (u.Quantity): Original frequencies/times
|
|
409
404
|
|
|
410
405
|
Raises:
|
|
411
|
-
IrregularSpacingException: If
|
|
406
|
+
IrregularSpacingException: If no grid of at most 1000 points per input holds every input
|
|
412
407
|
|
|
413
408
|
Returns:
|
|
414
409
|
SpequencyInfo: specs, missing_chan_idx
|
|
415
410
|
"""
|
|
416
|
-
|
|
417
|
-
# otherwise give an empty (or reversed) grid. Callers sort later, not here.
|
|
418
|
-
specs_arr = np.sort(specs.value.astype(np.longdouble))
|
|
411
|
+
specs_arr = np.sort(specs.value.astype(np.float64))
|
|
419
412
|
if len(specs_arr) < 2:
|
|
420
413
|
return SpequencyInfo(
|
|
421
414
|
specs_arr * specs.unit, np.zeros(len(specs_arr)).astype(bool)
|
|
422
415
|
)
|
|
423
416
|
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
max_channels = 1000 * len(specs_arr)
|
|
428
|
-
if span / step > max_channels:
|
|
417
|
+
spequencies = "times" if time_domain_mode else "frequencies"
|
|
418
|
+
grid = fit_grid(specs_arr, time_domain_mode) if np.all(np.diff(specs_arr)) else None
|
|
419
|
+
if grid is None:
|
|
429
420
|
msg = (
|
|
430
|
-
f"
|
|
431
|
-
f"
|
|
432
|
-
"Combine without blank channels."
|
|
421
|
+
f"No regular grid of at most {1000 * len(specs_arr)} points holds all "
|
|
422
|
+
f"{len(specs_arr)} {spequencies}, so blank channels would drop inputs. "
|
|
423
|
+
"Combine without blank channels, or regrid the inputs onto a common step."
|
|
433
424
|
)
|
|
434
425
|
raise IrregularSpacingException(msg)
|
|
435
|
-
# Create a new array with the fundamental grid step
|
|
436
|
-
new_specs = np_arange_fix(specs_arr[0], specs_arr[-1], step)
|
|
437
|
-
missing_chan_idx = np.logical_not(
|
|
438
|
-
isin_close(new_specs, specs_arr, time_domain_mode)
|
|
439
|
-
)
|
|
440
426
|
|
|
441
|
-
|
|
442
|
-
if
|
|
443
|
-
spequencies = "times" if time_domain_mode else "frequencies"
|
|
427
|
+
worst = int(np.argmax(grid.residuals))
|
|
428
|
+
if grid.residuals[worst] > 1e-12 * np.max(np.abs(specs_arr)):
|
|
444
429
|
msg = (
|
|
445
|
-
f"
|
|
446
|
-
f"{
|
|
447
|
-
"
|
|
448
|
-
"
|
|
430
|
+
f"The {spequencies} are not exactly on a regular grid of {grid.step} "
|
|
431
|
+
f"{specs.unit}: {specs_arr[worst]} {specs.unit} is "
|
|
432
|
+
f"{grid.residuals[worst]} {specs.unit} from "
|
|
433
|
+
"its plane. The cube labels each plane with its grid value, not the "
|
|
434
|
+
"input's own."
|
|
449
435
|
)
|
|
450
|
-
|
|
436
|
+
logger.warning(msg)
|
|
437
|
+
|
|
438
|
+
new_specs = grid.start + np.arange(grid.index[-1] + 1) * grid.step
|
|
439
|
+
missing_chan_idx = np.ones(len(new_specs), dtype=bool)
|
|
440
|
+
missing_chan_idx[grid.index] = False
|
|
451
441
|
|
|
452
442
|
return SpequencyInfo(new_specs * specs.unit, missing_chan_idx)
|
|
453
443
|
|
|
@@ -533,6 +523,7 @@ async def create_output_cube_coro(
|
|
|
533
523
|
time_domain_mode: bool = False,
|
|
534
524
|
bounding_box: BoundingBox | None = None,
|
|
535
525
|
float_length: FLOAT_LENGTH | None = None,
|
|
526
|
+
spec_error: float | None = None,
|
|
536
527
|
) -> InitResult:
|
|
537
528
|
"""Generate the output header and write a dummy cube to disk based on properties of the
|
|
538
529
|
inpute data. The output cube written here has a correctly formed header and a pre-zerod
|
|
@@ -549,6 +540,7 @@ async def create_output_cube_coro(
|
|
|
549
540
|
time_domain_mode (bool, optional): Whether to join images via the DATE-OBS (e.g. time) axis. If False cube joined along frequency axis. Defaults to False.
|
|
550
541
|
bounding_box (BoundingBox | None, optional): Whether a trimming operation should be applied to input images. If a BoundingBox is supplied this will be used to update the reference pixel position and data shape indicators. Defaults to None.
|
|
551
542
|
float_length (Literal[32, 64] | None, optional): The precision of the output data. If None drawn from input data. Otherwise values accepted are 32 and 64. Defaults to None.
|
|
543
|
+
spec_error (float | None, optional): Largest distance of an input from its plane on a gridded axis, in Hz or s. Written as CSYERi if given. Defaults to None.
|
|
552
544
|
|
|
553
545
|
Returns:
|
|
554
546
|
InitResult: Details of the output cube, including the output header
|
|
@@ -651,6 +643,12 @@ async def create_output_cube_coro(
|
|
|
651
643
|
del new_header[f"CUNIT{fits_idx}"]
|
|
652
644
|
new_header[f"CTYPE{fits_idx}"] = "CHAN"
|
|
653
645
|
new_header[f"CRVAL{fits_idx}"] = 1
|
|
646
|
+
elif spec_error is not None:
|
|
647
|
+
new_header[f"CSYER{fits_idx}"] = spec_error
|
|
648
|
+
new_header["HISTORY"] = (
|
|
649
|
+
f"fitscube placed the inputs on a regular {ctype} grid. Each is within "
|
|
650
|
+
f"CSYER{fits_idx} of its plane."
|
|
651
|
+
)
|
|
654
652
|
|
|
655
653
|
# Add extra transform fields for consistency. A CD matrix replaces CDELT,
|
|
656
654
|
# so it must carry the step itself.
|
|
@@ -1278,6 +1276,10 @@ async def combine_fits_coro(
|
|
|
1278
1276
|
new_sort_idx = np.argsort(specs)
|
|
1279
1277
|
specs = specs[new_sort_idx]
|
|
1280
1278
|
missing_chan_idx = missing_chan_idx[new_sort_idx]
|
|
1279
|
+
spec_error = None
|
|
1280
|
+
if create_blanks:
|
|
1281
|
+
populated = specs[~missing_chan_idx]
|
|
1282
|
+
spec_error = float(np.max(np.abs(np.sort(file_specs) - populated)).value)
|
|
1281
1283
|
|
|
1282
1284
|
has_beams = check_for_any_beam(file_list=file_list)
|
|
1283
1285
|
zero_beam_idx = np.zeros(len(specs), dtype=bool)
|
|
@@ -1350,6 +1352,7 @@ async def combine_fits_coro(
|
|
|
1350
1352
|
time_domain_mode=time_domain_mode,
|
|
1351
1353
|
bounding_box=final_bounding_box,
|
|
1352
1354
|
float_length=float_length,
|
|
1355
|
+
spec_error=spec_error,
|
|
1353
1356
|
)
|
|
1354
1357
|
|
|
1355
1358
|
if (missing_chan_idx | zero_beam_idx).any() and new_header["BITPIX"] > 0:
|
|
@@ -41,8 +41,6 @@ class CompressedPlane(NamedTuple):
|
|
|
41
41
|
"""ZSCALE of each tile, empty unless quantized"""
|
|
42
42
|
zeros: list[float]
|
|
43
43
|
"""ZZERO of each tile, empty unless quantized"""
|
|
44
|
-
has_blanks: bool
|
|
45
|
-
"""Whether quantized tiles marked blank (NaN) pixels with ZBLANK"""
|
|
46
44
|
|
|
47
45
|
|
|
48
46
|
def compress_cube(
|
|
@@ -128,6 +126,7 @@ def gzip2_tiles(
|
|
|
128
126
|
GZIP_2 gzips the big-endian bytes of the tile, shuffled so that all the first
|
|
129
127
|
bytes of each value come first, then all the second bytes, and so on.
|
|
130
128
|
"""
|
|
129
|
+
# A view, not a copy, for a plane read straight from a FITS file
|
|
131
130
|
big_endian = plane.astype(plane.dtype.newbyteorder(">"), copy=False)
|
|
132
131
|
itemsize = big_endian.dtype.itemsize
|
|
133
132
|
tiles = []
|
|
@@ -176,7 +175,6 @@ def astropy_compress_plane(
|
|
|
176
175
|
},
|
|
177
176
|
scales=list(table.data["ZSCALE"]) if "ZSCALE" in names else [],
|
|
178
177
|
zeros=list(table.data["ZZERO"]) if "ZZERO" in names else [],
|
|
179
|
-
has_blanks="ZBLANK" in table.header,
|
|
180
178
|
)
|
|
181
179
|
return compressed, table.header.copy()
|
|
182
180
|
|
|
@@ -203,10 +201,15 @@ def compress_plane(
|
|
|
203
201
|
heap={"COMPRESSED_DATA": gzip2_tiles(plane, tile_shape, gzip_level)},
|
|
204
202
|
scales=[],
|
|
205
203
|
zeros=[],
|
|
206
|
-
has_blanks=False,
|
|
207
204
|
)
|
|
205
|
+
# astropy wants native byte order; only this path pays for the copy
|
|
208
206
|
compressed, _ = astropy_compress_plane(
|
|
209
|
-
plane
|
|
207
|
+
plane.astype(plane.dtype.newbyteorder("="), copy=False),
|
|
208
|
+
fits.Header(),
|
|
209
|
+
compression_type,
|
|
210
|
+
quantize_level,
|
|
211
|
+
tile_shape,
|
|
212
|
+
dither_seed,
|
|
210
213
|
)
|
|
211
214
|
return compressed
|
|
212
215
|
|
|
@@ -218,7 +221,6 @@ def table_header(
|
|
|
218
221
|
heap_lengths: dict[str, int],
|
|
219
222
|
heap_size: int,
|
|
220
223
|
dither_seed: int,
|
|
221
|
-
has_blanks: bool,
|
|
222
224
|
) -> fits.Header:
|
|
223
225
|
"""Header of the whole cube's compressed table, built from the first plane's."""
|
|
224
226
|
header = template.copy()
|
|
@@ -238,7 +240,9 @@ def table_header(
|
|
|
238
240
|
header["NAXIS1"] = row_width
|
|
239
241
|
if "ZDITHER0" in header:
|
|
240
242
|
header["ZDITHER0"] = dither_seed
|
|
241
|
-
if
|
|
243
|
+
if "ZQUANTIZ" in header:
|
|
244
|
+
# Always set, so the header never grows past the placeholder written before
|
|
245
|
+
# the tiles that turn out to hold blanks
|
|
242
246
|
header["ZBLANK"] = np.iinfo(np.int32).min
|
|
243
247
|
return header
|
|
244
248
|
|
|
@@ -263,7 +267,11 @@ def compressed_planes(
|
|
|
263
267
|
gzip_level: int,
|
|
264
268
|
max_workers: int,
|
|
265
269
|
) -> Iterator[CompressedPlane]:
|
|
266
|
-
"""Compress every plane of `image` across threads, yielding them in order.
|
|
270
|
+
"""Compress every plane of `image` across threads, yielding them in order.
|
|
271
|
+
|
|
272
|
+
Planes are handed over as read, big-endian, which is the order GZIP_2 shuffles:
|
|
273
|
+
the lossless path never copies a plane.
|
|
274
|
+
"""
|
|
267
275
|
with ThreadPoolExecutor(max_workers) as pool:
|
|
268
276
|
pending: deque[Future[CompressedPlane]] = deque()
|
|
269
277
|
for plane_index, index in enumerate(np.ndindex(image.shape[:-2])):
|
|
@@ -271,7 +279,7 @@ def compressed_planes(
|
|
|
271
279
|
pending.append(
|
|
272
280
|
pool.submit(
|
|
273
281
|
compress_plane,
|
|
274
|
-
plane
|
|
282
|
+
plane,
|
|
275
283
|
compression_type,
|
|
276
284
|
quantize_level,
|
|
277
285
|
tile_shape,
|
|
@@ -389,19 +397,16 @@ def tile_compress_cube(
|
|
|
389
397
|
# Placeholders: the header and table are rewritten once the heap is known,
|
|
390
398
|
# and stay the same size because only fixed-width card values change
|
|
391
399
|
handle.write(
|
|
392
|
-
table_header(
|
|
393
|
-
template, cube_shape, nrows, heap_lengths, 0, dither_seed, False
|
|
394
|
-
)
|
|
400
|
+
table_header(template, cube_shape, nrows, heap_lengths, 0, dither_seed)
|
|
395
401
|
.tostring()
|
|
396
402
|
.encode()
|
|
397
403
|
)
|
|
398
404
|
handle.write(table.tobytes())
|
|
405
|
+
table_end = handle.tell()
|
|
399
406
|
|
|
400
407
|
heap_size = 0
|
|
401
|
-
has_blanks = False
|
|
402
408
|
row = 0
|
|
403
409
|
for plane in itertools.chain([first], planes):
|
|
404
|
-
has_blanks |= plane.has_blanks
|
|
405
410
|
for tile_index in range(tiles_per_plane):
|
|
406
411
|
for name in heap_lengths:
|
|
407
412
|
data = plane.heap[name][tile_index]
|
|
@@ -424,12 +429,14 @@ def tile_compress_cube(
|
|
|
424
429
|
heap_lengths,
|
|
425
430
|
heap_size,
|
|
426
431
|
dither_seed,
|
|
427
|
-
has_blanks,
|
|
428
432
|
)
|
|
429
433
|
.tostring()
|
|
430
434
|
.encode()
|
|
431
435
|
)
|
|
432
436
|
handle.write(table.tobytes())
|
|
437
|
+
if handle.tell() != table_end:
|
|
438
|
+
msg = "The final table header differs in size from its placeholder"
|
|
439
|
+
raise RuntimeError(msg)
|
|
433
440
|
|
|
434
441
|
for hdu in hdul[1:]:
|
|
435
442
|
fits.append(partial_path, hdu.data, hdu.header)
|
|
@@ -11,7 +11,7 @@ authors = [
|
|
|
11
11
|
description = "A package to produce produce FITS cubes."
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
license.file = "LICENSE"
|
|
14
|
-
requires-python = ">=3.
|
|
14
|
+
requires-python = ">=3.11"
|
|
15
15
|
classifiers = [
|
|
16
16
|
"Development Status :: 1 - Planning",
|
|
17
17
|
"Intended Audience :: Science/Research",
|
|
@@ -21,7 +21,6 @@ classifiers = [
|
|
|
21
21
|
"Programming Language :: Python",
|
|
22
22
|
"Programming Language :: Python :: 3",
|
|
23
23
|
"Programming Language :: Python :: 3 :: Only",
|
|
24
|
-
"Programming Language :: Python :: 3.10",
|
|
25
24
|
"Programming Language :: Python :: 3.11",
|
|
26
25
|
"Programming Language :: Python :: 3.12",
|
|
27
26
|
"Programming Language :: Python :: 3.13",
|
|
@@ -31,7 +30,7 @@ classifiers = [
|
|
|
31
30
|
]
|
|
32
31
|
dynamic = ["version"]
|
|
33
32
|
dependencies = [
|
|
34
|
-
"astropy>=
|
|
33
|
+
"astropy>=7.2", # thread-safe disable_image_compression, numpy >= 2.4
|
|
35
34
|
"numpy>=2",
|
|
36
35
|
"tqdm",
|
|
37
36
|
"radio_beam",
|
|
@@ -107,7 +106,7 @@ report.exclude_also = [
|
|
|
107
106
|
|
|
108
107
|
[tool.mypy]
|
|
109
108
|
files = ["fitscube", "tests"]
|
|
110
|
-
python_version = "3.
|
|
109
|
+
python_version = "3.11"
|
|
111
110
|
warn_unused_configs = true
|
|
112
111
|
strict = true
|
|
113
112
|
enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
|
|
@@ -175,7 +174,7 @@ isort.required-imports = ["from __future__ import annotations"]
|
|
|
175
174
|
|
|
176
175
|
|
|
177
176
|
[tool.pylint]
|
|
178
|
-
py-version = "3.
|
|
177
|
+
py-version = "3.11"
|
|
179
178
|
ignore-paths = [".*/_version.py"]
|
|
180
179
|
reports.output-format = "colorized"
|
|
181
180
|
similarities.ignore-imports = "yes"
|
|
@@ -579,3 +579,33 @@ def test_float_length_must_be_fits_float(
|
|
|
579
579
|
get_parser().parse_args(
|
|
580
580
|
[*map(str, file_list), str(out_cube), "--floating", str(float_length)]
|
|
581
581
|
)
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
@pytest.mark.parametrize("jitter", [0.0, 0.3])
|
|
585
|
+
def test_grid_offset_is_recorded(tmp_path: Path, jitter: float) -> None:
|
|
586
|
+
"""CSYERi holds how far an input sits from the plane it was written to."""
|
|
587
|
+
freqs = (1e9 + np.array([0.0, 1.0, 3.0]) * 1e6 + [0.0, jitter, 0.0]) * u.Hz
|
|
588
|
+
file_list = [
|
|
589
|
+
make_plane(tmp_path / f"offset_{i}.fits", freq, value=float(i))
|
|
590
|
+
for i, freq in enumerate(freqs)
|
|
591
|
+
]
|
|
592
|
+
out_cube = tmp_path / "cube.fits"
|
|
593
|
+
combine_fits(
|
|
594
|
+
file_list=file_list, out_cube=out_cube, create_blanks=True, overwrite=True
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
header = fits.getheader(out_cube)
|
|
598
|
+
assert header["CTYPE4"] == "FREQ"
|
|
599
|
+
# The grid is fitted to all inputs, so the offset is shared out among them
|
|
600
|
+
assert header["CSYER4"] <= jitter
|
|
601
|
+
assert (header["CSYER4"] > 0.1) == (jitter > 0)
|
|
602
|
+
assert "CSYER4" in str(header["HISTORY"])
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def test_no_grid_offset_without_blanks(tmp_path: Path, specs: u.Quantity) -> None:
|
|
606
|
+
file_list = [
|
|
607
|
+
make_plane(tmp_path / f"plain_{i}.fits", spec) for i, spec in enumerate(specs)
|
|
608
|
+
]
|
|
609
|
+
out_cube = tmp_path / "cube.fits"
|
|
610
|
+
combine_fits(file_list=file_list, out_cube=out_cube, overwrite=True)
|
|
611
|
+
assert "CSYER4" not in fits.getheader(out_cube)
|
|
@@ -2,16 +2,22 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import tracemalloc
|
|
6
|
+
from collections.abc import Callable, Iterator
|
|
5
7
|
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
6
9
|
|
|
7
10
|
import numpy as np
|
|
8
11
|
import pytest
|
|
9
12
|
from astropy.io import fits
|
|
13
|
+
from fitscube import compress as compress_module
|
|
10
14
|
from fitscube.cli import cli
|
|
11
15
|
from fitscube.combine_fits import combine_fits
|
|
12
16
|
from fitscube.compress import (
|
|
17
|
+
CompressedPlane,
|
|
13
18
|
CompressMethod,
|
|
14
19
|
compress_cube,
|
|
20
|
+
compress_plane,
|
|
15
21
|
decompress_cube,
|
|
16
22
|
tile_compress_cube,
|
|
17
23
|
)
|
|
@@ -134,6 +140,29 @@ def test_tile_compress_is_the_same_for_any_worker_count(tmp_path: Path) -> None:
|
|
|
134
140
|
assert outputs[0] == outputs[1]
|
|
135
141
|
|
|
136
142
|
|
|
143
|
+
def test_gzip2_shuffles_a_big_endian_plane_without_copying_it() -> None:
|
|
144
|
+
"""Planes come off the file big-endian, the order GZIP_2 shuffles, so no copy is needed"""
|
|
145
|
+
plane = np.full((256, 1024), 1.5, dtype=">f4")
|
|
146
|
+
|
|
147
|
+
def compress(plane: np.ndarray) -> CompressedPlane:
|
|
148
|
+
return compress_plane(plane, "GZIP_2", 0.0, (16, 1024), 1, 6)
|
|
149
|
+
|
|
150
|
+
tracemalloc.start()
|
|
151
|
+
compressed = compress(plane)
|
|
152
|
+
_, peak = tracemalloc.get_traced_memory()
|
|
153
|
+
tracemalloc.stop()
|
|
154
|
+
assert peak < plane.nbytes / 2
|
|
155
|
+
|
|
156
|
+
# The control: a native-order plane has to be copied back to big-endian
|
|
157
|
+
native = plane.astype("<f4")
|
|
158
|
+
tracemalloc.start()
|
|
159
|
+
from_native = compress(native)
|
|
160
|
+
_, native_peak = tracemalloc.get_traced_memory()
|
|
161
|
+
tracemalloc.stop()
|
|
162
|
+
assert native_peak >= plane.nbytes
|
|
163
|
+
assert from_native == compressed
|
|
164
|
+
|
|
165
|
+
|
|
137
166
|
def test_tile_compress_refuses_compressed_input(tmp_path: Path) -> None:
|
|
138
167
|
write_noise_cube(tmp_path / "cube.fits")
|
|
139
168
|
compressed = tile_compress_cube(tmp_path / "cube.fits")
|
|
@@ -232,3 +261,44 @@ def test_requantizing_dithers_afresh(tmp_path: Path) -> None:
|
|
|
232
261
|
)
|
|
233
262
|
|
|
234
263
|
assert fits.getheader(second, 1, disable_image_compression=True)["ZDITHER0"] != seed
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
@pytest.mark.parametrize("extra_cards", range(36))
|
|
267
|
+
def test_tile_rice_header_never_grows(tmp_path: Path, extra_cards: int) -> None:
|
|
268
|
+
"""Blanks only past the first tile must not grow the header over its placeholder"""
|
|
269
|
+
rng = np.random.default_rng(1)
|
|
270
|
+
data = rng.normal(0, 1e-3, (3, 40, 48)).astype(np.float32)
|
|
271
|
+
data[-1, -8:] = np.nan
|
|
272
|
+
header = fits.Header()
|
|
273
|
+
for i in range(extra_cards):
|
|
274
|
+
header.add_comment(f"padding {i}")
|
|
275
|
+
fits.PrimaryHDU(data, header).writeto(tmp_path / "cube.fits")
|
|
276
|
+
|
|
277
|
+
compressed = compress_cube(tmp_path / "cube.fits", method="tile-rice")
|
|
278
|
+
|
|
279
|
+
got = fits.getdata(compressed, 1)
|
|
280
|
+
np.testing.assert_array_equal(np.isnan(got), np.isnan(data))
|
|
281
|
+
finite = np.isfinite(data)
|
|
282
|
+
assert np.std(got[finite] - data[finite]) < 1e-3 / 4 / np.sqrt(12) * 1.2
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def test_tile_compress_refuses_a_grown_header(
|
|
286
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
287
|
+
) -> None:
|
|
288
|
+
"""A final header longer than its placeholder errors rather than corrupting the file"""
|
|
289
|
+
write_noise_cube(tmp_path / "cube.fits")
|
|
290
|
+
original = compress_module.table_header
|
|
291
|
+
|
|
292
|
+
def grown(*args: Any, **kwargs: Any) -> fits.Header:
|
|
293
|
+
header = original(*args, **kwargs)
|
|
294
|
+
for i in range(36):
|
|
295
|
+
header.add_comment(f"padding {i}")
|
|
296
|
+
return header
|
|
297
|
+
|
|
298
|
+
calls: Iterator[Callable[..., fits.Header]] = iter([original, grown])
|
|
299
|
+
monkeypatch.setattr(
|
|
300
|
+
compress_module, "table_header", lambda *a, **k: next(calls)(*a, **k)
|
|
301
|
+
)
|
|
302
|
+
with pytest.raises(RuntimeError, match="placeholder"):
|
|
303
|
+
tile_compress_cube(tmp_path / "cube.fits", remove=False)
|
|
304
|
+
assert not (tmp_path / "cube.fits.fz").exists()
|