fitscube 2.10.0__tar.gz → 2.12.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. {fitscube-2.10.0 → fitscube-2.12.0}/.github/workflows/cd.yml +1 -1
  2. {fitscube-2.10.0 → fitscube-2.12.0}/.github/workflows/ci.yml +2 -2
  3. {fitscube-2.10.0 → fitscube-2.12.0}/PKG-INFO +16 -4
  4. {fitscube-2.10.0 → fitscube-2.12.0}/README.md +13 -0
  5. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/_version.py +2 -2
  6. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/combine_fits.py +89 -86
  7. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/compress.py +22 -15
  8. {fitscube-2.10.0 → fitscube-2.12.0}/pyproject.toml +4 -5
  9. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_combine_guards.py +30 -0
  10. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_compress.py +70 -0
  11. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_frequencies.py +20 -9
  12. fitscube-2.12.0/tests/test_racs_all.py +112 -0
  13. {fitscube-2.10.0 → fitscube-2.12.0}/uv.lock +71 -445
  14. {fitscube-2.10.0 → fitscube-2.12.0}/.github/CONTRIBUTING.md +0 -0
  15. {fitscube-2.10.0 → fitscube-2.12.0}/.github/dependabot.yml +0 -0
  16. {fitscube-2.10.0 → fitscube-2.12.0}/.github/release.yml +0 -0
  17. {fitscube-2.10.0 → fitscube-2.12.0}/.gitignore +0 -0
  18. {fitscube-2.10.0 → fitscube-2.12.0}/.pre-commit-config.yaml +0 -0
  19. {fitscube-2.10.0 → fitscube-2.12.0}/CHANGELOG.md +0 -0
  20. {fitscube-2.10.0 → fitscube-2.12.0}/LICENSE +0 -0
  21. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/__init__.py +0 -0
  22. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/asyncio.py +0 -0
  23. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/bounding_box.py +0 -0
  24. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/cli.py +0 -0
  25. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/exceptions.py +0 -0
  26. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/extract.py +0 -0
  27. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/logging.py +0 -0
  28. {fitscube-2.10.0 → fitscube-2.12.0}/fitscube/version.pyi +0 -0
  29. {fitscube-2.10.0 → fitscube-2.12.0}/noxfile.py +0 -0
  30. {fitscube-2.10.0 → fitscube-2.12.0}/tests/__init__.py +0 -0
  31. {fitscube-2.10.0 → fitscube-2.12.0}/tests/conftest.py +0 -0
  32. {fitscube-2.10.0 → fitscube-2.12.0}/tests/data/cube.zip +0 -0
  33. {fitscube-2.10.0 → fitscube-2.12.0}/tests/data/images.zip +0 -0
  34. {fitscube-2.10.0 → fitscube-2.12.0}/tests/data/time_images.zip +0 -0
  35. {fitscube-2.10.0 → fitscube-2.12.0}/tests/data/timecube.zip +0 -0
  36. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_bb.py +0 -0
  37. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_combine.py +0 -0
  38. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_combine_layouts.py +0 -0
  39. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_extract.py +0 -0
  40. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_package.py +0 -0
  41. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_times.py +0 -0
  42. {fitscube-2.10.0 → fitscube-2.12.0}/tests/test_zero_beams.py +0 -0
@@ -29,7 +29,7 @@ jobs:
29
29
  with:
30
30
  fetch-depth: 0
31
31
 
32
- - uses: hynek/build-and-inspect-python-package@v3.0.1
32
+ - uses: hynek/build-and-inspect-python-package@v3.0.2
33
33
 
34
34
  publish:
35
35
  needs: [dist]
@@ -34,7 +34,7 @@ jobs:
34
34
  strategy:
35
35
  fail-fast: false
36
36
  matrix:
37
- python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
37
+ python-version: ["3.11", "3.12", "3.13", "3.14"]
38
38
  runs-on: [ubuntu-latest]
39
39
 
40
40
  steps:
@@ -55,6 +55,6 @@ jobs:
55
55
  --durations=20
56
56
 
57
57
  - name: Upload coverage report
58
- uses: codecov/codecov-action@v7.0.0
58
+ uses: codecov/codecov-action@v7.1.1
59
59
  with:
60
60
  token: ${{ secrets.CODECOV_TOKEN }}
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: fitscube
3
- Version: 2.10.0
3
+ Version: 2.12.0
4
4
  Summary: A package to produce produce FITS cubes.
5
5
  Project-URL: Homepage, https://github.com/AlecThomson/fitscube
6
6
  Project-URL: Bug Tracker, https://github.com/AlecThomson/fitscube/issues
@@ -37,15 +37,14 @@ Classifier: Operating System :: OS Independent
37
37
  Classifier: Programming Language :: Python
38
38
  Classifier: Programming Language :: Python :: 3
39
39
  Classifier: Programming Language :: Python :: 3 :: Only
40
- Classifier: Programming Language :: Python :: 3.10
41
40
  Classifier: Programming Language :: Python :: 3.11
42
41
  Classifier: Programming Language :: Python :: 3.12
43
42
  Classifier: Programming Language :: Python :: 3.13
44
43
  Classifier: Programming Language :: Python :: 3.14
45
44
  Classifier: Topic :: Scientific/Engineering
46
45
  Classifier: Typing :: Typed
47
- Requires-Python: >=3.10
48
- Requires-Dist: astropy>=6.0
46
+ Requires-Python: >=3.11
47
+ Requires-Dist: astropy>=7.2
49
48
  Requires-Dist: numpy>=2
50
49
  Requires-Dist: radio-beam
51
50
  Requires-Dist: tqdm
@@ -329,6 +328,19 @@ Quantise once. Every decompress and recompress with `tile-rice` adds a new error
329
328
  unless given `allow_requantize=True`. Smooth images such as weights do not shrink under
330
329
  RICE; use `tile-gzip` for them.
331
330
 
331
+ ## Blank channels
332
+
333
+ `--create-blanks` puts the inputs on one evenly spaced axis, with NaN planes
334
+ where there is no input. It picks the widest step that holds every input, so
335
+ images of uneven width, or images either side of a gap between bands (as in
336
+ RACS-low3, -mid and -high together), still make a single cube.
337
+
338
+ An input counts as on the grid if it is within ~1 Hz at 1 GHz of a plane (or
339
+ ~0.5 s for times). If any input is off by more than float rounding, `fitscube`
340
+ logs a warning, and the cube's `CSYERi` (the FITS systematic error of the axis)
341
+ records the largest offset. A grid of more than 1000 planes per input, or one
342
+ that cannot hold every input, is refused.
343
+
332
344
  ## Zero-sized beams
333
345
 
334
346
  wsclean writes `BMAJ = BMIN = 0` into an image when no PSF was fitted for that
@@ -256,6 +256,19 @@ Quantise once. Every decompress and recompress with `tile-rice` adds a new error
256
256
  unless given `allow_requantize=True`. Smooth images such as weights do not shrink under
257
257
  RICE; use `tile-gzip` for them.
258
258
 
259
+ ## Blank channels
260
+
261
+ `--create-blanks` puts the inputs on one evenly spaced axis, with NaN planes
262
+ where there is no input. It picks the widest step that holds every input, so
263
+ images of uneven width, or images either side of a gap between bands (as in
264
+ RACS-low3, -mid and -high together), still make a single cube.
265
+
266
+ An input counts as on the grid if it is within ~1 Hz at 1 GHz of a plane (or
267
+ ~0.5 s for times). If any input is off by more than float rounding, `fitscube`
268
+ logs a warning, and the cube's `CSYERi` (the FITS systematic error of the axis)
269
+ records the largest offset. A grid of more than 1000 planes per input, or one
270
+ that cannot hold every input, is refused.
271
+
259
272
  ## Zero-sized beams
260
273
 
261
274
  wsclean writes `BMAJ = BMIN = 0` into an image when no PSF was fitted for that
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '2.10.0'
22
- __version_tuple__ = version_tuple = (2, 10, 0)
21
+ __version__ = version = '2.12.0'
22
+ __version_tuple__ = version_tuple = (2, 12, 0)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -29,7 +29,7 @@ from astropy.io.fits.verify import VerifyWarning
29
29
  from astropy.table import Table
30
30
  from astropy.time import Time
31
31
  from astropy.wcs import WCS, WCSSUB_STOKES
32
- from numpy.typing import ArrayLike, NDArray
32
+ from numpy.typing import NDArray
33
33
  from radio_beam import Beam, Beams
34
34
  from radio_beam.beam import NoBeamException
35
35
  from tqdm.asyncio import tqdm
@@ -324,130 +324,120 @@ async def check_matching_scaling_coro(
324
324
  check_matching_scaling = sync_wrapper(check_matching_scaling_coro)
325
325
 
326
326
 
327
- # https://stackoverflow.com/a/66082278
328
- def np_arange_fix(start: float, stop: float, step: float) -> NDArray[np.float64]:
329
- n = (stop - start) / step + 1
330
- x = n - int(n)
331
- stop += step * max(0.1, x) if x < 0.5 else 0
332
- return np.arange(start, stop, step)
333
-
334
-
335
- def isin_close(
336
- element: NDArray[np.floating[Any]],
337
- test_element: ArrayLike,
338
- time_domain_mode: bool = False,
339
- ) -> NDArray[np.bool_]:
340
- """Check if element is in test_element, within a tolerance.
341
-
342
- Args:
343
- element (NDArray[np.floating[Any]]): Element to check
344
- test_element (ArrayLike): Element to check against
345
-
346
- Returns:
347
- NDArray[np.bool_]: Boolean array
348
- """
327
+ def match_tolerance(
328
+ values: NDArray[np.floating[Any]], time_domain_mode: bool = False
329
+ ) -> NDArray[np.floating[Any]]:
330
+ """How far a value may sit from a grid point and still be on it."""
349
331
  if time_domain_mode:
350
332
  # ~0.5 s for MJD seconds, so integrations of a second or more are told apart
351
- rtol = 1e-10
352
- atol = 1e-9
333
+ rtol, atol = 1e-10, 1e-9
353
334
  else:
354
335
  # ~1 Hz at 1 GHz. The np.isclose default of rtol=1e-5 (10 kHz) would
355
336
  # match several kHz-wide zoom channels and snap off-grid inputs.
356
- rtol = 1e-9
357
- atol = 1e-5
358
- return np.isclose(element[:, None], test_element, rtol=rtol, atol=atol).any(1)
337
+ rtol, atol = 1e-9, 1e-5
338
+ return atol + rtol * np.abs(values)
339
+
340
+
341
+ class GridFit(NamedTuple):
342
+ """A regular grid through a set of sorted values."""
359
343
 
344
+ start: float
345
+ """Value of the first grid point"""
346
+ step: float
347
+ """Grid step"""
348
+ index: NDArray[np.int64]
349
+ """Grid point of each value"""
350
+ residuals: NDArray[np.float64]
351
+ """Distance of each value from its grid point"""
360
352
 
361
- def grid_step(diffs: ArrayLike) -> float:
362
- """Recover the fundamental channel step from the observed diffs.
363
353
 
364
- On a regular grid with missing channels every diff is an integer multiple
365
- of the channel width, so the gcd of the diffs recovers that width even when
366
- no two surviving channels are adjacent -- the case where ``min(diffs)``
367
- overestimates the step (e.g. present channels 0, 3, 5 give ``min == 2`` but
368
- the true step is 1). Falls back to ``min(diffs)`` if the gcd is numerically
369
- unstable or does not divide every diff, so the common case is unchanged (any
370
- surviving adjacent pair makes the gcd equal the minimum diff exactly). Also
371
- falls back when the gcd is implausibly small: diffs of 1e6 and 1e6 + 2 have a
372
- float gcd of 2.
354
+ def fit_grid(
355
+ specs: NDArray[np.floating[Any]], time_domain_mode: bool = False
356
+ ) -> GridFit | None:
357
+ """Find the widest regular grid that holds every sorted value.
358
+
359
+ The step divides the smallest gap, so steps of that gap over 1, 2, 3...
360
+ are tried in turn. Each is fitted to all the values, and the first that
361
+ puts every value on a grid point within the match tolerance wins.
373
362
 
374
363
  Args:
375
- diffs (ArrayLike): Differences between consecutive spec values.
364
+ specs (NDArray[np.floating[Any]]): Sorted, distinct values
365
+ time_domain_mode (bool, optional): Use the tolerance for times. Defaults to False.
376
366
 
377
367
  Returns:
378
- float: Estimated regular grid step.
368
+ GridFit | None: The grid, or None if none of at most 1000 points per value fits
379
369
  """
380
- diffs = np.abs(np.asarray(diffs, dtype=np.longdouble))
381
- min_diff = np.min(diffs)
382
- tol = np.max(diffs) * 1e-6
383
-
384
- g = diffs[0]
385
- for d in diffs[1:]:
386
- a, b = (g, d) if g >= d else (d, g)
387
- # Tolerant Euclid: stop once the remainder is within the noise floor.
388
- while b > tol:
389
- a, b = b, a % b
390
- g = a
370
+ offsets = specs - specs[0]
371
+ span = offsets[-1]
372
+ min_gap = np.min(np.diff(specs))
373
+ tol = match_tolerance(specs, time_domain_mode)
374
+ max_channels = 1000 * len(specs)
375
+
376
+ divisor = 1
377
+ while span * divisor / min_gap < max_channels:
378
+ index = np.round(offsets * divisor / min_gap)
379
+ step, start = np.polyfit(index, offsets, 1)
380
+ residuals = np.abs(offsets - start - index * step)
381
+ if np.all(residuals <= tol):
382
+ return GridFit(
383
+ start=float(specs[0] + start),
384
+ step=float(step),
385
+ index=index.astype(np.int64),
386
+ residuals=residuals,
387
+ )
388
+ divisor += 1
391
389
 
392
- divides_all = np.all(np.abs(np.round(diffs / g) - diffs / g) <= 1e-3)
393
- # A gcd far below every diff is float noise, not a real channel width.
394
- if g <= tol or not divides_all or g * 16 < min_diff:
395
- return float(min_diff)
396
- return float(g)
390
+ return None
397
391
 
398
392
 
399
393
  def even_spacing(specs: u.Quantity, time_domain_mode: bool = False) -> SpequencyInfo:
400
394
  """Make the frequencies or times evenly spaced.
401
395
 
402
- A regular grid cannot always hold every input - an irregular spacing whose
403
- step cannot be recovered leaves inputs sitting between grid points, and equal
404
- values collapse onto one. Those inputs would be dropped from the cube without
405
- trace, so this raises rather than writing a cube that is missing data.
396
+ A regular grid cannot always hold every input - an irregular spacing leaves
397
+ inputs sitting between grid points, and equal values collapse onto one.
398
+ Those inputs would be dropped from the cube without trace, so this raises
399
+ rather than writing a cube that is missing data. Inputs that sit on the grid
400
+ only within the match tolerance are kept, with a warning.
406
401
 
407
402
  Args:
408
403
  specs (u.Quantity): Original frequencies/times
409
404
 
410
405
  Raises:
411
- IrregularSpacingException: If a grid through the inputs would drop any of them, or is implausibly large
406
+ IrregularSpacingException: If no grid of at most 1000 points per input holds every input
412
407
 
413
408
  Returns:
414
409
  SpequencyInfo: specs, missing_chan_idx
415
410
  """
416
- # The grid is built from the first and last value, so an unsorted input would
417
- # otherwise give an empty (or reversed) grid. Callers sort later, not here.
418
- specs_arr = np.sort(specs.value.astype(np.longdouble))
411
+ specs_arr = np.sort(specs.value.astype(np.float64))
419
412
  if len(specs_arr) < 2:
420
413
  return SpequencyInfo(
421
414
  specs_arr * specs.unit, np.zeros(len(specs_arr)).astype(bool)
422
415
  )
423
416
 
424
- diffs = np.diff(specs_arr)
425
- step = grid_step(diffs)
426
- span = specs_arr[-1] - specs_arr[0]
427
- max_channels = 1000 * len(specs_arr)
428
- if span / step > max_channels:
417
+ spequencies = "times" if time_domain_mode else "frequencies"
418
+ grid = fit_grid(specs_arr, time_domain_mode) if np.all(np.diff(specs_arr)) else None
419
+ if grid is None:
429
420
  msg = (
430
- f"A regular grid of step {step} {specs.unit} over {len(specs_arr)} values "
431
- f"spanning {span} {specs.unit} would need {span / step + 1:.0f} channels. "
432
- "Combine without blank channels."
421
+ f"No regular grid of at most {1000 * len(specs_arr)} points holds all "
422
+ f"{len(specs_arr)} {spequencies}, so blank channels would drop inputs. "
423
+ "Combine without blank channels, or regrid the inputs onto a common step."
433
424
  )
434
425
  raise IrregularSpacingException(msg)
435
- # Create a new array with the fundamental grid step
436
- new_specs = np_arange_fix(specs_arr[0], specs_arr[-1], step)
437
- missing_chan_idx = np.logical_not(
438
- isin_close(new_specs, specs_arr, time_domain_mode)
439
- )
440
426
 
441
- populated = int(np.logical_not(missing_chan_idx).sum())
442
- if populated != len(specs_arr):
443
- spequencies = "times" if time_domain_mode else "frequencies"
427
+ worst = int(np.argmax(grid.residuals))
428
+ if grid.residuals[worst] > 1e-12 * np.max(np.abs(specs_arr)):
444
429
  msg = (
445
- f"Cannot place all {len(specs_arr)} {spequencies} on a regular grid of "
446
- f"{step} {specs.unit} - only {populated} of {len(new_specs)} grid points "
447
- "are matched, so blank channels would drop inputs. Combine without "
448
- "blank channels, or regrid the inputs onto a common step."
430
+ f"The {spequencies} are not exactly on a regular grid of {grid.step} "
431
+ f"{specs.unit}: {specs_arr[worst]} {specs.unit} is "
432
+ f"{grid.residuals[worst]} {specs.unit} from "
433
+ "its plane. The cube labels each plane with its grid value, not the "
434
+ "input's own."
449
435
  )
450
- raise IrregularSpacingException(msg)
436
+ logger.warning(msg)
437
+
438
+ new_specs = grid.start + np.arange(grid.index[-1] + 1) * grid.step
439
+ missing_chan_idx = np.ones(len(new_specs), dtype=bool)
440
+ missing_chan_idx[grid.index] = False
451
441
 
452
442
  return SpequencyInfo(new_specs * specs.unit, missing_chan_idx)
453
443
 
@@ -533,6 +523,7 @@ async def create_output_cube_coro(
533
523
  time_domain_mode: bool = False,
534
524
  bounding_box: BoundingBox | None = None,
535
525
  float_length: FLOAT_LENGTH | None = None,
526
+ spec_error: float | None = None,
536
527
  ) -> InitResult:
537
528
  """Generate the output header and write a dummy cube to disk based on properties of the
538
529
  inpute data. The output cube written here has a correctly formed header and a pre-zerod
@@ -549,6 +540,7 @@ async def create_output_cube_coro(
549
540
  time_domain_mode (bool, optional): Whether to join images via the DATE-OBS (e.g. time) axis. If False cube joined along frequency axis. Defaults to False.
550
541
  bounding_box (BoundingBox | None, optional): Whether a trimming operation should be applied to input images. If a BoundingBox is supplied this will be used to update the reference pixel position and data shape indicators. Defaults to None.
551
542
  float_length (Literal[32, 64] | None, optional): The precision of the output data. If None drawn from input data. Otherwise values accepted are 32 and 64. Defaults to None.
543
+ spec_error (float | None, optional): Largest distance of an input from its plane on a gridded axis, in Hz or s. Written as CSYERi if given. Defaults to None.
552
544
 
553
545
  Returns:
554
546
  InitResult: Details of the output cube, including the output header
@@ -651,6 +643,12 @@ async def create_output_cube_coro(
651
643
  del new_header[f"CUNIT{fits_idx}"]
652
644
  new_header[f"CTYPE{fits_idx}"] = "CHAN"
653
645
  new_header[f"CRVAL{fits_idx}"] = 1
646
+ elif spec_error is not None:
647
+ new_header[f"CSYER{fits_idx}"] = spec_error
648
+ new_header["HISTORY"] = (
649
+ f"fitscube placed the inputs on a regular {ctype} grid. Each is within "
650
+ f"CSYER{fits_idx} of its plane."
651
+ )
654
652
 
655
653
  # Add extra transform fields for consistency. A CD matrix replaces CDELT,
656
654
  # so it must carry the step itself.
@@ -1278,6 +1276,10 @@ async def combine_fits_coro(
1278
1276
  new_sort_idx = np.argsort(specs)
1279
1277
  specs = specs[new_sort_idx]
1280
1278
  missing_chan_idx = missing_chan_idx[new_sort_idx]
1279
+ spec_error = None
1280
+ if create_blanks:
1281
+ populated = specs[~missing_chan_idx]
1282
+ spec_error = float(np.max(np.abs(np.sort(file_specs) - populated)).value)
1281
1283
 
1282
1284
  has_beams = check_for_any_beam(file_list=file_list)
1283
1285
  zero_beam_idx = np.zeros(len(specs), dtype=bool)
@@ -1350,6 +1352,7 @@ async def combine_fits_coro(
1350
1352
  time_domain_mode=time_domain_mode,
1351
1353
  bounding_box=final_bounding_box,
1352
1354
  float_length=float_length,
1355
+ spec_error=spec_error,
1353
1356
  )
1354
1357
 
1355
1358
  if (missing_chan_idx | zero_beam_idx).any() and new_header["BITPIX"] > 0:
@@ -41,8 +41,6 @@ class CompressedPlane(NamedTuple):
41
41
  """ZSCALE of each tile, empty unless quantized"""
42
42
  zeros: list[float]
43
43
  """ZZERO of each tile, empty unless quantized"""
44
- has_blanks: bool
45
- """Whether quantized tiles marked blank (NaN) pixels with ZBLANK"""
46
44
 
47
45
 
48
46
  def compress_cube(
@@ -128,6 +126,7 @@ def gzip2_tiles(
128
126
  GZIP_2 gzips the big-endian bytes of the tile, shuffled so that all the first
129
127
  bytes of each value come first, then all the second bytes, and so on.
130
128
  """
129
+ # A view, not a copy, for a plane read straight from a FITS file
131
130
  big_endian = plane.astype(plane.dtype.newbyteorder(">"), copy=False)
132
131
  itemsize = big_endian.dtype.itemsize
133
132
  tiles = []
@@ -176,7 +175,6 @@ def astropy_compress_plane(
176
175
  },
177
176
  scales=list(table.data["ZSCALE"]) if "ZSCALE" in names else [],
178
177
  zeros=list(table.data["ZZERO"]) if "ZZERO" in names else [],
179
- has_blanks="ZBLANK" in table.header,
180
178
  )
181
179
  return compressed, table.header.copy()
182
180
 
@@ -203,10 +201,15 @@ def compress_plane(
203
201
  heap={"COMPRESSED_DATA": gzip2_tiles(plane, tile_shape, gzip_level)},
204
202
  scales=[],
205
203
  zeros=[],
206
- has_blanks=False,
207
204
  )
205
+ # astropy wants native byte order; only this path pays for the copy
208
206
  compressed, _ = astropy_compress_plane(
209
- plane, fits.Header(), compression_type, quantize_level, tile_shape, dither_seed
207
+ plane.astype(plane.dtype.newbyteorder("="), copy=False),
208
+ fits.Header(),
209
+ compression_type,
210
+ quantize_level,
211
+ tile_shape,
212
+ dither_seed,
210
213
  )
211
214
  return compressed
212
215
 
@@ -218,7 +221,6 @@ def table_header(
218
221
  heap_lengths: dict[str, int],
219
222
  heap_size: int,
220
223
  dither_seed: int,
221
- has_blanks: bool,
222
224
  ) -> fits.Header:
223
225
  """Header of the whole cube's compressed table, built from the first plane's."""
224
226
  header = template.copy()
@@ -238,7 +240,9 @@ def table_header(
238
240
  header["NAXIS1"] = row_width
239
241
  if "ZDITHER0" in header:
240
242
  header["ZDITHER0"] = dither_seed
241
- if has_blanks:
243
+ if "ZQUANTIZ" in header:
244
+ # Always set, so the header never grows past the placeholder written before
245
+ # the tiles that turn out to hold blanks
242
246
  header["ZBLANK"] = np.iinfo(np.int32).min
243
247
  return header
244
248
 
@@ -263,7 +267,11 @@ def compressed_planes(
263
267
  gzip_level: int,
264
268
  max_workers: int,
265
269
  ) -> Iterator[CompressedPlane]:
266
- """Compress every plane of `image` across threads, yielding them in order."""
270
+ """Compress every plane of `image` across threads, yielding them in order.
271
+
272
+ Planes are handed over as read, big-endian, which is the order GZIP_2 shuffles:
273
+ the lossless path never copies a plane.
274
+ """
267
275
  with ThreadPoolExecutor(max_workers) as pool:
268
276
  pending: deque[Future[CompressedPlane]] = deque()
269
277
  for plane_index, index in enumerate(np.ndindex(image.shape[:-2])):
@@ -271,7 +279,7 @@ def compressed_planes(
271
279
  pending.append(
272
280
  pool.submit(
273
281
  compress_plane,
274
- plane.astype(plane.dtype.newbyteorder("="), copy=False),
282
+ plane,
275
283
  compression_type,
276
284
  quantize_level,
277
285
  tile_shape,
@@ -389,19 +397,16 @@ def tile_compress_cube(
389
397
  # Placeholders: the header and table are rewritten once the heap is known,
390
398
  # and stay the same size because only fixed-width card values change
391
399
  handle.write(
392
- table_header(
393
- template, cube_shape, nrows, heap_lengths, 0, dither_seed, False
394
- )
400
+ table_header(template, cube_shape, nrows, heap_lengths, 0, dither_seed)
395
401
  .tostring()
396
402
  .encode()
397
403
  )
398
404
  handle.write(table.tobytes())
405
+ table_end = handle.tell()
399
406
 
400
407
  heap_size = 0
401
- has_blanks = False
402
408
  row = 0
403
409
  for plane in itertools.chain([first], planes):
404
- has_blanks |= plane.has_blanks
405
410
  for tile_index in range(tiles_per_plane):
406
411
  for name in heap_lengths:
407
412
  data = plane.heap[name][tile_index]
@@ -424,12 +429,14 @@ def tile_compress_cube(
424
429
  heap_lengths,
425
430
  heap_size,
426
431
  dither_seed,
427
- has_blanks,
428
432
  )
429
433
  .tostring()
430
434
  .encode()
431
435
  )
432
436
  handle.write(table.tobytes())
437
+ if handle.tell() != table_end:
438
+ msg = "The final table header differs in size from its placeholder"
439
+ raise RuntimeError(msg)
433
440
 
434
441
  for hdu in hdul[1:]:
435
442
  fits.append(partial_path, hdu.data, hdu.header)
@@ -11,7 +11,7 @@ authors = [
11
11
  description = "A package to produce produce FITS cubes."
12
12
  readme = "README.md"
13
13
  license.file = "LICENSE"
14
- requires-python = ">=3.10"
14
+ requires-python = ">=3.11"
15
15
  classifiers = [
16
16
  "Development Status :: 1 - Planning",
17
17
  "Intended Audience :: Science/Research",
@@ -21,7 +21,6 @@ classifiers = [
21
21
  "Programming Language :: Python",
22
22
  "Programming Language :: Python :: 3",
23
23
  "Programming Language :: Python :: 3 :: Only",
24
- "Programming Language :: Python :: 3.10",
25
24
  "Programming Language :: Python :: 3.11",
26
25
  "Programming Language :: Python :: 3.12",
27
26
  "Programming Language :: Python :: 3.13",
@@ -31,7 +30,7 @@ classifiers = [
31
30
  ]
32
31
  dynamic = ["version"]
33
32
  dependencies = [
34
- "astropy>=6.0", # StokesCoord
33
+ "astropy>=7.2", # thread-safe disable_image_compression, numpy >= 2.4
35
34
  "numpy>=2",
36
35
  "tqdm",
37
36
  "radio_beam",
@@ -107,7 +106,7 @@ report.exclude_also = [
107
106
 
108
107
  [tool.mypy]
109
108
  files = ["fitscube", "tests"]
110
- python_version = "3.10"
109
+ python_version = "3.11"
111
110
  warn_unused_configs = true
112
111
  strict = true
113
112
  enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
@@ -175,7 +174,7 @@ isort.required-imports = ["from __future__ import annotations"]
175
174
 
176
175
 
177
176
  [tool.pylint]
178
- py-version = "3.10"
177
+ py-version = "3.11"
179
178
  ignore-paths = [".*/_version.py"]
180
179
  reports.output-format = "colorized"
181
180
  similarities.ignore-imports = "yes"
@@ -579,3 +579,33 @@ def test_float_length_must_be_fits_float(
579
579
  get_parser().parse_args(
580
580
  [*map(str, file_list), str(out_cube), "--floating", str(float_length)]
581
581
  )
582
+
583
+
584
+ @pytest.mark.parametrize("jitter", [0.0, 0.3])
585
+ def test_grid_offset_is_recorded(tmp_path: Path, jitter: float) -> None:
586
+ """CSYERi holds how far an input sits from the plane it was written to."""
587
+ freqs = (1e9 + np.array([0.0, 1.0, 3.0]) * 1e6 + [0.0, jitter, 0.0]) * u.Hz
588
+ file_list = [
589
+ make_plane(tmp_path / f"offset_{i}.fits", freq, value=float(i))
590
+ for i, freq in enumerate(freqs)
591
+ ]
592
+ out_cube = tmp_path / "cube.fits"
593
+ combine_fits(
594
+ file_list=file_list, out_cube=out_cube, create_blanks=True, overwrite=True
595
+ )
596
+
597
+ header = fits.getheader(out_cube)
598
+ assert header["CTYPE4"] == "FREQ"
599
+ # The grid is fitted to all inputs, so the offset is shared out among them
600
+ assert header["CSYER4"] <= jitter
601
+ assert (header["CSYER4"] > 0.1) == (jitter > 0)
602
+ assert "CSYER4" in str(header["HISTORY"])
603
+
604
+
605
+ def test_no_grid_offset_without_blanks(tmp_path: Path, specs: u.Quantity) -> None:
606
+ file_list = [
607
+ make_plane(tmp_path / f"plain_{i}.fits", spec) for i, spec in enumerate(specs)
608
+ ]
609
+ out_cube = tmp_path / "cube.fits"
610
+ combine_fits(file_list=file_list, out_cube=out_cube, overwrite=True)
611
+ assert "CSYER4" not in fits.getheader(out_cube)
@@ -2,16 +2,22 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ import tracemalloc
6
+ from collections.abc import Callable, Iterator
5
7
  from pathlib import Path
8
+ from typing import Any
6
9
 
7
10
  import numpy as np
8
11
  import pytest
9
12
  from astropy.io import fits
13
+ from fitscube import compress as compress_module
10
14
  from fitscube.cli import cli
11
15
  from fitscube.combine_fits import combine_fits
12
16
  from fitscube.compress import (
17
+ CompressedPlane,
13
18
  CompressMethod,
14
19
  compress_cube,
20
+ compress_plane,
15
21
  decompress_cube,
16
22
  tile_compress_cube,
17
23
  )
@@ -134,6 +140,29 @@ def test_tile_compress_is_the_same_for_any_worker_count(tmp_path: Path) -> None:
134
140
  assert outputs[0] == outputs[1]
135
141
 
136
142
 
143
+ def test_gzip2_shuffles_a_big_endian_plane_without_copying_it() -> None:
144
+ """Planes come off the file big-endian, the order GZIP_2 shuffles, so no copy is needed"""
145
+ plane = np.full((256, 1024), 1.5, dtype=">f4")
146
+
147
+ def compress(plane: np.ndarray) -> CompressedPlane:
148
+ return compress_plane(plane, "GZIP_2", 0.0, (16, 1024), 1, 6)
149
+
150
+ tracemalloc.start()
151
+ compressed = compress(plane)
152
+ _, peak = tracemalloc.get_traced_memory()
153
+ tracemalloc.stop()
154
+ assert peak < plane.nbytes / 2
155
+
156
+ # The control: a native-order plane has to be copied back to big-endian
157
+ native = plane.astype("<f4")
158
+ tracemalloc.start()
159
+ from_native = compress(native)
160
+ _, native_peak = tracemalloc.get_traced_memory()
161
+ tracemalloc.stop()
162
+ assert native_peak >= plane.nbytes
163
+ assert from_native == compressed
164
+
165
+
137
166
  def test_tile_compress_refuses_compressed_input(tmp_path: Path) -> None:
138
167
  write_noise_cube(tmp_path / "cube.fits")
139
168
  compressed = tile_compress_cube(tmp_path / "cube.fits")
@@ -232,3 +261,44 @@ def test_requantizing_dithers_afresh(tmp_path: Path) -> None:
232
261
  )
233
262
 
234
263
  assert fits.getheader(second, 1, disable_image_compression=True)["ZDITHER0"] != seed
264
+
265
+
266
+ @pytest.mark.parametrize("extra_cards", range(36))
267
+ def test_tile_rice_header_never_grows(tmp_path: Path, extra_cards: int) -> None:
268
+ """Blanks only past the first tile must not grow the header over its placeholder"""
269
+ rng = np.random.default_rng(1)
270
+ data = rng.normal(0, 1e-3, (3, 40, 48)).astype(np.float32)
271
+ data[-1, -8:] = np.nan
272
+ header = fits.Header()
273
+ for i in range(extra_cards):
274
+ header.add_comment(f"padding {i}")
275
+ fits.PrimaryHDU(data, header).writeto(tmp_path / "cube.fits")
276
+
277
+ compressed = compress_cube(tmp_path / "cube.fits", method="tile-rice")
278
+
279
+ got = fits.getdata(compressed, 1)
280
+ np.testing.assert_array_equal(np.isnan(got), np.isnan(data))
281
+ finite = np.isfinite(data)
282
+ assert np.std(got[finite] - data[finite]) < 1e-3 / 4 / np.sqrt(12) * 1.2
283
+
284
+
285
+ def test_tile_compress_refuses_a_grown_header(
286
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
287
+ ) -> None:
288
+ """A final header longer than its placeholder errors rather than corrupting the file"""
289
+ write_noise_cube(tmp_path / "cube.fits")
290
+ original = compress_module.table_header
291
+
292
+ def grown(*args: Any, **kwargs: Any) -> fits.Header:
293
+ header = original(*args, **kwargs)
294
+ for i in range(36):
295
+ header.add_comment(f"padding {i}")
296
+ return header
297
+
298
+ calls: Iterator[Callable[..., fits.Header]] = iter([original, grown])
299
+ monkeypatch.setattr(
300
+ compress_module, "table_header", lambda *a, **k: next(calls)(*a, **k)
301
+ )
302
+ with pytest.raises(RuntimeError, match="placeholder"):
303
+ tile_compress_cube(tmp_path / "cube.fits", remove=False)
304
+ assert not (tmp_path / "cube.fits.fz").exists()