xradio 0.0.56__py3-none-any.whl → 0.0.58__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. xradio/__init__.py +2 -2
  2. xradio/_utils/_casacore/casacore_from_casatools.py +12 -2
  3. xradio/_utils/_casacore/tables.py +1 -0
  4. xradio/_utils/coord_math.py +22 -23
  5. xradio/_utils/dict_helpers.py +76 -11
  6. xradio/_utils/schema.py +5 -2
  7. xradio/_utils/zarr/common.py +1 -73
  8. xradio/image/_util/_casacore/xds_from_casacore.py +49 -33
  9. xradio/image/_util/_casacore/xds_to_casacore.py +41 -14
  10. xradio/image/_util/_fits/xds_from_fits.py +146 -35
  11. xradio/image/_util/casacore.py +4 -3
  12. xradio/image/_util/common.py +4 -4
  13. xradio/image/_util/image_factory.py +8 -8
  14. xradio/image/image.py +45 -5
  15. xradio/measurement_set/__init__.py +19 -9
  16. xradio/measurement_set/_utils/__init__.py +1 -3
  17. xradio/measurement_set/_utils/_msv2/__init__.py +0 -0
  18. xradio/measurement_set/_utils/_msv2/_tables/read.py +17 -76
  19. xradio/measurement_set/_utils/_msv2/_tables/read_main_table.py +2 -685
  20. xradio/measurement_set/_utils/_msv2/conversion.py +123 -145
  21. xradio/measurement_set/_utils/_msv2/create_antenna_xds.py +9 -16
  22. xradio/measurement_set/_utils/_msv2/create_field_and_source_xds.py +125 -221
  23. xradio/measurement_set/_utils/_msv2/msv2_to_msv4_meta.py +1 -2
  24. xradio/measurement_set/_utils/_msv2/msv4_info_dicts.py +8 -7
  25. xradio/measurement_set/_utils/_msv2/msv4_sub_xdss.py +27 -72
  26. xradio/measurement_set/_utils/_msv2/partition_queries.py +1 -261
  27. xradio/measurement_set/_utils/_msv2/subtables.py +0 -107
  28. xradio/measurement_set/_utils/_utils/interpolate.py +60 -0
  29. xradio/measurement_set/_utils/_zarr/encoding.py +2 -7
  30. xradio/measurement_set/convert_msv2_to_processing_set.py +0 -2
  31. xradio/measurement_set/load_processing_set.py +2 -2
  32. xradio/measurement_set/measurement_set_xdt.py +14 -14
  33. xradio/measurement_set/open_processing_set.py +1 -3
  34. xradio/measurement_set/processing_set_xdt.py +41 -835
  35. xradio/measurement_set/schema.py +95 -122
  36. xradio/schema/check.py +91 -97
  37. xradio/schema/dataclass.py +159 -22
  38. xradio/schema/export.py +99 -0
  39. xradio/schema/metamodel.py +51 -16
  40. xradio/schema/typing.py +5 -5
  41. {xradio-0.0.56.dist-info → xradio-0.0.58.dist-info}/METADATA +2 -1
  42. xradio-0.0.58.dist-info/RECORD +65 -0
  43. {xradio-0.0.56.dist-info → xradio-0.0.58.dist-info}/WHEEL +1 -1
  44. xradio/image/_util/fits.py +0 -13
  45. xradio/measurement_set/_utils/_msv2/_tables/load.py +0 -66
  46. xradio/measurement_set/_utils/_msv2/_tables/load_main_table.py +0 -490
  47. xradio/measurement_set/_utils/_msv2/_tables/read_subtables.py +0 -398
  48. xradio/measurement_set/_utils/_msv2/_tables/write.py +0 -323
  49. xradio/measurement_set/_utils/_msv2/_tables/write_exp_api.py +0 -388
  50. xradio/measurement_set/_utils/_msv2/chunks.py +0 -115
  51. xradio/measurement_set/_utils/_msv2/descr.py +0 -165
  52. xradio/measurement_set/_utils/_msv2/msv2_msv3.py +0 -7
  53. xradio/measurement_set/_utils/_msv2/partitions.py +0 -392
  54. xradio/measurement_set/_utils/_utils/cds.py +0 -40
  55. xradio/measurement_set/_utils/_utils/xds_helper.py +0 -404
  56. xradio/measurement_set/_utils/_zarr/read.py +0 -263
  57. xradio/measurement_set/_utils/_zarr/write.py +0 -329
  58. xradio/measurement_set/_utils/msv2.py +0 -106
  59. xradio/measurement_set/_utils/zarr.py +0 -133
  60. xradio-0.0.56.dist-info/RECORD +0 -78
  61. {xradio-0.0.56.dist-info → xradio-0.0.58.dist-info}/licenses/LICENSE.txt +0 -0
  62. {xradio-0.0.56.dist-info → xradio-0.0.58.dist-info}/top_level.txt +0 -0
@@ -5,6 +5,7 @@ from typing import Union
5
5
  import dask
6
6
  import dask.array as da
7
7
  import numpy as np
8
+ import psutil
8
9
  import xarray as xr
9
10
  from astropy import units as u
10
11
  from astropy.io import fits
@@ -13,7 +14,7 @@ from astropy.time import Time
13
14
  from xradio._utils.coord_math import _deg_to_rad
14
15
  from xradio._utils.dict_helpers import (
15
16
  make_quantity,
16
- make_frequency_reference_dict,
17
+ make_spectral_coord_reference_dict,
17
18
  make_skycoord_dict,
18
19
  make_time_measure_dict,
19
20
  )
@@ -35,20 +36,31 @@ from xradio.image._util.common import (
35
36
 
36
37
 
37
38
  def _fits_image_to_xds(
38
- img_full_path: str, chunks: dict, verbose: bool, do_sky_coords: bool
39
+ img_full_path: str,
40
+ chunks: dict,
41
+ verbose: bool,
42
+ do_sky_coords: bool,
43
+ compute_mask: bool,
39
44
  ) -> dict:
40
45
  """
46
+ compute_mask : bool, optional
47
+ If True (default), compute and attach valid data masks to the xds.
48
+ If False, skip mask generation for performance. It is solely the responsibility
49
+ of the user to ensure downstream apps can handle NaN values; do not
50
+ ask package developers to add this non-standard behavior.
51
+
41
52
  TODO: complete documentation
42
53
  Create an xds without any pixel data from metadata from the specified FITS image
43
54
  """
44
- # memmap = True allows only part of data to be loaded into memory
45
55
  # may also need to pass mode='denywrite'
46
56
  # https://stackoverflow.com/questions/35759713/astropy-io-fits-read-row-from-large-fits-file-with-mutliple-hdus
47
- hdulist = fits.open(img_full_path, memmap=True)
48
- attrs, helpers, header = _fits_header_to_xds_attrs(hdulist)
49
- hdulist.close()
50
- # avoid keeping reference to mem-mapped fits file
51
- del hdulist
57
+ try:
58
+ hdulist = fits.open(img_full_path, memmap=True)
59
+ attrs, helpers, header = _fits_header_to_xds_attrs(hdulist, compute_mask)
60
+ finally:
61
+ hdulist.close()
62
+ # avoid keeping reference to mem-mapped fits file
63
+ del hdulist
52
64
  xds = _create_coords(helpers, header, do_sky_coords)
53
65
  sphr_dims = helpers["sphr_dims"]
54
66
  ary = _read_image_array(img_full_path, chunks, helpers, verbose)
@@ -86,10 +98,10 @@ def _add_freq_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
86
98
  meta["rest_frequency"] = make_quantity(helpers["restfreq"], "Hz")
87
99
  meta["rest_frequencies"] = [meta["rest_frequency"]]
88
100
  meta["type"] = "frequency"
89
- meta["wave_unit"] = ["mm"]
101
+ meta["wave_units"] = "mm"
90
102
  freq_axis = helpers["freq_axis"]
91
- meta["reference_value"] = make_frequency_reference_dict(
92
- helpers["crval"][freq_axis], ["Hz"], helpers["specsys"]
103
+ meta["reference_frequency"] = make_spectral_coord_reference_dict(
104
+ helpers["crval"][freq_axis], "Hz", helpers["specsys"]
93
105
  )
94
106
  # meta["cdelt"] = helpers["cdelt"][freq_axis]
95
107
  if not meta:
@@ -102,7 +114,7 @@ def _add_freq_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
102
114
 
103
115
  def _add_vel_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
104
116
  vel_coord = xds.coords["velocity"]
105
- meta = {"units": ["m/s"]}
117
+ meta = {"units": "m/s"}
106
118
  if helpers["has_freq"]:
107
119
  meta["doppler_type"] = helpers.get("doppler", "RADIO")
108
120
  else:
@@ -159,9 +171,7 @@ def _xds_direction_attrs_from_header(helpers: dict, header) -> dict:
159
171
  helpers["ref_sys"] = ref_sys
160
172
  helpers["ref_eqx"] = ref_eqx
161
173
  # fits does not support conversion frames
162
- direction["reference"] = make_skycoord_dict(
163
- [0.0, 0.0], units=["rad", "rad"], frame=ref_sys
164
- )
174
+ direction["reference"] = make_skycoord_dict([0.0, 0.0], units="rad", frame=ref_sys)
165
175
  dir_axes = helpers["dir_axes"]
166
176
  ddata = []
167
177
  dunits = []
@@ -239,16 +249,43 @@ def _get_telescope_metadata(helpers: dict, header) -> dict:
239
249
  r = np.sqrt(np.sum(xyz * xyz))
240
250
  lat = np.arcsin(z / r)
241
251
  long = np.arctan2(y, x)
242
- tel["location"] = {
252
+ tel["direction"] = {
253
+ "attrs": {
254
+ "coordinate_system": "geocentric",
255
+ # I haven't seen a FITS keyword for reference frame of telescope posiiton
256
+ "frame": "ITRF",
257
+ "origin_object_name": "earth",
258
+ "type": "location",
259
+ "units": "rad",
260
+ },
261
+ "data": np.array([long, lat]),
262
+ "dims": ["ellipsoid_dir_label"],
263
+ "coords": {
264
+ "ellipsoid_dir_label": {
265
+ "dims": ["ellipsoid_dir_label"],
266
+ "data": ["lon", "lat"],
267
+ }
268
+ },
269
+ }
270
+ tel["distance"] = {
243
271
  "attrs": {
244
272
  "coordinate_system": "geocentric",
245
273
  # I haven't seen a FITS keyword for reference frame of telescope posiiton
246
274
  "frame": "ITRF",
247
275
  "origin_object_name": "earth",
248
276
  "type": "location",
249
- "units": ["rad", "rad", "m"],
277
+ "units": "m",
278
+ },
279
+ "data": np.array([r]),
280
+ "dims": ["ellipsoid_dis_label"],
281
+ "coords": {
282
+ "ellipsoid_dis_label": {
283
+ "dims": ["ellipsoid_dis_label"],
284
+ "data": [
285
+ "dist",
286
+ ],
287
+ }
250
288
  },
251
- "data": np.array([long, lat, r]),
252
289
  }
253
290
  return tel
254
291
 
@@ -266,9 +303,7 @@ def _compute_pointing_center(helpers: dict, header) -> dict:
266
303
  pc_lat = float(header[f"CRVAL{t_axes[1]}"]) * unit[1]
267
304
  pc_long = pc_long.to(u.rad).value
268
305
  pc_lat = pc_lat.to(u.rad).value
269
- return make_skycoord_dict(
270
- [pc_long, pc_lat], units=["rad", "rad"], frame=helpers["ref_sys"]
271
- )
306
+ return make_skycoord_dict([pc_long, pc_lat], units="rad", frame=helpers["ref_sys"])
272
307
 
273
308
 
274
309
  def _user_attrs_from_header(header) -> dict:
@@ -367,12 +402,41 @@ def _create_dim_map(helpers: dict, header) -> dict:
367
402
  return dim_map
368
403
 
369
404
 
370
- def _fits_header_to_xds_attrs(hdulist: fits.hdu.hdulist.HDUList) -> tuple:
405
+ def _fits_header_to_xds_attrs(
406
+ hdulist: fits.hdu.hdulist.HDUList, compute_mask: bool
407
+ ) -> tuple:
408
+ # First: Guard for unsupported compressed images
409
+ for i, hdu in enumerate(hdulist):
410
+ if isinstance(hdu, fits.CompImageHDU):
411
+ raise RuntimeError(
412
+ f"HDU {i}, name={hdu.name} is a CompImageHDU, which is not supported "
413
+ "for memory-mapping. "
414
+ "Cannot memory-map compressed FITS image (CompImageHDU). "
415
+ "Workaround: decompress the FITS using tools like `funpack`, `cfitsio`, "
416
+ "or Astropy's `.scale()`/`.copy()` workflows"
417
+ )
371
418
  primary = None
419
+ # FIXME beams is set but never actually used in this function. What's up with that?
372
420
  beams = None
373
421
  for hdu in hdulist:
374
422
  if hdu.name == "PRIMARY":
375
423
  primary = hdu
424
+ # Memory map support check
425
+ # avoid possibly non-existent hdu.scale_type attribute check and check header instead
426
+ header = hdu.header
427
+ scale = hdu.header.get("BSCALE", 1.0)
428
+ zero = hdu.header.get("BZERO", 0.0)
429
+ if not (scale == 1.0 and zero == 0.0):
430
+ raise RuntimeError(
431
+ "Cannot memory-map scaled FITS data (BSCALE/BZERO set). "
432
+ f"BZERO={zero}, BSCALE={scale}. "
433
+ "Workaround: remove scaling with Astropy's"
434
+ " `HDU.data = HDU.data * BSCALE + BZERO` and save a new file"
435
+ )
436
+ # NOTE: check for primary.data size being too large removed, since
437
+ # data is read in chunks, so no danger of exhausting memory
438
+ # NOTE: sanity-check for ndarray type has been removed to avoid
439
+ # forcing eager memory load of possibly very large data array.
376
440
  elif hdu.name == "BEAMS":
377
441
  beams = hdu
378
442
  else:
@@ -402,13 +466,57 @@ def _fits_header_to_xds_attrs(hdulist: fits.hdu.hdulist.HDUList) -> tuple:
402
466
  raise RuntimeError("Could not find both direction axes")
403
467
  if dir_axes is not None:
404
468
  attrs["direction"] = _xds_direction_attrs_from_header(helpers, header)
405
- # FIXME read fits data in chunks in case all data too large to hold in memory
406
- helpers["has_mask"] = da.any(da.isnan(primary.data)).compute()
469
+ helpers["has_mask"] = False
470
+ if compute_mask:
471
+ # 🧠 Why the primary.data reference here is Safe (does not cause
472
+ # an eager read of entire data array)
473
+ # primary.data is a memory-mapped array (because fits.open(..., memmap=True)
474
+ # is used upstream)
475
+ # da.from_array(...) wraps this without reading it immediately
476
+ # The actual read occurs inside:
477
+ # .map_blocks(...).any().compute()
478
+ # ...and that triggers blockwise loading via Dask → safe and parallel
479
+ # 💡 Gotcha
480
+ # What would be dangerous:
481
+ # arr = np.isnan(primary.data).any()
482
+ # That would pull the whole array into memory. But we're not doing that.
483
+ data_dask = da.from_array(primary.data, chunks="auto")
484
+ # The following code black has corner case exposure, although the guard should
485
+ # eliminate it. But there is a cleaner, dask-y way that should work that we implement
486
+ # next, with cautions
487
+ # def chunk_has_nan(block):
488
+ # if not isinstance(block, np.ndarray) or block.size == 0:
489
+ # return False
490
+ # return np.isnan(block).any()
491
+ # helpers["has_mask"] = data_dask.map_blocks(chunk_has_nan, dtype=bool).any().compute()
492
+ # ✅ Option: np.isnan(data_dask).any().compute()
493
+ # 🔒 Pros:
494
+ # Cleaner and shorter (no custom function)
495
+ # Handles all chunk shapes robustly — no risk of empty inputs
496
+ # Uses Dask’s own optimized blockwise operations under the hood
497
+ # ⚠️ Cons:
498
+ # Might trigger more eager computation if Dask can't optimize well:
499
+ # If chunks are misaligned or small, Dask might combine many or materialize more blocks than needed
500
+ # Especially on large images, it could bump memory pressure slightly
501
+ # But since we already call .compute(), we will load some block data no matter
502
+ # what — this just changes how much and how smartly.
503
+ # ✅ Verdict for compute_mask
504
+ # Because this is explicitly for computing a global has-NaN flag (not building the
505
+ # dataset), recommend:
506
+ # helpers["has_mask"] = np.isnan(data_dask).any().compute()
507
+ # It's concise, robust to shape edge cases, and still parallelized.
508
+ # We can always revisit it later if perf becomes a concern — and even then,
509
+ # it's likely a matter of tuning chunks= manually rather than the expression itself.
510
+ #
511
+ # This compute will normally be done in parallel
512
+ helpers["has_mask"] = np.isnan(data_dask).any().compute()
407
513
  beam = _beam_attr_from_header(helpers, header)
408
514
  if beam != "mb":
409
515
  helpers["beam"] = beam
410
516
  if "BITPIX" in header:
411
517
  v = abs(header["BITPIX"])
518
+ if v == 16:
519
+ helpers["dtype"] = "int16"
412
520
  if v == 32:
413
521
  helpers["dtype"] = "float32"
414
522
  elif v == 64:
@@ -490,8 +598,8 @@ def _create_coords(
490
598
  cdelt=pick(helpers["cdelt"]),
491
599
  cunit=pick(helpers["cunit"]),
492
600
  )
601
+ helpers["cunit"] = my_ret["units"]
493
602
  for j, i in enumerate(dir_axes):
494
- helpers["cunit"][i] = my_ret["unit"][j]
495
603
  helpers["crval"][i] = my_ret["ref_val"][j]
496
604
  helpers["cdelt"][i] = my_ret["inc"][j]
497
605
  coords[my_ret["axis_name"][0]] = (["l", "m"], my_ret["value"][0])
@@ -566,9 +674,9 @@ def _get_freq_values(helpers: dict) -> list:
566
674
  freq, vel = _freq_from_vel(
567
675
  crval, cdelt, crpix, cunit, "Z", helpers["shape"][v_idx], restfreq
568
676
  )
569
- helpers["velocity"] = vel["value"] * u.Unit(vel["unit"])
570
- helpers["crval"][v_idx] = (freq["crval"] * u.Unit(freq["unit"])).to(u.Hz).value
571
- helpers["cdelt"][v_idx] = (freq["cdelt"] * u.Unit(freq["unit"])).to(u.Hz).value
677
+ helpers["velocity"] = vel["value"] * u.Unit(vel["units"])
678
+ helpers["crval"][v_idx] = (freq["crval"] * u.Unit(freq["units"])).to(u.Hz).value
679
+ helpers["cdelt"][v_idx] = (freq["cdelt"] * u.Unit(freq["units"])).to(u.Hz).value
572
680
  return list(freq["value"])
573
681
  else:
574
682
  return [1420e6]
@@ -587,6 +695,9 @@ def _get_velocity_values(helpers: dict) -> list:
587
695
  return v
588
696
 
589
697
 
698
+ # FIXME change namee, even if there is only a single beam, we make a
699
+ # multi beam array using it. If we have a beam, it will always be
700
+ # "mutltibeam" is name is redundant and confusing
590
701
  def _do_multibeam(xds: xr.Dataset, imname: str) -> xr.Dataset:
591
702
  """Only run if we are sure there are multiple beams"""
592
703
  hdulist = fits.open(imname)
@@ -821,12 +932,12 @@ def _get_transpose_list(helpers: dict) -> tuple:
821
932
 
822
933
  def _read_image_chunk(img_full_path, shapes: tuple, starts: tuple) -> np.ndarray:
823
934
  hdulist = fits.open(img_full_path, memmap=True)
824
- s = []
825
- for start, length in zip(starts, shapes):
826
- s.append(slice(start, start + length))
827
- t = tuple(s)
828
- z = hdulist[0].data[t]
935
+ hdu = hdulist[0]
936
+ # Chunk slice
937
+ slices = tuple(
938
+ slice(start, start + length) for start, length in zip(starts, shapes)
939
+ )
940
+ chunk = hdu.data[slices]
829
941
  hdulist.close()
830
- # delete to avoid having a reference to a mem-mapped hdulist
831
942
  del hdulist
832
- return z
943
+ return chunk
@@ -46,8 +46,8 @@ def _load_casa_image_block(infile: str, block_des: dict, do_sky_coords) -> xr.Da
46
46
  cshape = casa_image.shape()
47
47
  ret = _casa_image_to_xds_coords(image_full_path, False, do_sky_coords)
48
48
  xds = ret["xds"].isel(block_des)
49
- nchan = ret["xds"].dims["frequency"]
50
- npol = ret["xds"].dims["polarization"]
49
+ nchan = ret["xds"].sizes["frequency"]
50
+ npol = ret["xds"].sizes["polarization"]
51
51
  starts, shapes, slices = _get_starts_shapes_slices(block_des, coords, cshape)
52
52
  dimorder = _get_xds_dim_order(ret["sphr_dims"])
53
53
  transpose_list, new_axes = _get_transpose_list(coords)
@@ -105,7 +105,7 @@ def _read_casa_image(
105
105
  xds = _add_mask(xds, m.upper(), ary, dimorder)
106
106
  xds.attrs = _casa_image_to_xds_attrs(img_full_path)
107
107
  beam = _get_beam(
108
- img_full_path, xds.dims["frequency"], xds.dims["polarization"], True
108
+ img_full_path, xds.sizes["frequency"], xds.sizes["polarization"], True
109
109
  )
110
110
  if beam is not None:
111
111
  xds["BEAM"] = beam
@@ -133,6 +133,7 @@ def _xds_to_casa_image(xds: xr.Dataset, imagename: str) -> None:
133
133
  lockoptions={"option": "permanentwait"},
134
134
  ack=False,
135
135
  )
136
+
136
137
  tb.putkeyword("coords", coord)
137
138
  tb.putkeyword("imageinfo", ii)
138
139
  if units:
@@ -110,7 +110,7 @@ def _numpy_arrayize_dv(xds: xr.Dataset) -> xr.Dataset:
110
110
  def _default_freq_info() -> dict:
111
111
  return {
112
112
  "rest_frequency": make_quantity(1420405751.7860003, "Hz"),
113
- "type": "frequency",
113
+ "type": "spectral_coord",
114
114
  "frame": "lsrk",
115
115
  "units": "Hz",
116
116
  "waveUnit": "mm",
@@ -141,7 +141,7 @@ def _freq_from_vel(
141
141
  vel = vel * u.Unit(cunit)
142
142
  v_dict = {
143
143
  "value": vel.value,
144
- "unit": cunit,
144
+ "units": cunit,
145
145
  "crval": crval,
146
146
  "cdelt": cdelt,
147
147
  "crpix": crpix,
@@ -154,7 +154,7 @@ def _freq_from_vel(
154
154
  fcdelt = -restfreq / _c / (crval * vel.unit / _c + 1) ** 2 * cdelt * vel.unit
155
155
  f_dict = {
156
156
  "value": freq.value,
157
- "unit": "Hz",
157
+ "units": "Hz",
158
158
  "crval": fcrval.to(u.Hz).value,
159
159
  "cdelt": fcdelt.to(u.Hz).value,
160
160
  "crpix": crpix,
@@ -180,7 +180,7 @@ def _compute_world_sph_dims(
180
180
  "axis_name": [None, None],
181
181
  "ref_val": [None, None],
182
182
  "inc": [None, None],
183
- "unit": ["rad", "rad"],
183
+ "units": "rad",
184
184
  "value": [None, None],
185
185
  }
186
186
  for i in range(2):
@@ -6,7 +6,7 @@ from typing import List, Union
6
6
  from .common import _c, _compute_world_sph_dims, _l_m_attr_notes
7
7
  from xradio._utils.coord_math import _deg_to_rad
8
8
  from xradio._utils.dict_helpers import (
9
- make_frequency_reference_dict,
9
+ make_spectral_coord_reference_dict,
10
10
  make_quantity,
11
11
  make_skycoord_dict,
12
12
  make_time_coord_attrs,
@@ -50,24 +50,24 @@ def _add_common_attrs(
50
50
  cell_size: Union[List[float], np.ndarray],
51
51
  projection: str,
52
52
  ) -> xr.Dataset:
53
- xds.time.attrs = make_time_coord_attrs(units=["d"], scale="utc", time_format="mjd")
53
+ xds.time.attrs = make_time_coord_attrs(units="d", scale="utc", time_format="mjd")
54
54
  freq_vals = np.array(xds.frequency)
55
55
  xds.frequency.attrs = {
56
56
  "observer": spectral_reference.lower(),
57
- "reference_value": make_frequency_reference_dict(
57
+ "reference_frequency": make_spectral_coord_reference_dict(
58
58
  value=freq_vals[len(freq_vals) // 2].item(),
59
- units=["Hz"],
59
+ units="Hz",
60
60
  observer=spectral_reference.lower(),
61
61
  ),
62
62
  "rest_frequencies": make_quantity(restfreq, "Hz"),
63
63
  "rest_frequency": make_quantity(restfreq, "Hz"),
64
- "type": "frequency",
65
- "units": ["Hz"],
66
- "wave_unit": ["mm"],
64
+ "type": "spectral_coord",
65
+ "units": "Hz",
66
+ "wave_units": "mm",
67
67
  }
68
68
  xds.velocity.attrs = {"doppler_type": "radio", "type": "doppler", "units": "m/s"}
69
69
  reference = make_skycoord_dict(
70
- data=phase_center, units=["rad", "rad"], frame=direction_reference
70
+ data=phase_center, units="rad", frame=direction_reference
71
71
  )
72
72
  reference["attrs"].update({"equinox": "j2000.0"})
73
73
  xds.attrs = {
xradio/image/image.py CHANGED
@@ -15,13 +15,15 @@ import xarray as xr
15
15
  # from .._utils.zarr.common import _load_no_dask_zarr
16
16
 
17
17
  from ._util.casacore import _load_casa_image_block, _xds_to_casa_image
18
- from ._util.fits import _read_fits_image
18
+
19
+ # from ._util.fits import _read_fits_image
19
20
  from ._util.image_factory import (
20
21
  _make_empty_aperture_image,
21
22
  _make_empty_lmuv_image,
22
23
  _make_empty_sky_image,
23
24
  )
24
25
  from ._util.zarr import _load_image_from_zarr_no_dask, _xds_from_zarr, _xds_to_zarr
26
+ from ._util._fits.xds_from_fits import _fits_image_to_xds
25
27
 
26
28
  warnings.filterwarnings("ignore", category=FutureWarning)
27
29
 
@@ -32,12 +34,37 @@ def read_image(
32
34
  verbose: bool = False,
33
35
  do_sky_coords: bool = True,
34
36
  selection: dict = {},
37
+ compute_mask: bool = True,
35
38
  ) -> xr.Dataset:
36
39
  """
37
40
  Convert CASA, FITS, or zarr image to xradio image xds format
38
41
  ngCASA image spec is located at
39
42
  https://docs.google.com/spreadsheets/d/1WW0Gl6z85cJVPgtdgW4dxucurHFa06OKGjgoK8OREFA/edit#gid=1719181934
40
43
 
44
+ Notes on FITS compatibility and memory mapping:
45
+
46
+ This function relies on Astropy's `memmap=True` to avoid loading full image data into memory.
47
+ However, not all FITS files support memory-mapped reads.
48
+
49
+ ⚠️ The following FITS types are incompatible with memory mapping:
50
+
51
+ 1. Compressed images (`CompImageHDU`)
52
+ = Workaround: decompress the FITS using tools like `funpack`, `cfitsio`,
53
+ or Astropy's `.scale()`/`.copy()` workflows
54
+ 2. Some scaled images (using BSCALE/BZERO headers)
55
+ ✅ Supported:
56
+ - Files with no BSCALE/BZERO headers (or BSCALE=1.0 and BZERO=0.0)
57
+ - Uncompressed, unscaled primary HDUs
58
+ ⚠️ Unsupported: Files with BSCALE ≠ 1.0 or BZERO ≠ 0.0
59
+ - These require data rescaling in memory, which disables lazy access
60
+ - Attempting to slice such arrays forces eager read of the full dataset
61
+ - Workaround: remove scaling with Astropy's
62
+ `HDU.data = HDU.data * BSCALE + BZERO` and save a new file
63
+
64
+ These cases will raise `RuntimeError` to prevent silent eager loads that can exhaust memory.
65
+
66
+ If you encounter such an error, consider preprocessing the file to make it memory-mappable.
67
+
41
68
  Parameters
42
69
  ----------
43
70
  infile : str
@@ -69,11 +96,19 @@ def read_image(
69
96
  the selection, and the end pixel is not. An empty dictionary (the
70
97
  default) indicates that the entire image should be returned. Currently
71
98
  only supported for images stored in zarr format.
72
-
99
+ compute_mask : bool, optional
100
+ If True (default), compute and attach valid data masks when converting from FITS to xds.
101
+ If False, skip mask computation entirely. This may improve performance if the mask
102
+ is not required for subsequent processing. It may, however, result in unpredictable behavior
103
+ for applications that are not designed to handle missing data. It is the user's responsibility,
104
+ not the software's, to ensure that the mask is computed if it is necessary. Currently only
105
+ implemented for FITS images.
73
106
  Returns
74
107
  -------
75
108
  xarray.Dataset
76
109
  """
110
+ # from ._util.casacore import _read_casa_image
111
+ # return _read_casa_image(infile, chunks, verbose, do_sky_coords)
77
112
  emsgs = []
78
113
  do_casa = True
79
114
  try:
@@ -92,9 +127,10 @@ def read_image(
92
127
  except Exception as e:
93
128
  emsgs.append(f"image format appears not to be casacore: {e.args}")
94
129
  # next statement is for debug, comment when done debugging
95
- # return _read_fits_image(infile, chunks, verbose, do_sky_coords)
130
+ # return _fits_image_to_xds(infile, chunks, verbose, do_sky_coords, compute_mask)
96
131
  try:
97
- return _read_fits_image(infile, chunks, verbose, do_sky_coords)
132
+ img_full_path = os.path.expanduser(infile)
133
+ return _fits_image_to_xds(infile, chunks, verbose, do_sky_coords, compute_mask)
98
134
  except Exception as e:
99
135
  emsgs.append(f"image format appears not to be fits {e.args}")
100
136
  # when done debuggin comment out next line
@@ -111,7 +147,7 @@ def read_image(
111
147
  raise RuntimeError("\n".join(emsgs))
112
148
 
113
149
 
114
- def load_image(infile: str, block_des: dict = {}, do_sky_coords=True) -> xr.Dataset:
150
+ def load_image(infile: str, block_des: dict = None, do_sky_coords=True) -> xr.Dataset:
115
151
  """
116
152
  Load an image or portion of an image (subimage) into memory with data variables
117
153
  being converted from dask to numpy arrays and coordinate arrays being converted
@@ -144,6 +180,10 @@ def load_image(infile: str, block_des: dict = {}, do_sky_coords=True) -> xr.Data
144
180
  """
145
181
  do_casa = True
146
182
  emsgs = []
183
+
184
+ if block_des is None:
185
+ block_des = {}
186
+
147
187
  selection = copy.deepcopy(block_des) if block_des else block_des
148
188
  if selection:
149
189
  for k, v in selection.items():
@@ -4,13 +4,11 @@ convert, and retrieve information from Processing Set and Measurement Sets nodes
4
4
  Processing Set DataTree
5
5
  """
6
6
 
7
- from .processing_set_xdt import *
7
+ import toolviper.utils.logger as _logger
8
+
9
+ from .processing_set_xdt import ProcessingSetXdt
8
10
  from .open_processing_set import open_processing_set
9
- from .load_processing_set import load_processing_set # , ProcessingSetIterator
10
- from .convert_msv2_to_processing_set import (
11
- convert_msv2_to_processing_set,
12
- estimate_conversion_memory_and_cores,
13
- )
11
+ from .load_processing_set import load_processing_set
14
12
  from .measurement_set_xdt import MeasurementSetXdt
15
13
  from .schema import SpectrumXds, VisibilityXds
16
14
 
@@ -19,9 +17,21 @@ __all__ = [
19
17
  "MeasurementSetXdt",
20
18
  "open_processing_set",
21
19
  "load_processing_set",
22
- "ProcessingSetIterator",
23
- "convert_msv2_to_processing_set",
24
- "estimate_conversion_memory_and_cores",
25
20
  "SpectrumXds",
26
21
  "VisibilityXds",
27
22
  ]
23
+
24
+ try:
25
+ from .convert_msv2_to_processing_set import (
26
+ convert_msv2_to_processing_set,
27
+ estimate_conversion_memory_and_cores,
28
+ )
29
+ except ModuleNotFoundError as exc:
30
+ _logger.warning(
31
+ "Could not import the function to convert from MSv2 to MSv4. "
32
+ f"That functionality will not be available. Details: {exc}"
33
+ )
34
+ else:
35
+ __all__.extend(
36
+ ["convert_msv2_to_processing_set", "estimate_conversion_memory_and_cores"]
37
+ )
@@ -1,5 +1,3 @@
1
- from . import msv2
2
- from . import zarr
3
1
  from . import _utils
4
2
 
5
- __all__ = ["msv2", "zarr", "_utils"]
3
+ __all__ = ["_utils"]
File without changes