xradio 0.0.56__py3-none-any.whl → 0.0.59__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xradio/__init__.py +2 -2
- xradio/_utils/_casacore/casacore_from_casatools.py +12 -2
- xradio/_utils/_casacore/tables.py +1 -0
- xradio/_utils/coord_math.py +22 -23
- xradio/_utils/dict_helpers.py +76 -11
- xradio/_utils/schema.py +5 -2
- xradio/_utils/zarr/common.py +1 -73
- xradio/image/_util/_casacore/xds_from_casacore.py +49 -33
- xradio/image/_util/_casacore/xds_to_casacore.py +41 -14
- xradio/image/_util/_fits/xds_from_fits.py +146 -35
- xradio/image/_util/casacore.py +4 -3
- xradio/image/_util/common.py +4 -4
- xradio/image/_util/image_factory.py +8 -8
- xradio/image/image.py +45 -5
- xradio/measurement_set/__init__.py +19 -9
- xradio/measurement_set/_utils/__init__.py +1 -3
- xradio/measurement_set/_utils/_msv2/__init__.py +0 -0
- xradio/measurement_set/_utils/_msv2/_tables/read.py +17 -76
- xradio/measurement_set/_utils/_msv2/_tables/read_main_table.py +2 -685
- xradio/measurement_set/_utils/_msv2/conversion.py +174 -156
- xradio/measurement_set/_utils/_msv2/create_antenna_xds.py +9 -16
- xradio/measurement_set/_utils/_msv2/create_field_and_source_xds.py +128 -222
- xradio/measurement_set/_utils/_msv2/msv2_to_msv4_meta.py +1 -2
- xradio/measurement_set/_utils/_msv2/msv4_info_dicts.py +8 -7
- xradio/measurement_set/_utils/_msv2/msv4_sub_xdss.py +31 -74
- xradio/measurement_set/_utils/_msv2/partition_queries.py +1 -261
- xradio/measurement_set/_utils/_msv2/subtables.py +0 -107
- xradio/measurement_set/_utils/_utils/interpolate.py +60 -0
- xradio/measurement_set/_utils/_zarr/encoding.py +2 -7
- xradio/measurement_set/convert_msv2_to_processing_set.py +0 -2
- xradio/measurement_set/load_processing_set.py +2 -2
- xradio/measurement_set/measurement_set_xdt.py +20 -16
- xradio/measurement_set/open_processing_set.py +1 -3
- xradio/measurement_set/processing_set_xdt.py +54 -841
- xradio/measurement_set/schema.py +122 -132
- xradio/schema/check.py +95 -101
- xradio/schema/dataclass.py +159 -22
- xradio/schema/export.py +99 -0
- xradio/schema/metamodel.py +51 -16
- xradio/schema/typing.py +5 -5
- xradio/sphinx/schema_table.py +41 -77
- {xradio-0.0.56.dist-info → xradio-0.0.59.dist-info}/METADATA +20 -5
- xradio-0.0.59.dist-info/RECORD +65 -0
- {xradio-0.0.56.dist-info → xradio-0.0.59.dist-info}/WHEEL +1 -1
- xradio/image/_util/fits.py +0 -13
- xradio/measurement_set/_utils/_msv2/_tables/load.py +0 -66
- xradio/measurement_set/_utils/_msv2/_tables/load_main_table.py +0 -490
- xradio/measurement_set/_utils/_msv2/_tables/read_subtables.py +0 -398
- xradio/measurement_set/_utils/_msv2/_tables/write.py +0 -323
- xradio/measurement_set/_utils/_msv2/_tables/write_exp_api.py +0 -388
- xradio/measurement_set/_utils/_msv2/chunks.py +0 -115
- xradio/measurement_set/_utils/_msv2/descr.py +0 -165
- xradio/measurement_set/_utils/_msv2/msv2_msv3.py +0 -7
- xradio/measurement_set/_utils/_msv2/partitions.py +0 -392
- xradio/measurement_set/_utils/_utils/cds.py +0 -40
- xradio/measurement_set/_utils/_utils/xds_helper.py +0 -404
- xradio/measurement_set/_utils/_zarr/read.py +0 -263
- xradio/measurement_set/_utils/_zarr/write.py +0 -329
- xradio/measurement_set/_utils/msv2.py +0 -106
- xradio/measurement_set/_utils/zarr.py +0 -133
- xradio-0.0.56.dist-info/RECORD +0 -78
- {xradio-0.0.56.dist-info → xradio-0.0.59.dist-info}/licenses/LICENSE.txt +0 -0
- {xradio-0.0.56.dist-info → xradio-0.0.59.dist-info}/top_level.txt +0 -0
|
@@ -5,6 +5,7 @@ from typing import Union
|
|
|
5
5
|
import dask
|
|
6
6
|
import dask.array as da
|
|
7
7
|
import numpy as np
|
|
8
|
+
import psutil
|
|
8
9
|
import xarray as xr
|
|
9
10
|
from astropy import units as u
|
|
10
11
|
from astropy.io import fits
|
|
@@ -13,7 +14,7 @@ from astropy.time import Time
|
|
|
13
14
|
from xradio._utils.coord_math import _deg_to_rad
|
|
14
15
|
from xradio._utils.dict_helpers import (
|
|
15
16
|
make_quantity,
|
|
16
|
-
|
|
17
|
+
make_spectral_coord_reference_dict,
|
|
17
18
|
make_skycoord_dict,
|
|
18
19
|
make_time_measure_dict,
|
|
19
20
|
)
|
|
@@ -35,20 +36,31 @@ from xradio.image._util.common import (
|
|
|
35
36
|
|
|
36
37
|
|
|
37
38
|
def _fits_image_to_xds(
|
|
38
|
-
img_full_path: str,
|
|
39
|
+
img_full_path: str,
|
|
40
|
+
chunks: dict,
|
|
41
|
+
verbose: bool,
|
|
42
|
+
do_sky_coords: bool,
|
|
43
|
+
compute_mask: bool,
|
|
39
44
|
) -> dict:
|
|
40
45
|
"""
|
|
46
|
+
compute_mask : bool, optional
|
|
47
|
+
If True (default), compute and attach valid data masks to the xds.
|
|
48
|
+
If False, skip mask generation for performance. It is solely the responsibility
|
|
49
|
+
of the user to ensure downstream apps can handle NaN values; do not
|
|
50
|
+
ask package developers to add this non-standard behavior.
|
|
51
|
+
|
|
41
52
|
TODO: complete documentation
|
|
42
53
|
Create an xds without any pixel data from metadata from the specified FITS image
|
|
43
54
|
"""
|
|
44
|
-
# memmap = True allows only part of data to be loaded into memory
|
|
45
55
|
# may also need to pass mode='denywrite'
|
|
46
56
|
# https://stackoverflow.com/questions/35759713/astropy-io-fits-read-row-from-large-fits-file-with-mutliple-hdus
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
57
|
+
try:
|
|
58
|
+
hdulist = fits.open(img_full_path, memmap=True)
|
|
59
|
+
attrs, helpers, header = _fits_header_to_xds_attrs(hdulist, compute_mask)
|
|
60
|
+
finally:
|
|
61
|
+
hdulist.close()
|
|
62
|
+
# avoid keeping reference to mem-mapped fits file
|
|
63
|
+
del hdulist
|
|
52
64
|
xds = _create_coords(helpers, header, do_sky_coords)
|
|
53
65
|
sphr_dims = helpers["sphr_dims"]
|
|
54
66
|
ary = _read_image_array(img_full_path, chunks, helpers, verbose)
|
|
@@ -86,10 +98,10 @@ def _add_freq_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
|
|
|
86
98
|
meta["rest_frequency"] = make_quantity(helpers["restfreq"], "Hz")
|
|
87
99
|
meta["rest_frequencies"] = [meta["rest_frequency"]]
|
|
88
100
|
meta["type"] = "frequency"
|
|
89
|
-
meta["
|
|
101
|
+
meta["wave_units"] = "mm"
|
|
90
102
|
freq_axis = helpers["freq_axis"]
|
|
91
|
-
meta["
|
|
92
|
-
helpers["crval"][freq_axis],
|
|
103
|
+
meta["reference_frequency"] = make_spectral_coord_reference_dict(
|
|
104
|
+
helpers["crval"][freq_axis], "Hz", helpers["specsys"]
|
|
93
105
|
)
|
|
94
106
|
# meta["cdelt"] = helpers["cdelt"][freq_axis]
|
|
95
107
|
if not meta:
|
|
@@ -102,7 +114,7 @@ def _add_freq_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
|
|
|
102
114
|
|
|
103
115
|
def _add_vel_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
|
|
104
116
|
vel_coord = xds.coords["velocity"]
|
|
105
|
-
meta = {"units":
|
|
117
|
+
meta = {"units": "m/s"}
|
|
106
118
|
if helpers["has_freq"]:
|
|
107
119
|
meta["doppler_type"] = helpers.get("doppler", "RADIO")
|
|
108
120
|
else:
|
|
@@ -159,9 +171,7 @@ def _xds_direction_attrs_from_header(helpers: dict, header) -> dict:
|
|
|
159
171
|
helpers["ref_sys"] = ref_sys
|
|
160
172
|
helpers["ref_eqx"] = ref_eqx
|
|
161
173
|
# fits does not support conversion frames
|
|
162
|
-
direction["reference"] = make_skycoord_dict(
|
|
163
|
-
[0.0, 0.0], units=["rad", "rad"], frame=ref_sys
|
|
164
|
-
)
|
|
174
|
+
direction["reference"] = make_skycoord_dict([0.0, 0.0], units="rad", frame=ref_sys)
|
|
165
175
|
dir_axes = helpers["dir_axes"]
|
|
166
176
|
ddata = []
|
|
167
177
|
dunits = []
|
|
@@ -239,16 +249,43 @@ def _get_telescope_metadata(helpers: dict, header) -> dict:
|
|
|
239
249
|
r = np.sqrt(np.sum(xyz * xyz))
|
|
240
250
|
lat = np.arcsin(z / r)
|
|
241
251
|
long = np.arctan2(y, x)
|
|
242
|
-
tel["
|
|
252
|
+
tel["direction"] = {
|
|
253
|
+
"attrs": {
|
|
254
|
+
"coordinate_system": "geocentric",
|
|
255
|
+
# I haven't seen a FITS keyword for reference frame of telescope posiiton
|
|
256
|
+
"frame": "ITRF",
|
|
257
|
+
"origin_object_name": "earth",
|
|
258
|
+
"type": "location",
|
|
259
|
+
"units": "rad",
|
|
260
|
+
},
|
|
261
|
+
"data": np.array([long, lat]),
|
|
262
|
+
"dims": ["ellipsoid_dir_label"],
|
|
263
|
+
"coords": {
|
|
264
|
+
"ellipsoid_dir_label": {
|
|
265
|
+
"dims": ["ellipsoid_dir_label"],
|
|
266
|
+
"data": ["lon", "lat"],
|
|
267
|
+
}
|
|
268
|
+
},
|
|
269
|
+
}
|
|
270
|
+
tel["distance"] = {
|
|
243
271
|
"attrs": {
|
|
244
272
|
"coordinate_system": "geocentric",
|
|
245
273
|
# I haven't seen a FITS keyword for reference frame of telescope posiiton
|
|
246
274
|
"frame": "ITRF",
|
|
247
275
|
"origin_object_name": "earth",
|
|
248
276
|
"type": "location",
|
|
249
|
-
"units":
|
|
277
|
+
"units": "m",
|
|
278
|
+
},
|
|
279
|
+
"data": np.array([r]),
|
|
280
|
+
"dims": ["ellipsoid_dis_label"],
|
|
281
|
+
"coords": {
|
|
282
|
+
"ellipsoid_dis_label": {
|
|
283
|
+
"dims": ["ellipsoid_dis_label"],
|
|
284
|
+
"data": [
|
|
285
|
+
"dist",
|
|
286
|
+
],
|
|
287
|
+
}
|
|
250
288
|
},
|
|
251
|
-
"data": np.array([long, lat, r]),
|
|
252
289
|
}
|
|
253
290
|
return tel
|
|
254
291
|
|
|
@@ -266,9 +303,7 @@ def _compute_pointing_center(helpers: dict, header) -> dict:
|
|
|
266
303
|
pc_lat = float(header[f"CRVAL{t_axes[1]}"]) * unit[1]
|
|
267
304
|
pc_long = pc_long.to(u.rad).value
|
|
268
305
|
pc_lat = pc_lat.to(u.rad).value
|
|
269
|
-
return make_skycoord_dict(
|
|
270
|
-
[pc_long, pc_lat], units=["rad", "rad"], frame=helpers["ref_sys"]
|
|
271
|
-
)
|
|
306
|
+
return make_skycoord_dict([pc_long, pc_lat], units="rad", frame=helpers["ref_sys"])
|
|
272
307
|
|
|
273
308
|
|
|
274
309
|
def _user_attrs_from_header(header) -> dict:
|
|
@@ -367,12 +402,41 @@ def _create_dim_map(helpers: dict, header) -> dict:
|
|
|
367
402
|
return dim_map
|
|
368
403
|
|
|
369
404
|
|
|
370
|
-
def _fits_header_to_xds_attrs(
|
|
405
|
+
def _fits_header_to_xds_attrs(
|
|
406
|
+
hdulist: fits.hdu.hdulist.HDUList, compute_mask: bool
|
|
407
|
+
) -> tuple:
|
|
408
|
+
# First: Guard for unsupported compressed images
|
|
409
|
+
for i, hdu in enumerate(hdulist):
|
|
410
|
+
if isinstance(hdu, fits.CompImageHDU):
|
|
411
|
+
raise RuntimeError(
|
|
412
|
+
f"HDU {i}, name={hdu.name} is a CompImageHDU, which is not supported "
|
|
413
|
+
"for memory-mapping. "
|
|
414
|
+
"Cannot memory-map compressed FITS image (CompImageHDU). "
|
|
415
|
+
"Workaround: decompress the FITS using tools like `funpack`, `cfitsio`, "
|
|
416
|
+
"or Astropy's `.scale()`/`.copy()` workflows"
|
|
417
|
+
)
|
|
371
418
|
primary = None
|
|
419
|
+
# FIXME beams is set but never actually used in this function. What's up with that?
|
|
372
420
|
beams = None
|
|
373
421
|
for hdu in hdulist:
|
|
374
422
|
if hdu.name == "PRIMARY":
|
|
375
423
|
primary = hdu
|
|
424
|
+
# Memory map support check
|
|
425
|
+
# avoid possibly non-existent hdu.scale_type attribute check and check header instead
|
|
426
|
+
header = hdu.header
|
|
427
|
+
scale = hdu.header.get("BSCALE", 1.0)
|
|
428
|
+
zero = hdu.header.get("BZERO", 0.0)
|
|
429
|
+
if not (scale == 1.0 and zero == 0.0):
|
|
430
|
+
raise RuntimeError(
|
|
431
|
+
"Cannot memory-map scaled FITS data (BSCALE/BZERO set). "
|
|
432
|
+
f"BZERO={zero}, BSCALE={scale}. "
|
|
433
|
+
"Workaround: remove scaling with Astropy's"
|
|
434
|
+
" `HDU.data = HDU.data * BSCALE + BZERO` and save a new file"
|
|
435
|
+
)
|
|
436
|
+
# NOTE: check for primary.data size being too large removed, since
|
|
437
|
+
# data is read in chunks, so no danger of exhausting memory
|
|
438
|
+
# NOTE: sanity-check for ndarray type has been removed to avoid
|
|
439
|
+
# forcing eager memory load of possibly very large data array.
|
|
376
440
|
elif hdu.name == "BEAMS":
|
|
377
441
|
beams = hdu
|
|
378
442
|
else:
|
|
@@ -402,13 +466,57 @@ def _fits_header_to_xds_attrs(hdulist: fits.hdu.hdulist.HDUList) -> tuple:
|
|
|
402
466
|
raise RuntimeError("Could not find both direction axes")
|
|
403
467
|
if dir_axes is not None:
|
|
404
468
|
attrs["direction"] = _xds_direction_attrs_from_header(helpers, header)
|
|
405
|
-
|
|
406
|
-
|
|
469
|
+
helpers["has_mask"] = False
|
|
470
|
+
if compute_mask:
|
|
471
|
+
# 🧠 Why the primary.data reference here is Safe (does not cause
|
|
472
|
+
# an eager read of entire data array)
|
|
473
|
+
# primary.data is a memory-mapped array (because fits.open(..., memmap=True)
|
|
474
|
+
# is used upstream)
|
|
475
|
+
# da.from_array(...) wraps this without reading it immediately
|
|
476
|
+
# The actual read occurs inside:
|
|
477
|
+
# .map_blocks(...).any().compute()
|
|
478
|
+
# ...and that triggers blockwise loading via Dask → safe and parallel
|
|
479
|
+
# 💡 Gotcha
|
|
480
|
+
# What would be dangerous:
|
|
481
|
+
# arr = np.isnan(primary.data).any()
|
|
482
|
+
# That would pull the whole array into memory. But we're not doing that.
|
|
483
|
+
data_dask = da.from_array(primary.data, chunks="auto")
|
|
484
|
+
# The following code black has corner case exposure, although the guard should
|
|
485
|
+
# eliminate it. But there is a cleaner, dask-y way that should work that we implement
|
|
486
|
+
# next, with cautions
|
|
487
|
+
# def chunk_has_nan(block):
|
|
488
|
+
# if not isinstance(block, np.ndarray) or block.size == 0:
|
|
489
|
+
# return False
|
|
490
|
+
# return np.isnan(block).any()
|
|
491
|
+
# helpers["has_mask"] = data_dask.map_blocks(chunk_has_nan, dtype=bool).any().compute()
|
|
492
|
+
# ✅ Option: np.isnan(data_dask).any().compute()
|
|
493
|
+
# 🔒 Pros:
|
|
494
|
+
# Cleaner and shorter (no custom function)
|
|
495
|
+
# Handles all chunk shapes robustly — no risk of empty inputs
|
|
496
|
+
# Uses Dask’s own optimized blockwise operations under the hood
|
|
497
|
+
# ⚠️ Cons:
|
|
498
|
+
# Might trigger more eager computation if Dask can't optimize well:
|
|
499
|
+
# If chunks are misaligned or small, Dask might combine many or materialize more blocks than needed
|
|
500
|
+
# Especially on large images, it could bump memory pressure slightly
|
|
501
|
+
# But since we already call .compute(), we will load some block data no matter
|
|
502
|
+
# what — this just changes how much and how smartly.
|
|
503
|
+
# ✅ Verdict for compute_mask
|
|
504
|
+
# Because this is explicitly for computing a global has-NaN flag (not building the
|
|
505
|
+
# dataset), recommend:
|
|
506
|
+
# helpers["has_mask"] = np.isnan(data_dask).any().compute()
|
|
507
|
+
# It's concise, robust to shape edge cases, and still parallelized.
|
|
508
|
+
# We can always revisit it later if perf becomes a concern — and even then,
|
|
509
|
+
# it's likely a matter of tuning chunks= manually rather than the expression itself.
|
|
510
|
+
#
|
|
511
|
+
# This compute will normally be done in parallel
|
|
512
|
+
helpers["has_mask"] = np.isnan(data_dask).any().compute()
|
|
407
513
|
beam = _beam_attr_from_header(helpers, header)
|
|
408
514
|
if beam != "mb":
|
|
409
515
|
helpers["beam"] = beam
|
|
410
516
|
if "BITPIX" in header:
|
|
411
517
|
v = abs(header["BITPIX"])
|
|
518
|
+
if v == 16:
|
|
519
|
+
helpers["dtype"] = "int16"
|
|
412
520
|
if v == 32:
|
|
413
521
|
helpers["dtype"] = "float32"
|
|
414
522
|
elif v == 64:
|
|
@@ -490,8 +598,8 @@ def _create_coords(
|
|
|
490
598
|
cdelt=pick(helpers["cdelt"]),
|
|
491
599
|
cunit=pick(helpers["cunit"]),
|
|
492
600
|
)
|
|
601
|
+
helpers["cunit"] = my_ret["units"]
|
|
493
602
|
for j, i in enumerate(dir_axes):
|
|
494
|
-
helpers["cunit"][i] = my_ret["unit"][j]
|
|
495
603
|
helpers["crval"][i] = my_ret["ref_val"][j]
|
|
496
604
|
helpers["cdelt"][i] = my_ret["inc"][j]
|
|
497
605
|
coords[my_ret["axis_name"][0]] = (["l", "m"], my_ret["value"][0])
|
|
@@ -566,9 +674,9 @@ def _get_freq_values(helpers: dict) -> list:
|
|
|
566
674
|
freq, vel = _freq_from_vel(
|
|
567
675
|
crval, cdelt, crpix, cunit, "Z", helpers["shape"][v_idx], restfreq
|
|
568
676
|
)
|
|
569
|
-
helpers["velocity"] = vel["value"] * u.Unit(vel["
|
|
570
|
-
helpers["crval"][v_idx] = (freq["crval"] * u.Unit(freq["
|
|
571
|
-
helpers["cdelt"][v_idx] = (freq["cdelt"] * u.Unit(freq["
|
|
677
|
+
helpers["velocity"] = vel["value"] * u.Unit(vel["units"])
|
|
678
|
+
helpers["crval"][v_idx] = (freq["crval"] * u.Unit(freq["units"])).to(u.Hz).value
|
|
679
|
+
helpers["cdelt"][v_idx] = (freq["cdelt"] * u.Unit(freq["units"])).to(u.Hz).value
|
|
572
680
|
return list(freq["value"])
|
|
573
681
|
else:
|
|
574
682
|
return [1420e6]
|
|
@@ -587,6 +695,9 @@ def _get_velocity_values(helpers: dict) -> list:
|
|
|
587
695
|
return v
|
|
588
696
|
|
|
589
697
|
|
|
698
|
+
# FIXME change namee, even if there is only a single beam, we make a
|
|
699
|
+
# multi beam array using it. If we have a beam, it will always be
|
|
700
|
+
# "mutltibeam" is name is redundant and confusing
|
|
590
701
|
def _do_multibeam(xds: xr.Dataset, imname: str) -> xr.Dataset:
|
|
591
702
|
"""Only run if we are sure there are multiple beams"""
|
|
592
703
|
hdulist = fits.open(imname)
|
|
@@ -821,12 +932,12 @@ def _get_transpose_list(helpers: dict) -> tuple:
|
|
|
821
932
|
|
|
822
933
|
def _read_image_chunk(img_full_path, shapes: tuple, starts: tuple) -> np.ndarray:
|
|
823
934
|
hdulist = fits.open(img_full_path, memmap=True)
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
935
|
+
hdu = hdulist[0]
|
|
936
|
+
# Chunk slice
|
|
937
|
+
slices = tuple(
|
|
938
|
+
slice(start, start + length) for start, length in zip(starts, shapes)
|
|
939
|
+
)
|
|
940
|
+
chunk = hdu.data[slices]
|
|
829
941
|
hdulist.close()
|
|
830
|
-
# delete to avoid having a reference to a mem-mapped hdulist
|
|
831
942
|
del hdulist
|
|
832
|
-
return
|
|
943
|
+
return chunk
|
xradio/image/_util/casacore.py
CHANGED
|
@@ -46,8 +46,8 @@ def _load_casa_image_block(infile: str, block_des: dict, do_sky_coords) -> xr.Da
|
|
|
46
46
|
cshape = casa_image.shape()
|
|
47
47
|
ret = _casa_image_to_xds_coords(image_full_path, False, do_sky_coords)
|
|
48
48
|
xds = ret["xds"].isel(block_des)
|
|
49
|
-
nchan = ret["xds"].
|
|
50
|
-
npol = ret["xds"].
|
|
49
|
+
nchan = ret["xds"].sizes["frequency"]
|
|
50
|
+
npol = ret["xds"].sizes["polarization"]
|
|
51
51
|
starts, shapes, slices = _get_starts_shapes_slices(block_des, coords, cshape)
|
|
52
52
|
dimorder = _get_xds_dim_order(ret["sphr_dims"])
|
|
53
53
|
transpose_list, new_axes = _get_transpose_list(coords)
|
|
@@ -105,7 +105,7 @@ def _read_casa_image(
|
|
|
105
105
|
xds = _add_mask(xds, m.upper(), ary, dimorder)
|
|
106
106
|
xds.attrs = _casa_image_to_xds_attrs(img_full_path)
|
|
107
107
|
beam = _get_beam(
|
|
108
|
-
img_full_path, xds.
|
|
108
|
+
img_full_path, xds.sizes["frequency"], xds.sizes["polarization"], True
|
|
109
109
|
)
|
|
110
110
|
if beam is not None:
|
|
111
111
|
xds["BEAM"] = beam
|
|
@@ -133,6 +133,7 @@ def _xds_to_casa_image(xds: xr.Dataset, imagename: str) -> None:
|
|
|
133
133
|
lockoptions={"option": "permanentwait"},
|
|
134
134
|
ack=False,
|
|
135
135
|
)
|
|
136
|
+
|
|
136
137
|
tb.putkeyword("coords", coord)
|
|
137
138
|
tb.putkeyword("imageinfo", ii)
|
|
138
139
|
if units:
|
xradio/image/_util/common.py
CHANGED
|
@@ -110,7 +110,7 @@ def _numpy_arrayize_dv(xds: xr.Dataset) -> xr.Dataset:
|
|
|
110
110
|
def _default_freq_info() -> dict:
|
|
111
111
|
return {
|
|
112
112
|
"rest_frequency": make_quantity(1420405751.7860003, "Hz"),
|
|
113
|
-
"type": "
|
|
113
|
+
"type": "spectral_coord",
|
|
114
114
|
"frame": "lsrk",
|
|
115
115
|
"units": "Hz",
|
|
116
116
|
"waveUnit": "mm",
|
|
@@ -141,7 +141,7 @@ def _freq_from_vel(
|
|
|
141
141
|
vel = vel * u.Unit(cunit)
|
|
142
142
|
v_dict = {
|
|
143
143
|
"value": vel.value,
|
|
144
|
-
"
|
|
144
|
+
"units": cunit,
|
|
145
145
|
"crval": crval,
|
|
146
146
|
"cdelt": cdelt,
|
|
147
147
|
"crpix": crpix,
|
|
@@ -154,7 +154,7 @@ def _freq_from_vel(
|
|
|
154
154
|
fcdelt = -restfreq / _c / (crval * vel.unit / _c + 1) ** 2 * cdelt * vel.unit
|
|
155
155
|
f_dict = {
|
|
156
156
|
"value": freq.value,
|
|
157
|
-
"
|
|
157
|
+
"units": "Hz",
|
|
158
158
|
"crval": fcrval.to(u.Hz).value,
|
|
159
159
|
"cdelt": fcdelt.to(u.Hz).value,
|
|
160
160
|
"crpix": crpix,
|
|
@@ -180,7 +180,7 @@ def _compute_world_sph_dims(
|
|
|
180
180
|
"axis_name": [None, None],
|
|
181
181
|
"ref_val": [None, None],
|
|
182
182
|
"inc": [None, None],
|
|
183
|
-
"
|
|
183
|
+
"units": "rad",
|
|
184
184
|
"value": [None, None],
|
|
185
185
|
}
|
|
186
186
|
for i in range(2):
|
|
@@ -6,7 +6,7 @@ from typing import List, Union
|
|
|
6
6
|
from .common import _c, _compute_world_sph_dims, _l_m_attr_notes
|
|
7
7
|
from xradio._utils.coord_math import _deg_to_rad
|
|
8
8
|
from xradio._utils.dict_helpers import (
|
|
9
|
-
|
|
9
|
+
make_spectral_coord_reference_dict,
|
|
10
10
|
make_quantity,
|
|
11
11
|
make_skycoord_dict,
|
|
12
12
|
make_time_coord_attrs,
|
|
@@ -50,24 +50,24 @@ def _add_common_attrs(
|
|
|
50
50
|
cell_size: Union[List[float], np.ndarray],
|
|
51
51
|
projection: str,
|
|
52
52
|
) -> xr.Dataset:
|
|
53
|
-
xds.time.attrs = make_time_coord_attrs(units=
|
|
53
|
+
xds.time.attrs = make_time_coord_attrs(units="d", scale="utc", time_format="mjd")
|
|
54
54
|
freq_vals = np.array(xds.frequency)
|
|
55
55
|
xds.frequency.attrs = {
|
|
56
56
|
"observer": spectral_reference.lower(),
|
|
57
|
-
"
|
|
57
|
+
"reference_frequency": make_spectral_coord_reference_dict(
|
|
58
58
|
value=freq_vals[len(freq_vals) // 2].item(),
|
|
59
|
-
units=
|
|
59
|
+
units="Hz",
|
|
60
60
|
observer=spectral_reference.lower(),
|
|
61
61
|
),
|
|
62
62
|
"rest_frequencies": make_quantity(restfreq, "Hz"),
|
|
63
63
|
"rest_frequency": make_quantity(restfreq, "Hz"),
|
|
64
|
-
"type": "
|
|
65
|
-
"units":
|
|
66
|
-
"
|
|
64
|
+
"type": "spectral_coord",
|
|
65
|
+
"units": "Hz",
|
|
66
|
+
"wave_units": "mm",
|
|
67
67
|
}
|
|
68
68
|
xds.velocity.attrs = {"doppler_type": "radio", "type": "doppler", "units": "m/s"}
|
|
69
69
|
reference = make_skycoord_dict(
|
|
70
|
-
data=phase_center, units=
|
|
70
|
+
data=phase_center, units="rad", frame=direction_reference
|
|
71
71
|
)
|
|
72
72
|
reference["attrs"].update({"equinox": "j2000.0"})
|
|
73
73
|
xds.attrs = {
|
xradio/image/image.py
CHANGED
|
@@ -15,13 +15,15 @@ import xarray as xr
|
|
|
15
15
|
# from .._utils.zarr.common import _load_no_dask_zarr
|
|
16
16
|
|
|
17
17
|
from ._util.casacore import _load_casa_image_block, _xds_to_casa_image
|
|
18
|
-
|
|
18
|
+
|
|
19
|
+
# from ._util.fits import _read_fits_image
|
|
19
20
|
from ._util.image_factory import (
|
|
20
21
|
_make_empty_aperture_image,
|
|
21
22
|
_make_empty_lmuv_image,
|
|
22
23
|
_make_empty_sky_image,
|
|
23
24
|
)
|
|
24
25
|
from ._util.zarr import _load_image_from_zarr_no_dask, _xds_from_zarr, _xds_to_zarr
|
|
26
|
+
from ._util._fits.xds_from_fits import _fits_image_to_xds
|
|
25
27
|
|
|
26
28
|
warnings.filterwarnings("ignore", category=FutureWarning)
|
|
27
29
|
|
|
@@ -32,12 +34,37 @@ def read_image(
|
|
|
32
34
|
verbose: bool = False,
|
|
33
35
|
do_sky_coords: bool = True,
|
|
34
36
|
selection: dict = {},
|
|
37
|
+
compute_mask: bool = True,
|
|
35
38
|
) -> xr.Dataset:
|
|
36
39
|
"""
|
|
37
40
|
Convert CASA, FITS, or zarr image to xradio image xds format
|
|
38
41
|
ngCASA image spec is located at
|
|
39
42
|
https://docs.google.com/spreadsheets/d/1WW0Gl6z85cJVPgtdgW4dxucurHFa06OKGjgoK8OREFA/edit#gid=1719181934
|
|
40
43
|
|
|
44
|
+
Notes on FITS compatibility and memory mapping:
|
|
45
|
+
|
|
46
|
+
This function relies on Astropy's `memmap=True` to avoid loading full image data into memory.
|
|
47
|
+
However, not all FITS files support memory-mapped reads.
|
|
48
|
+
|
|
49
|
+
⚠️ The following FITS types are incompatible with memory mapping:
|
|
50
|
+
|
|
51
|
+
1. Compressed images (`CompImageHDU`)
|
|
52
|
+
= Workaround: decompress the FITS using tools like `funpack`, `cfitsio`,
|
|
53
|
+
or Astropy's `.scale()`/`.copy()` workflows
|
|
54
|
+
2. Some scaled images (using BSCALE/BZERO headers)
|
|
55
|
+
✅ Supported:
|
|
56
|
+
- Files with no BSCALE/BZERO headers (or BSCALE=1.0 and BZERO=0.0)
|
|
57
|
+
- Uncompressed, unscaled primary HDUs
|
|
58
|
+
⚠️ Unsupported: Files with BSCALE ≠ 1.0 or BZERO ≠ 0.0
|
|
59
|
+
- These require data rescaling in memory, which disables lazy access
|
|
60
|
+
- Attempting to slice such arrays forces eager read of the full dataset
|
|
61
|
+
- Workaround: remove scaling with Astropy's
|
|
62
|
+
`HDU.data = HDU.data * BSCALE + BZERO` and save a new file
|
|
63
|
+
|
|
64
|
+
These cases will raise `RuntimeError` to prevent silent eager loads that can exhaust memory.
|
|
65
|
+
|
|
66
|
+
If you encounter such an error, consider preprocessing the file to make it memory-mappable.
|
|
67
|
+
|
|
41
68
|
Parameters
|
|
42
69
|
----------
|
|
43
70
|
infile : str
|
|
@@ -69,11 +96,19 @@ def read_image(
|
|
|
69
96
|
the selection, and the end pixel is not. An empty dictionary (the
|
|
70
97
|
default) indicates that the entire image should be returned. Currently
|
|
71
98
|
only supported for images stored in zarr format.
|
|
72
|
-
|
|
99
|
+
compute_mask : bool, optional
|
|
100
|
+
If True (default), compute and attach valid data masks when converting from FITS to xds.
|
|
101
|
+
If False, skip mask computation entirely. This may improve performance if the mask
|
|
102
|
+
is not required for subsequent processing. It may, however, result in unpredictable behavior
|
|
103
|
+
for applications that are not designed to handle missing data. It is the user's responsibility,
|
|
104
|
+
not the software's, to ensure that the mask is computed if it is necessary. Currently only
|
|
105
|
+
implemented for FITS images.
|
|
73
106
|
Returns
|
|
74
107
|
-------
|
|
75
108
|
xarray.Dataset
|
|
76
109
|
"""
|
|
110
|
+
# from ._util.casacore import _read_casa_image
|
|
111
|
+
# return _read_casa_image(infile, chunks, verbose, do_sky_coords)
|
|
77
112
|
emsgs = []
|
|
78
113
|
do_casa = True
|
|
79
114
|
try:
|
|
@@ -92,9 +127,10 @@ def read_image(
|
|
|
92
127
|
except Exception as e:
|
|
93
128
|
emsgs.append(f"image format appears not to be casacore: {e.args}")
|
|
94
129
|
# next statement is for debug, comment when done debugging
|
|
95
|
-
# return
|
|
130
|
+
# return _fits_image_to_xds(infile, chunks, verbose, do_sky_coords, compute_mask)
|
|
96
131
|
try:
|
|
97
|
-
|
|
132
|
+
img_full_path = os.path.expanduser(infile)
|
|
133
|
+
return _fits_image_to_xds(infile, chunks, verbose, do_sky_coords, compute_mask)
|
|
98
134
|
except Exception as e:
|
|
99
135
|
emsgs.append(f"image format appears not to be fits {e.args}")
|
|
100
136
|
# when done debuggin comment out next line
|
|
@@ -111,7 +147,7 @@ def read_image(
|
|
|
111
147
|
raise RuntimeError("\n".join(emsgs))
|
|
112
148
|
|
|
113
149
|
|
|
114
|
-
def load_image(infile: str, block_des: dict =
|
|
150
|
+
def load_image(infile: str, block_des: dict = None, do_sky_coords=True) -> xr.Dataset:
|
|
115
151
|
"""
|
|
116
152
|
Load an image or portion of an image (subimage) into memory with data variables
|
|
117
153
|
being converted from dask to numpy arrays and coordinate arrays being converted
|
|
@@ -144,6 +180,10 @@ def load_image(infile: str, block_des: dict = {}, do_sky_coords=True) -> xr.Data
|
|
|
144
180
|
"""
|
|
145
181
|
do_casa = True
|
|
146
182
|
emsgs = []
|
|
183
|
+
|
|
184
|
+
if block_des is None:
|
|
185
|
+
block_des = {}
|
|
186
|
+
|
|
147
187
|
selection = copy.deepcopy(block_des) if block_des else block_des
|
|
148
188
|
if selection:
|
|
149
189
|
for k, v in selection.items():
|
|
@@ -4,13 +4,11 @@ convert, and retrieve information from Processing Set and Measurement Sets nodes
|
|
|
4
4
|
Processing Set DataTree
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
import toolviper.utils.logger as _logger
|
|
8
|
+
|
|
9
|
+
from .processing_set_xdt import ProcessingSetXdt
|
|
8
10
|
from .open_processing_set import open_processing_set
|
|
9
|
-
from .load_processing_set import load_processing_set
|
|
10
|
-
from .convert_msv2_to_processing_set import (
|
|
11
|
-
convert_msv2_to_processing_set,
|
|
12
|
-
estimate_conversion_memory_and_cores,
|
|
13
|
-
)
|
|
11
|
+
from .load_processing_set import load_processing_set
|
|
14
12
|
from .measurement_set_xdt import MeasurementSetXdt
|
|
15
13
|
from .schema import SpectrumXds, VisibilityXds
|
|
16
14
|
|
|
@@ -19,9 +17,21 @@ __all__ = [
|
|
|
19
17
|
"MeasurementSetXdt",
|
|
20
18
|
"open_processing_set",
|
|
21
19
|
"load_processing_set",
|
|
22
|
-
"ProcessingSetIterator",
|
|
23
|
-
"convert_msv2_to_processing_set",
|
|
24
|
-
"estimate_conversion_memory_and_cores",
|
|
25
20
|
"SpectrumXds",
|
|
26
21
|
"VisibilityXds",
|
|
27
22
|
]
|
|
23
|
+
|
|
24
|
+
try:
|
|
25
|
+
from .convert_msv2_to_processing_set import (
|
|
26
|
+
convert_msv2_to_processing_set,
|
|
27
|
+
estimate_conversion_memory_and_cores,
|
|
28
|
+
)
|
|
29
|
+
except ModuleNotFoundError as exc:
|
|
30
|
+
_logger.warning(
|
|
31
|
+
"Could not import the function to convert from MSv2 to MSv4. "
|
|
32
|
+
f"That functionality will not be available. Details: {exc}"
|
|
33
|
+
)
|
|
34
|
+
else:
|
|
35
|
+
__all__.extend(
|
|
36
|
+
["convert_msv2_to_processing_set", "estimate_conversion_memory_and_cores"]
|
|
37
|
+
)
|
|
File without changes
|