xradio 0.0.51__py3-none-any.whl → 0.0.53__py3-none-any.whl

xradio/image/_util/_fits/xds_from_fits.py

@@ -17,9 +17,10 @@ from ..common import (
 )
 from xradio._utils.coord_math import _deg_to_rad
 from xradio._utils.dict_helpers import (
-    make_quantity,
     make_frequency_reference_dict,
+    make_quantity,
     make_skycoord_dict,
+    make_time_measure_dict,
 )
 import copy
 import dask
@@ -49,7 +50,7 @@ def _fits_image_to_xds(
     sphr_dims = helpers["sphr_dims"]
     ary = _read_image_array(img_full_path, chunks, helpers, verbose)
     dim_order = _get_xds_dim_order(sphr_dims)
-    xds = _add_sky_or_aperture(xds, ary, dim_order, helpers, sphr_dims)
+    xds = _add_sky_or_aperture(xds, ary, dim_order, header, helpers, sphr_dims)
     xds.attrs = attrs
     xds = _add_coord_attrs(xds, helpers)
     if helpers["has_multibeam"]:
@@ -70,13 +71,7 @@ def _add_coord_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
 
 def _add_time_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
     time_coord = xds.coords["time"]
-    meta = copy.deepcopy(helpers["obsdate"])
-    del meta["value"]
-    # meta["units"] = [ meta["units"] ]
-    # meta['format'] = 'MJD'
-    # meta['time_scale'] = meta['refer']
-    # del meta['refer']
-    time_coord.attrs = meta
+    time_coord.attrs = copy.deepcopy(helpers["obsdate"]["attrs"])
     xds.assign_coords(time=time_coord)
     return xds
 
@@ -87,8 +82,6 @@ def _add_freq_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
     if helpers["has_freq"]:
         meta["rest_frequency"] = make_quantity(helpers["restfreq"], "Hz")
         meta["rest_frequencies"] = [meta["rest_frequency"]]
-        # meta["frame"] = helpers["specsys"]
-        # meta["units"] = "Hz"
         meta["type"] = "frequency"
         meta["wave_unit"] = "mm"
         freq_axis = helpers["freq_axis"]
@@ -128,16 +121,6 @@ def _add_l_m_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
 
 
 def _add_lin_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
-    """
-    if helpers["sphr_dims"]:
-        for i, name in zip(helpers["dir_axes"], helpers["sphr_axis_names"]):
-            meta = {
-                "units": "rad",
-                "crval": helpers["crval"][i],
-                "cdelt": helpers["cdelt"][i],
-            }
-            xds.coords[name].attrs = meta
-    """
     if not helpers["sphr_dims"]:
         for i, j in zip(helpers["dir_axes"], ("u", "v")):
             meta = {
@@ -167,7 +150,7 @@ def _xds_direction_attrs_from_header(helpers: dict, header) -> dict:
     direction["projection"] = p0
     helpers["projection"] = p0
     ref_sys = header["RADESYS"]
-    ref_eqx = header["EQUINOX"]
+    ref_eqx = None if ref_sys.upper() == "ICRS" else header["EQUINOX"]
     if ref_sys == "FK5" and ref_eqx == 2000:
         ref_eqx = "J2000.0"
     helpers["ref_sys"] = ref_sys
@@ -176,15 +159,6 @@ def _xds_direction_attrs_from_header(helpers: dict, header) -> dict:
     direction["reference"] = make_skycoord_dict(
         [0.0, 0.0], units=["rad", "rad"], frame=ref_sys
     )
-    """
-    direction["reference"] = {
-        "type": "sky_coord",
-        "frame": ref_sys,
-        "equinox": ref_eqx,
-        "units": ["rad", "rad"],
-        "value": [0.0, 0.0],
-    }
-    """
     dir_axes = helpers["dir_axes"]
     ddata = []
     dunits = []
@@ -194,9 +168,10 @@ def _xds_direction_attrs_from_header(helpers: dict, header) -> dict:
         ddata.append(x.value)
         # direction["reference"]["value"][i] = x.value
         x = helpers["cdelt"][i] * u.Unit(_get_unit(helpers["cunit"][i]))
-        dunits.append(x.to("rad"))
+        dunits.append("rad")
     direction["reference"] = make_skycoord_dict(ddata, units=dunits, frame=ref_sys)
-    direction["reference"]["attrs"]["equinox"] = ref_eqx.lower()
+    if ref_eqx is not None:
+        direction["reference"]["attrs"]["equinox"] = ref_eqx.lower()
     direction["latpole"] = make_quantity(
         header["LATPOLE"] * _deg_to_rad, "rad", dims=["l", "m"]
     )
@@ -261,18 +236,21 @@ def _get_telescope_metadata(helpers: dict, header) -> dict:
         r = np.sqrt(np.sum(xyz * xyz))
         lat = np.arcsin(z / r)
         long = np.arctan2(y, x)
-        tel["position"] = {
-            "type": "position",
-            # I haven't seen a FITS keyword for reference frame of telescope posiiton
-            "ellipsoid": "GRS80",
-            "units": ["rad", "rad", "m"],
-            "value": np.array([long, lat, r]),
+        tel["location"] = {
+            "attrs": {
+                "coordinate_system": "geocentric",
+                # I haven't seen a FITS keyword for reference frame of telescope posiiton
+                "frame": "ITRF",
+                "origin_object_name": "earth",
+                "type": "location",
+                "units": ["rad", "rad", "m"],
+            },
+            "data": np.array([long, lat, r]),
         }
-        helpers["tel_pos"] = tel["position"]
     return tel
 
 
-def _pointing_center_to_metadata(helpers: dict, header) -> dict:
+def _compute_pointing_center(helpers: dict, header) -> dict:
     # Neither helpers or header is modified
     t_axes = helpers["t_axes"]
     long_unit = header[f"CUNIT{t_axes[0]}"]
@@ -285,7 +263,9 @@ def _pointing_center_to_metadata(helpers: dict, header) -> dict:
     pc_lat = float(header[f"CRVAL{t_axes[1]}"]) * unit[1]
     pc_long = pc_long.to(u.rad).value
     pc_lat = pc_lat.to(u.rad).value
-    return {"value": np.array([pc_long, pc_lat]), "initial": True}
+    return make_skycoord_dict(
+        [pc_long, pc_lat], units=["rad", "rad"], frame=helpers["ref_sys"]
+    )
 
 
 def _user_attrs_from_header(header) -> dict:
@@ -294,6 +274,9 @@ def _user_attrs_from_header(header) -> dict:
         "ALTRPIX",
         "ALTRVAL",
         "BITPIX",
+        "BMAJ",
+        "BMIN",
+        "BPA",
         "BSCALE",
         "BTYPE",
         "BUNIT",
@@ -319,21 +302,20 @@ def _user_attrs_from_header(header) -> dict:
     ]
     regex = r"|".join(
         [
-            "^NAXIS\\d?$",
-            "^CRVAL\\d$",
-            "^CRPIX\\d$",
-            "^CTYPE\\d$",
-            "^CDELT\\d$",
-            "^CUNIT\\d$",
-            "^OBSGEO-(X|Y|Z)$",
-            "^P(C|V)\\d_\\d$",
+            r"^NAXIS\d?$",
+            r"^CRVAL\d$",
+            r"^CRPIX\d$",
+            r"^CTYPE\d$",
+            r"^CDELT\d$",
+            r"^CUNIT\d$",
+            r"^OBSGEO-(X|Y|Z)$",
+            r"^P(C|V)0?\d_0?\d",
         ]
     )
     user = {}
     for k, v in header.items():
-        if re.search(regex, k) or k in exclude:
-            continue
-        user[k.lower()] = v
+        if not (re.search(regex, k) or k in exclude):
+            user[k.lower()] = v
     return user
 
 
@@ -382,7 +364,7 @@ def _create_dim_map(helpers: dict, header) -> dict:
     return dim_map
 
 
-def _fits_header_to_xds_attrs(hdulist: fits.hdu.hdulist.HDUList) -> dict:
+def _fits_header_to_xds_attrs(hdulist: fits.hdu.hdulist.HDUList) -> tuple:
     primary = None
     beams = None
     for hdu in hdulist:
@@ -418,9 +400,7 @@ def _fits_header_to_xds_attrs(hdulist: fits.hdu.hdulist.HDUList) -> dict:
     if dir_axes is not None:
         attrs["direction"] = _xds_direction_attrs_from_header(helpers, header)
     # FIXME read fits data in chunks in case all data too large to hold in memory
-    has_mask = da.any(da.isnan(primary.data)).compute()
-    attrs["active_mask"] = "MASK0" if has_mask else None
-    helpers["has_mask"] = has_mask
+    helpers["has_mask"] = da.any(da.isnan(primary.data)).compute()
     beam = _beam_attr_from_header(helpers, header)
     if beam != "mb":
         helpers["beam"] = beam
@@ -432,24 +412,15 @@ def _fits_header_to_xds_attrs(hdulist: fits.hdu.hdulist.HDUList) -> dict:
             helpers["dtype"] = "float64"
         else:
             raise RuntimeError(f'Unhandled data type {header["BITPIX"]}')
-    helpers["btype"] = header["BTYPE"] if "BTYPE" in header else None
-    helpers["bunit"] = header["BUNIT"] if "BUNIT" in header else None
-    attrs["object_name"] = header["OBJECT"] if "OBJECT" in header else None
-    obsdate = {}
-    obsdate["type"] = "time"
-    obsdate["value"] = Time(header["DATE-OBS"], format="isot").mjd
-    obsdate["units"] = ["d"]
-    obsdate["scale"] = header["TIMESYS"]
-    obsdate["format"] = "MJD"
-    attrs["obsdate"] = obsdate
-    helpers["obsdate"] = obsdate
-    attrs["observer"] = header["OBSERVER"]
-    attrs["pointing_center"] = _pointing_center_to_metadata(helpers, header)
-    attrs["description"] = None
-    attrs["telescope"] = _get_telescope_metadata(helpers, header)
+    helpers["obsdate"] = make_time_measure_dict(
+        data=Time(header["DATE-OBS"], format="isot").mjd,
+        units=["d"],
+        scale=header["TIMESYS"],
+        time_format="MJD",
+    )
+
     # TODO complete _make_history_xds when spec has been finalized
     # attrs['history'] = _make_history_xds(header)
-    attrs["user"] = _user_attrs_from_header(header)
     return attrs, helpers, header
 
 
@@ -532,7 +503,7 @@ def _create_coords(
 
 
 def _get_time_values(helpers):
-    return [helpers["obsdate"]["value"]]
+    return [helpers["obsdate"]["data"]]
 
 
 def _get_pol_values(helpers):
@@ -710,14 +681,22 @@ def _add_sky_or_aperture(
     xds: xr.Dataset,
     ary: Union[np.ndarray, da.array],
     dim_order: list,
+    header,
     helpers: dict,
     has_sph_dims: bool,
 ) -> xr.Dataset:
     xda = xr.DataArray(ary, dims=dim_order)
-    image_type = helpers["btype"]
-    unit = helpers["bunit"]
-    xda.attrs[_image_type] = image_type
-    xda.attrs["units"] = unit
+    for h, a in zip(
+        ["BUNIT", "BTYPE", "OBJECT", "OBSERVER"],
+        ["units", _image_type, "object_name", "observer"],
+    ):
+        if h in header:
+            xda.attrs[a] = header[h]
+    xda.attrs["obsdate"] = helpers["obsdate"].copy()
+    xda.attrs["pointing_center"] = _compute_pointing_center(helpers, header)
+    xda.attrs["telescope"] = _get_telescope_metadata(helpers, header)
+    xda.attrs["description"] = None
+    xda.attrs["user"] = _user_attrs_from_header(header)
     name = "SKY" if has_sph_dims else "APERTURE"
     xda = xda.rename(name)
     xds[xda.name] = xda
@@ -727,6 +706,9 @@ def _add_sky_or_aperture(
         mask.attrs = {}
         mask = mask.rename("MASK0")
         xds["MASK0"] = mask
+        xda.attrs["active_mask"] = "MASK0"
+    xda = xda.rename(name)
+    xds[xda.name] = xda
     return xds
 
 

xradio/image/_util/_zarr/common.py

@@ -13,4 +13,4 @@ _np_types = {
     "int64": np.int64,
 }
 
-_top_level_sub_xds = "_attrs_xds"
+_top_level_sub_xds = "_attrs_xds_"

xradio/image/_util/_zarr/xds_from_zarr.py

@@ -12,9 +12,9 @@ from xradio._utils.zarr.common import _get_file_system_and_items
 
 
 def _read_zarr(
-    zarr_store: str, output: dict, selection: dict = {}
+    zarr_store: str, id_dict: dict, selection: dict = {}
 ) -> (xr.Dataset, bool):
-    # supported key/values in output are:
+    # supported key/values in id_dict are:
     # "dv"
     #    what data variables should be returned as.
     #    "numpy": numpy arrays
@@ -29,27 +29,27 @@ def _read_zarr(
     do_dask = False
     do_numpy = False
     do_np_coords = False
-    if "dv" in output:
-        dv = output["dv"]
+    if "dv" in id_dict:
+        dv = id_dict["dv"]
         if dv in ["dask", "numpy"]:
             do_dask = dv == "dask"
             do_numpy = not do_dask
         else:
             raise ValueError(
-                f"Unsupported value {output[dv]} for output[dv]. "
+                f"Unsupported value {id_dict[dv]} for id_dict[dv]. "
                 "Supported values are 'dask' and 'numpy'"
             )
-    if "coords" in output:
-        c = output["coords"]
+    if "coords" in id_dict:
+        c = id_dict["coords"]
         if c == "numpy":
             do_np_coords = True
         else:
             raise ValueError(
-                f"Unexpected value {c} for output[coords]. "
+                f"Unexpected value {c} for id_dict[coords]. "
                 "The supported value is 'numpy'"
             )
     # do not pass selection, because that is only for the top level data vars
-    xds = _decode(xds, zarr_store, output)
+    xds = _decode(xds, zarr_store, id_dict)
     if do_np_coords:
         xds = _coords_to_numpy(xds)
     if do_dask:
@@ -59,11 +59,9 @@ def _read_zarr(
     return xds
 
 
-def _decode(xds: xr.Dataset, zarr_store: str, output: dict) -> (xr.Dataset, bool):
+def _decode(xds: xr.Dataset, zarr_store: str, id_dict: dict) -> xr.Dataset:
     xds.attrs = _decode_dict(xds.attrs, "")
-    sub_xdses = _decode_sub_xdses(zarr_store, output)
-    for k, v in sub_xdses.items():
-        xds.attrs[k] = v
+    _decode_sub_xdses(xds, zarr_store, id_dict)
     return xds
 
 
@@ -85,26 +83,45 @@ def _decode_dict(my_dict: dict, top_key: str) -> dict:
     return my_dict
 
 
-def _decode_sub_xdses(zarr_store: str, output: dict) -> dict:
-    sub_xdses = {}
+def _decode_sub_xdses(xarrayObj, top_dir: str, id_dict: dict) -> None:
+    # FIXME this also needs to support S3
+    # determine immediate subdirs of zarr_store
+    entries = os.scandir(top_dir)
+    for d in entries:
+        path = os.path.join(top_dir, d.name)
+        if os.path.isdir(path):
+            if d.name.startswith(_top_level_sub_xds):
+                ky = d.name[len(_top_level_sub_xds) :]
+                xarrayObj.attrs[ky] = _read_zarr(path, id_dict)
+                # TODO if attrs that are xdses have attrs that are xdses ...
+            else:
+                # descend into the directory
+                _decode_sub_xdses(xarrayObj[d.name], path, id_dict)
 
-    fs, store_contents = _get_file_system_and_items(zarr_store)
 
+"""
+def _decode_sub_xdses(zarr_store: str, id_dict: dict) -> dict:
+    sub_xdses = {}
+    fs, store_contents = _get_file_system_and_items(zarr_store)
     if isinstance(fs, s3fs.core.S3FileSystem):
         # could we just use the items as returned from the helper function..?
         store_tree = fs.walk(zarr_store, topdown=True)
+        # Q: what is prepend_s3 used for? In this version it is defined but not used.
         prepend_s3 = "s3://"
     else:
         store_tree = os.walk(zarr_store, topdown=True)
         prepend_s3 = ""
-
     for root, dirs, files in store_tree:
+        relpath = os.path.relpath(root, zarr_store)
+        print("rpath", relpath)
         for d in dirs:
             if d.startswith(_top_level_sub_xds):
-                xds = _read_zarr(os.sep.join([root, d]), output)
+                xds = _read_zarr(os.sep.join([root, d]), id_dict)
                 # for k, v in xds.data_vars.items():
                 #    xds = xds.drop_vars([k]).assign({k: v.compute()})
-                ky = d[len(_top_level_sub_xds) + 1 :]
+                ky = d[len(_top_level_sub_xds) :]
                 sub_xdses[ky] = xds
-
+    print(f"Sub xdses: {sub_xdses.keys()}")
+    print("return")
     return sub_xdses
+"""

xradio/image/_util/_zarr/xds_to_zarr.py

@@ -1,3 +1,5 @@
+import dask.array as da
+import logging
 import numpy as np
 import xarray as xr
 import os
@@ -5,45 +7,59 @@ from .common import _np_types, _top_level_sub_xds
 
 
 def _write_zarr(xds: xr.Dataset, zarr_store: str):
+    max_chunk_size = 0.95 * 2**30
+    for dv in xds.data_vars:
+        obj = xds[dv]
+        if isinstance(obj, xr.core.dataarray.DataArray) and isinstance(
+            obj.data, da.Array
+        ):
+            # get chunk size to make sure it is small enough to be compressed
+            ary = obj.data
+            chunk_size_bytes = np.prod(ary.chunksize) * np.dtype(ary.dtype).itemsize
+            if chunk_size_bytes > max_chunk_size:
+                raise ValueError(
+                    f"Chunk size of {chunk_size_bytes/1e9} GB for data variable {dv} "
+                    "bytes is too large for compression. To fix this, "
+                    "reduce the chunk size of the dask array in the data variable "
+                    f"by at least a factor of {chunk_size_bytes/max_chunk_size}."
+                )
     xds_copy = xds.copy(deep=True)
-    xds_copy, xds_dict = _encode(xds_copy)
+    sub_xds_dict = _encode(xds_copy, zarr_store)
     z_obj = xds_copy.to_zarr(store=zarr_store, compute=True)
-    _write_sub_xdses(zarr_store, xds_dict, _top_level_sub_xds)
+    if sub_xds_dict:
+        _write_sub_xdses(sub_xds_dict)
 
 
-def _encode(xds: xr.Dataset):
+def _encode(xds: xr.Dataset, top_path: str) -> dict:
     # encode attrs
-    xds.attrs, xds_dict = _encode_dict(xds.attrs)
-    return xds, xds_dict
+    sub_xds_dict = {}
+    _encode_dict(xds.attrs, top_path, sub_xds_dict)
+    for dv in xds.data_vars:
+        _encode_dict(xds[dv].attrs, os.sep.join([top_path, dv]), sub_xds_dict)
+    logging.debug(f"Encoded sub_xds_dict: {sub_xds_dict}")
+    return sub_xds_dict
 
 
-def _encode_dict(my_dict: dict, top_key="") -> tuple:
-    xds_dict = {}
+def _encode_dict(my_dict: dict, top_path: str, sub_xds_dict) -> tuple:
     del_keys = []
     for k, v in my_dict.items():
         if isinstance(v, dict):
-            z = os.sep.join([top_key, k]) if top_key else k
-            my_dict[k], ret_xds_dict = _encode_dict(v, z)
-            if ret_xds_dict:
-                xds_dict[k] = ret_xds_dict
+            z = os.sep.join([top_path, k])
+            _encode_dict(v, z, sub_xds_dict)
         elif isinstance(v, np.ndarray):
             my_dict[k] = {}
             my_dict[k]["_type"] = "numpy.ndarray"
             my_dict[k]["_value"] = v.tolist()
             my_dict[k]["_dtype"] = str(v.dtype)
         elif isinstance(v, xr.Dataset):
-            xds_dict[k] = v.copy(deep=True)
+            sub_xds_dict[os.sep.join([top_path, f"{_top_level_sub_xds}{k}"])] = v.copy(
+                deep=True
+            )
             del_keys.append(k)
     for k in del_keys:
         del my_dict[k]
-    return my_dict, xds_dict
 
 
-def _write_sub_xdses(zarr_store: str, xds_dict: dict, path: str):
-    for k, v in xds_dict.items():
-        my_path = f"{path}_{k}" if path else f"{k}"
-        if isinstance(v, dict):
-            _write_sub_xdses(zarr_store, xds_dict[k], my_path)
-        elif isinstance(v, xr.Dataset):
-            zs = os.sep.join([zarr_store, my_path])
-            z_obj = v.to_zarr(store=zs, compute=True)
+def _write_sub_xdses(sub_xds: dict):
+    for k, v in sub_xds.items():
+        z_obj = v.to_zarr(store=k, compute=True)

xradio/image/_util/casacore.py

@@ -48,7 +48,9 @@ def _load_casa_image_block(infile: str, block_des: dict, do_sky_coords) -> xr.Da
     block = _get_persistent_block(
         image_full_path, shapes, starts, dimorder, transpose_list, new_axes
     )
-    xds = _add_sky_or_aperture(xds, block, dimorder, image_full_path, ret["sphr_dims"])
+    xds = _add_sky_or_aperture(
+        xds, block, dimorder, image_full_path, ret["sphr_dims"], True
+    )
     mymasks = _get_mask_names(image_full_path)
     for m in mymasks:
         full_path = os.sep.join([image_full_path, m])
@@ -57,7 +59,7 @@ def _load_casa_image_block(infile: str, block_des: dict, do_sky_coords) -> xr.Da
         )
         # data vars are all caps by convention
         xds = _add_mask(xds, m.upper(), block, dimorder)
-    xds.attrs = _casa_image_to_xds_attrs(image_full_path, True)
+    xds.attrs = _casa_image_to_xds_attrs(image_full_path)
     mb = _multibeam_array(xds, image_full_path, False)
     if mb is not None:
         selectors = {}
@@ -86,6 +88,7 @@ def _read_casa_image(
         dimorder,
         img_full_path,
         ret["sphr_dims"],
+        history,
     )
     if masks:
         mymasks = _get_mask_names(img_full_path)
@@ -93,7 +96,7 @@ def _read_casa_image(
             ary = _read_image_array(img_full_path, chunks, mask=m, verbose=verbose)
             # data var names are all caps by convention
             xds = _add_mask(xds, m.upper(), ary, dimorder)
-    xds.attrs = _casa_image_to_xds_attrs(img_full_path, history)
+    xds.attrs = _casa_image_to_xds_attrs(img_full_path)
     mb = _multibeam_array(xds, img_full_path, True)
     if mb is not None:
         xds["BEAM"] = mb

xradio/image/image.py

@@ -7,6 +7,9 @@ import warnings
 from typing import List, Union
 import copy
 import numpy as np
+import os
+import shutil
+import toolviper.utils.logger as logger
 import xarray as xr
 
 # from .._utils.zarr.common import _load_no_dask_zarr
@@ -182,27 +185,36 @@ def load_image(infile: str, block_des: dict = {}, do_sky_coords=True) -> xr.Data
 
 
 def write_image(
-    xds: xr.Dataset, imagename: str, out_format: str = "casa", overwrite=False
+    xds: xr.Dataset, imagename: str, out_format: str = "casa", overwrite: bool = False
 ) -> None:
     """
     Convert an xds image to CASA or zarr image.
     xds : xarray.Dataset
         XDS to convert
     imagename : str
-        Path to output CASA image
+        Path to output image
     out_format : str
         Format of output image, currently "casa" and "zarr" are supported
+    overwrite : bool
+        If True, overwrite existing image. Default is False.
     Returns
     -------
     None
     """
+    if os.path.exists(imagename):
+        if overwrite:
+            logger.warning(
+                f"Because overwrite=True, removing existing path {imagename}"
+            )
+            if os.path.isdir(imagename):
+                shutil.rmtree(imagename)
+            else:
+                os.remove(imagename)
+        else:
+            raise FileExistsError(
+                f"Path {imagename} already exists. Set overwrite=True to remove it."
+            )
     my_format = out_format.lower()
-
-    if overwrite:
-        import os
-
-        os.system("rm -rf " + imagename)
-
     if my_format == "casa":
         _xds_to_casa_image(xds, imagename)
     elif my_format == "zarr":

xradio/measurement_set/_utils/_msv2/conversion.py

@@ -16,6 +16,7 @@ from xradio.measurement_set._utils._msv2.msv4_sub_xdss import (
     create_pointing_xds,
     create_system_calibration_xds,
     create_weather_xds,
+    create_phased_array_xds,
 )
 from .msv4_info_dicts import create_info_dicts
 from xradio.measurement_set.schema import MSV4_SCHEMA_VERSION
@@ -1243,6 +1244,14 @@ def convert_and_write_partition(
                 + str(time.time() - start)
             )
 
+        # Create phased array xds
+        phased_array_xds = create_phased_array_xds(
+            in_file,
+            ant_xds.antenna_name,
+            ant_xds.receptor_label,
+            ant_xds.polarization_type,
+        )
+
         start = time.time()
 
         # Time and frequency should always be increasing
@@ -1358,6 +1367,9 @@ def convert_and_write_partition(
         if weather_xds:
             ms_xdt["/weather_xds"] = weather_xds
 
+        if phased_array_xds:
+            ms_xdt["/phased_array_xds"] = phased_array_xds
+
         if storage_backend == "zarr":
             ms_xdt.to_zarr(store=os.path.join(out_file, ms_v4_name))
         elif storage_backend == "netcdf":

xradio/measurement_set/_utils/_msv2/create_field_and_source_xds.py

@@ -17,7 +17,8 @@ from xradio.measurement_set._utils._msv2._tables.read import (
     make_taql_where_between_min_max,
     load_generic_table,
 )
-from xradio._utils.list_and_array import cast_to_str
+from xradio._utils.list_and_array import cast_to_str, get_pad_value
+
 from xradio._utils.coord_math import (
     convert_to_si_units,
     add_position_offsets,
@@ -547,10 +548,16 @@ def pad_missing_sources(
         for source_id in unique_source_ids
         if source_id not in source_xds.coords["SOURCE_ID"]
     ]
+    if len(missing_source_ids) < 1:
+        return source_xds
 
     # would like to use the new-ish xr.pad, but it creates issues with indices/coords and is
     # also not free of overheads, as it for example changes all numeric types to float64
-    missing_source_xds = xr.full_like(source_xds.isel(SOURCE_ID=0), fill_value=np.nan)
+    fill_value = {
+        var_name: get_pad_value(var.dtype)
+        for var_name, var in source_xds.data_vars.items()
+    }
+    missing_source_xds = xr.full_like(source_xds.isel(SOURCE_ID=0), fill_value)
     pad_str = "Unknown"
     pad_str_type = "<U9"
     for var in missing_source_xds.data_vars: