xradio 0.0.41__tar.gz → 0.0.42__tar.gz

{xradio-0.0.41/src/xradio.egg-info → xradio-0.0.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xradio
-Version: 0.0.41
+Version: 0.0.42
 Summary:  Xarray Radio Astronomy Data IO
 Author-email: Jan-Willem Steeb <jsteeb@nrao.edu>
 License: BSD 3-Clause License
@@ -78,18 +78,21 @@ Requires-Dist: pandoc; extra == "docs"
 # xradio
 Xarray Radio Astronomy Data IO is still in development.
 
-[![Python 3.9 3.10 3.11](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11-blue)](https://www.python.org/downloads/release/python-380/)
+[![Python 3.9 3.10 3.11 3.12](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%203.12-blue)](https://www.python.org/downloads/release/python-380/)
 
 # Installing
-It is recommended to use the [conda](https://docs.conda.io/projects/conda/en/latest/) environment manager to create a clean, self-contained runtime where xradio and all its dependencies can be installed:
+It is recommended to use the conda environment manager from [miniforge](https://github.com/conda-forge/miniforge) to create a clean, self-contained runtime where XRADIO and all its dependencies can be installed:
 ```sh
-conda create --name xradio python=3.11 --no-default-packages
+conda create --name xradio python=3.12 --no-default-packages
 conda activate xradio
-
 ```
 > 📝 On macOS it is required to pre-install `python-casacore` using `conda install -c conda-forge python-casacore`.
 
-Making xradio available for download from conda-forge directly is pending, so until then the current recommendation is to sully that pristine environment by calling pip [from within conda](https://www.anaconda.com/blog/using-pip-in-a-conda-environment), like this:
+XRADIO can now be installed using:
 ```sh
 pip install xradio
 ```
+This will also install the minimal dependencies for XRADIO. To install the minimal dependencies and the interactive components (JupyterLab) use:
+```sh
+pip install "xradio[interactive]"
+```

xradio-0.0.42/README.md

@@ -0,0 +1,21 @@
+# xradio
+Xarray Radio Astronomy Data IO is still in development.
+
+[![Python 3.9 3.10 3.11 3.12](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%203.12-blue)](https://www.python.org/downloads/release/python-380/)
+
+# Installing
+It is recommended to use the conda environment manager from [miniforge](https://github.com/conda-forge/miniforge) to create a clean, self-contained runtime where XRADIO and all its dependencies can be installed:
+```sh
+conda create --name xradio python=3.12 --no-default-packages
+conda activate xradio
+```
+> 📝 On macOS it is required to pre-install `python-casacore` using `conda install -c conda-forge python-casacore`.
+
+XRADIO can now be installed using:
+```sh
+pip install xradio
+```
+This will also install the minimal dependencies for XRADIO. To install the minimal dependencies and the interactive components (JupyterLab) use:
+```sh
+pip install "xradio[interactive]"
+```

{xradio-0.0.41 → xradio-0.0.42}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "xradio"
-version = "0.0.41"
+version = "0.0.42"
 description = " Xarray Radio Astronomy Data IO"
 authors = [
     {name = "Jan-Willem Steeb", email="jsteeb@nrao.edu"},

xradio-0.0.42/src/xradio/_utils/coord_math.py

@@ -0,0 +1,100 @@
+import numpy as np
+
+_deg_to_rad = np.pi / 180
+
+
+def haversine(ra1, dec1, ra2, dec2):
+    """
+    Calculate the great circle distance between two points
+    on the celestial sphere specified in radians.
+    """
+    # Haversine formula
+    d_ra = ra2 - ra1
+    d_dec = dec2 - dec1
+    a = np.sin(d_dec / 2) ** 2 + np.cos(dec1) * np.cos(dec2) * np.sin(d_ra / 2) ** 2
+    c = 2 * np.arcsin(np.sqrt(a))
+    return c
+
+
+def add_position_offsets(dv_1, dv_2):
+    # Fun with angles: We are adding angles together. We need to make sure that the results are between -pi and pi.
+    new_pos = dv_1 + dv_2
+
+    new_pos = wrap_to_pi(new_pos)
+
+    # while np.any(new_pos[:, 0] > np.pi) or np.any(new_pos[:, 0] < -np.pi):
+    #     new_pos[:, 0] = np.where(
+    #         new_pos[:, 0] > np.pi, new_pos[:, 0] - 2 * np.pi, new_pos[:, 0]
+    #     )
+    #     new_pos[:, 0] = np.where(
+    #         new_pos[:, 0] < -np.pi, new_pos[:, 0] + 2 * np.pi, new_pos[:, 0]
+    #     )
+
+    # while np.any(new_pos[:, 1] > np.pi / 2) or np.any(new_pos[:, 1] < -np.pi / 2):
+    #     new_pos[:, 1] = np.where(
+    #         new_pos[:, 1] > np.pi / 2, new_pos[:, 1] - np.pi, new_pos[:, 1]
+    #     )
+    #     new_pos[:, 1] = np.where(
+    #         new_pos[:, 1] < -np.pi / 2, new_pos[:, 1] + np.pi, new_pos[:, 1]
+    #     )
+
+    return new_pos
+
+
+def wrap_to_pi(angles):
+    """
+    Wraps an array of angles in radians to the range (-π, π].
+
+    Parameters:
+    -----------
+    angles : array_like
+        Input array of angles in radians.
+
+    Returns:
+    --------
+    array_like
+        The input angles wrapped to the range (-π, π].
+
+    Examples:
+    ---------
+    >>> wrap_to_pi(np.array([0, np.pi, -np.pi, 3*np.pi, -3*np.pi]))
+    array([ 0.        ,  3.14159265, -3.14159265,  3.14159265, -3.14159265])
+    """
+    return (angles + np.pi) % (2 * np.pi) - np.pi
+
+
+def wrap_to_pi(angles):
+    return (angles + np.pi) % (2 * np.pi) - np.pi
+
+
+def convert_to_si_units(xds):
+    for data_var in xds.data_vars:
+        if "units" in xds[data_var].attrs:
+            for u_i, u in enumerate(xds[data_var].attrs["units"]):
+                if u == "km":
+                    xds[data_var][..., u_i] = xds[data_var][..., u_i] * 1e3
+                    xds[data_var].attrs["units"][u_i] = "m"
+                if u == "km/s":
+                    xds[data_var][..., u_i] = xds[data_var][..., u_i] * 1e3
+                    xds[data_var].attrs["units"][u_i] = "m/s"
+                if u == "deg":
+                    xds[data_var][..., u_i] = xds[data_var][..., u_i] * np.pi / 180
+                    xds[data_var].attrs["units"][u_i] = "rad"
+                if u == "Au" or u == "AU":
+                    xds[data_var][..., u_i] = xds[data_var][..., u_i] * 149597870700
+                    xds[data_var].attrs["units"][u_i] = "m"
+                if u == "Au/d" or u == "AU/d":
+                    xds[data_var][..., u_i] = (
+                        xds[data_var][..., u_i] * 149597870700 / 86400
+                    )
+                    xds[data_var].attrs["units"][u_i] = "m/s"
+                if u == "arcsec":
+                    xds[data_var][..., u_i] = xds[data_var][..., u_i] * np.pi / 648000
+                    xds[data_var].attrs["units"][u_i] = "rad"
+                if u == "hPa":
+                    xds[data_var][..., u_i] = xds[data_var][..., u_i] * 100.0
+                    xds[data_var].attrs["units"][u_i] = "Pa"
+                if u == "m-2":
+                    # IONOS_ELECTRON sometimes has "m-2" instead of "/m^2"
+                    xds[data_var].attrs["units"][u_i] = "/m^2"
+    return xds

{xradio-0.0.41 → xradio-0.0.42}/src/xradio/_utils/list_and_array.py

@@ -2,6 +2,55 @@
 
 import numpy as np
 import xarray as xr
+import pandas as pd
+
+
+def cast_to_str(x):
+    if isinstance(x, list):
+        return x[0]
+    else:
+        return x
+
+
+def get_pad_value(col_dtype: np.dtype) -> object:
+    """
+    Produce a padding/missing/nan value appropriate for a casacore data column
+    (for when we need to pad data vars coming from columns with rows of
+    variable size array values)
+
+    Parameters
+    ----------
+    col_dtype : dtype
+        dtype of data being loaded from a table column
+
+    Returns
+    -------
+    object
+        pad value ("missing" / "fill") for the type given
+    """
+    # Fill values for missing/NaN data in integer variables, based on usual
+    # numpy fill values. See https://github.com/numpy/numpy/issues/21166,
+    # https://github.com/casangi/xradio/issues/219, https://github.com/casangi/xradio/pull/177
+    fill_value_int32 = np.int32(-2147483648)
+    fill_value_int64 = np.int64(-9223372036854775808)
+
+    if col_dtype == np.int32:
+        return fill_value_int32
+    elif col_dtype == np.int64 or col_dtype == "int":
+        return fill_value_int64
+    elif np.issubdtype(col_dtype, np.floating):
+        return np.nan
+    elif np.issubdtype(col_dtype, np.complexfloating):
+        return complex(np.nan, np.nan)
+    elif np.issubdtype(col_dtype, np.bool_):
+        return False
+    elif np.issubdtype(col_dtype, str):
+        return ""
+    else:
+        raise RuntimeError(
+            "Padding / missing value not defined for the type requested: "
+            f"{col_dtype} (of type: {type(col_dtype)})"
+        )
 
 
 def to_list(x):
@@ -45,10 +94,6 @@ def check_if_consistent(array: np.ndarray, array_name: str) -> np.ndarray:
     return array_unique[0]
 
 
-import numpy as np
-import pandas as pd
-
-
 def unique_1d(array: np.ndarray) -> np.ndarray:
     """
     Optimised version of np.unique for 1D arrays.

{xradio-0.0.41 → xradio-0.0.42}/src/xradio/_utils/schema.py

@@ -93,7 +93,7 @@ def convert_generic_xds_to_xradio_schema(
 
 
 def column_description_casacore_to_msv4_measure(
-    casacore_column_description, ref_code=None, time_format="UNIX"
+    casacore_column_description, ref_code=None, time_format="unix"
 ):
     import numpy as np
 
@@ -136,9 +136,14 @@ def column_description_casacore_to_msv4_measure(
                 casa_ref = msv4_measure_conversion["Ref_map"][casa_ref]
             else:
                 logger.debug(
-                    f"Unknown reference frame for {measinfo['type']} "
-                    f"measure, using verbatim: {casa_ref}"
+                    f"No translation defined for casacore reference frame ({measinfo['type']} "
+                    f"measure), using verbatim: {casa_ref}"
                 )
+                if measinfo["type"] == "direction" and casa_ref == "B1950_VLA":
+                    raise RuntimeError(
+                        f"The direction reference frame {casa_ref} is not supported. "
+                        "Please use CASA to convert the frame to supported frames like J2000 or ICRS."
+                    )
 
             msv4_measure[msv4_measure_conversion["Ref"]] = casa_ref
 
@@ -158,30 +163,45 @@ casacore_to_msv4_measure_type = {
     "quanta": {
         "type": "quantity",
     },
-    "direction": {"type": "sky_coord", "Ref": "frame", "Ref_map": {"J2000": "fk5"}},
-    "epoch": {"type": "time", "Ref": "scale", "Ref_map": {"UTC": "utc"}},
+    "direction": {
+        "type": "sky_coord",
+        "Ref": "frame",
+        "Ref_map": {
+            "AZELGEO": "altaz",
+            "ICRS": "icrs",
+            "J2000": "fk5",
+        },
+    },
+    "epoch": {
+        "type": "time",
+        "Ref": "scale",
+        "Ref_map": {
+            "UTC": "utc",
+            "TAI": "tai",
+        },
+    },
     "frequency": {
         "type": "spectral_coord",
-        "Ref": "frame",
+        "Ref": "observer",
         "Ref_map": {
             "REST": "REST",
-            "LSRK": "LSRK",
-            "LSRD": "LSRD",
+            "LSRK": "lsrk",
+            "LSRD": "lsrd",
             "BARY": "BARY",
             "GEO": "GEO",
             "TOPO": "TOPO",
-            "GALACTO": "GALACTO",
-            "LGROUP": "LGROUP",
-            "CMB": "CMB",
-            "Undefined": "Undefined",
-        },
+        },  # The frames/observer we are not sure if/how to translate to astropy are uppercase
     },
     "position": {
-        "type": "earth_location",
-        "Ref": "ellipsoid",
+        "type": "location",
+        "Ref": "frame",
         "Ref_map": {"ITRF": "GRS80"},
     },
-    "uvw": {"type": "uvw", "Ref": "frame", "Ref_map": {"ITRF": "GRS80"}},
+    "uvw": {
+        "type": "uvw",
+        "Ref": "frame",
+        "Ref_map": {"J2000": "fk5", "APP": "APP"},
+    },
     "radialvelocity": {"type": "quantity"},
 }
 

{xradio-0.0.41 → xradio-0.0.42}/src/xradio/image/_util/_casacore/xds_from_casacore.py

@@ -30,7 +30,7 @@ from ..common import (
     _l_m_attr_notes,
 )
 from ...._utils._casacore.tables import extract_table_attributes, open_table_ro
-from ...._utils.common import _deg_to_rad
+from xradio._utils.coord_math import _deg_to_rad
 
 """
 def _add_coord_attrs(xds: xr.Dataset, icoords: dict, dir_axes: list) -> xr.Dataset:
@@ -204,7 +204,7 @@ def _casa_image_to_xds_attrs(img_full_path: str, history: bool = True) -> dict:
         if k.startswith("telescope"):
             if k == "telescope":
                 telescope["name"] = coord_dict[k]
-            else:
+            elif k in coord_dict:
                 telescope["position"] = coord_dict[k]
                 telescope["position"]["ellipsoid"] = telescope["position"]["refer"]
                 if telescope["position"]["refer"] == "ITRF":
@@ -362,7 +362,7 @@ def _convert_direction_system(
                 'This corresponds to FK5(equinox="J2000") in astropy. '
                 "Metadata will be written appropriately"
             )
-        return ("FK5", "J2000")
+        return ("FK5", "J2000.0")
     elif casa_system == "B1950":
         if verbose:
             logger.info(
@@ -370,9 +370,9 @@ def _convert_direction_system(
                 'This corresponds to FK4(equinox="B1950") in astropy. '
                 "Metadata will be written appropriately"
             )
-        return ("FK4", "B1950")
+        return ("FK4", "B1950.0")
     elif casa_system in ("GALACTIC", "ICRS"):
-        return (casa_system, None)
+        return (casa_system.lower(), None)
     else:
         raise Exception(
             f"astropy does not support frame {casa_system} and this "

{xradio-0.0.41 → xradio-0.0.42}/src/xradio/image/_util/_casacore/xds_to_casacore.py

@@ -100,17 +100,18 @@ def _coord_dict_from_xds(xds: xr.Dataset) -> dict:
     obsdate["m0"]["value"] = xds.coords["time"].values[0]
     coord["obsdate"] = obsdate
     coord["pointingcenter"] = xds.attrs[_pointing_center].copy()
-    telpos = {}
-    telpos["refer"] = xds.attrs["telescope"]["position"]["ellipsoid"]
-    if xds.attrs["telescope"]["position"]["ellipsoid"] == "GRS80":
-        telpos["refer"] = "ITRF"
-    for i in range(3):
-        telpos[f"m{i}"] = {
-            "unit": xds.attrs["telescope"]["position"]["units"][i],
-            "value": xds.attrs["telescope"]["position"]["value"][i],
-        }
-    telpos["type"] = "position"
-    coord["telescopeposition"] = telpos
+    if "position" in xds.attrs["telescope"]:
+        telpos = {}
+        telpos["refer"] = xds.attrs["telescope"]["position"]["ellipsoid"]
+        if xds.attrs["telescope"]["position"]["ellipsoid"] == "GRS80":
+            telpos["refer"] = "ITRF"
+        for i in range(3):
+            telpos[f"m{i}"] = {
+                "unit": xds.attrs["telescope"]["position"]["units"][i],
+                "value": xds.attrs["telescope"]["position"]["value"][i],
+            }
+        telpos["type"] = "position"
+        coord["telescopeposition"] = telpos
     if "l" in xds.coords:
         coord["direction0"] = _compute_direction_dict(xds)
     else:

{xradio-0.0.41 → xradio-0.0.42}/src/xradio/image/_util/_fits/xds_from_fits.py

@@ -15,7 +15,7 @@ from ..common import (
     _image_type,
     _l_m_attr_notes,
 )
-from ...._utils.common import _deg_to_rad
+from xradio._utils.coord_math import _deg_to_rad
 import copy
 import dask
 import dask.array as da
@@ -166,7 +166,7 @@ def _xds_direction_attrs_from_header(helpers: dict, header) -> dict:
     ref_sys = header["RADESYS"]
     ref_eqx = header["EQUINOX"]
     if ref_sys == "FK5" and ref_eqx == 2000:
-        ref_eqx = "J2000"
+        ref_eqx = "J2000.0"
     helpers["ref_sys"] = ref_sys
     helpers["ref_eqx"] = ref_eqx
     # fits does not support conversion frames
@@ -237,21 +237,22 @@ def _get_telescope_metadata(helpers: dict, header) -> dict:
     # The helpers dict is modified in place. header is not modified
     tel = {}
     tel["name"] = header["TELESCOP"]
-    x = header["OBSGEO-X"]
-    y = header["OBSGEO-Y"]
-    z = header["OBSGEO-Z"]
-    xyz = np.array([x, y, z])
-    r = np.sqrt(np.sum(xyz * xyz))
-    lat = np.arcsin(z / r)
-    long = np.arctan2(y, x)
-    tel["position"] = {
-        "type": "position",
-        # I haven't seen a FITS keyword for reference frame of telescope posiiton
-        "ellipsoid": "GRS80",
-        "units": ["rad", "rad", "m"],
-        "value": np.array([long, lat, r]),
-    }
-    helpers["tel_pos"] = tel["position"]
+    if "OBSGEO-x" in header:
+        x = header["OBSGEO-X"]
+        y = header["OBSGEO-Y"]
+        z = header["OBSGEO-Z"]
+        xyz = np.array([x, y, z])
+        r = np.sqrt(np.sum(xyz * xyz))
+        lat = np.arcsin(z / r)
+        long = np.arctan2(y, x)
+        tel["position"] = {
+            "type": "position",
+            # I haven't seen a FITS keyword for reference frame of telescope posiiton
+            "ellipsoid": "GRS80",
+            "units": ["rad", "rad", "m"],
+            "value": np.array([long, lat, r]),
+        }
+        helpers["tel_pos"] = tel["position"]
     return tel
 
 

{xradio-0.0.41 → xradio-0.0.42}/src/xradio/image/_util/_zarr/zarr_low_level.py

@@ -148,7 +148,9 @@ def write_binary_blob_to_disk(arr, file_path, compressor):
 
 
 def write_to_lustre_chunked(
-    file_path, compressed_arr, chunk_size=1024 * 1024 * 128
+    file_path,
+    compressed_arr,
+    chunk_size=1024 * 1024 * 128,
 ):  # 128 MiB chunks
     """
     Writes compressed data to a Lustre file path with chunking.
@@ -158,10 +160,18 @@ def write_to_lustre_chunked(
         compressed_arr: Compressed data array to write.
         chunk_size: Size of each data chunk in bytes (default: 128 MiB).
     """
-    with open(file_path, "wb") as f:
-        for i in range(0, len(compressed_arr), chunk_size):
-            chunk = compressed_arr[i : i + chunk_size]
-            f.write(chunk)
+    fs, items = _get_file_system_and_items(file_path.rsplit("/", 1)[0])
+
+    if isinstance(fs, s3fs.core.S3FileSystem):
+        with fs.open(file_path, "wb") as f:
+            for i in range(0, len(compressed_arr), chunk_size):
+                chunk = compressed_arr[i : i + chunk_size]
+                f.write(chunk)
+    else:
+        with open(file_path, "wb") as f:
+            for i in range(0, len(compressed_arr), chunk_size):
+                chunk = compressed_arr[i : i + chunk_size]
+                f.write(chunk)
 
 
 def read_binary_blob_from_disk(file_path, compressor, dtype=np.float64):
@@ -237,6 +247,7 @@ def write_json_file(data, file_path):
     Returns:
     - None
     """
+
     with open(file_path, "w") as file:
         json.dump(
             data,
@@ -385,13 +396,19 @@ def write_chunk(img_xds, meta, parallel_dims_chunk_id, compressor, image_file):
         else:
             array = img_xds[data_variable_name].values
 
-        z_chunk = zarr.open(
-            os.path.join(image_file, data_variable_name, chunk_name),
-            mode="a",
-            shape=meta["shape"],
-            chunks=meta["chunks"],
-            dtype=meta["dtype"],
+        write_binary_blob_to_disk(
+            array,
+            file_path=os.path.join(image_file, data_variable_name, chunk_name),
             compressor=compressor,
         )
 
-        return z_chunk
+        # z_chunk = zarr.open(
+        #     os.path.join(image_file, data_variable_name, chunk_name),
+        #     mode="a",
+        #     shape=meta["shape"],
+        #     chunks=meta["chunks"],
+        #     dtype=meta["dtype"],
+        #     compressor=compressor,
+        # )
+
+        # return z_chunk

{xradio-0.0.41 → xradio-0.0.42}/src/xradio/image/_util/common.py

@@ -5,7 +5,7 @@ import dask.array as da
 import numpy as np
 from typing import Dict, List
 import xarray as xr
-from ..._utils.common import _deg_to_rad
+from xradio._utils.coord_math import _deg_to_rad
 
 _c = 2.99792458e08 * u.m / u.s
 # OPTICAL = Z

{xradio-0.0.41 → xradio-0.0.42}/src/xradio/image/_util/image_factory.py

@@ -3,7 +3,7 @@ import numpy as np
 import xarray as xr
 from typing import List, Union
 from .common import _c, _compute_world_sph_dims, _l_m_attr_notes
-from ..._utils.common import _deg_to_rad
+from xradio._utils.coord_math import _deg_to_rad
 
 
 def _input_checks(

{xradio-0.0.41/src/xradio/correlated_data → xradio-0.0.42/src/xradio/measurement_set}/__init__.py

@@ -1,15 +1,18 @@
 from .processing_set import ProcessingSet
 from .open_processing_set import open_processing_set
-from .load_processing_set import load_processing_set
+from .load_processing_set import load_processing_set, ProcessingSetIterator
 from .convert_msv2_to_processing_set import convert_msv2_to_processing_set
+from .measurement_set_xds import MeasurementSetXds
 
-from .schema import VisibilityXds
+from .schema import SpectrumXds, VisibilityXds
 
 __all__ = [
+    "ProcessingSet",
+    "MeasurementSetXds",
     "open_processing_set",
     "load_processing_set",
+    "ProcessingSetIterator",
     "convert_msv2_to_processing_set",
+    "SpectrumXds",
     "VisibilityXds",
-    "PointingXds",
-    "AntennaXds",
 ]

xradio-0.0.42/src/xradio/measurement_set/_utils/__init__.py

@@ -0,0 +1,5 @@
+from . import msv2
+from . import zarr
+from . import _utils
+
+__all__ = ["msv2", "zarr", "_utils"]

{xradio-0.0.41/src/xradio/correlated_data/_utils/_ms → xradio-0.0.42/src/xradio/measurement_set/_utils/_msv2}/_tables/load_main_table.py

@@ -12,7 +12,7 @@ from .read_main_table import get_partition_ids, redim_id_data_vars, rename_vars
 from .read import add_units_measures, convert_casacore_time, extract_table_attributes
 from .write import revert_time
 from .table_query import open_query, open_table_ro
-from xradio.correlated_data._utils._ms._tables.read_main_table import (
+from xradio.measurement_set._utils._ms._tables.read_main_table import (
     get_baselines,
     get_baseline_indices,
 )

{xradio-0.0.41/src/xradio/correlated_data/_utils/_ms → xradio-0.0.42/src/xradio/measurement_set/_utils/_msv2}/_tables/read.py

@@ -12,7 +12,7 @@ import astropy.units
 from casacore import tables
 
 from .table_query import open_query, open_table_ro
-from ....._utils.common import get_pad_value
+from xradio._utils.list_and_array import get_pad_value
 
 CASACORE_TO_PD_TIME_CORRECTION = 3_506_716_800.0
 SECS_IN_DAY = 86400