xradio 0.0.38__tar.gz → 0.0.40__tar.gz

{xradio-0.0.38/src/xradio.egg-info → xradio-0.0.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xradio
-Version: 0.0.38
+Version: 0.0.40
 Summary:  Xarray Radio Astronomy Data IO
 Author-email: Jan-Willem Steeb <jsteeb@nrao.edu>
 License: BSD 3-Clause License
@@ -44,9 +44,10 @@ Requires-Dist: astropy
 Requires-Dist: dask
 Requires-Dist: distributed
 Requires-Dist: graphviper
+Requires-Dist: toolviper
 Requires-Dist: matplotlib
 Requires-Dist: numba>=0.57.0
-Requires-Dist: numpy<2.0.0
+Requires-Dist: numpy
 Requires-Dist: prettytable
 Requires-Dist: pytest
 Requires-Dist: pytest-cov

{xradio-0.0.38 → xradio-0.0.40}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "xradio"
-version = "0.0.38"
+version = "0.0.40"
 description = " Xarray Radio Astronomy Data IO"
 authors = [
     {name = "Jan-Willem Steeb", email="jsteeb@nrao.edu"},
@@ -14,9 +14,10 @@ dependencies = [
     'dask',
     'distributed',
     'graphviper',
+    'toolviper',
     'matplotlib',
     'numba>=0.57.0',
-    'numpy<2.0.0',
+    'numpy',
     'prettytable',
     'pytest',
     'pytest-cov',

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/__init__.py

@@ -1,5 +1,5 @@
 import os
-from graphviper.utils.logger import setup_logger
+from toolviper.utils.logger import setup_logger
 
 # _logger_name = "xradio"
 # if os.getenv("VIPER_LOGGER_NAME") != _logger_name:

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/_utils/common.py

@@ -10,6 +10,47 @@ def cast_to_str(x):
         return x
 
 
+def get_pad_value(col_dtype: np.dtype) -> object:
+    """
+    Produce a padding/missing/nan value appropriate for a casacore data column
+    (for when we need to pad data vars coming from columns with rows of
+    variable size array values)
+
+    Parameters
+    ----------
+    col_dtype : dtype
+        dtype of data being loaded from a table column
+
+    Returns
+    -------
+    object
+        pad value ("missing" / "fill") for the type given
+    """
+    # Fill values for missing/NaN data in integer variables, based on usual
+    # numpy fill values. See https://github.com/numpy/numpy/issues/21166,
+    # https://github.com/casangi/xradio/issues/219, https://github.com/casangi/xradio/pull/177
+    fill_value_int32 = np.int32(-2147483648)
+    fill_value_int64 = np.int64(-9223372036854775808)
+
+    if col_dtype == np.int32:
+        return fill_value_int32
+    elif col_dtype == np.int64 or col_dtype == "int":
+        return fill_value_int64
+    elif np.issubdtype(col_dtype, np.floating):
+        return np.nan
+    elif np.issubdtype(col_dtype, np.complexfloating):
+        return complex(np.nan, np.nan)
+    elif np.issubdtype(col_dtype, np.bool_):
+        return False
+    elif np.issubdtype(col_dtype, str):
+        return ""
+    else:
+        raise RuntimeError(
+            "Padding / missing value not defined for the type requested: "
+            f"{col_dtype} (of type: {type(col_dtype)})"
+        )
+
+
 def convert_to_si_units(xds):
     for data_var in xds.data_vars:
         if "units" in xds[data_var].attrs:

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/_utils/schema.py

@@ -1,12 +1,12 @@
-import graphviper.utils.logger as logger
+import toolviper.utils.logger as logger
 import xarray as xr
 
 
 def convert_generic_xds_to_xradio_schema(
     generic_xds: xr.Dataset,
     msv4_xds: xr.Dataset,
-    to_new_data_variables: dict,
-    to_new_coords: dict,
+    to_new_data_variables: dict[str, list],
+    to_new_coords: dict[str, list],
 ) -> xr.Dataset:
     """Converts a generic xarray Dataset to the xradio schema.
 
@@ -55,6 +55,7 @@ def convert_generic_xds_to_xradio_schema(
         "column_descriptions"
     ]
     coords = {}
+    coord_attrs = {}
 
     name_keys = list(generic_xds.data_vars.keys()) + list(generic_xds.coords.keys())
 
@@ -80,7 +81,14 @@ def convert_generic_xds_to_xradio_schema(
                 new_coord[1],
                 generic_xds[key].data,
             )
+
+            if msv4_measure:
+                coord_attrs[new_coord[0]] = msv4_measure
+
     msv4_xds = msv4_xds.assign_coords(coords)
+    for coord, coord_attrs in coord_attrs.items():
+        msv4_xds.coords[coord].attrs.update(coord_attrs)
+
     return msv4_xds
 
 
@@ -102,6 +110,9 @@ def column_description_casacore_to_msv4_measure(
             casacore_column_description["keywords"]["QuantumUnits"]
         )
 
+        # Beware: casa_ref won't be found in cases such as the custom
+        # 'NRAO_GBT_USER/NRAO_GBT_USER_DIR_REF' in POINTING
+        casa_ref = None
         # Reference frame to convert?
         if "Ref" in msv4_measure_conversion:
             # Find reference frame

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/image/_util/_casacore/xds_from_casacore.py

@@ -5,7 +5,7 @@ from typing import List, Tuple, Union
 
 import dask
 import dask.array as da
-import graphviper.utils.logger as logger
+import toolviper.utils.logger as logger
 import numpy as np
 import xarray as xr
 from astropy import units as u

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/image/_util/_zarr/xds_from_zarr.py

@@ -1,7 +1,7 @@
 import copy
 import dask.array as da
 
-# import graphviper.utils.logger as logger
+# import toolviper.utils.logger as logger
 import numpy as np
 import os
 import xarray as xr

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/image/_util/_zarr/zarr_low_level.py

@@ -112,7 +112,7 @@ def write_binary_blob_to_disk(arr, file_path, compressor):
     Returns:
     - None
     """
-    import graphviper.utils.logger as logger
+    import toolviper.utils.logger as logger
 
     # Encode the NumPy array using the codec
     logger.debug("1. Before compressor " + file_path)

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/vis/_vis_utils/_ms/_tables/load.py

@@ -1,8 +1,10 @@
 from typing import Tuple
 
-from casacore import tables
 import numpy as np
 
+from casacore import tables
+from ....._utils.common import get_pad_value
+
 
 def load_col_chunk(
     tb_tool: tables.table,
@@ -53,11 +55,8 @@ def load_col_chunk(
         data = tb_tool.getcolslice(col, (d1[0], d2[0]), (d1[1], d2[1]), [], 0, -1)
 
     # full data is the maximum of the data shape and chunk shape dimensions
-    policy = "warn"
-    if np.issubdtype(data.dtype, np.integer):
-        policy = "ignore"
-    with np.errstate(invalid=policy):
-        chunk = np.full(cshape, np.nan, dtype=data.dtype)
+    fill_value = get_pad_value(data.dtype)
+    chunk = np.full(cshape, np.fill_value, dtype=data.dtype)
     if len(didxs) > 0:
         chunk[tidxs[didxs], bidxs[didxs]] = data[didxs]
 

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/vis/_vis_utils/_ms/_tables/load_main_table.py

@@ -1,4 +1,4 @@
-import graphviper.utils.logger as logger
+import toolviper.utils.logger as logger
 from typing import Dict, List, Tuple, Union
 
 import pandas as pd

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/vis/_vis_utils/_ms/_tables/read.py

@@ -1,4 +1,4 @@
-import graphviper.utils.logger as logger
+import toolviper.utils.logger as logger
 import os
 from pathlib import Path
 import re
@@ -12,6 +12,7 @@ import astropy.units
 from casacore import tables
 
 from .table_query import open_query, open_table_ro
+from ....._utils.common import get_pad_value
 
 CASACORE_TO_PD_TIME_CORRECTION = 3_506_716_800.0
 SECS_IN_DAY = 86400
@@ -393,35 +394,6 @@ def make_freq_attrs(spw_xds: xr.Dataset, spw_id: int) -> Dict[str, Any]:
     return cf_attrs
 
 
-def get_pad_nan(col: np.ndarray) -> np.ndarray:
-    """
-    Produce a padding/nan value appropriate for a data column
-    (for when we need to pad data vars coming from columns with rows of
-    variable size array values)
-
-    Parameters
-    ----------
-    col : np.ndarray
-        data being loaded from a table column
-
-    Returns
-    -------
-    np.ndarray
-        nan ("nan") value for the type of the input column
-    """
-    # This is causing frequent warnings for integers. Cast of nan to "int nan"
-    # produces -2147483648 but also seems to trigger a
-    # "RuntimeWarning: invalid value encountered in cast" (new in numpy>=1.24)
-    policy = "warn"
-    col_type = np.array(col).dtype
-    if np.issubdtype(col_type, np.integer):
-        policy = "ignore"
-    with np.errstate(invalid=policy):
-        pad_nan = np.array([np.nan]).astype(col_type)[0]
-
-    return pad_nan
-
-
 def redimension_ms_subtable(xds: xr.Dataset, subt_name: str) -> xr.Dataset:
     """
     Expand a MeasurementSet subtable xds from single dimension (row)
@@ -471,13 +443,12 @@ def redimension_ms_subtable(xds: xr.Dataset, subt_name: str) -> xr.Dataset:
         #   (ANTENNA_ID=0, TIME=0) and no other columns to figure out the right IDs, such
         #   as "NS_WX_STATION_ID" or similar. (example: X425.pm04.scan4.ms)
         # - Some GBT MSs have duplicated (ANTENNA_ID=0, TIME=xxx). (example: analytic_variable.ms)
-        with np.errstate(invalid="ignore"):
-            rxds = (
-                rxds.set_index(row=key_dims)
-                .drop_duplicates("row")
-                .unstack("row")
-                .transpose(*key_dims, ...)
-            )
+        rxds = (
+            rxds.set_index(row=key_dims)
+            .drop_duplicates("row")
+            .unstack("row")
+            .transpose(*key_dims, ...)
+        )
         # unstack changes type to float when it needs to introduce NaNs, so
         # we need to reset to the original type.
         for var in rxds.data_vars:
@@ -999,13 +970,47 @@ def raw_col_data_to_coords_vars(
     return array_type, array_data
 
 
+def get_pad_value_in_tablerow_column(trows: tables.tablerow, col: str) -> object:
+    """
+    Gets the pad value for the type of a column (IMPORTANTLY) as froun in the
+    the type specified in the row / column value dict returned by tablerow.
+    This can differ from the type of the column as given in the casacore
+    column descriptions. See https://github.com/casangi/xradio/issues/242.
+
+    Parameters
+    ----------
+    trows : tables.tablerow
+        list of rows from a table as loaded by tables.row()
+    col: str
+        get the pad value for this column
+
+    Returns
+    -------
+    object
+        pad value as produced by get_pad_value for the appropriate data type from
+    tablerow
+    """
+    col_value = trows[0][col]
+    if isinstance(col_value, np.ndarray):
+        col_dtype = col_value.dtype
+    elif isinstance(col_value, list):
+        col_dtype = type(col_value[0])
+    else:
+        raise RuntimeError(
+            "Found unexpected type (not np.array or list) in column value of "
+            f"first row of column {col}: {col_value}"
+        )
+
+    return get_pad_value(col_dtype)
+
+
 def handle_variable_col_issues(
     inpath: str, col: str, col_type: str, trows: tables.tablerow
 ) -> np.ndarray:
     """
-    load variable-size array columns, padding with nans wherever
-    needed. This happens for example often in the SPECTRAL_WINDOW
-    table (CHAN_WIDTH, EFFECTIVE_BW, etc.).
+    load variable-size array columns, padding with missing/fill/nans
+    wherever needed. This happens for example often in the
+    SPECTRAL_WINDOW table (CHAN_WIDTH, EFFECTIVE_BW, etc.).
     Also handle exceptions gracefully when trying to load the rows.
 
     Parameters
@@ -1015,7 +1020,7 @@ def handle_variable_col_issues(
     col : str
         column being loaded
     col_type : str
-        type of the column cell values
+        type of the column cell values (as numpy dtype string)
     trows : tables.tablerow
         rows from a table as loaded by tables.row()
 
@@ -1030,7 +1035,8 @@ def handle_variable_col_issues(
 
     mshape = np.array(max([np.array(row[col]).shape for row in trows]))
     try:
-        pad_nan = get_pad_nan(np.array((), dtype=col_type))
+        pad_val = None
+        pad_val = get_pad_value_in_tablerow_column(trows, col)
 
         # TODO
         # benchmark np.stack() performance
@@ -1044,13 +1050,13 @@ def handle_variable_col_issues(
                     ),
                     [(0, ss) for ss in mshape - np.array(row[col]).shape],
                     "constant",
-                    constant_values=pad_nan,
+                    constant_values=pad_val,
                 )
                 for row in trows
             ]
         )
     except Exception as exc:
-        msg = f"{inpath}: failed to load data for column {col}: {exc}"
+        msg = f"{inpath}: failed to load data for column {col}, with {pad_val=}: {exc}"
         if col in known_misbehaving_cols:
             logger.debug(msg)
         else:
@@ -1178,12 +1184,8 @@ def read_col_chunk(
             elif len(cshape) == 4:  # DATA and FLAG
                 data = query.getcolslice(col, (d1[0], d2[0]), (d1[1], d2[1]), [], 0, -1)
 
-    policy = "warn"
-    if np.issubdtype(data.dtype, np.integer):
-        policy = "ignore"
-    with np.errstate(invalid=policy):
-        # full data is the maximum of the data shape and chunk shape dimensions
-        fulldata = np.full(cshape, np.nan, dtype=data.dtype)
+    fill_value = get_pad_value(data.dtype)
+    fulldata = np.full(cshape, fill_value, dtype=data.dtype)
 
     if len(didxs) > 0:
         fulldata[tidxs[didxs], bidxs[didxs]] = data[didxs]
@@ -1250,9 +1252,11 @@ def read_col_conversion(
 
     # Get dtype of the column. Only read first row from disk
     col_dtype = np.array(tb_tool.col(col)[0]).dtype
+    # Use a custom/safe fill value (https://github.com/casangi/xradio/issues/219)
+    fill_value = get_pad_value(col_dtype)
 
     # Construct a numpy array to populate. `data` has shape (n_times, n_baselines, n_frequencies, n_polarizations)
-    data = np.full(cshape + extra_dimensions, np.nan, dtype=col_dtype)
+    data = np.full(cshape + extra_dimensions, fill_value, dtype=col_dtype)
 
     # Use built-in casacore table iterator to populate the data column by unique times.
     if use_table_iter:
@@ -1261,7 +1265,9 @@ def read_col_conversion(
             num_rows = ts.nrows()
 
             # Create small temporary array to store the partial column
-            tmp_arr = np.full((num_rows,) + extra_dimensions, np.nan, dtype=col_dtype)
+            tmp_arr = np.full(
+                (num_rows,) + extra_dimensions, fill_value, dtype=col_dtype
+            )
 
             # Note we don't use `getcol()` because it's less safe. See:
             # https://github.com/casacore/python-casacore/issues/130#issuecomment-463202373

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/vis/_vis_utils/_ms/_tables/read_main_table.py

@@ -1,4 +1,4 @@
-import graphviper.utils.logger as logger
+import toolviper.utils.logger as logger
 from typing import Any, Dict, List, Tuple, Union
 
 import dask, dask.array

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/vis/_vis_utils/_ms/_tables/read_subtables.py

@@ -1,4 +1,4 @@
-import graphviper.utils.logger as logger
+import toolviper.utils.logger as logger
 from pathlib import Path
 from typing import Dict, Tuple, Union
 

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/vis/_vis_utils/_ms/_tables/write.py

@@ -1,4 +1,4 @@
-import graphviper.utils.logger as logger, os
+import toolviper.utils.logger as logger, os
 from typing import Tuple
 
 import numpy as np

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/vis/_vis_utils/_ms/conversion.py

@@ -2,7 +2,7 @@ import numcodecs
 import time
 from .._zarr.encoding import add_encoding
 from typing import Dict, Union
-import graphviper.utils.logger as logger
+import toolviper.utils.logger as logger
 import os
 import pathlib
 

{xradio-0.0.38 → xradio-0.0.40}/src/xradio/vis/_vis_utils/_ms/create_antenna_xds.py

@@ -1,4 +1,4 @@
-import graphviper.utils.logger as logger
+import toolviper.utils.logger as logger
 import time
 from typing import Tuple, Union