xradio 0.0.37__tar.gz → 0.0.39__tar.gz

{xradio-0.0.37/src/xradio.egg-info → xradio-0.0.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xradio
-Version: 0.0.37
+Version: 0.0.39
 Summary:  Xarray Radio Astronomy Data IO
 Author-email: Jan-Willem Steeb <jsteeb@nrao.edu>
 License: BSD 3-Clause License
@@ -37,17 +37,16 @@ License: BSD 3-Clause License
         OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
         OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         
-Requires-Python: <3.12,>=3.9
+Requires-Python: <3.13,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: astropy
-Requires-Dist: casadata
 Requires-Dist: dask
 Requires-Dist: distributed
 Requires-Dist: graphviper
 Requires-Dist: matplotlib
 Requires-Dist: numba>=0.57.0
-Requires-Dist: numpy<2.0.0
+Requires-Dist: numpy
 Requires-Dist: prettytable
 Requires-Dist: pytest
 Requires-Dist: pytest-cov
@@ -58,7 +57,7 @@ Requires-Dist: tqdm
 Requires-Dist: xarray
 Requires-Dist: zarr
 Requires-Dist: pyarrow
-Requires-Dist: python_casacore>=3.5.2; sys_platform != "darwin"
+Requires-Dist: python_casacore>=3.6.1; sys_platform != "darwin"
 Requires-Dist: typing_extensions; python_version < "3.10"
 Requires-Dist: typeguard
 Provides-Extra: docs

{xradio-0.0.37 → xradio-0.0.39}/pyproject.toml

@@ -1,23 +1,22 @@
 [project]
 name = "xradio"
-version = "0.0.37"
+version = "0.0.39"
 description = " Xarray Radio Astronomy Data IO"
 authors = [
     {name = "Jan-Willem Steeb", email="jsteeb@nrao.edu"},
 ]
 license = {file = "LICENSE.txt"}
 readme = "README.md"
-requires-python = ">= 3.9, < 3.12"
+requires-python = ">= 3.9, < 3.13"
 
 dependencies = [
     'astropy',
-    'casadata',
     'dask',
     'distributed',
     'graphviper',
     'matplotlib',
     'numba>=0.57.0',
-    'numpy<2.0.0',
+    'numpy',
     'prettytable',
     'pytest',
     'pytest-cov',
@@ -28,7 +27,7 @@ dependencies = [
     'xarray',
     'zarr',
     'pyarrow',
-    'python_casacore>=3.5.2; sys_platform != "darwin" ',
+    'python_casacore>=3.6.1; sys_platform != "darwin" ',
     'typing_extensions; python_version < "3.10"',
     'typeguard'
 ]

{xradio-0.0.37 → xradio-0.0.39}/src/xradio/_utils/common.py

@@ -10,6 +10,47 @@ def cast_to_str(x):
         return x
 
 
+def get_pad_value(col_dtype: np.dtype) -> object:
+    """
+    Produce a padding/missing/nan value appropriate for a casacore data column
+    (for when we need to pad data vars coming from columns with rows of
+    variable size array values)
+
+    Parameters
+    ----------
+    col_dtype : dtype
+        dtype of data being loaded from a table column
+
+    Returns
+    -------
+    object
+        pad value ("missing" / "fill") for the type given
+    """
+    # Fill values for missing/NaN data in integer variables, based on usual
+    # numpy fill values. See https://github.com/numpy/numpy/issues/21166,
+    # https://github.com/casangi/xradio/issues/219, https://github.com/casangi/xradio/pull/177
+    fill_value_int32 = np.int32(-2147483648)
+    fill_value_int64 = np.int64(-9223372036854775808)
+
+    if col_dtype == np.int32:
+        return fill_value_int32
+    elif col_dtype == np.int64 or col_dtype == "int":
+        return fill_value_int64
+    elif np.issubdtype(col_dtype, np.floating):
+        return np.nan
+    elif np.issubdtype(col_dtype, np.complexfloating):
+        return complex(np.nan, np.nan)
+    elif np.issubdtype(col_dtype, np.bool_):
+        return False
+    elif np.issubdtype(col_dtype, str):
+        return ""
+    else:
+        raise RuntimeError(
+            "Padding / missing value not defined for the type requested: "
+            f"{col_dtype} (of type: {type(col_dtype)})"
+        )
+
+
 def convert_to_si_units(xds):
     for data_var in xds.data_vars:
         if "units" in xds[data_var].attrs:

{xradio-0.0.37 → xradio-0.0.39}/src/xradio/vis/_vis_utils/_ms/_tables/load.py

@@ -1,8 +1,10 @@
 from typing import Tuple
 
-from casacore import tables
 import numpy as np
 
+from casacore import tables
+from ....._utils.common import get_pad_value
+
 
 def load_col_chunk(
     tb_tool: tables.table,
@@ -53,11 +55,8 @@ def load_col_chunk(
         data = tb_tool.getcolslice(col, (d1[0], d2[0]), (d1[1], d2[1]), [], 0, -1)
 
     # full data is the maximum of the data shape and chunk shape dimensions
-    policy = "warn"
-    if np.issubdtype(data.dtype, np.integer):
-        policy = "ignore"
-    with np.errstate(invalid=policy):
-        chunk = np.full(cshape, np.nan, dtype=data.dtype)
+    fill_value = get_pad_value(data.dtype)
+    chunk = np.full(cshape, np.fill_value, dtype=data.dtype)
     if len(didxs) > 0:
         chunk[tidxs[didxs], bidxs[didxs]] = data[didxs]
 

{xradio-0.0.37 → xradio-0.0.39}/src/xradio/vis/_vis_utils/_ms/_tables/read.py

@@ -12,6 +12,7 @@ import astropy.units
 from casacore import tables
 
 from .table_query import open_query, open_table_ro
+from ....._utils.common import get_pad_value
 
 CASACORE_TO_PD_TIME_CORRECTION = 3_506_716_800.0
 SECS_IN_DAY = 86400
@@ -393,35 +394,6 @@ def make_freq_attrs(spw_xds: xr.Dataset, spw_id: int) -> Dict[str, Any]:
     return cf_attrs
 
 
-def get_pad_nan(col: np.ndarray) -> np.ndarray:
-    """
-    Produce a padding/nan value appropriate for a data column
-    (for when we need to pad data vars coming from columns with rows of
-    variable size array values)
-
-    Parameters
-    ----------
-    col : np.ndarray
-        data being loaded from a table column
-
-    Returns
-    -------
-    np.ndarray
-        nan ("nan") value for the type of the input column
-    """
-    # This is causing frequent warnings for integers. Cast of nan to "int nan"
-    # produces -2147483648 but also seems to trigger a
-    # "RuntimeWarning: invalid value encountered in cast" (new in numpy>=1.24)
-    policy = "warn"
-    col_type = np.array(col).dtype
-    if np.issubdtype(col_type, np.integer):
-        policy = "ignore"
-    with np.errstate(invalid=policy):
-        pad_nan = np.array([np.nan]).astype(col_type)[0]
-
-    return pad_nan
-
-
 def redimension_ms_subtable(xds: xr.Dataset, subt_name: str) -> xr.Dataset:
     """
     Expand a MeasurementSet subtable xds from single dimension (row)
@@ -471,13 +443,12 @@ def redimension_ms_subtable(xds: xr.Dataset, subt_name: str) -> xr.Dataset:
         #   (ANTENNA_ID=0, TIME=0) and no other columns to figure out the right IDs, such
         #   as "NS_WX_STATION_ID" or similar. (example: X425.pm04.scan4.ms)
         # - Some GBT MSs have duplicated (ANTENNA_ID=0, TIME=xxx). (example: analytic_variable.ms)
-        with np.errstate(invalid="ignore"):
-            rxds = (
-                rxds.set_index(row=key_dims)
-                .drop_duplicates("row")
-                .unstack("row")
-                .transpose(*key_dims, ...)
-            )
+        rxds = (
+            rxds.set_index(row=key_dims)
+            .drop_duplicates("row")
+            .unstack("row")
+            .transpose(*key_dims, ...)
+        )
         # unstack changes type to float when it needs to introduce NaNs, so
         # we need to reset to the original type.
         for var in rxds.data_vars:
@@ -999,13 +970,47 @@ def raw_col_data_to_coords_vars(
     return array_type, array_data
 
 
+def get_pad_value_in_tablerow_column(trows: tables.tablerow, col: str) -> object:
+    """
+    Gets the pad value for the type of a column (IMPORTANTLY) as froun in the
+    the type specified in the row / column value dict returned by tablerow.
+    This can differ from the type of the column as given in the casacore
+    column descriptions. See https://github.com/casangi/xradio/issues/242.
+
+    Parameters
+    ----------
+    trows : tables.tablerow
+        list of rows from a table as loaded by tables.row()
+    col: str
+        get the pad value for this column
+
+    Returns
+    -------
+    object
+        pad value as produced by get_pad_value for the appropriate data type from
+    tablerow
+    """
+    col_value = trows[0][col]
+    if isinstance(col_value, np.ndarray):
+        col_dtype = col_value.dtype
+    elif isinstance(col_value, list):
+        col_dtype = type(col_value[0])
+    else:
+        raise RuntimeError(
+            "Found unexpected type (not np.array or list) in column value of "
+            f"first row of column {col}: {col_value}"
+        )
+
+    return get_pad_value(col_dtype)
+
+
 def handle_variable_col_issues(
     inpath: str, col: str, col_type: str, trows: tables.tablerow
 ) -> np.ndarray:
     """
-    load variable-size array columns, padding with nans wherever
-    needed. This happens for example often in the SPECTRAL_WINDOW
-    table (CHAN_WIDTH, EFFECTIVE_BW, etc.).
+    load variable-size array columns, padding with missing/fill/nans
+    wherever needed. This happens for example often in the
+    SPECTRAL_WINDOW table (CHAN_WIDTH, EFFECTIVE_BW, etc.).
     Also handle exceptions gracefully when trying to load the rows.
 
     Parameters
@@ -1015,7 +1020,7 @@ def handle_variable_col_issues(
     col : str
         column being loaded
     col_type : str
-        type of the column cell values
+        type of the column cell values (as numpy dtype string)
     trows : tables.tablerow
         rows from a table as loaded by tables.row()
 
@@ -1030,7 +1035,8 @@ def handle_variable_col_issues(
 
     mshape = np.array(max([np.array(row[col]).shape for row in trows]))
     try:
-        pad_nan = get_pad_nan(np.array((), dtype=col_type))
+        pad_val = None
+        pad_val = get_pad_value_in_tablerow_column(trows, col)
 
         # TODO
         # benchmark np.stack() performance
@@ -1044,13 +1050,13 @@ def handle_variable_col_issues(
                     ),
                     [(0, ss) for ss in mshape - np.array(row[col]).shape],
                     "constant",
-                    constant_values=pad_nan,
+                    constant_values=pad_val,
                 )
                 for row in trows
             ]
         )
     except Exception as exc:
-        msg = f"{inpath}: failed to load data for column {col}: {exc}"
+        msg = f"{inpath}: failed to load data for column {col}, with {pad_val=}: {exc}"
         if col in known_misbehaving_cols:
             logger.debug(msg)
         else:
@@ -1178,12 +1184,8 @@ def read_col_chunk(
             elif len(cshape) == 4:  # DATA and FLAG
                 data = query.getcolslice(col, (d1[0], d2[0]), (d1[1], d2[1]), [], 0, -1)
 
-    policy = "warn"
-    if np.issubdtype(data.dtype, np.integer):
-        policy = "ignore"
-    with np.errstate(invalid=policy):
-        # full data is the maximum of the data shape and chunk shape dimensions
-        fulldata = np.full(cshape, np.nan, dtype=data.dtype)
+    fill_value = get_pad_value(data.dtype)
+    fulldata = np.full(cshape, fill_value, dtype=data.dtype)
 
     if len(didxs) > 0:
         fulldata[tidxs[didxs], bidxs[didxs]] = data[didxs]
@@ -1250,9 +1252,11 @@ def read_col_conversion(
 
     # Get dtype of the column. Only read first row from disk
     col_dtype = np.array(tb_tool.col(col)[0]).dtype
+    # Use a custom/safe fill value (https://github.com/casangi/xradio/issues/219)
+    fill_value = get_pad_value(col_dtype)
 
     # Construct a numpy array to populate. `data` has shape (n_times, n_baselines, n_frequencies, n_polarizations)
-    data = np.full(cshape + extra_dimensions, np.nan, dtype=col_dtype)
+    data = np.full(cshape + extra_dimensions, fill_value, dtype=col_dtype)
 
     # Use built-in casacore table iterator to populate the data column by unique times.
     if use_table_iter:
@@ -1261,7 +1265,9 @@ def read_col_conversion(
             num_rows = ts.nrows()
 
             # Create small temporary array to store the partial column
-            tmp_arr = np.full((num_rows,) + extra_dimensions, np.nan, dtype=col_dtype)
+            tmp_arr = np.full(
+                (num_rows,) + extra_dimensions, fill_value, dtype=col_dtype
+            )
 
             # Note we don't use `getcol()` because it's less safe. See:
             # https://github.com/casacore/python-casacore/issues/130#issuecomment-463202373

{xradio-0.0.37 → xradio-0.0.39}/src/xradio/vis/_vis_utils/_ms/conversion.py

@@ -976,11 +976,7 @@ def convert_and_write_partition(
 
             file_name = os.path.join(
                 out_file,
-                pathlib.Path(out_file)
-                .name.replace(".vis.zarr", "")
-                .replace(".zarr", "")
-                + "_"
-                + str(ms_v4_id),
+                pathlib.Path(in_file).name.replace(".ms", "") + "_" + str(ms_v4_id),
             )
 
             if "line_name" in field_and_source_xds.coords:

{xradio-0.0.37 → xradio-0.0.39}/src/xradio/vis/_vis_utils/_utils/xds_helper.py

@@ -1,4 +1,4 @@
-from importlib_metadata import version
+from importlib.metadata import version
 import graphviper.utils.logger as logger, multiprocessing, psutil
 from typing import Any, Dict, List, Tuple, Union
 
@@ -7,6 +7,7 @@ import xarray as xr
 
 from .cds import CASAVisSet
 from .stokes_types import stokes_types
+from ...._utils.common import get_pad_value
 
 
 def make_coords(
@@ -195,7 +196,8 @@ def expand_xds(xds: xr.Dataset) -> xr.Dataset:
     except Exception as exc:
         logger.warning(
             f"WARNING: Cannot expand rows to (time, baseline), "
-            f"possibly duplicate values in (time, baseline). Exception: {exc}"
+            f"possibly duplicate values in (time, baseline). Exception: {exc}."
+            f"\nDataset: {txds=}"
         )
         txds = xds.copy()
 
@@ -214,10 +216,8 @@ def flatten_xds(xds: xr.Dataset) -> xr.Dataset:
     Returns
     -------
     xr.Dataset
+        Dataset in flat form (back to 'row' dimension as read by casacore tables)
     """
-    # known invalid cast warning when casting to integer
-    with np.errstate(invalid="ignore"):
-        nan_int = np.array([np.nan]).astype("int32")[0]
     txds = xds.copy()
 
     # flatten the time x baseline dimensions of main table
@@ -226,14 +226,12 @@ def flatten_xds(xds: xr.Dataset) -> xr.Dataset:
         # compute for issue https://github.com/hainegroup/oceanspy/issues/332
         # drop=True silently does compute (or at least used to)
 
-        # Skip this step for now since on Mac nan_int=0. See issue https://github.com/casangi/xradio/issues/219
-        # txds = txds.where(
-        #     ((txds.STATE_ID != nan_int) & (txds.FIELD_ID != nan_int)).compute(),
-        #     drop=True,
-        # )  # .unify_chunks()
-
+        fill_value_int32 = get_pad_value(np.int32)
         txds = txds.where(
-            ~np.isnan(txds["EXPOSURE"]).compute(),
+            (
+                (txds.STATE_ID != fill_value_int32)
+                & (txds.FIELD_ID != fill_value_int32)
+            ).compute(),
             drop=True,
         )  # .unify_chunks()
 

{xradio-0.0.37 → xradio-0.0.39/src/xradio.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xradio
-Version: 0.0.37
+Version: 0.0.39
 Summary:  Xarray Radio Astronomy Data IO
 Author-email: Jan-Willem Steeb <jsteeb@nrao.edu>
 License: BSD 3-Clause License
@@ -37,17 +37,16 @@ License: BSD 3-Clause License
         OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
         OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         
-Requires-Python: <3.12,>=3.9
+Requires-Python: <3.13,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: astropy
-Requires-Dist: casadata
 Requires-Dist: dask
 Requires-Dist: distributed
 Requires-Dist: graphviper
 Requires-Dist: matplotlib
 Requires-Dist: numba>=0.57.0
-Requires-Dist: numpy<2.0.0
+Requires-Dist: numpy
 Requires-Dist: prettytable
 Requires-Dist: pytest
 Requires-Dist: pytest-cov
@@ -58,7 +57,7 @@ Requires-Dist: tqdm
 Requires-Dist: xarray
 Requires-Dist: zarr
 Requires-Dist: pyarrow
-Requires-Dist: python_casacore>=3.5.2; sys_platform != "darwin"
+Requires-Dist: python_casacore>=3.6.1; sys_platform != "darwin"
 Requires-Dist: typing_extensions; python_version < "3.10"
 Requires-Dist: typeguard
 Provides-Extra: docs

{xradio-0.0.37 → xradio-0.0.39}/src/xradio.egg-info/requires.txt

@@ -1,11 +1,10 @@
 astropy
-casadata
 dask
 distributed
 graphviper
 matplotlib
 numba>=0.57.0
-numpy<2.0.0
+numpy
 prettytable
 pytest
 pytest-cov
@@ -22,7 +21,7 @@ typeguard
 typing_extensions
 
 [:sys_platform != "darwin"]
-python_casacore>=3.5.2
+python_casacore>=3.6.1
 
 [docs]
 jupyterlab