xradio 0.0.27__tar.gz → 0.0.29__tar.gz

{xradio-0.0.27 → xradio-0.0.29}/LICENSE.txt

@@ -1,6 +1,11 @@
 BSD 3-Clause License
 
-Copyright (c) 2023, Associated Universities, Inc.
+All works in this repository are copyrighted 2024.  
+For inquiries contact  Associated Universities, Inc., 2650 Park Tower Drive Vienna, VA 22180, USA.
+
+Portions of this repository are copyrighted by the following entities:
+1. AUI/NRAO, ESO, NAOJ, in the framework of the ALMA partnership.
+2. AUI/NRAO in the framework of the ngVLA project.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

{xradio-0.0.27/src/xradio.egg-info → xradio-0.0.29}/PKG-INFO

@@ -1,11 +1,16 @@
 Metadata-Version: 2.1
 Name: xradio
-Version: 0.0.27
-Summary: Xarray Radio Astronomy Data IO
+Version: 0.0.29
+Summary:  Xarray Radio Astronomy Data IO
 Author-email: Jan-Willem Steeb <jsteeb@nrao.edu>
 License: BSD 3-Clause License
         
-        Copyright (c) 2023, Associated Universities, Inc.
+        All works in this repository are copyrighted 2024.  
+        For inquiries contact  Associated Universities, Inc., 2650 Park Tower Drive Vienna, VA 22180, USA.
+        
+        Portions of this repository are copyrighted by the following entities:
+        1. AUI/NRAO, ESO, NAOJ, in the framework of the ALMA partnership.
+        2. AUI/NRAO in the framework of the ngVLA project.
         
         Redistribution and use in source and binary forms, with or without
         modification, are permitted provided that the following conditions are met:
@@ -42,7 +47,7 @@ Requires-Dist: distributed
 Requires-Dist: graphviper
 Requires-Dist: matplotlib
 Requires-Dist: numba>=0.57.0
-Requires-Dist: numpy
+Requires-Dist: numpy<2.0.0
 Requires-Dist: prettytable
 Requires-Dist: pytest
 Requires-Dist: pytest-cov
@@ -52,12 +57,12 @@ Requires-Dist: scipy
 Requires-Dist: tqdm
 Requires-Dist: xarray
 Requires-Dist: zarr
-Requires-Dist: bokeh
-Requires-Dist: jupyterlab
 Requires-Dist: pyarrow
 Requires-Dist: python_casacore>=3.5.2; sys_platform != "darwin"
 Requires-Dist: typing_extensions; python_version < "3.10"
+Requires-Dist: typeguard
 Provides-Extra: docs
+Requires-Dist: jupyterlab; extra == "docs"
 Requires-Dist: ipykernel; extra == "docs"
 Requires-Dist: ipympl; extra == "docs"
 Requires-Dist: ipython; extra == "docs"

{xradio-0.0.27 → xradio-0.0.29}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "xradio"
-version = "0.0.27"
-description = "Xarray Radio Astronomy Data IO"
+version = "0.0.29"
+description = " Xarray Radio Astronomy Data IO"
 authors = [
     {name = "Jan-Willem Steeb", email="jsteeb@nrao.edu"},
 ]
@@ -17,7 +17,7 @@ dependencies = [
     'graphviper',
     'matplotlib',
     'numba>=0.57.0',
-    'numpy',
+    'numpy<2.0.0',
     'prettytable',
     'pytest',
     'pytest-cov',
@@ -27,15 +27,15 @@ dependencies = [
     'tqdm',
     'xarray',
     'zarr',
-    'bokeh',
-    'jupyterlab',
     'pyarrow',
     'python_casacore>=3.5.2; sys_platform != "darwin" ',
-    'typing_extensions; python_version < "3.10"'
+    'typing_extensions; python_version < "3.10"',
+    'typeguard'
 ]
 
 [project.optional-dependencies]
 docs = [
+    'jupyterlab',
     'ipykernel',
     'ipympl',
     'ipython',
@@ -49,3 +49,20 @@ docs = [
     'twine',
     'pandoc'
 ]
+
+[tool.pytest.ini_options]
+addopts = [
+    "--import-mode=importlib",
+    "--strict-markers"
+]
+testpaths = [
+    "tests"
+]
+pythonpath = [
+    "."
+]
+
+[tool.black]
+line-length = 88
+target-version = ['py39']
+include = '\.py$'

xradio-0.0.29/src/xradio/__init__.py

@@ -0,0 +1,13 @@
+import os
+from graphviper.utils.logger import setup_logger
+
+_logger_name = "xradio"
+if os.getenv("VIPER_LOGGER_NAME") != _logger_name:
+    os.environ["VIPER_LOGGER_NAME"] = _logger_name
+    setup_logger(
+        logger_name="xradio",
+        log_to_term=True,
+        log_to_file=False,  # True
+        log_file="xradio-logfile",
+        log_level="DEBUG",
+    )

xradio-0.0.29/src/xradio/_utils/array.py

@@ -0,0 +1,90 @@
+"""Contains optimised functions to be used within other modules."""
+
+import numpy as np
+
+
+def check_if_consistent(array: np.ndarray, array_name: str) -> np.ndarray:
+    """_summary_
+
+    Parameters
+    ----------
+    col : _type_
+        _description_
+    col_name : _type_
+        _description_
+
+    Returns
+    -------
+    _type_
+        _description_
+    """
+    if array.ndim == 0:
+        return array.item()
+
+    array_unique = unique_1d(array)
+    assert len(array_unique) == 1, array_name + " is not consistent."
+    return array_unique[0]
+
+
+import numpy as np
+import pandas as pd
+
+
+def unique_1d(array: np.ndarray) -> np.ndarray:
+    """
+    Optimised version of np.unique for 1D arrays.
+
+    Parameters
+    ----------
+    array : np.ndarray
+        a 1D array of values.
+
+    Returns
+    -------
+    np.ndarray
+        a sorted array of unique values.
+
+    """
+    return np.sort(pd.unique(array))
+
+
+def pairing_function(antenna_pairs: np.ndarray) -> np.ndarray:
+    """
+    Pairing function to convert each array pair to a single value.
+
+    This custom pairing function will only work if the maximum value is less
+    than 2**20 and less than 2,048 if using signed 32-bit integers.
+
+    Parameters
+    ----------
+    antenna_pairs : np.ndarray
+        a 2D array containing antenna 1 and antenna
+        2 ids, which forms a baseline.
+
+    Returns
+    -------
+    np.ndarray
+        a 1D array of the paired values.
+
+    """
+    return antenna_pairs[:, 0] * 2**20 + antenna_pairs[:, 1]
+
+
+def inverse_pairing_function(paired_array: np.ndarray) -> np.ndarray:
+    """
+    Inverse pairing function to convert each paired value to an antenna pair.
+
+    This inverse pairing function is the inverse of the custom pairing function.
+
+    Parameters
+    ----------
+    paired_array : np.ndarray
+        a 1D array of the paired values.
+
+    Returns
+    -------
+    np.ndarray
+        a 2D array containing antenna 1 and antenna 2 ids.
+
+    """
+    return np.column_stack(np.divmod(paired_array, 2**20))

{xradio-0.0.27 → xradio-0.0.29}/src/xradio/_utils/zarr/common.py

@@ -2,9 +2,54 @@ import copy
 import xarray as xr
 import zarr
 import s3fs
+import os
+from botocore.exceptions import NoCredentialsError
+
+
+def _get_ms_stores_and_file_system(ps_store: str):
+
+    if os.path.isdir(ps_store):
+        # default to assuming the data are accessible on local file system
+        items = os.listdir(ps_store)
+        file_system = os
+
+    elif ps_store.startswith("s3"):
+        # only if not found locally, check if dealing with an S3 bucket URL
+        # if not ps_store.endswith("/"):
+        #     # just for consistency, as there is no os.path equivalent in s3fs
+        #     ps_store = ps_store + "/"
+
+        try:
+            # initialize the S3 "file system", first attempting to use pre-configured credentials
+            file_system = s3fs.S3FileSystem(anon=False, requester_pays=False)
+            items = [
+                bd.split(sep="/")[-1]
+                for bd in file_system.listdir(ps_store, detail=False)
+            ]
+
+        except (NoCredentialsError, PermissionError) as e:
+            # only public, read-only buckets will be accessible
+            # we will want to add messaging and error handling here
+            file_system = s3fs.S3FileSystem(anon=True)
+            items = [
+                bd.split(sep="/")[-1]
+                for bd in file_system.listdir(ps_store, detail=False)
+            ]
+    else:
+        raise (
+            FileNotFoundError,
+            f"Could not find {ps_store} either locally or in the cloud.",
+        )
+
+    items = [
+        item for item in items if not item.startswith(".")
+    ]  # Mac OS likes to place hidden files in the directory (.DStore).
+    return file_system, items
 
 
-def _open_dataset(store, xds_isel=None, data_variables=None, load=False, **kwargs):
+def _open_dataset(
+    store, file_system=os, xds_isel=None, data_variables=None, load=False
+):
     """
 
     Parameters
@@ -26,8 +71,8 @@ def _open_dataset(store, xds_isel=None, data_variables=None, load=False, **kwarg
 
     import dask
 
-    if "s3" in kwargs.keys():
-        mapping = s3fs.S3Map(root=store, s3=kwargs["s3"], check=False)
+    if isinstance(file_system, s3fs.core.S3FileSystem):
+        mapping = s3fs.S3Map(root=store, s3=file_system, check=False)
         xds = xr.open_zarr(store=mapping)
     else:
         xds = xr.open_zarr(store)

{xradio-0.0.27 → xradio-0.0.29}/src/xradio/image/_util/_fits/xds_from_fits.py

@@ -101,7 +101,7 @@ def _add_vel_attrs(xds: xr.Dataset, helpers: dict) -> xr.Dataset:
     vel_coord = xds.coords["velocity"]
     meta = {"units": "m/s"}
     if helpers["has_freq"]:
-        meta["doppler_type"] = helpers["doppler"]
+        meta["doppler_type"] = helpers.get("doppler", "RADIO")
     else:
         meta["doppler_type"] = _doppler_types[0]
     meta["type"] = "doppler"
@@ -570,8 +570,8 @@ def _get_freq_values(helpers: dict) -> list:
             crval=helpers["crval"][freq_idx],
             crpix=helpers["crpix"][freq_idx],
             cdelt=helpers["cdelt"][freq_idx],
-            cunit=helpers["cunit"][freq_idx],
         )
+        cunit = helpers["cunit"][freq_idx]
         helpers["frequency"] = vals * u.Unit(cunit)
         return vals
     elif "VOPT" in ctype:
@@ -606,9 +606,9 @@ def _get_velocity_values(helpers: dict) -> list:
         return helpers["velocity"].to(u.m / u.s).value
     elif "frequency" in helpers:
         v = _compute_velocity_values(
-            restfreq=helpers["reestfreq"],
+            restfreq=helpers["restfreq"],
             freq_values=helpers["frequency"].to("Hz").value,
-            doppler=helpers["doppler"],
+            doppler=helpers.get("doppler", "RADIO"),
         )
         helpers["velocity"] = v * (u.m / u.s)
         return v
@@ -796,7 +796,12 @@ def _get_transpose_list(helpers: dict) -> tuple:
             transpose_list[4] = i
             not_covered.remove("m")
             not_covered.remove("v")
-        elif b.startswith("frequency") or b.startswith("vopt") or b.startswith("vrad"):
+        elif (
+            b.startswith("frequency")
+            or b.startswith("freq")
+            or b.startswith("vopt")
+            or b.startswith("vrad")
+        ):
             transpose_list[2] = i
             not_covered.remove("f")
         elif b.startswith("stok"):

{xradio-0.0.27 → xradio-0.0.29}/src/xradio/image/_util/_zarr/zarr_low_level.py

@@ -110,53 +110,56 @@ def write_binary_blob_to_disk(arr, file_path, compressor):
     - None
     """
     import graphviper.utils.logger as logger
+
     # Encode the NumPy array using the codec
-    logger.debug('1. Before compressor ' + file_path)
+    logger.debug("1. Before compressor " + file_path)
     compressed_arr = compressor.encode(np.ascontiguousarray(arr))
 
-    logger.debug('2. Before makedir')
+    logger.debug("2. Before makedir")
     # Ensure the directory exists before saving the file
     os.makedirs(os.path.dirname(file_path), exist_ok=True)
 
     arr_len = len(compressed_arr)
-    logger.debug('3. Before write the len is: ' + str(arr_len))
-    #Save the compressed array to disk
+    logger.debug("3. Before write the len is: " + str(arr_len))
+    # Save the compressed array to disk
     # with open(file_path, "wb") as file:
     #     file.write(compressed_arr)
-    
-    logger.debug('4. Using new writer: ' + str(arr_len))
+
+    logger.debug("4. Using new writer: " + str(arr_len))
     write_to_lustre_chunked(file_path, compressed_arr)
-        
+
     # /.lustre/aoc/sciops/pford/CHILES/cube_image/uid___A002_Xee7674_X2844_Cube_3.img.zarr/SKY/0.0.110.0.0
     # 348192501 bytes
     # 332.0622453689575 M
-    
+
     # from io import BufferedWriter
     # # Calculate buffer size based on compressed_arr size (adjust multiplier)
     # buffer_size = min(len(compressed_arr), 1024 * 1024 * 4)  # Max 4 MB buffer
     # with BufferedWriter(open(file_path, "wb"), buffer_size) as f:
     #     f.write(compressed_arr)
-    #     f.flush()  # Ensure data gets written to disk 
-    
-    
-    logger.debug('4. Write completed')
+    #     f.flush()  # Ensure data gets written to disk
+
+    logger.debug("4. Write completed")
 
     # print(f"Compressed array saved to {file_path}")
 
 
-def write_to_lustre_chunked(file_path, compressed_arr, chunk_size=1024 * 1024 * 128):  # 128 MiB chunks
-  """
-  Writes compressed data to a Lustre file path with chunking.
+def write_to_lustre_chunked(
+    file_path, compressed_arr, chunk_size=1024 * 1024 * 128
+):  # 128 MiB chunks
+    """
+    Writes compressed data to a Lustre file path with chunking.
+
+    Args:
+        file_path: Path to the file for writing.
+        compressed_arr: Compressed data array to write.
+        chunk_size: Size of each data chunk in bytes (default: 128 MiB).
+    """
+    with open(file_path, "wb") as f:
+        for i in range(0, len(compressed_arr), chunk_size):
+            chunk = compressed_arr[i : i + chunk_size]
+            f.write(chunk)
 
-  Args:
-      file_path: Path to the file for writing.
-      compressed_arr: Compressed data array to write.
-      chunk_size: Size of each data chunk in bytes (default: 128 MiB).
-  """
-  with open(file_path, "wb") as f:
-    for i in range(0, len(compressed_arr), chunk_size):
-      chunk = compressed_arr[i:i + chunk_size]
-      f.write(chunk)
 
 def read_binary_blob_from_disk(file_path, compressor, dtype=np.float64):
     """

{xradio-0.0.27 → xradio-0.0.29}/src/xradio/image/_util/common.py

@@ -33,7 +33,10 @@ def _convert_beam_to_rad(beam: dict) -> dict:
     """
     mybeam = {}
     for k in beam:
-        q = u.quantity.Quantity(f"{beam[k]['value']}{beam[k]['unit']}")
+        try:
+            q = u.quantity.Quantity(f"{beam[k]['value']}{beam[k]['unit']}")
+        except:
+            q = u.quantity.Quantity(f"{beam[k]['value']}{beam[k]['units']}")
         q = q.to("rad")
         j = "pa" if k == "positionangle" else k
         mybeam[j] = {"type": "quantity", "value": q.value, "units": "rad"}

{xradio-0.0.27 → xradio-0.0.29}/src/xradio/image/_util/zarr.py

@@ -25,7 +25,10 @@ def _xds_from_zarr(
 
 
 def _load_image_from_zarr_no_dask(zarr_file: str, selection: dict) -> xr.Dataset:
-    image_xds = _open_dataset(zarr_file, selection, load=True)
+    # At the moment image module does not support S3 file system. file_system=os is hardcoded.
+    image_xds = _open_dataset(
+        store=zarr_file, file_system=os, xds_isel=selection, load=True
+    )
     for h in ["HISTORY", "_attrs_xds_history"]:
         history = os.sep.join([zarr_file, h])
         if os.path.isdir(history):

xradio-0.0.29/src/xradio/schema/__init__.py

@@ -0,0 +1,35 @@
+from .dataclass import (
+    xarray_dataclass_to_array_schema,
+    xarray_dataclass_to_dataset_schema,
+    xarray_dataclass_to_dict_schema,
+)
+from .bases import (
+    xarray_dataarray_schema,
+    xarray_dataset_schema,
+    dict_schema,
+)
+from .check import (
+    SchemaIssue,
+    SchemaIssues,
+    check_array,
+    check_dataset,
+    check_dict,
+    schema_checked,
+)
+
+__all__ = [
+    "xarray_dataclass_to_array_schema",
+    "xarray_dataclass_to_dataset_schema",
+    "xarray_dataclass_to_dict_schema",
+    "AsDataArray",
+    "AsDataset",
+    "AsDict",
+    "xarray_dataarray_schema",
+    "xarray_dataset_schema",
+    "SchemaIssue",
+    "SchemaIssues",
+    "check_array",
+    "check_dataset",
+    "check_dict",
+    "schema_checked",
+]