pyadps 0.2.0b0__py3-none-any.whl → 0.3.0b0__py3-none-any.whl

pyadps/utils/signal_quality.py

@@ -1,20 +1,47 @@
 import numpy as np
 from pyadps.utils.plotgen import PlotNoise
+from pyadps.utils.readrdi import ReadFile
 
 
 def qc_check(var, mask, cutoff=0):
     """
-    The module returns the modified mask file after checking the cutoff criteria.
-    All values less than the cuttoff are masked.
+    Perform a quality control check on the provided data and update the mask 
+    based on a cutoff threshold. Values in `var` that are less than the cutoff 
+    are marked as invalid in the mask.
 
-    Args:
-        var (numpy.ndarray):
-        mask (numpy.ndarray): A mask file having same array size as var
-        cutoff (int): Default cutoff is 0
+    Parameters
+    ----------
+    var : numpy.ndarray
+        The input array containing data to be checked against the cutoff.
+    mask : numpy.ndarray
+        An integer array of the same shape as `var`, where `1` indicates 
+        invalid data and `0` indicates valid data.
+    cutoff : int, optional
+        The threshold value for quality control. Any value in `var` less than 
+        or equal to this cutoff will be marked as invalid in the mask. Default is 0.
 
-    Returns:
-        mask (numpy.ndarray): Modified mask file based on cutoff
+    Returns
+    -------
+    numpy.ndarray
+        An updated integer mask array of the same shape as `var`, with `1` 
+        indicating invalid data and `0` indicating valid data.
+
+    Notes
+    -----
+    - The function modifies the `mask` by applying the cutoff condition. 
+      Values in `var` that are less than or equal to the cutoff will be 
+      marked as invalid (`1`), while all other values will remain valid (`0`).
+    - Ensure that `var` and `mask` are compatible in shape for element-wise 
+      operations.
+
+    Example
+    -------
+    >>> import pyadps
+    >>> ds = pyadps.Readfile('dummy.000')
+    >>> var = ds.echo.data
+    >>> mask = qc_check(var, mask, cutoff=40)
     """
+
     shape = np.shape(var)
     if len(shape) == 2:
         mask[var[:, :] < cutoff] = 1
@@ -22,16 +49,138 @@ def qc_check(var, mask, cutoff=0):
         beam = shape[0]
         for i in range(beam):
             mask[var[i, :, :] < cutoff] = 1
-    values, counts = np.unique(mask, return_counts=True)
+    # values, counts = np.unique(mask, return_counts=True)
     # print(values, counts, np.round(counts[1] * 100 / np.sum(counts)))
     return mask
 
 
-cor_check = qc_check
-echo_check = qc_check
+def correlation_check(ds, mask, cutoff=64):
+    """
+    Perform an correlation check on the provided variable and update the 
+    mask to mark valid and invalid values based on a cutoff threshold.
+
+    Parameters
+    ----------
+    ds : pyadps.dataset
+        The input pyadps dataframe containing correlation data to be checked.
+        Accepts 2-D or 3-D masks.
+    mask : numpy.ndarray
+        An integer array of the same shape as `var`, where `1` indicates invalid 
+        data or masked data and `0` indicates valid data.
+    cutoff : float, optional
+        The threshold value for echo intensity. Any value in `ds.correlation.data` below 
+        this cutoff will be considered invalid and marked as `1` in the mask. 
+        Default is 64.
+
+    Returns
+    -------
+    numpy.ndarray
+        An updated integer mask array of the same shape as `var`, with `1` 
+        indicating invalid or masked data (within the cutoff limit) and `0` indicating 
+        valid.
+
+    Notes
+    -----
+    - The function modifies the `mask` based on the cutoff condition. Valid 
+      values in `var` retain their corresponding mask value as `0`, while 
+      invalid values or previously masked elements are marked as `1`.
+      operations.
+
+    Example
+    -------
+    >>> import pyadps
+    >>> ds = pyadps.Readfile('dummy.000')
+    >>> outmask = correlation_check(ds, mask, cutoff=9999)
+    """
+    correlation = ds.correlation.data
+    mask = qc_check(correlation, mask, cutoff=cutoff)
+    return mask
+
+def echo_check(ds, mask, cutoff=40):
+    """
+    Perform an echo intensity check on the provided variable and update the 
+    mask to mark valid and invalid values based on a cutoff threshold.
+
+    Parameters
+    ----------
+    ds : pyadps.dataset
+        The input pyadps dataframe containing echo intensity data to be checked.
+        Accepts 2-D or 3-D masks.
+    mask : numpy.ndarray
+        An integer array of the same shape as `var`, where `1` indicates invalid 
+        data or masked data and `0` indicates valid data.
+    cutoff : float, optional
+        The threshold value for echo intensity. Any value in `ds.echo.data` below 
+        this cutoff will be considered invalid and marked as `1` in the mask. 
+        Default is 40.
+
+    Returns
+    -------
+    numpy.ndarray
+        An updated integer mask array of the same shape as `var`, with `1` 
+        indicating invalid or masked data (within the cutoff limit) and `0` indicating 
+        valid.
 
+    Notes
+    -----
+    - The function modifies the `mask` based on the cutoff condition. Valid 
+      values in `var` retain their corresponding mask value as `0`, while 
+      invalid values or previously masked elements are marked as `1`.
+    - Ensure that `var` and `mask` are compatible in shape for element-wise 
+      operations.
 
-def ev_check(var, mask, cutoff=9999):
+    Example
+    -------
+    >>> import pyadps
+    >>> ds = pyadps.Readfile('dummy.000')
+    >>> outmask = echo_check(ds, mask, cutoff=9999)
+    """
+
+    echo = ds.echo.data
+    mask = qc_check(echo, mask, cutoff=cutoff)
+    return mask
+
+
+def ev_check(ds, mask, cutoff=9999):
+    """
+    Perform an error velocity check on the provided variable and update the 
+    mask to mark valid and invalid values based on a cutoff threshold.
+
+    Parameters
+    ----------
+    ds : pyadps.dataset
+        The input pyadps dataframe containing error velocity data to be checked.
+    mask : numpy.ndarray
+        An integer array of the same shape as `var`, where `1` indicates invalid 
+        data or masked data and `0` indicates valid data.
+    cutoff : float, optional
+        The threshold value for error velocity. Any value in `var` exceeding 
+        this cutoff will be considered invalid and marked as `0` in the mask. 
+        Default is 9999.
+
+    Returns
+    -------
+    numpy.ndarray
+        An updated integer mask array of the same shape as `var`, with `1` 
+        indicating invalid or masked data (within the cutoff limit) and `0` indicating 
+        valid.
+
+    Notes
+    -----
+    - The function modifies the `mask` based on the cutoff condition. Valid 
+      values in `var` retain their corresponding mask value as `0`, while 
+      invalid values or previously masked elements are marked as `1`.
+    - Ensure that `var` and `mask` are compatible in shape for element-wise 
+      operations.
+
+    Example
+    -------
+    >>> import pyadps
+    >>> ds = pyadps.Readfile('dummy.000')
+    >>> outmask = ev_check(ds, mask, cutoff=9999)
+    """
+
+    var = ds.velocity.data[3, :, :]
     shape = np.shape(var)
     var = abs(var)
     if len(shape) == 2:
@@ -40,24 +189,110 @@ def ev_check(var, mask, cutoff=9999):
         beam = shape[2]
         for i in range(beam):
             mask[(var[i, :, :] >= cutoff) & (var[i, :, :] < 32768)] = 1
-    values, counts = np.unique(mask, return_counts=True)
-    # print(values, counts, np.round(counts[1] * 100 / np.sum(counts)))
     return mask
 
 
-def pg_check(pgood, mask, cutoff=0, threebeam=True):
+def pg_check(ds, mask, cutoff=0, threebeam=True):
+    """
+    Perform a percent-good check on the provided data and update the mask 
+    to mark valid and invalid values based on a cutoff threshold.
+
+    Parameters
+    ----------
+    ds : pyadps.dataset 
+        The input pyadps dataframe containing percent-good data, where values range from 
+        0 to 100 (maximum percent good).
+    mask : numpy.ndarray
+        An integer array of the same shape as `pgood`, where `1` indicates 
+        invalid data and `0` indicates valid data.
+    cutoff : float, optional
+        The threshold value for percent good. Any value in `pgood` greater than 
+        or equal to this cutoff will be considered valid (marked as `0`), 
+        while values not exceeding the cutoff are marked as invalid (`1`). 
+        Default is 0.
+    threebeam : bool, optional
+        If `True`, sums up Percent Good 1 and Percent Good 4 for the check. 
+
+    Returns
+    -------
+    numpy.ndarray
+        An updated integer mask array of the same shape as `pgood`, with `1` 
+        indicating invalid data and `0` indicating valid data.
+
+    Notes
+    -----
+    - The function modifies the `mask` based on the cutoff condition. Valid 
+      values in `pgood` are marked as `0`, while invalid values are marked 
+      as `1` in the mask.
+    - Ensure that `pgood` and `mask` are compatible in shape for element-wise 
+      operations.
+    - If `threebeam` is `True`, the logic may be adjusted to allow partial 
+      validity based on specific criteria.
+
+    Example
+    -------
+    >>> import pyadps
+    >>> ds = pyadps.Readfile('dummy.000')
+    >>> outmask = pg_check(ds, mask, cutoff=50, threebeam=True)
+    """
+
+    pgood = ds.percentgood.data
     if threebeam:
         pgood1 = pgood[0, :, :] + pgood[3, :, :]
     else:
         pgood1 = pgood[:, :, :]
 
     mask[pgood1[:, :] < cutoff] = 1
-    values, counts = np.unique(mask, return_counts=True)
-    # print(values, counts, np.round(counts[1] * 100 / np.sum(counts)))
     return mask
 
 
-def false_target(echo, mask, cutoff=255, threebeam=True):
+def false_target(ds, mask, cutoff=255, threebeam=True):
+    """
+    Apply a false target detection algorithm based on echo intensity values. 
+    This function identifies invalid or false targets in the data and updates 
+    the mask accordingly based on a specified cutoff threshold.
+
+    Parameters
+    ----------
+    ds : pyadps.dataset 
+        The input pyadps dataframe containing echo intensity values, which are used to 
+        detect false targets.
+    mask : numpy.ndarray
+        An integer array of the same shape as `echo`, where `1` indicates 
+        invalid or false target data and `0` indicates valid data.
+    cutoff : int, optional
+        The threshold value for echo intensity. Any value in `echo` greater 
+        than or equal to this cutoff will be considered a false target (invalid), 
+        marked as `1` in the mask. Default is 255.
+    threebeam : bool, optional
+        If `True`, applies a relaxed check that considers data valid even 
+        when only three beams report valid data. Default is `True`.
+
+    Returns
+    -------
+    numpy.ndarray
+        An updated integer mask array of the same shape as `echo`, with `1` 
+        indicating false target or invalid data and `0` indicating valid data.
+
+    Notes
+    -----
+    - The function modifies the `mask` by applying the cutoff condition. 
+      Echo values greater than or equal to the cutoff are marked as false 
+      targets (`1`), while values below the cutoff are considered valid (`0`).
+    - If `threebeam` is `True`, a more lenient check may be applied to handle 
+      data with fewer valid beams.
+    - Ensure that `echo` and `mask` are compatible in shape for element-wise 
+      operations.
+
+    Example
+    -------
+    >>> import pyadps
+    >>> ds = pyadps.Readfile('dummy.000')
+    >>> mask = false_target(echo, mask, cutoff=255)
+    """
+
+    echo = ds.echo.data
+
     shape = np.shape(echo)
     for i in range(shape[1]):
         for j in range(shape[2]):
@@ -74,10 +309,49 @@ def false_target(echo, mask, cutoff=255, threebeam=True):
     return mask
 
 
-def default_mask(flobj, velocity):
-    cells = flobj.field()["Cells"]
-    beams = flobj.field()["Beams"]
-    ensembles = flobj.ensembles
+def default_mask(ds):
+    """
+    Create a default 2-D mask file based on the velocity data.
+    This function generates a mask where values are marked as valid or invalid 
+    based on the missing values from the velocity data.
+
+    Parameters
+    ----------
+    ds : pyadps.dataset or numpy.ndarray
+         A pyadps data frame is used to extract velocity and dimensions for the mask.
+         If numpy.ndarray, enter the values for beams, cells and ensembles.
+
+    Returns
+    -------
+    numpy.ndarray
+        A mask array of the same shape as `velocity`, where `1` indicates invalid 
+        data  and `0` indicates valid data.
+
+    Notes
+    -----
+    - The function uses the velocity data along with the information from the 
+      Fixed Leader object to determine which values are valid and which are invalid.
+
+    Example
+    -------
+    >>> import pyadps
+    >>> ds = pyadps.ReadFile('demo.000')
+    >>> mask = pyadps.default_mask(ds)
+    """
+    if isinstance(ds, ReadFile) or ds.__class__.__name__ == "ReadFile":
+        flobj = ds.fixedleader
+        velocity = ds.velocity.data
+        cells = flobj.field()["Cells"]
+        beams = flobj.field()["Beams"]
+        ensembles = flobj.ensembles
+    elif isinstance(ds, np.ndarray) and ds.ndim == 3:
+        velocity = ds
+        beams = ds.shape[0]
+        cells = ds.shape[1]
+        ensembles = ds.shape[2] 
+    else:
+        raise ValueError("Input must be a 3-D numpy array or a PyADPS instance")
+
     mask = np.zeros((cells, ensembles))
     # Ignore mask for error velocity
     for i in range(beams - 1):
@@ -85,7 +359,53 @@ def default_mask(flobj, velocity):
     return mask
 
 
-def qc_prompt(flobj, name, data=None):
+def qc_prompt(ds, name, data=None):
+    """
+    Prompt the user to confirm or adjust the quality control threshold for a specific 
+    parameter based on predefined ranges. The function provides an interactive interface 
+    for the user to adjust thresholds for various quality control criteria, with options 
+    for certain thresholds like "Echo Intensity Thresh" to check the noise floor.
+
+    Parameters
+    ----------
+    flobj : FixedLeader
+        An instance of the FixedLeader class that holds metadata and configuration 
+        data. The `flobj` is used to retrieve the current threshold values based on 
+        the provided parameter name.
+    name : str
+        The name of the parameter for which the threshold is being adjusted. Examples 
+        include "Echo Intensity Thresh", "Correlation Thresh", "Percent Good Min", etc.
+    data : numpy.ndarray, optional
+        The data associated with the threshold. This is required for parameters like 
+        "Echo Intensity Thresh" where a noise floor check might be performed. Default is None.
+
+    Returns
+    -------
+    int
+        The updated threshold value, either the default or the new value entered by the user.
+
+    Notes
+    -----
+    - The function will prompt the user to change the threshold for the given `name` parameter.
+    - For certain parameters, the user may be asked if they would like to check the noise floor 
+      (for example, for "Echo Intensity Thresh"). This triggers the display of a plot and lets 
+      the user select a new threshold.
+    - The function ensures that the new threshold is within the acceptable range for each parameter.
+    - The default thresholds are provided if the user chooses not to change them.
+
+    Example
+    -------
+    >>> import pyadps
+    >>> ds = pyadps.ReadFile('demo.000')
+    >>> name = "Echo Intensity Thresh"
+    >>> threshold = qc_prompt(ds, name, data)
+    The default threshold for echo intensity thresh is 0
+    Would you like to change the threshold [y/n]: y
+    Would you like to check the noise floor [y/n]: y
+    Threshold changed to 50
+    """
+
+    flobj = ds.fixedleader
     cutoff = 0
     if name == "Echo Intensity Thresh":
         cutoff = 0

pyadps/utils/velocity_test.py

@@ -1,9 +1,24 @@
 from itertools import groupby
+from pygeomag import GeoMag
 
 import requests
 import numpy as np
 import scipy as sp
 
+
+def magdec(glat, glon, alt, time):
+    # Selecting COF file According to given year
+    if time >= 2010 and time < 2030:
+        var = 2010 + (int(time) - 2010) // 5 * 5
+        file_name = "wmm/WMM_{}.COF".format(str(var))
+        geo_mag = GeoMag(coefficients_file=file_name)
+    else:
+        geo_mag = GeoMag("wmm/WMM_2025.COF")
+    result = geo_mag.calculate(glat=glat, glon=glon, alt=alt, time=time)
+
+    return [[result.d]]
+
+
 def wmm2020api(lat1, lon1, year):
     """
     This function uses the WMM2020 API to retrieve the magnetic field values at a given location
@@ -18,37 +33,60 @@ def wmm2020api(lat1, lon1, year):
     Returns:
         mag -> magnetic declination at the given location in degree.
     """
-    baseurl = "https://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?"
+    baseurl_wmm = (
+        "https://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?"
+    )
+    baseurl_igrf = (
+        "https://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?"
+    )
+    baseurl_emm = "https://emmcalc.geomag.info/?magneticcomponent=d&"
     key = "zNEw7"
-    resultFormat="json"
-    url = "{}lat1={}&lon1={}&key={}&startYear{}&resultFormat={}".format(baseurl, lat1, lon1, key, year, resultFormat)
+    resultFormat = "json"
+    if year >= 2025:
+        baseurl = baseurl_wmm
+        model = "WMM"
+    elif year >= 2019:
+        baseurl = baseurl_wmm
+        model = "IGRF"
+    elif year >= 2000:
+        baseurl = baseurl_emm
+        model = "EMM"
+    elif year >= 1590:
+        baseurl = baseurl_igrf
+        model = "IGRF"
+    url = "{}model={}&lat1={}&lon1={}&key={}&startYear={}&resultFormat={}".format(
+        baseurl, model, lat1, lon1, key, year, resultFormat
+    )
     response = requests.get(url)
     data = response.json()
     results = data["result"][0]
     mag = [[results["declination"]]]
-    
+
     return mag
 
-def magnetic_declination(lat, lon, depth, year):
-    """
-    The function  calculates the magnetic declination at a given location and depth.
-    using a local installation of wmm2020 model.
 
+# Commentin magnetic_declination model since the method is no longer using.
+# def magnetic_declination(lat, lon, depth, year):
+#     """
+#     The function  calculates the magnetic declination at a given location and depth.
+#     using a local installation of wmm2020 model.
 
-    Args:
-        lat (parameter, float): Latitude in decimals
-        lon (parameter, float): Longitude in decimals
-        depth (parameter, float): depth in m
-        year (parameter, integer): Year
 
-    Returns:
-        mag: Magnetic declination (degrees)
-    """
-    import wmm2020
-    mag = wmm2020.wmm(lat, lon, depth, year)
-    mag = mag.decl.data
+#     Args:
+#         lat (parameter, float): Latitude in decimals
+#         lon (parameter, float): Longitude in decimals
+#         depth (parameter, float): depth in m
+#         year (parameter, integer): Year
+
+#     Returns:
+#         mag: Magnetic declination (degrees)
+#     """
+#     import wmm2020
+#     mag = wmm2020.wmm(lat, lon, depth, year)
+#     mag = mag.decl.data
+
+#     return  mag
 
-    return  mag
 
 def velocity_modifier(velocity, mag):
     """
@@ -64,12 +102,17 @@ def velocity_modifier(velocity, mag):
     """
     mag = np.deg2rad(mag[0][0])
     velocity = np.where(velocity == -32768, np.nan, velocity)
-    velocity[0, :, :] = velocity[0, :, :] * np.cos(mag) + velocity[1, :, :] * np.sin(mag)
-    velocity[1, :, :] = -1 * velocity[0, :, :] * np.sin(mag) + velocity[1, :, :] * np.cos(mag)
+    velocity[0, :, :] = velocity[0, :, :] * np.cos(mag) + velocity[1, :, :] * np.sin(
+        mag
+    )
+    velocity[1, :, :] = -1 * velocity[0, :, :] * np.sin(mag) + velocity[
+        1, :, :
+    ] * np.cos(mag)
     velocity = np.where(velocity == np.nan, -32768, velocity)
 
     return velocity
 
+
 def velocity_cutoff(velocity, mask, cutoff=250):
     """
     Masks all velocities above a cutoff. Note that
@@ -89,7 +132,7 @@ def velocity_cutoff(velocity, mask, cutoff=250):
     return mask
 
 
-def despike(velocity, mask, kernal_size=13, cutoff=150):
+def despike(velocity, mask, kernal_size=13, cutoff=3):
     """
     Function to remove anomalous spikes in the data over a period of time.
     A median filter is used to despike the data.
@@ -97,19 +140,24 @@ def despike(velocity, mask, kernal_size=13, cutoff=150):
     Args:
         velocity (numpy array, integer): Velocity(depth, time) in mm/s
         mask (numpy array, integer): Mask file
-        kernal_size (paramater, integer): Number of ensembles over which the spike has to be checked
-        cutoff (parameter, integer): [TODO:description]
+        kernal_size (paramater, integer): Window size for rolling median filter
+        cutoff (parameter, integer): Number of standard deviations to identify spikes
 
     Returns:
         mask
     """
-    cutoff = cutoff * 10
     velocity = np.where(velocity == -32768, np.nan, velocity)
     shape = np.shape(velocity)
     for j in range(shape[0]):
+        # Apply median filter
         filt = sp.signal.medfilt(velocity[j, :], kernal_size)
+        # Calculate absolute deviation from the rolling median
         diff = np.abs(velocity[j, :] - filt)
-        mask[j, :] = np.where(diff < cutoff, mask[j, :], 1)
+        # Calculate threshold for spikes based on standard deviation
+        std_dev = np.nanstd(diff)
+        spike_threshold = cutoff * std_dev
+        # Apply mask after identifying spikes
+        mask[j, :] = np.where(diff < spike_threshold, mask[j, :], 1)
     return mask
 
 

pyadps/utils/writenc.py

@@ -244,7 +244,7 @@ def vlead_nc(infile, outfile, time, axis_option=None, attributes=None, t0="hours
     outnc.close()
 
 
-def finalnc(outfile, depth, time, data, t0="hours since 2000-01-01", attributes=None):
+def finalnc(outfile, depth, final_mask, time, data, t0="hours since 2000-01-01", attributes=None):
     """
     Function to create the processed NetCDF file.
 
@@ -268,6 +268,7 @@ def finalnc(outfile, depth, time, data, t0="hours since 2000-01-01", attributes=
     if np.all(depth[:-1] >= depth[1:]):
         depth = depth[::-1]
         data = data[:, ::-1, :]
+        final_mask = final_mask[::-1, :]
 
     ncfile = nc4.Dataset(outfile, mode="w", format="NETCDF4")
     # Check if depth is scalar or array
@@ -307,6 +308,9 @@ def finalnc(outfile, depth, time, data, t0="hours since 2000-01-01", attributes=
     evel.units = "cm/s"
     evel.long_name = "error_velocity"
 
+    mvel = ncfile.createVariable("mask", np.float32, ("time", "depth"), fill_value=fill)
+    mvel.long_name = "Velocity Mask (1: bad value, 0: good value)"
+
     nctime = pd2nctime(time, t0)
     # write data
     z[:] = depth
@@ -315,10 +319,13 @@ def finalnc(outfile, depth, time, data, t0="hours since 2000-01-01", attributes=
     vvel[:, :] = data[1, :, :].T
     wvel[:, :] = data[2, :, :].T
     evel[:, :] = data[3, :, :].T
+    mvel[:, :] = final_mask.T
     
     # Add global attributes if provided
     if attributes:
         for key, value in attributes.items():
             setattr(ncfile, key, str(value))  # Store attributes as strings
+    
+    ncfile.mask_applied = "True"
 
     ncfile.close()