pyadps 0.1.0__py3-none-any.whl → 0.1.0b0__py3-none-any.whl

pyadps/utils/signal_quality.py

@@ -1,47 +1,20 @@
 import numpy as np
 from pyadps.utils.plotgen import PlotNoise
-from pyadps.utils.readrdi import ReadFile
 
 
 def qc_check(var, mask, cutoff=0):
     """
-    Perform a quality control check on the provided data and update the mask 
-    based on a cutoff threshold. Values in `var` that are less than the cutoff 
-    are marked as invalid in the mask.
+    The module returns the modified mask file after checking the cutoff criteria.
+    All values less than the cuttoff are masked.
 
-    Parameters
-    ----------
-    var : numpy.ndarray
-        The input array containing data to be checked against the cutoff.
-    mask : numpy.ndarray
-        An integer array of the same shape as `var`, where `1` indicates 
-        invalid data and `0` indicates valid data.
-    cutoff : int, optional
-        The threshold value for quality control. Any value in `var` less than 
-        or equal to this cutoff will be marked as invalid in the mask. Default is 0.
+    Args:
+        var (numpy.ndarray):
+        mask (numpy.ndarray): A mask file having same array size as var
+        cutoff (int): Default cutoff is 0
 
-    Returns
-    -------
-    numpy.ndarray
-        An updated integer mask array of the same shape as `var`, with `1` 
-        indicating invalid data and `0` indicating valid data.
-
-    Notes
-    -----
-    - The function modifies the `mask` by applying the cutoff condition. 
-      Values in `var` that are less than or equal to the cutoff will be 
-      marked as invalid (`1`), while all other values will remain valid (`0`).
-    - Ensure that `var` and `mask` are compatible in shape for element-wise 
-      operations.
-
-    Example
-    -------
-    >>> import pyadps
-    >>> ds = pyadps.Readfile('dummy.000')
-    >>> var = ds.echo.data
-    >>> mask = qc_check(var, mask, cutoff=40)
+    Returns:
+        mask (numpy.ndarray): Modified mask file based on cutoff
     """
-
     shape = np.shape(var)
     if len(shape) == 2:
         mask[var[:, :] < cutoff] = 1
@@ -49,138 +22,16 @@ def qc_check(var, mask, cutoff=0):
         beam = shape[0]
         for i in range(beam):
             mask[var[i, :, :] < cutoff] = 1
-    # values, counts = np.unique(mask, return_counts=True)
+    values, counts = np.unique(mask, return_counts=True)
     # print(values, counts, np.round(counts[1] * 100 / np.sum(counts)))
     return mask
 
 
-def correlation_check(ds, mask, cutoff=64):
-    """
-    Perform an correlation check on the provided variable and update the 
-    mask to mark valid and invalid values based on a cutoff threshold.
-
-    Parameters
-    ----------
-    ds : pyadps.dataset
-        The input pyadps dataframe containing correlation data to be checked.
-        Accepts 2-D or 3-D masks.
-    mask : numpy.ndarray
-        An integer array of the same shape as `var`, where `1` indicates invalid 
-        data or masked data and `0` indicates valid data.
-    cutoff : float, optional
-        The threshold value for echo intensity. Any value in `ds.correlation.data` below 
-        this cutoff will be considered invalid and marked as `1` in the mask. 
-        Default is 64.
-
-    Returns
-    -------
-    numpy.ndarray
-        An updated integer mask array of the same shape as `var`, with `1` 
-        indicating invalid or masked data (within the cutoff limit) and `0` indicating 
-        valid.
-
-    Notes
-    -----
-    - The function modifies the `mask` based on the cutoff condition. Valid 
-      values in `var` retain their corresponding mask value as `0`, while 
-      invalid values or previously masked elements are marked as `1`.
-      operations.
-
-    Example
-    -------
-    >>> import pyadps
-    >>> ds = pyadps.Readfile('dummy.000')
-    >>> outmask = correlation_check(ds, mask, cutoff=9999)
-    """
-    correlation = ds.correlation.data
-    mask = qc_check(correlation, mask, cutoff=cutoff)
-    return mask
-
-def echo_check(ds, mask, cutoff=40):
-    """
-    Perform an echo intensity check on the provided variable and update the 
-    mask to mark valid and invalid values based on a cutoff threshold.
-
-    Parameters
-    ----------
-    ds : pyadps.dataset
-        The input pyadps dataframe containing echo intensity data to be checked.
-        Accepts 2-D or 3-D masks.
-    mask : numpy.ndarray
-        An integer array of the same shape as `var`, where `1` indicates invalid 
-        data or masked data and `0` indicates valid data.
-    cutoff : float, optional
-        The threshold value for echo intensity. Any value in `ds.echo.data` below 
-        this cutoff will be considered invalid and marked as `1` in the mask. 
-        Default is 40.
-
-    Returns
-    -------
-    numpy.ndarray
-        An updated integer mask array of the same shape as `var`, with `1` 
-        indicating invalid or masked data (within the cutoff limit) and `0` indicating 
-        valid.
+cor_check = qc_check
+echo_check = qc_check
 
-    Notes
-    -----
-    - The function modifies the `mask` based on the cutoff condition. Valid 
-      values in `var` retain their corresponding mask value as `0`, while 
-      invalid values or previously masked elements are marked as `1`.
-    - Ensure that `var` and `mask` are compatible in shape for element-wise 
-      operations.
 
-    Example
-    -------
-    >>> import pyadps
-    >>> ds = pyadps.Readfile('dummy.000')
-    >>> outmask = echo_check(ds, mask, cutoff=9999)
-    """
-
-    echo = ds.echo.data
-    mask = qc_check(echo, mask, cutoff=cutoff)
-    return mask
-
-
-def ev_check(ds, mask, cutoff=9999):
-    """
-    Perform an error velocity check on the provided variable and update the 
-    mask to mark valid and invalid values based on a cutoff threshold.
-
-    Parameters
-    ----------
-    ds : pyadps.dataset
-        The input pyadps dataframe containing error velocity data to be checked.
-    mask : numpy.ndarray
-        An integer array of the same shape as `var`, where `1` indicates invalid 
-        data or masked data and `0` indicates valid data.
-    cutoff : float, optional
-        The threshold value for error velocity. Any value in `var` exceeding 
-        this cutoff will be considered invalid and marked as `0` in the mask. 
-        Default is 9999.
-
-    Returns
-    -------
-    numpy.ndarray
-        An updated integer mask array of the same shape as `var`, with `1` 
-        indicating invalid or masked data (within the cutoff limit) and `0` indicating 
-        valid.
-
-    Notes
-    -----
-    - The function modifies the `mask` based on the cutoff condition. Valid 
-      values in `var` retain their corresponding mask value as `0`, while 
-      invalid values or previously masked elements are marked as `1`.
-    - Ensure that `var` and `mask` are compatible in shape for element-wise 
-      operations.
-
-    Example
-    -------
-    >>> import pyadps
-    >>> ds = pyadps.Readfile('dummy.000')
-    >>> outmask = ev_check(ds, mask, cutoff=9999)
-    """
-
-    var = ds.velocity.data[3, :, :]
+def ev_check(var, mask, cutoff=9999):
     shape = np.shape(var)
     var = abs(var)
     if len(shape) == 2:
@@ -189,110 +40,24 @@ def ev_check(ds, mask, cutoff=9999):
         beam = shape[2]
         for i in range(beam):
             mask[(var[i, :, :] >= cutoff) & (var[i, :, :] < 32768)] = 1
+    values, counts = np.unique(mask, return_counts=True)
+    # print(values, counts, np.round(counts[1] * 100 / np.sum(counts)))
     return mask
 
 
-def pg_check(ds, mask, cutoff=0, threebeam=True):
-    """
-    Perform a percent-good check on the provided data and update the mask 
-    to mark valid and invalid values based on a cutoff threshold.
-
-    Parameters
-    ----------
-    ds : pyadps.dataset 
-        The input pyadps dataframe containing percent-good data, where values range from 
-        0 to 100 (maximum percent good).
-    mask : numpy.ndarray
-        An integer array of the same shape as `pgood`, where `1` indicates 
-        invalid data and `0` indicates valid data.
-    cutoff : float, optional
-        The threshold value for percent good. Any value in `pgood` greater than 
-        or equal to this cutoff will be considered valid (marked as `0`), 
-        while values not exceeding the cutoff are marked as invalid (`1`). 
-        Default is 0.
-    threebeam : bool, optional
-        If `True`, sums up Percent Good 1 and Percent Good 4 for the check. 
-
-    Returns
-    -------
-    numpy.ndarray
-        An updated integer mask array of the same shape as `pgood`, with `1` 
-        indicating invalid data and `0` indicating valid data.
-
-    Notes
-    -----
-    - The function modifies the `mask` based on the cutoff condition. Valid 
-      values in `pgood` are marked as `0`, while invalid values are marked 
-      as `1` in the mask.
-    - Ensure that `pgood` and `mask` are compatible in shape for element-wise 
-      operations.
-    - If `threebeam` is `True`, the logic may be adjusted to allow partial 
-      validity based on specific criteria.
-
-    Example
-    -------
-    >>> import pyadps
-    >>> ds = pyadps.Readfile('dummy.000')
-    >>> outmask = pg_check(ds, mask, cutoff=50, threebeam=True)
-    """
-
-    pgood = ds.percentgood.data
+def pg_check(pgood, mask, cutoff=0, threebeam=True):
     if threebeam:
         pgood1 = pgood[0, :, :] + pgood[3, :, :]
     else:
-        pgood1 = pgood[3, :, :]
+        pgood1 = pgood[:, :, :]
 
     mask[pgood1[:, :] < cutoff] = 1
+    values, counts = np.unique(mask, return_counts=True)
+    # print(values, counts, np.round(counts[1] * 100 / np.sum(counts)))
     return mask
 
 
-def false_target(ds, mask, cutoff=255, threebeam=True):
-    """
-    Apply a false target detection algorithm based on echo intensity values. 
-    This function identifies invalid or false targets in the data and updates 
-    the mask accordingly based on a specified cutoff threshold.
-
-    Parameters
-    ----------
-    ds : pyadps.dataset 
-        The input pyadps dataframe containing echo intensity values, which are used to 
-        detect false targets.
-    mask : numpy.ndarray
-        An integer array of the same shape as `echo`, where `1` indicates 
-        invalid or false target data and `0` indicates valid data.
-    cutoff : int, optional
-        The threshold value for echo intensity. Any value in `echo` greater 
-        than or equal to this cutoff will be considered a false target (invalid), 
-        marked as `1` in the mask. Default is 255.
-    threebeam : bool, optional
-        If `True`, applies a relaxed check that considers data valid even 
-        when only three beams report valid data. Default is `True`.
-
-    Returns
-    -------
-    numpy.ndarray
-        An updated integer mask array of the same shape as `echo`, with `1` 
-        indicating false target or invalid data and `0` indicating valid data.
-
-    Notes
-    -----
-    - The function modifies the `mask` by applying the cutoff condition. 
-      Echo values greater than or equal to the cutoff are marked as false 
-      targets (`1`), while values below the cutoff are considered valid (`0`).
-    - If `threebeam` is `True`, a more lenient check may be applied to handle 
-      data with fewer valid beams.
-    - Ensure that `echo` and `mask` are compatible in shape for element-wise 
-      operations.
-
-    Example
-    -------
-    >>> import pyadps
-    >>> ds = pyadps.Readfile('dummy.000')
-    >>> mask = false_target(echo, mask, cutoff=255)
-    """
-
-    echo = ds.echo.data
-
+def false_target(echo, mask, cutoff=255, threebeam=True):
     shape = np.shape(echo)
     for i in range(shape[1]):
         for j in range(shape[2]):
@@ -309,49 +74,10 @@ def false_target(ds, mask, cutoff=255, threebeam=True):
     return mask
 
 
-def default_mask(ds):
-    """
-    Create a default 2-D mask file based on the velocity data.
-    This function generates a mask where values are marked as valid or invalid 
-    based on the missing values from the velocity data.
-
-    Parameters
-    ----------
-    ds : pyadps.dataset or numpy.ndarray
-         A pyadps data frame is used to extract velocity and dimensions for the mask.
-         If numpy.ndarray, enter the values for beams, cells and ensembles.
-
-    Returns
-    -------
-    numpy.ndarray
-        A mask array of the same shape as `velocity`, where `1` indicates invalid 
-        data  and `0` indicates valid data.
-
-    Notes
-    -----
-    - The function uses the velocity data along with the information from the 
-      Fixed Leader object to determine which values are valid and which are invalid.
-
-    Example
-    -------
-    >>> import pyadps
-    >>> ds = pyadps.ReadFile('demo.000')
-    >>> mask = pyadps.default_mask(ds)
-    """
-    if isinstance(ds, ReadFile) or ds.__class__.__name__ == "ReadFile":
-        flobj = ds.fixedleader
-        velocity = ds.velocity.data
-        cells = flobj.field()["Cells"]
-        beams = flobj.field()["Beams"]
-        ensembles = flobj.ensembles
-    elif isinstance(ds, np.ndarray) and ds.ndim == 3:
-        velocity = ds
-        beams = ds.shape[0]
-        cells = ds.shape[1]
-        ensembles = ds.shape[2] 
-    else:
-        raise ValueError("Input must be a 3-D numpy array or a PyADPS instance")
-
+def default_mask(flobj, velocity):
+    cells = flobj.field()["Cells"]
+    beams = flobj.field()["Beams"]
+    ensembles = flobj.ensembles
     mask = np.zeros((cells, ensembles))
     # Ignore mask for error velocity
     for i in range(beams - 1):
@@ -359,53 +85,7 @@ def default_mask(ds):
     return mask
 
 
-def qc_prompt(ds, name, data=None):
-    """
-    Prompt the user to confirm or adjust the quality control threshold for a specific 
-    parameter based on predefined ranges. The function provides an interactive interface 
-    for the user to adjust thresholds for various quality control criteria, with options 
-    for certain thresholds like "Echo Intensity Thresh" to check the noise floor.
-
-    Parameters
-    ----------
-    flobj : FixedLeader
-        An instance of the FixedLeader class that holds metadata and configuration 
-        data. The `flobj` is used to retrieve the current threshold values based on 
-        the provided parameter name.
-    name : str
-        The name of the parameter for which the threshold is being adjusted. Examples 
-        include "Echo Intensity Thresh", "Correlation Thresh", "Percent Good Min", etc.
-    data : numpy.ndarray, optional
-        The data associated with the threshold. This is required for parameters like 
-        "Echo Intensity Thresh" where a noise floor check might be performed. Default is None.
-
-    Returns
-    -------
-    int
-        The updated threshold value, either the default or the new value entered by the user.
-
-    Notes
-    -----
-    - The function will prompt the user to change the threshold for the given `name` parameter.
-    - For certain parameters, the user may be asked if they would like to check the noise floor 
-      (for example, for "Echo Intensity Thresh"). This triggers the display of a plot and lets 
-      the user select a new threshold.
-    - The function ensures that the new threshold is within the acceptable range for each parameter.
-    - The default thresholds are provided if the user chooses not to change them.
-
-    Example
-    -------
-    >>> import pyadps
-    >>> ds = pyadps.ReadFile('demo.000')
-    >>> name = "Echo Intensity Thresh"
-    >>> threshold = qc_prompt(ds, name, data)
-    The default threshold for echo intensity thresh is 0
-    Would you like to change the threshold [y/n]: y
-    Would you like to check the noise floor [y/n]: y
-    Threshold changed to 50
-    """
-
-    flobj = ds.fixedleader
+def qc_prompt(flobj, name, data=None):
     cutoff = 0
     if name == "Echo Intensity Thresh":
         cutoff = 0

pyadps/utils/velocity_test.py

@@ -1,106 +1,28 @@
 from itertools import groupby
-from pygeomag import GeoMag
 
-import requests
 import numpy as np
 import scipy as sp
+import wmm2020
 
 
-def magdec(glat, glon, alt, time):
-    # Selecting COF file According to given year
-    if time >= 2010 and time < 2030:
-        var = 2010 + (int(time) - 2010) // 5 * 5
-        file_name = "wmm/WMM_{}.COF".format(str(var))
-        geo_mag = GeoMag(coefficients_file=file_name)
-    else:
-        geo_mag = GeoMag("wmm/WMM_2025.COF")
-    result = geo_mag.calculate(glat=glat, glon=glon, alt=alt, time=time)
-
-    return [[result.d]]
-
-
-def wmm2020api(lat1, lon1, year):
-    """
-    This function uses the WMM2020 API to retrieve the magnetic field values at a given location
-    The API need latitude, longitude and year to perform the calculation. The key in the function
-    must be updated time to time since the API is subjected to timely updates and the key may change.
-
-    Args:
-        Latitude (float)
-        Longitude (float)
-        startYear (int)
-
-    Returns:
-        mag -> magnetic declination at the given location in degree.
-    """
-    baseurl_wmm = (
-        "https://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?"
-    )
-    baseurl_igrf = (
-        "https://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?"
-    )
-    baseurl_emm = "https://emmcalc.geomag.info/?magneticcomponent=d&"
-    key = "zNEw7"
-    resultFormat = "json"
-    if year >= 2025:
-        baseurl = baseurl_wmm
-        model = "WMM"
-    elif year >= 2019:
-        baseurl = baseurl_wmm
-        model = "IGRF"
-    elif year >= 2000:
-        baseurl = baseurl_emm
-        model = "EMM"
-    elif year >= 1590:
-        baseurl = baseurl_igrf
-        model = "IGRF"
-    url = "{}model={}&lat1={}&lon1={}&key={}&startYear={}&resultFormat={}".format(
-        baseurl, model, lat1, lon1, key, year, resultFormat
-    )
-    response = requests.get(url)
-    data = response.json()
-    results = data["result"][0]
-    mag = [[results["declination"]]]
-
-    return mag
-
-
-# Commentin magnetic_declination model since the method is no longer using.
-# def magnetic_declination(lat, lon, depth, year):
-#     """
-#     The function  calculates the magnetic declination at a given location and depth.
-#     using a local installation of wmm2020 model.
-
-
-#     Args:
-#         lat (parameter, float): Latitude in decimals
-#         lon (parameter, float): Longitude in decimals
-#         depth (parameter, float): depth in m
-#         year (parameter, integer): Year
-
-#     Returns:
-#         mag: Magnetic declination (degrees)
-#     """
-#     import wmm2020
-#     mag = wmm2020.wmm(lat, lon, depth, year)
-#     mag = mag.decl.data
-
-#     return  mag
-
-
-def velocity_modifier(velocity, mag):
+def magnetic_declination(velocity, lat, lon, depth, year):
     """
     The function uses magnetic declination from wmm2020 to correct
     the horizontal velocities
 
     Args:
-    velocity (numpy array): velocity array
-    mag: magnetic declination  (degrees)
+        velocity (numpy array): velocity(beam, depth, time)
+        lat (parameter, float): Latitude in decimals
+        lon (parameter, float): Longitude in decimals
+        depth (parameter, float): depth in m
+        year (parameter, integer): Year
 
     Returns:
         velocity (numpy array): Rotated velocity using magnetic declination
+        mag: Magnetic declination (degrees)
     """
-    mag = np.deg2rad(mag[0][0])
+    mag = wmm2020.wmm(lat, lon, depth, year)
+    mag = np.deg2rad(mag.decl.data)
     velocity = np.where(velocity == -32768, np.nan, velocity)
     velocity[0, :, :] = velocity[0, :, :] * np.cos(mag) + velocity[1, :, :] * np.sin(
         mag
@@ -110,7 +32,7 @@ def velocity_modifier(velocity, mag):
     ] * np.cos(mag)
     velocity = np.where(velocity == np.nan, -32768, velocity)
 
-    return velocity
+    return velocity, np.rad2deg(mag)
 
 
 def velocity_cutoff(velocity, mask, cutoff=250):
@@ -132,7 +54,7 @@ def velocity_cutoff(velocity, mask, cutoff=250):
     return mask
 
 
-def despike(velocity, mask, kernal_size=13, cutoff=3):
+def despike(velocity, mask, kernal_size=13, cutoff=150):
     """
     Function to remove anomalous spikes in the data over a period of time.
     A median filter is used to despike the data.
@@ -140,24 +62,19 @@ def despike(velocity, mask, kernal_size=13, cutoff=3):
     Args:
         velocity (numpy array, integer): Velocity(depth, time) in mm/s
         mask (numpy array, integer): Mask file
-        kernal_size (paramater, integer): Window size for rolling median filter
-        cutoff (parameter, integer): Number of standard deviations to identify spikes
+        kernal_size (paramater, integer): Number of ensembles over which the spike has to be checked
+        cutoff (parameter, integer): [TODO:description]
 
     Returns:
         mask
     """
+    cutoff = cutoff * 10
     velocity = np.where(velocity == -32768, np.nan, velocity)
     shape = np.shape(velocity)
     for j in range(shape[0]):
-        # Apply median filter
         filt = sp.signal.medfilt(velocity[j, :], kernal_size)
-        # Calculate absolute deviation from the rolling median
         diff = np.abs(velocity[j, :] - filt)
-        # Calculate threshold for spikes based on standard deviation
-        std_dev = np.nanstd(diff)
-        spike_threshold = cutoff * std_dev
-        # Apply mask after identifying spikes
-        mask[j, :] = np.where(diff < spike_threshold, mask[j, :], 1)
+        mask[j, :] = np.where(diff < cutoff, mask[j, :], 1)
     return mask