pyadps 0.2.0b0__py3-none-any.whl → 0.3.0__py3-none-any.whl

pyadps/utils/velocity_test.py

@@ -1,9 +1,24 @@
 from itertools import groupby
+from pygeomag import GeoMag
 
 import requests
 import numpy as np
 import scipy as sp
 
+
+def magdec(glat, glon, alt, time):
+    # Selecting COF file According to given year
+    if time >= 2010 and time < 2030:
+        var = 2010 + (int(time) - 2010) // 5 * 5
+        file_name = "wmm/WMM_{}.COF".format(str(var))
+        geo_mag = GeoMag(coefficients_file=file_name)
+    else:
+        geo_mag = GeoMag("wmm/WMM_2025.COF")
+    result = geo_mag.calculate(glat=glat, glon=glon, alt=alt, time=time)
+
+    return [[result.d]]
+
+
 def wmm2020api(lat1, lon1, year):
     """
     This function uses the WMM2020 API to retrieve the magnetic field values at a given location
@@ -18,37 +33,60 @@ def wmm2020api(lat1, lon1, year):
     Returns:
         mag -> magnetic declination at the given location in degree.
     """
-    baseurl = "https://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?"
+    baseurl_wmm = (
+        "https://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?"
+    )
+    baseurl_igrf = (
+        "https://www.ngdc.noaa.gov/geomag-web/calculators/calculateDeclination?"
+    )
+    baseurl_emm = "https://emmcalc.geomag.info/?magneticcomponent=d&"
     key = "zNEw7"
-    resultFormat="json"
-    url = "{}lat1={}&lon1={}&key={}&startYear{}&resultFormat={}".format(baseurl, lat1, lon1, key, year, resultFormat)
+    resultFormat = "json"
+    if year >= 2025:
+        baseurl = baseurl_wmm
+        model = "WMM"
+    elif year >= 2019:
+        baseurl = baseurl_wmm
+        model = "IGRF"
+    elif year >= 2000:
+        baseurl = baseurl_emm
+        model = "EMM"
+    elif year >= 1590:
+        baseurl = baseurl_igrf
+        model = "IGRF"
+    url = "{}model={}&lat1={}&lon1={}&key={}&startYear={}&resultFormat={}".format(
+        baseurl, model, lat1, lon1, key, year, resultFormat
+    )
     response = requests.get(url)
     data = response.json()
     results = data["result"][0]
     mag = [[results["declination"]]]
-    
+
     return mag
 
-def magnetic_declination(lat, lon, depth, year):
-    """
-    The function  calculates the magnetic declination at a given location and depth.
-    using a local installation of wmm2020 model.
 
+# Commentin magnetic_declination model since the method is no longer using.
+# def magnetic_declination(lat, lon, depth, year):
+#     """
+#     The function  calculates the magnetic declination at a given location and depth.
+#     using a local installation of wmm2020 model.
 
-    Args:
-        lat (parameter, float): Latitude in decimals
-        lon (parameter, float): Longitude in decimals
-        depth (parameter, float): depth in m
-        year (parameter, integer): Year
 
-    Returns:
-        mag: Magnetic declination (degrees)
-    """
-    import wmm2020
-    mag = wmm2020.wmm(lat, lon, depth, year)
-    mag = mag.decl.data
+#     Args:
+#         lat (parameter, float): Latitude in decimals
+#         lon (parameter, float): Longitude in decimals
+#         depth (parameter, float): depth in m
+#         year (parameter, integer): Year
+
+#     Returns:
+#         mag: Magnetic declination (degrees)
+#     """
+#     import wmm2020
+#     mag = wmm2020.wmm(lat, lon, depth, year)
+#     mag = mag.decl.data
+
+#     return  mag
 
-    return  mag
 
 def velocity_modifier(velocity, mag):
     """
@@ -64,12 +102,17 @@ def velocity_modifier(velocity, mag):
     """
     mag = np.deg2rad(mag[0][0])
     velocity = np.where(velocity == -32768, np.nan, velocity)
-    velocity[0, :, :] = velocity[0, :, :] * np.cos(mag) + velocity[1, :, :] * np.sin(mag)
-    velocity[1, :, :] = -1 * velocity[0, :, :] * np.sin(mag) + velocity[1, :, :] * np.cos(mag)
+    velocity[0, :, :] = velocity[0, :, :] * np.cos(mag) + velocity[1, :, :] * np.sin(
+        mag
+    )
+    velocity[1, :, :] = -1 * velocity[0, :, :] * np.sin(mag) + velocity[
+        1, :, :
+    ] * np.cos(mag)
     velocity = np.where(velocity == np.nan, -32768, velocity)
 
     return velocity
 
+
 def velocity_cutoff(velocity, mask, cutoff=250):
     """
     Masks all velocities above a cutoff. Note that
@@ -89,7 +132,7 @@ def velocity_cutoff(velocity, mask, cutoff=250):
     return mask
 
 
-def despike(velocity, mask, kernal_size=13, cutoff=150):
+def despike(velocity, mask, kernel_size=13, cutoff=3):
     """
     Function to remove anomalous spikes in the data over a period of time.
     A median filter is used to despike the data.
@@ -97,26 +140,31 @@ def despike(velocity, mask, kernal_size=13, cutoff=150):
     Args:
         velocity (numpy array, integer): Velocity(depth, time) in mm/s
         mask (numpy array, integer): Mask file
-        kernal_size (paramater, integer): Number of ensembles over which the spike has to be checked
-        cutoff (parameter, integer): [TODO:description]
+        kernel_size (paramater, integer): Window size for rolling median filter
+        cutoff (parameter, integer): Number of standard deviations to identify spikes
 
     Returns:
         mask
     """
-    cutoff = cutoff * 10
     velocity = np.where(velocity == -32768, np.nan, velocity)
     shape = np.shape(velocity)
     for j in range(shape[0]):
-        filt = sp.signal.medfilt(velocity[j, :], kernal_size)
+        # Apply median filter
+        filt = sp.signal.medfilt(velocity[j, :], kernel_size)
+        # Calculate absolute deviation from the rolling median
         diff = np.abs(velocity[j, :] - filt)
-        mask[j, :] = np.where(diff < cutoff, mask[j, :], 1)
+        # Calculate threshold for spikes based on standard deviation
+        std_dev = np.nanstd(diff)
+        spike_threshold = cutoff * std_dev
+        # Apply mask after identifying spikes
+        mask[j, :] = np.where(diff < spike_threshold, mask[j, :], 1)
     return mask
 
 
 def flatline(
     velocity,
     mask,
-    kernal_size=4,
+    kernel_size=4,
     cutoff=1,
 ):
     """
@@ -126,7 +174,7 @@ def flatline(
     Args:
         velocity (numpy arrray, integer): Velocity (depth, time)
         mask (numpy  array, integer): Mask file
-        kernal_size (parameter, integer): No. of ensembles over which flatline has to be detected
+        kernel_size (parameter, integer): No. of ensembles over which flatline has to be detected
         cutoff (parameter, integer): Permitted deviation in velocity
 
     Returns:
@@ -143,7 +191,7 @@ def flatline(
         for k, g in groupby(dummymask):
             # subset_size = sum(1 for i in g)
             subset_size = len(list(g))
-            if k == 1 and subset_size >= kernal_size:
+            if k == 1 and subset_size >= kernel_size:
                 mask[j, index : index + subset_size] = 1
             index = index + subset_size
         dummymask = np.zeros(shape[1])

pyadps/utils/writenc.py

@@ -17,7 +17,6 @@ from netCDF4 import date2num
 from pyadps.utils import readrdi as rd
 
 
-
 def pd2nctime(time, t0="hours since 2000-01-01"):
     """
     Function to convert pandas datetime format to netcdf datetime format.
@@ -68,7 +67,14 @@ def flead_ncatt(fl_obj, ncfile_id, ens=0):
             setattr(ncfile_id, format_key, format(value))
 
 
-def rawnc(infile, outfile, time, axis_option=None, attributes=None, t0="hours since 2000-01-01"):
+def rawnc(
+    infile,
+    outfile,
+    axis_option="time",
+    fixensemble=True,
+    attributes=None,
+    t0="hours since 2000-01-01",
+):
     """
     rawnc is a function to create netcdf file. Stores 3-D data types like
     velocity, echo, correlation, and percent good.
@@ -85,7 +91,12 @@ def rawnc(infile, outfile, time, axis_option=None, attributes=None, t0="hours si
 
     outnc = nc4.Dataset(outfile, "w", format="NETCDF4")
 
-    flead = rd.FixedLeader(infile)
+    ds = rd.ReadFile(infile)
+    if fixensemble and not ds.isFixedEnsemble:
+        ds.fixensemble()
+
+    head = ds.fileheader
+    flead = ds.fixedleader
     cell_list = flead.fleader["Cells"]
     beam_list = flead.fleader["Beams"]
 
@@ -96,8 +107,11 @@ def rawnc(infile, outfile, time, axis_option=None, attributes=None, t0="hours si
         primary_axis = "ensemble"
         ensemble = outnc.createVariable("ensemble", "i4", ("ensemble",))
         ensemble.axis = "T"
+        # Add ensemble
+        total_ensembles = ds.ensembles
+        ensemble = np.arange(1, total_ensembles + 1, 1)
     elif axis_option == "time":
-        tsize = len(time)
+        tsize = ds.ensembles
         outnc.createDimension("time", tsize)
         primary_axis = "time"
         time_var = outnc.createVariable("time", "i4", ("time",))
@@ -106,6 +120,7 @@ def rawnc(infile, outfile, time, axis_option=None, attributes=None, t0="hours si
         time_var.long_name = "time"
 
         # Convert time_data to numerical format
+        time = ds.time
         nctime = pd2nctime(time, t0)
         time_var[:] = nctime
 
@@ -128,7 +143,7 @@ def rawnc(infile, outfile, time, axis_option=None, attributes=None, t0="hours si
     cell[:] = np.arange(1, max(cell_list) + 1, 1)
     beam[:] = np.arange(1, max(beam_list) + 1, 1)
 
-    varlist = rd.FileHeader(infile).data_types(1)
+    varlist = head.data_types(1)
     varlist.remove("Fixed Leader")
     varlist.remove("Variable Leader")
 
@@ -139,10 +154,8 @@ def rawnc(infile, outfile, time, axis_option=None, attributes=None, t0="hours si
             varid[i] = outnc.createVariable(
                 item, "i2", (primary_axis, "cell", "beam"), fill_value=-32768
             )
-            # varid[i].missing_value = -32768
-            vel = getattr(rd, item)
-            var = vel(infile).data
-            # var = rd.variables(infile, item)
+            vel = getattr(ds, item.lower())
+            var = vel.data
 
         else:
             # Unsigned integers might be assigned for future netcdf versions
@@ -150,34 +163,34 @@ def rawnc(infile, outfile, time, axis_option=None, attributes=None, t0="hours si
             varid[i] = outnc.createVariable(
                 format_item, "i2", (primary_axis, "cell", "beam")
             )
-            datatype = getattr(rd, format_item)
-            var = np.array(datatype(infile).data, dtype="int16")
-            # var = np.array(rd.variables(infile, item), dtype="int16")
+            datatype = getattr(ds, format_item.lower())
+            var = np.array(datatype.data, dtype="int16")
 
         vshape = var.T.shape
-        if i == 0:
-            if primary_axis == "time":
-                time[:] = np.arange(1, vshape[0] + 1, 1)
-            elif primary_axis == "ensemble":
-                ensemble[:] = np.arange(1, vshape[0] + 1, 1)
-            else:
-                raise ValueError(f"Invalid axis_option: {axis_option}.")
 
         varid[i][0 : vshape[0], 0 : vshape[1], 0 : vshape[2]] = var.T
-        
+
     # Add global attributes if provided
     if attributes:
         for key, value in attributes.items():
-            setattr(outnc, key, str(value))  # Convert to string to store in NetCDF metadata
+            setattr(
+                outnc, key, str(value)
+            )  # Convert to string to store in NetCDF metadata
 
     # outnc.history = "Created " + time.ctime(time.time())
     flead_ncatt(flead, outnc)
-    
 
     outnc.close()
 
 
-def vlead_nc(infile, outfile, time, axis_option=None, attributes=None, t0="hours since 2000-01-01"):
+def flead_nc(
+    infile,
+    outfile,
+    axis_option="time",
+    fixensemble=True,
+    attributes=None,
+    t0="hours since 2000-01-01",
+):
     """
     Function to create ncfile containing Variable Leader.
 
@@ -187,6 +200,11 @@ def vlead_nc(infile, outfile, time, axis_option=None, attributes=None, t0="hours
     """
     outnc = nc4.Dataset(outfile, "w", format="NETCDF4")
 
+    # Read Binary File
+    ds = rd.ReadFile(infile)
+    if fixensemble:
+        ds.fixensemble()
+    flead = ds.fixedleader
     # Dimensions
     # Define the primary axis based on axis_option
     if axis_option == "ensemble":
@@ -194,8 +212,12 @@ def vlead_nc(infile, outfile, time, axis_option=None, attributes=None, t0="hours
         primary_axis = "ensemble"
         ensemble = outnc.createVariable("ensemble", "i4", ("ensemble",))
         ensemble.axis = "T"
+        # Add ensemble
+        total_ensembles = ds.ensembles
+        ensemble = np.arange(1, total_ensembles + 1, 1)
+
     elif axis_option == "time":
-        tsize = len(time)
+        tsize = ds.ensembles
         outnc.createDimension("time", tsize)
         primary_axis = "time"
         time_var = outnc.createVariable("time", "i4", ("time",))
@@ -204,6 +226,7 @@ def vlead_nc(infile, outfile, time, axis_option=None, attributes=None, t0="hours
         time_var.long_name = "time"
 
         # Convert time_data to numerical format
+        time = ds.time
         nctime = pd2nctime(time, t0)
         time_var[:] = nctime
 
@@ -211,8 +234,81 @@ def vlead_nc(infile, outfile, time, axis_option=None, attributes=None, t0="hours
         raise ValueError(f"Invalid axis_option: {axis_option}.")
 
     # Variables
+    fdict = flead.fleader
+    varid = [0] * len(fdict)
+
+    i = 0
+
+    for key, values in fdict.items():
+        format_item = key.replace(" ", "_")
+        varid[i] = outnc.createVariable(
+            format_item, "i4", primary_axis, fill_value=-32768
+        )
+        var = values
+        vshape = var.shape
+
+        varid[i][0 : vshape[0]] = var
+        i += 1
 
-    vlead = rd.VariableLeader(infile)
+    # Add global attributes if provided
+    if attributes:
+        for key, value in attributes.items():
+            setattr(outnc, key, str(value))  # Store attributes as strings
+
+    outnc.close()
+
+
+def vlead_nc(
+    infile,
+    outfile,
+    axis_option="time",
+    fixensemble=True,
+    attributes=None,
+    t0="hours since 2000-01-01",
+):
+    """
+    Function to create ncfile containing Variable Leader.
+
+    Args:
+        infile (string): Input file path including filename
+        outfile (string): Output file path including filename
+    """
+    # Create output NetCDF File
+    outnc = nc4.Dataset(outfile, "w", format="NETCDF4")
+
+    # Read Binary File
+    ds = rd.ReadFile(infile)
+    if fixensemble:
+        ds.fixensemble()
+    vlead = ds.variableleader
+    # Dimensions
+    # Define the primary axis based on axis_option
+    if axis_option == "ensemble":
+        outnc.createDimension("ensemble", None)
+        primary_axis = "ensemble"
+        ensemble = outnc.createVariable("ensemble", "i4", ("ensemble",))
+        ensemble.axis = "T"
+        # Add ensemble
+        total_ensembles = ds.ensembles
+        ensemble = np.arange(1, total_ensembles + 1, 1)
+    elif axis_option == "time":
+        tsize = ds.ensembles
+        outnc.createDimension("time", tsize)
+        primary_axis = "time"
+        time_var = outnc.createVariable("time", "i4", ("time",))
+        time_var.axis = "T"
+        time_var.units = t0
+        time_var.long_name = "time"
+
+        # Convert time_data to numerical format
+        time = ds.time
+        nctime = pd2nctime(time, t0)
+        time_var[:] = nctime
+
+    else:
+        raise ValueError(f"Invalid axis_option: {axis_option}.")
+
+    # Read Data
     vdict = vlead.vleader
     varid = [0] * len(vdict)
 
@@ -224,18 +320,12 @@ def vlead_nc(infile, outfile, time, axis_option=None, attributes=None, t0="hours
             format_item, "i4", primary_axis, fill_value=-32768
         )
         var = values
-        vshape = var.shape
-        if i == 0:
-            if primary_axis == "time":
-                time[:] = np.arange(1, vshape[0] + 1, 1)
-            elif primary_axis == "ensemble":
-                ensemble[:] = np.arange(1, vshape[0] + 1, 1)
-            else:
-                raise ValueError(f"Invalid axis_option: {axis_option}.")
+        # vshape = var.shape
 
-        varid[i][0 : vshape[0]] = var
+        varid[i][0 : ds.ensembles] = var
+        # varid[i][0 : vshape[0]] = var
         i += 1
-        
+
     # Add global attributes if provided
     if attributes:
         for key, value in attributes.items():
@@ -244,7 +334,18 @@ def vlead_nc(infile, outfile, time, axis_option=None, attributes=None, t0="hours
     outnc.close()
 
 
-def finalnc(outfile, depth, time, data, t0="hours since 2000-01-01", attributes=None):
+def finalnc(
+    outfile,
+    depth,
+    final_mask,
+    final_echo,
+    final_corr,
+    final_pgood,
+    time,
+    data,
+    t0="hours since 2000-01-01",
+    attributes=None,
+):
     """
     Function to create the processed NetCDF file.
 
@@ -259,15 +360,19 @@ def finalnc(outfile, depth, time, data, t0="hours since 2000-01-01", attributes=
 
     # Change velocity to cm/s
     data = data.astype(np.float64)
-    data[data > fill] /= 10 
+    data[data > fill] /= 10
 
     # Change depth to positive
-    depth = abs(depth)
+    # depth = abs(depth)
 
     # Reverse the arrays if depth in descending order
     if np.all(depth[:-1] >= depth[1:]):
         depth = depth[::-1]
         data = data[:, ::-1, :]
+        final_mask = final_mask[::-1, :]
+        final_echo = final_echo[:, ::-1, :]
+        final_corr = final_corr[:, ::-1, :]
+        final_pgood = final_pgood[:, ::-1, :]
 
     ncfile = nc4.Dataset(outfile, mode="w", format="NETCDF4")
     # Check if depth is scalar or array
@@ -307,6 +412,69 @@ def finalnc(outfile, depth, time, data, t0="hours since 2000-01-01", attributes=
     evel.units = "cm/s"
     evel.long_name = "error_velocity"
 
+    mvel = ncfile.createVariable("mask", np.float32, ("time", "depth"), fill_value=fill)
+    mvel.long_name = "Velocity Mask (1: bad value, 0: good value)"
+
+    echo1 = ncfile.createVariable(
+        "echo1", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    echo1.long_name = "Echo intensity Beam 1"
+
+    echo2 = ncfile.createVariable(
+        "echo2", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    echo2.long_name = "Echo intensity Beam 2"
+
+    echo3 = ncfile.createVariable(
+        "echo3", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    echo3.long_name = "Echo intensity Beam 3"
+
+    echo4 = ncfile.createVariable(
+        "echo4", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    echo4.long_name = "Echo intensity Beam 4"
+
+    corr1 = ncfile.createVariable(
+        "corr1", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    corr1.long_name = "Beam 1 correlation"
+
+    corr2 = ncfile.createVariable(
+        "corr2", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    corr2.long_name = "Beam 2 correlation"
+
+    corr3 = ncfile.createVariable(
+        "corr3", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    corr3.long_name = "Beam 3 correlation"
+
+    corr4 = ncfile.createVariable(
+        "corr4", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    corr4.long_name = "Beam 4 correlation"
+
+    pgd1 = ncfile.createVariable(
+        "pgd1", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    pgd1.long_name = "Percent Good Beam 1"
+
+    pgd2 = ncfile.createVariable(
+        "pgd2", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    pgd2.long_name = "Percent Good Beam 2"
+
+    pgd3 = ncfile.createVariable(
+        "pgd3", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    pgd3.long_name = "Percent Good Beam 3"
+
+    pgd4 = ncfile.createVariable(
+        "pgd4", np.float32, ("time", "depth"), fill_value=-32768
+    )
+    pgd4.long_name = "Percent Good Beam 4"
+
     nctime = pd2nctime(time, t0)
     # write data
     z[:] = depth
@@ -315,10 +483,25 @@ def finalnc(outfile, depth, time, data, t0="hours since 2000-01-01", attributes=
     vvel[:, :] = data[1, :, :].T
     wvel[:, :] = data[2, :, :].T
     evel[:, :] = data[3, :, :].T
-    
+    mvel[:, :] = final_mask.T
+    echo1[:, :] = final_echo[0, :, :].T
+    echo2[:, :] = final_echo[1, :, :].T
+    echo3[:, :] = final_echo[2, :, :].T
+    echo4[:, :] = final_echo[3, :, :].T
+    corr1[:, :] = final_corr[0, :, :].T
+    corr2[:, :] = final_corr[1, :, :].T
+    corr3[:, :] = final_corr[2, :, :].T
+    corr4[:, :] = final_corr[3, :, :].T
+    pgd1[:, :] = final_pgood[0, :, :].T
+    pgd2[:, :] = final_pgood[1, :, :].T
+    pgd3[:, :] = final_pgood[2, :, :].T
+    pgd4[:, :] = final_pgood[3, :, :].T
+
     # Add global attributes if provided
     if attributes:
         for key, value in attributes.items():
             setattr(ncfile, key, str(value))  # Store attributes as strings
 
+    ncfile.mask_applied = "True"
+
     ncfile.close()