pyadps 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

pyadps/utils/plotgen.py

@@ -4,7 +4,7 @@ import matplotlib.pyplot as plt
 from matplotlib.widgets import Button, RadioButtons, Slider, TextBox
 from matplotlib.widgets import RectangleSelector
 
-mpl.use("TkAgg")
+# mpl.use("TkAgg")
 
 
 class CutBins:

pyadps/utils/pyreadrdi.py

@@ -3,17 +3,17 @@ pyreadrdi.py
 
 Module Overview
 ---------------
-This module provides functionalities to read and parse RDI ADCP files. 
-It includes functions for reading file headers, fixed and variable leaders, 
+This module provides functionalities to read and parse RDI ADCP files.
+It includes functions for reading file headers, fixed and variable leaders,
 and data types like velocity, correlation, echo intensity, and percent good.
-Currently reads only PD0 format. 
+Currently reads only PD0 format.
 
 Modules
 -------------------
 - fileheader: Function to read and parse the file header information.
 - fixedleader: Function to read and parse the fixed leader section of an RDI file.
 - variableleader: Function to read and parse the variable leader section of an RDI file.
-- datatype: Function to read and parse 3D data types. 
+- datatype: Function to read and parse 3D data types.
 - ErrorCode: Enum class to define and manage error codes for file operations.
 
 Creation Date
@@ -57,7 +57,7 @@ Examples
 >>> vel_data = datatype('example.rdi', "velocity")
 >>> vel_data = datatype('example.rdi', "echo", beam=4, cell=20)
 
-Other add-on functions and classes inlcude bcolors, safe_open, and ErrorCode. 
+Other add-on functions and classes inlcude bcolors, safe_open, and ErrorCode.
 Examples (add-on)
 -------------------
 >>> error = ErrorCode.FILE_NOT_FOUND
@@ -403,72 +403,85 @@ def fileheader(rdi_file):
     bfile.seek(0, 0)
     bskip = i = 0
     hid = [None] * 5
-    while byt := bfile.read(6):
-        hid[0], hid[1], hid[2], hid[3], hid[4] = unpack("<BBHBB", byt)
-        headerid = np.append(headerid, np.int8(hid[0]))
-        sourceid = np.append(sourceid, np.int16(hid[1]))
-        byte = np.append(byte, np.int16(hid[2]))
-        spare = np.append(spare, np.int16(hid[3]))
-        datatype = np.append(datatype, np.int16(hid[4]))
-
-        # dbyte = bfile.read(2 * datatype[i])
-        dbyte, error = safe_read(bfile, 2 * datatype[i])
-        if dbyte is None:
+    try:
+        while byt := bfile.read(6):
+            hid[0], hid[1], hid[2], hid[3], hid[4] = unpack("<BBHBB", byt)
+            headerid = np.append(headerid, np.int8(hid[0]))
+            sourceid = np.append(sourceid, np.int16(hid[1]))
+            byte = np.append(byte, np.int16(hid[2]))
+            spare = np.append(spare, np.int16(hid[3]))
+            datatype = np.append(datatype, np.int16(hid[4]))
+
+            # dbyte = bfile.read(2 * datatype[i])
+            dbyte, error = safe_read(bfile, 2 * datatype[i])
+            if dbyte is None:
+                if i == 0:
+                    error_code = error.code
+                    dummytuple = ([], [], [], [], [], ensemble, error_code)
+                    return dummytuple
+                else:
+                    break
+
+            # Check for id and datatype errors
             if i == 0:
-                error_code = error.code
-                dummytuple = ([], [], [], [], [], ensemble, error_code)
-                return dummytuple
+                if headerid[0] != 127 or sourceid[0] != 127:
+                    error = ErrorCode.WRONG_RDIFILE_TYPE
+                    print(bcolors.FAIL + error.message + bcolors.ENDC)
+                    error_code = error.code
+                    dummytuple = ([], [], [], [], [], ensemble, error_code)
+                    return dummytuple
             else:
-                break
+                if headerid[i] != 127 or sourceid[i] != 127:
+                    error = ErrorCode.ID_NOT_FOUND
+                    print(bcolors.FAIL + error.message)
+                    print(f"Ensembles reset to {i}" + bcolors.ENDC)
+                    break
+
+                if datatype[i] != datatype[i - 1]:
+                    error = ErrorCode.DATATYPE_MISMATCH
+                    print(bcolors.FAIL + error.message)
+                    print(f"Data Types for ensemble {i} is {datatype[i - 1]}.")
+                    print(f"Data Types for ensemble {i + 1} is {datatype[i]}.")
+                    print(f"Ensembles reset to {i}" + bcolors.ENDC)
+                    break
 
-        # Check for id and datatype errors
-        if i == 0:
-            if headerid[0] != 127 or sourceid[0] != 127:
-                error = ErrorCode.WRONG_RDIFILE_TYPE
-                print(bcolors.FAIL + error.message + bcolors.ENDC)
+            try:
+                data = unpack("H" * datatype[i], dbyte)
+                address_offset.append(data)
+            except:
+                error = ErrorCode.FILE_CORRUPTED
                 error_code = error.code
                 dummytuple = ([], [], [], [], [], ensemble, error_code)
                 return dummytuple
-        else:
-            if headerid[i] != 127 or sourceid[i] != 127:
-                error = ErrorCode.ID_NOT_FOUND
-                print(bcolors.FAIL + error.message + bcolors.ENDC)
-                break
-
-            if datatype[i] != datatype[i - 1]:
-                error = ErrorCode.DATATYPE_MISMATCH
-                print(bcolors.FAIL + error.message)
-                print(f"Data Types for ensemble {i} is {datatype[i - 1]}.")
-                print(f"Data Types for ensemble {i + 1} is {datatype[i]}.")
-                print(f"Ensembles reset to {i}" + bcolors.ENDC)
-                break
-
-        try:
-            data = unpack("H" * datatype[i], dbyte)
-            address_offset.append(data)
-        except:
-            error = ErrorCode.FILE_CORRUPTED
-            error_code = error.code
-            dummytuple = ([], [], [], [], [], ensemble, error_code)
-            return dummytuple
-
-        skip_array = [None] * datatype[i]
-        for dtype in range(datatype[i]):
-            bseek = int(bskip) + int(address_offset[i][dtype])
-            bfile.seek(bseek, 0)
-            readbyte = bfile.read(2)
-            skip_array[dtype] = int.from_bytes(
-                readbyte, byteorder="little", signed=False
-            )
-
-        dataid.append(skip_array)
-        # bytekip is the number of bytes to skip to reach
-        # an ensemble from beginning of file.
-        # ?? Should byteskip be from current position ??
-        bskip = int(bskip) + int(byte[i]) + 2
-        bfile.seek(bskip, 0)
-        byteskip = np.append(byteskip, np.int32(bskip))
-        i += 1
+
+            skip_array = [None] * datatype[i]
+            for dtype in range(datatype[i]):
+                bseek = int(bskip) + int(address_offset[i][dtype])
+                bfile.seek(bseek, 0)
+                readbyte = bfile.read(2)
+                skip_array[dtype] = int.from_bytes(
+                    readbyte, byteorder="little", signed=False
+                )
+
+            dataid.append(skip_array)
+            # bytekip is the number of bytes to skip to reach
+            # an ensemble from beginning of file.
+            # ?? Should byteskip be from current position ??
+            bskip = int(bskip) + int(byte[i]) + 2
+            bfile.seek(bskip, 0)
+            byteskip = np.append(byteskip, np.int32(bskip))
+            i += 1
+    except (ValueError, StructError, OverflowError) as e:
+        # except:
+        print(bcolors.WARNING + "WARNING: The file is broken.")
+        print(
+            f"Function `fileheader` unable to extract data for ensemble {i + 1}. Total ensembles reset to {i}."
+        )
+        print(bcolors.UNDERLINE + "Details from struct function" + bcolors.ENDC)
+        print(f"  Error Type: {type(e).__name__}")
+        print(f"  Error Details: {e}")
+        error = ErrorCode.FILE_CORRUPTED
+        ensemble = i
 
     ensemble = i
     bfile.close()
@@ -612,13 +625,14 @@ def fixedleader(rdi_file, byteskip=None, offset=None, idarray=None, ensemble=0):
 
                 bfile.seek(byteskip[i], 0)
 
-            except (ValueError, StructError) as e:
+            except (ValueError, StructError, OverflowError) as e:
                 print(bcolors.WARNING + "WARNING: The file is broken.")
                 print(
                     f"Function `fixedleader` unable to extract data for ensemble {i + 1}. Total ensembles reset to {i}."
                 )
-                print("Details from struct function:")
-                print(f"An error occurred: {e}" + bcolors.ENDC)
+                print(bcolors.UNDERLINE + "Details from struct function" + bcolors.ENDC)
+                print(f"  Error Type: {type(e).__name__}")
+                print(f"  Error Details: {e}")
                 error = ErrorCode.FILE_CORRUPTED
                 ensemble = i
 
@@ -794,13 +808,14 @@ def variableleader(rdi_file, byteskip=None, offset=None, idarray=None, ensemble=
 
                 bfile.seek(byteskip[i], 0)
 
-            except (ValueError, StructError) as e:
+            except (ValueError, StructError, OverflowError) as e:
                 print(bcolors.WARNING + "WARNING: The file is broken.")
                 print(
                     f"Function `variableleader` unable to extract data for ensemble {i + 1}. Total ensembles reset to {i}."
                 )
-                print("Details from struct function:")
-                print(f"An error occurred: {e}" + bcolors.ENDC)
+                print(bcolors.UNDERLINE + "Details from struct function" + bcolors.ENDC)
+                print(f"  Error Type: {type(e).__name__}")
+                print(f"  Error Details: {e}")
                 error = ErrorCode.FILE_CORRUPTED
                 ensemble = i
 
@@ -910,7 +925,9 @@ def datatype(
     # Velocity is 16 bits and all others are 8 bits.
     # Create empty array for the chosen variable name.
     if var_name == "velocity":
-        var_array = np.full((int(max(beam)), int(max(cell)), ensemble), -32768, dtype="int16")
+        var_array = np.full(
+            (int(max(beam)), int(max(cell)), ensemble), -32768, dtype="int16"
+        )
         bitstr = "<h"
         bitint = 2
     else:  # inserted
@@ -957,16 +974,28 @@ def datatype(
         return (var_array, error.code)
 
     # READ DATA
-    for i in range(ensemble):
-        bfile.seek(fbyteskip[i], 1)
-        bdata = bfile.read(2)
-        for cno in range(int(cell[i])):
-            for bno in range(int(beam[i])):
-                bdata = bfile.read(bitint)
-                varunpack = unpack(bitstr, bdata)
-                var_array[bno][cno][i] = varunpack[0]
-        bfile.seek(byteskip[i], 0)
-    bfile.close()
+    i = 0
+    try:
+        for i in range(ensemble):
+            bfile.seek(fbyteskip[i], 1)
+            bdata = bfile.read(2)
+            for cno in range(int(cell[i])):
+                for bno in range(int(beam[i])):
+                    bdata = bfile.read(bitint)
+                    varunpack = unpack(bitstr, bdata)
+                    var_array[bno][cno][i] = varunpack[0]
+            bfile.seek(byteskip[i], 0)
+        bfile.close()
+    except (ValueError, StructError, OverflowError) as e:
+        print(bcolors.WARNING + "WARNING: The file is broken.")
+        print(
+            f"Function `datatype` unable to extract {var_name} for ensemble {i + 1}. Total ensembles reset to {i}."
+        )
+        print(bcolors.UNDERLINE + "Details from struct function" + bcolors.ENDC)
+        print(f"  Error Type: {type(e).__name__}")
+        print(f"  Error Details: {e}")
+        error = ErrorCode.FILE_CORRUPTED
+        ensemble = i
 
-    data = var_array
+    data = var_array[:, :, :ensemble]
     return (data, ensemble, cell, beam, error_code)

pyadps/utils/readrdi.py

@@ -103,6 +103,7 @@ import sys
 import numpy as np
 import pandas as pd
 from pyadps.utils import pyreadrdi
+from pyadps.utils.pyreadrdi import bcolors
 
 
 class DotDict:
@@ -773,7 +774,7 @@ def vlead_dict(vid):
         "MPT Minute": "int16",
         "MPT Second": "int16",
         "MPT Hundredth": "int16",
-        "Hdg Std Dev": "int16",
+        "Head Std Dev": "int16",
         "Pitch Std Dev": "int16",
         "Roll Std Dev": "int16",
         "ADC Channel 0": "int16",
@@ -1358,7 +1359,7 @@ class ReadFile:
         The RDI ADCP binary file to be read.
     """
 
-    def __init__(self, filename):
+    def __init__(self, filename, is_fix_ensemble=True):
         """
         Initializes the ReadFile object and extracts data from the RDI ADCP binary file.
         """
@@ -1534,6 +1535,10 @@ class ReadFile:
         # Add attribute that lists all variables/functions
         self.list_vars = list(vars(self).keys())
 
+        # By default fix ensemble
+        if is_fix_ensemble and not self.isEnsembleEqual:
+            self.fixensemble()
+
     def _copy_attributes_from_var(self):
         for attr_name, attr_value in self.variableleader.__dict__.items():
             # Copy each attribute of var into self
@@ -1552,6 +1557,18 @@ class ReadFile:
             f"'{self.__class__.__name__}' object has no attribute '{name}'"
         )
 
+    def resize_fixedleader(self, newshape):
+        for key in self.fixedleader.fleader:
+            attr_name = key.lower().replace(" ", "_")
+            attr_obj = getattr(self.fixedleader, attr_name)
+            attr_obj.data = attr_obj.data[:newshape]
+
+    def resize_variableleader(self, newshape):
+        for key in self.variableleader.vleader:
+            attr_name = key.lower().replace(" ", "_")
+            attr_obj = getattr(self.variableleader, attr_name)
+            attr_obj.data = attr_obj.data[:newshape]
+
     def fixensemble(self, min_cutoff=0):
         """
         Fixes the ensemble size across all data types in the file if they differ.
@@ -1582,10 +1599,18 @@ class ReadFile:
             self.fileheader.dataid = self.fileheader.dataid[:minens, :]
             if "Fixed Leader" in datatype_array:
                 self.fixedleader.data = self.fixedleader.data[:, :minens]
+                self.fixedleader.fleader = {
+                    k: v[:minens] for k, v in self.fixedleader.fleader.items()
+                }
                 self.fixedleader.ensembles = minens
+                self.resize_fixedleader(minens)
             if "Variable Leader" in datatype_array:
                 self.variableleader.data = self.variableleader.data[:, :minens]
+                self.variableleader.vleader = {
+                    k: v[:minens] for k, v in self.variableleader.vleader.items()
+                }
                 self.variableleader.ensembles = minens
+                self.resize_variableleader(minens)
             if "Velocity" in datatype_array:
                 self.velocity.data = self.velocity.data[:, :, :minens]
                 self.velocity.ensembles = minens
@@ -1601,7 +1626,13 @@ class ReadFile:
             if "Status" in datatype_array:
                 self.status.data = self.status.data[:, :, :minens]
                 self.status.ensembles = minens
-            print(f"Ensembles fixed to {minens}. All data types have same ensembles.")
+
+            self.time = self.time[:minens]
+            print(
+                bcolors.OKBLUE
+                + f"Ensembles fixed to {minens}. All data types have same ensembles."
+                + bcolors.ENDC
+            )
         else:
             print(
                 "WARNING: No response was initiated. All data types have same ensemble."

pyadps/utils/signal_quality.py

@@ -54,7 +54,7 @@ def qc_check(var, mask, cutoff=0):
     return mask
 
 
-def correlation_check(ds, mask, cutoff=64):
+def correlation_check(ds, mask, cutoff,threebeam,beam_ignore=None):
     """
     Perform an correlation check on the provided variable and update the 
     mask to mark valid and invalid values based on a cutoff threshold.
@@ -93,10 +93,15 @@ def correlation_check(ds, mask, cutoff=64):
     >>> outmask = correlation_check(ds, mask, cutoff=9999)
     """
     correlation = ds.correlation.data
+    if threebeam ==True:
+        if beam_ignore == None:
+            correlation = correlation
+        else:
+            correlation = np.delete(correlation,beam_ignore,axis=0) 
     mask = qc_check(correlation, mask, cutoff=cutoff)
     return mask
 
-def echo_check(ds, mask, cutoff=40):
+def echo_check(ds, mask, cutoff, threebeam, beam_ignore=None):
     """
     Perform an echo intensity check on the provided variable and update the 
     mask to mark valid and invalid values based on a cutoff threshold.
@@ -137,6 +142,11 @@ def echo_check(ds, mask, cutoff=40):
     """
 
     echo = ds.echo.data
+    if threebeam ==True:
+        if beam_ignore == None:
+           echo  = echo
+        else:
+            echo = np.delete(echo,beam_ignore,axis=0)
     mask = qc_check(echo, mask, cutoff=cutoff)
     return mask
 
@@ -246,7 +256,7 @@ def pg_check(ds, mask, cutoff=0, threebeam=True):
     return mask
 
 
-def false_target(ds, mask, cutoff=255, threebeam=True):
+def false_target(ds, mask, cutoff=255, threebeam=True, beam_ignore=None):
     """
     Apply a false target detection algorithm based on echo intensity values. 
     This function identifies invalid or false targets in the data and updates 
@@ -292,12 +302,14 @@ def false_target(ds, mask, cutoff=255, threebeam=True):
     """
 
     echo = ds.echo.data
+    if beam_ignore != None:
+        echo = np.delete(echo,beam_ignore, axis=0)
 
     shape = np.shape(echo)
     for i in range(shape[1]):
         for j in range(shape[2]):
             x = np.sort(echo[:, i, j])
-            if threebeam:
+            if threebeam and beam_ignore is None:
                 if x[-1] - x[1] > cutoff:
                     mask[i, j] = 1
             else: