pyadps 0.2.1b0__py3-none-any.whl → 0.3.0__py3-none-any.whl

pyadps/utils/pyreadrdi.py

@@ -3,17 +3,17 @@ pyreadrdi.py
 
 Module Overview
 ---------------
-This module provides functionalities to read and parse RDI ADCP files. 
-It includes functions for reading file headers, fixed and variable leaders, 
+This module provides functionalities to read and parse RDI ADCP files.
+It includes functions for reading file headers, fixed and variable leaders,
 and data types like velocity, correlation, echo intensity, and percent good.
-Currently reads only PD0 format. 
+Currently reads only PD0 format.
 
 Modules
 -------------------
 - fileheader: Function to read and parse the file header information.
 - fixedleader: Function to read and parse the fixed leader section of an RDI file.
 - variableleader: Function to read and parse the variable leader section of an RDI file.
-- datatype: Function to read and parse 3D data types. 
+- datatype: Function to read and parse 3D data types.
 - ErrorCode: Enum class to define and manage error codes for file operations.
 
 Creation Date
@@ -22,15 +22,15 @@ Creation Date
 
 Last Modified Date
 --------------
-2024-09-01
+2025-10-01
 
 Version
 -------
-0.2.0
+0.3.0
 
 Author
 ------
-[P. Amol] <your.email@example.com>
+[P. Amol] <prakashamol@gmail.com>
 
 License
 -------
@@ -57,7 +57,7 @@ Examples
 >>> vel_data = datatype('example.rdi', "velocity")
 >>> vel_data = datatype('example.rdi', "echo", beam=4, cell=20)
 
-Other add-on functions and classes inlcude bcolors, safe_open, and ErrorCode. 
+Other add-on functions and classes inlcude bcolors, safe_open, and ErrorCode.
 Examples (add-on)
 -------------------
 >>> error = ErrorCode.FILE_NOT_FOUND
@@ -320,7 +320,7 @@ def safe_read(bfile, num_bytes):
 
         if len(readbytes) != num_bytes:
             print(f"Unexpected end of file: fewer than {num_bytes} bytes were read.")
-            return (readbytes, ErrorCode.FILE_CORRUPTED)
+            return (None, ErrorCode.FILE_CORRUPTED)
         else:
             return (readbytes, ErrorCode.SUCCESS)
 
@@ -403,65 +403,85 @@ def fileheader(rdi_file):
     bfile.seek(0, 0)
     bskip = i = 0
     hid = [None] * 5
-    while byt := bfile.read(6):
-        hid[0], hid[1], hid[2], hid[3], hid[4] = unpack("<BBHBB", byt)
-        headerid = np.append(headerid, np.int8(hid[0]))
-        sourceid = np.append(sourceid, np.int16(hid[1]))
-        byte = np.append(byte, np.int16(hid[2]))
-        spare = np.append(spare, np.int16(hid[3]))
-        datatype = np.append(datatype, np.int16(hid[4]))
-
-        # dbyte = bfile.read(2 * datatype[i])
-        dbyte, error = safe_read(bfile, 2 * datatype[i])
-        if dbyte is None:
+    try:
+        while byt := bfile.read(6):
+            hid[0], hid[1], hid[2], hid[3], hid[4] = unpack("<BBHBB", byt)
+            headerid = np.append(headerid, np.int8(hid[0]))
+            sourceid = np.append(sourceid, np.int16(hid[1]))
+            byte = np.append(byte, np.int16(hid[2]))
+            spare = np.append(spare, np.int16(hid[3]))
+            datatype = np.append(datatype, np.int16(hid[4]))
+
+            # dbyte = bfile.read(2 * datatype[i])
+            dbyte, error = safe_read(bfile, 2 * datatype[i])
+            if dbyte is None:
+                if i == 0:
+                    error_code = error.code
+                    dummytuple = ([], [], [], [], [], ensemble, error_code)
+                    return dummytuple
+                else:
+                    break
+
+            # Check for id and datatype errors
             if i == 0:
-                error_code = error.code
-                dummytuple = ([], [], [], [], [], ensemble, error_code)
-                return dummytuple
+                if headerid[0] != 127 or sourceid[0] != 127:
+                    error = ErrorCode.WRONG_RDIFILE_TYPE
+                    print(bcolors.FAIL + error.message + bcolors.ENDC)
+                    error_code = error.code
+                    dummytuple = ([], [], [], [], [], ensemble, error_code)
+                    return dummytuple
             else:
-                break
+                if headerid[i] != 127 or sourceid[i] != 127:
+                    error = ErrorCode.ID_NOT_FOUND
+                    print(bcolors.FAIL + error.message)
+                    print(f"Ensembles reset to {i}" + bcolors.ENDC)
+                    break
 
-        # Check for id and datatype errors
-        if i == 0:
-            if headerid[0] != 127 or sourceid[0] != 127:
-                error = ErrorCode.WRONG_RDIFILE_TYPE
-                print(bcolors.FAIL + error.message + bcolors.ENDC)
+                if datatype[i] != datatype[i - 1]:
+                    error = ErrorCode.DATATYPE_MISMATCH
+                    print(bcolors.FAIL + error.message)
+                    print(f"Data Types for ensemble {i} is {datatype[i - 1]}.")
+                    print(f"Data Types for ensemble {i + 1} is {datatype[i]}.")
+                    print(f"Ensembles reset to {i}" + bcolors.ENDC)
+                    break
+
+            try:
+                data = unpack("H" * datatype[i], dbyte)
+                address_offset.append(data)
+            except:
+                error = ErrorCode.FILE_CORRUPTED
                 error_code = error.code
+                dummytuple = ([], [], [], [], [], ensemble, error_code)
                 return dummytuple
-        else:
-            if headerid[i] != 127 or sourceid[i] != 127:
-                error = ErrorCode.ID_NOT_FOUND
-                print(bcolors.FAIL + error.message + bcolors.ENDC)
-                break
-
-            if datatype[i] != datatype[i - 1]:
-                error = ErrorCode.DATATYPE_MISMATCH
-                print(bcolors.FAIL + error.message)
-                print(f"Data Types for ensemble {i} is {datatype[i - 1]}.")
-                print(f"Data Types for ensemble {i + 1} is {datatype[i]}.")
-                print(f"Ensembles reset to {i}" + bcolors.ENDC)
-                break
-
-        data = unpack("H" * datatype[i], dbyte)
-        address_offset.append(data)
-
-        skip_array = [None] * datatype[i]
-        for dtype in range(datatype[i]):
-            bseek = int(bskip) + int(address_offset[i][dtype])
-            bfile.seek(bseek, 0)
-            readbyte = bfile.read(2)
-            skip_array[dtype] = int.from_bytes(
-                readbyte, byteorder="little", signed=False
-            )
-
-        dataid.append(skip_array)
-        # bytekip is the number of bytes to skip to reach
-        # an ensemble from beginning of file.
-        # ?? Should byteskip be from current position ??
-        bskip = int(bskip) + int(byte[i]) + 2
-        bfile.seek(bskip, 0)
-        byteskip = np.append(byteskip, np.int32(bskip))
-        i += 1
+
+            skip_array = [None] * datatype[i]
+            for dtype in range(datatype[i]):
+                bseek = int(bskip) + int(address_offset[i][dtype])
+                bfile.seek(bseek, 0)
+                readbyte = bfile.read(2)
+                skip_array[dtype] = int.from_bytes(
+                    readbyte, byteorder="little", signed=False
+                )
+
+            dataid.append(skip_array)
+            # bytekip is the number of bytes to skip to reach
+            # an ensemble from beginning of file.
+            # ?? Should byteskip be from current position ??
+            bskip = int(bskip) + int(byte[i]) + 2
+            bfile.seek(bskip, 0)
+            byteskip = np.append(byteskip, np.int32(bskip))
+            i += 1
+    except (ValueError, StructError, OverflowError) as e:
+        # except:
+        print(bcolors.WARNING + "WARNING: The file is broken.")
+        print(
+            f"Function `fileheader` unable to extract data for ensemble {i + 1}. Total ensembles reset to {i}."
+        )
+        print(bcolors.UNDERLINE + "Details from struct function" + bcolors.ENDC)
+        print(f"  Error Type: {type(e).__name__}")
+        print(f"  Error Details: {e}")
+        error = ErrorCode.FILE_CORRUPTED
+        ensemble = i
 
     ensemble = i
     bfile.close()
@@ -548,13 +568,24 @@ def fixedleader(rdi_file, byteskip=None, offset=None, idarray=None, ensemble=0):
         error.code = error_code
         error.message = error.get_message(error.code)
 
+    # Note: When processing data from ADCPs with older firmware,
+    # the instrument serial number may be missing. As a result,
+    # garbage value is recorded, which sometimes is too large for a standard 64-bit integer.
+    # The following variables are defined to replace garbage value with a missing value.
+    # Flag to track if a missing serial number is detected
+    is_serial_missing = False
+    # Define the maximum value for a standard signed int64
+    INT64_MAX = 2**63 - 1
+    # Define a missing value flag (0 is a safe unsigned integer choice)
+    MISSING_VALUE_FLAG = 0
+
     bfile.seek(0, 0)
     for i in range(ensemble):
         fbyteskip = None
         for count, item in enumerate(idarray[i]):
             if item in (0, 1):
-                fbyteskip = offset[1][count]
-        if fbyteskip == None:
+                fbyteskip = offset[0][count]
+        if fbyteskip is None:
             error = ErrorCode.ID_NOT_FOUND
             ensemble = i
             print(bcolors.WARNING + error.message)
@@ -596,6 +627,14 @@ def fixedleader(rdi_file, byteskip=None, offset=None, idarray=None, ensemble=0):
                 (fid[27][i], fid[28][i], fid[29][i]) = unpack("<BBH", bdata[38:42])
                 # CPU board serial number (Big Endian)
                 (fid[30][i]) = unpack(">Q", bdata[42:50])[0]
+                # Check for overflow only once to set the flag
+                if not is_serial_missing and fid[30][i] > INT64_MAX:
+                    print(
+                        bcolors.WARNING
+                        + "WARNING: Missing serial number detected (old firmware). Flagging for replacement."
+                        + bcolors.ENDC
+                    )
+                    is_serial_missing = True
                 # (fid[30][i], fid[31][i])= struct.unpack('>II', packed_data)
                 # fid[30][i] = int.from_bytes(bdata[42:50], byteorder="big", signed=False)
                 # System bandwidth, system power & Spare
@@ -605,13 +644,14 @@ def fixedleader(rdi_file, byteskip=None, offset=None, idarray=None, ensemble=0):
 
                 bfile.seek(byteskip[i], 0)
 
-            except (ValueError, StructError) as e:
+            except (ValueError, StructError, OverflowError) as e:
                 print(bcolors.WARNING + "WARNING: The file is broken.")
                 print(
                     f"Function `fixedleader` unable to extract data for ensemble {i + 1}. Total ensembles reset to {i}."
                 )
-                print("Details from struct function:")
-                print(f"An error occurred: {e}" + bcolors.ENDC)
+                print(bcolors.UNDERLINE + "Details from struct function" + bcolors.ENDC)
+                print(f"  Error Type: {type(e).__name__}")
+                print(f"  Error Details: {e}")
                 error = ErrorCode.FILE_CORRUPTED
                 ensemble = i
 
@@ -625,6 +665,20 @@ def fixedleader(rdi_file, byteskip=None, offset=None, idarray=None, ensemble=0):
                 ensemble = i
     bfile.close()
     error_code = error.code
+
+    if is_serial_missing:
+        print(
+            bcolors.OKBLUE
+            + "INFO: Replacing entire serial number array with missing value flag."
+            + bcolors.ENDC
+        )
+        # If Serial No. is missing, flag all data after Serial No.
+        fid[30] = [MISSING_VALUE_FLAG] * ensemble  # Serial No.
+        fid[31] = [MISSING_VALUE_FLAG] * ensemble  # System Bandwidth
+        fid[32] = [MISSING_VALUE_FLAG] * ensemble  # System Power
+        fid[33] = [MISSING_VALUE_FLAG] * ensemble  # Spare 2
+        fid[34] = [MISSING_VALUE_FLAG] * ensemble  # Instrument No
+        fid[35] = [MISSING_VALUE_FLAG] * ensemble  # Beam Angle
     fid = np.array(fid)
     data = fid[:, :ensemble]
     return (data, ensemble, error_code)
@@ -703,7 +757,7 @@ def variableleader(rdi_file, byteskip=None, offset=None, idarray=None, ensemble=
         fbyteskip = None
         for count, item in enumerate(idarray[i]):
             if item in (128, 129):
-                fbyteskip = offset[1][count]
+                fbyteskip = offset[0][count]
         if fbyteskip == None:
             error = ErrorCode.ID_NOT_FOUND
             ensemble = i
@@ -787,13 +841,14 @@ def variableleader(rdi_file, byteskip=None, offset=None, idarray=None, ensemble=
 
                 bfile.seek(byteskip[i], 0)
 
-            except (ValueError, StructError) as e:
+            except (ValueError, StructError, OverflowError) as e:
                 print(bcolors.WARNING + "WARNING: The file is broken.")
                 print(
                     f"Function `variableleader` unable to extract data for ensemble {i + 1}. Total ensembles reset to {i}."
                 )
-                print("Details from struct function:")
-                print(f"An error occurred: {e}" + bcolors.ENDC)
+                print(bcolors.UNDERLINE + "Details from struct function" + bcolors.ENDC)
+                print(f"  Error Type: {type(e).__name__}")
+                print(f"  Error Details: {e}")
                 error = ErrorCode.FILE_CORRUPTED
                 ensemble = i
 
@@ -883,7 +938,7 @@ def datatype(
 
     # These arguments are outputs of fixedleader function.
     # Makes the code faster if the fixedheader function is already executed.
-    if cell == 0 or beam == 0:
+    if isinstance(cell, (np.integer, int)) or isinstance(beam, (np.integer, int)):
         flead, ensemble, fl_error_code = fixedleader(
             filename,
             byteskip=byteskip,
@@ -891,22 +946,25 @@ def datatype(
             idarray=idarray,
             ensemble=ensemble,
         )
-        cell = int(flead[7][0])
-        beam = int(flead[6][0])
+        cell = []
+        beam = []
+        cell = flead[7][:]
+        beam = flead[6][:]
         if fl_error_code != 0:
             error_code = fl_error_code
     else:
-        cell = int(cell)
-        beam = int(beam)
-
+        cell = cell
+        beam = beam
     # Velocity is 16 bits and all others are 8 bits.
     # Create empty array for the chosen variable name.
     if var_name == "velocity":
-        var_array = np.zeros((beam, cell, ensemble), dtype="int16")
+        var_array = np.full(
+            (int(max(beam)), int(max(cell)), ensemble), -32768, dtype="int16"
+        )
         bitstr = "<h"
         bitint = 2
     else:  # inserted
-        var_array = np.zeros((beam, cell, ensemble), dtype="uint8")
+        var_array = np.zeros((int(max(beam)), int(max(cell)), ensemble), dtype="uint8")
         bitstr = "<B"
         bitint = 1
     # -----------------------------
@@ -935,7 +993,9 @@ def datatype(
     fbyteskip = None
     for count, item in enumerate(idarray[0][:]):
         if item in vid:
-            fbyteskip = offset[1][count]
+            fbyteskip = []
+            for i in range(ensemble):
+                fbyteskip.append(int(offset[i][count]))
             break
     if fbyteskip is None:
         print(
@@ -947,16 +1007,28 @@ def datatype(
         return (var_array, error.code)
 
     # READ DATA
-    for i in range(ensemble):
-        bfile.seek(fbyteskip, 1)
-        bdata = bfile.read(2)
-        for cno in range(cell):
-            for bno in range(beam):
-                bdata = bfile.read(bitint)
-                varunpack = unpack(bitstr, bdata)
-                var_array[bno][cno][i] = varunpack[0]
-        bfile.seek(byteskip[i], 0)
-    bfile.close()
+    i = 0
+    try:
+        for i in range(ensemble):
+            bfile.seek(fbyteskip[i], 1)
+            bdata = bfile.read(2)
+            for cno in range(int(cell[i])):
+                for bno in range(int(beam[i])):
+                    bdata = bfile.read(bitint)
+                    varunpack = unpack(bitstr, bdata)
+                    var_array[bno][cno][i] = varunpack[0]
+            bfile.seek(byteskip[i], 0)
+        bfile.close()
+    except (ValueError, StructError, OverflowError) as e:
+        print(bcolors.WARNING + "WARNING: The file is broken.")
+        print(
+            f"Function `datatype` unable to extract {var_name} for ensemble {i + 1}. Total ensembles reset to {i}."
+        )
+        print(bcolors.UNDERLINE + "Details from struct function" + bcolors.ENDC)
+        print(f"  Error Type: {type(e).__name__}")
+        print(f"  Error Details: {e}")
+        error = ErrorCode.FILE_CORRUPTED
+        ensemble = i
 
-    data = var_array
+    data = var_array[:, :, :ensemble]
     return (data, ensemble, cell, beam, error_code)