pyadps 0.2.0b0__py3-none-any.whl → 0.3.0__py3-none-any.whl

pyadps/utils/autoprocess.py

@@ -4,22 +4,28 @@ import os
 import numpy as np
 import pandas as pd
 import pyadps.utils.writenc as wr
+from netCDF4 import date2num
 from pyadps.utils import readrdi
-from pyadps.utils.profile_test import side_lobe_beam_angle
-from pyadps.utils.regrid import regrid2d, regrid3d
+from pyadps.utils.profile_test import side_lobe_beam_angle, manual_cut_bins
+from pyadps.utils.profile_test import regrid2d, regrid3d
 from pyadps.utils.signal_quality import (
     default_mask,
     ev_check,
     false_target,
     pg_check,
-    qc_check,
+    echo_check,
+    correlation_check,
 )
 from pyadps.utils.velocity_test import (
     despike,
     flatline,
-    magnetic_declination,
     velocity_cutoff,
+    magdec,
+    wmm2020api,
+    velocity_modifier,
 )
+from pyadps.utils.sensor_health import sound_speed_correction, tilt_sensor_check
+
 
 def main():
     # Get the config file
@@ -29,16 +35,34 @@ def main():
             autoprocess(filepath)
         else:
             print("File not found!")
-    except:
+    except Exception as e:
+        import traceback
+
         print("Error: Unable to process the data.")
+        traceback.print_exc()
 
-def autoprocess(filepath):
+
+def autoprocess(config_file, binary_file_path=None):
+    # Load configuration
     config = configparser.ConfigParser()
-    config.read(filepath)
-    input_file_name = config.get("FileSettings", "input_file_name")
-    input_file_path = config.get("FileSettings", "input_file_path")
 
-    full_input_file_path = os.path.join(input_file_path, input_file_name)
+    # Decode and parse the config file
+    # Check if config_file is a file-like object or a file path
+    if hasattr(config_file, "read"):
+        # If it's a file-like object, read its content
+        config_content = config_file.read().decode("utf-8")
+    else:
+        # If it's a file path, open the file and read its content
+        with open(config_file, "r", encoding="utf-8") as file:
+            config_content = file.read()
+    config.read_string(config_content)
+
+    if not binary_file_path:
+        input_file_name = config.get("FileSettings", "input_file_name")
+        input_file_path = config.get("FileSettings", "input_file_path")
+        full_input_file_path = os.path.join(input_file_path, input_file_name)
+    else:
+        full_input_file_path = binary_file_path
 
     print("File reading started. Please wait for a few seconds ...")
     ds = readrdi.ReadFile(full_input_file_path)
@@ -51,44 +75,173 @@ def autoprocess(filepath):
     echo = ds.echo.data
     correlation = ds.correlation.data
     pgood = ds.percentgood.data
+    roll = ds.roll.data
+    pitch = ds.pitch.data
+    sound = ds.speed_of_sound.data
+    depth = ds.depth_of_transducer.data
+    temperature = ds.temperature.data * ds.temperature.scale
+    salinity = ds.salinity.data * ds.salinity.scale
+    orientation = ds.fixedleader.system_configuration()["Beam Direction"]
     ensembles = header.ensembles
     cells = flobj.field()["Cells"]
-    fdata = flobj.fleader
-    vdata = vlobj.vleader
+    beams = flobj.field()["Beams"]
+    cell_size = flobj.field()["Depth Cell Len"]
+    bin1dist = flobj.field()["Bin 1 Dist"]
+    beam_angle = int(flobj.system_configuration()["Beam Angle"])
+
+    # Initialize mask
+    mask = default_mask(ds)
 
-    mask = default_mask(flobj, velocity)
-    print("Default Mask created.")
+    # Debugging statement
     x = np.arange(0, ensembles, 1)
-    y = np.arange(0, cells, 1)
-    depth = None
+
+    axis_option = config.get("DownloadOptions", "axis_option")
+
+    # Time Correction
+    isTimeAxisModified = config.getboolean("FixTime", "is_time_axis_modified")
+
+    if isTimeAxisModified:
+        isSnapTimeAxis = config.getboolean("FixTime", "is_snap_time_axis")
+        if isSnapTimeAxis:
+            time_snap_frequency = config.get("FixTime", "time_snap_frequency")
+            time_snap_tolerance = config.get("FixTime", "time_snap_tolerance")
+            time_target_minute = config.get("FixTime", "time_target_minute")
+            success, message = ds.snap_time_axis(
+                freq=time_snap_frequency,
+                tolerance=time_snap_tolerance,
+                target_minute=time_target_minute,
+            )
+            if success:
+                print(message)
+
+        isTimeGapFilled = config.getboolean("FixTime", "is_time_gap_filled")
+        if isTimeGapFilled:
+            success, message = ds.fill_time_axis()
+            if success:
+                print(message)
+
+    # Sensor Test
+    isSensorTest = config.getboolean("SensorTest", "sensor_test")
+    if isSensorTest:
+        isDepthModified = config.getboolean("SensorTest", "is_depth_modified")
+        if isDepthModified:
+            depth_option = config.get("SensorTest", "depth_input_option")
+            if depth_option == "Fixed Value":
+                fixed_depth = config.getfloat("SensorTest", "fixed_depth")
+                depth = np.full(ensembles, fixed_depth)
+                depth *= 10
+            elif depth_option == "File Upload":
+                depth_file_path = config.get("SensorTest", "depth_file_path")
+                df = pd.read_csv(depth_file_path)
+                depth = np.squeeze(df)
+                if len(depth) != ensembles:
+                    print("""
+                          Error: Uploaded file ensembles and 
+                          actual ensembles mismatch
+                          """)
+                else:
+                    print("Depth file uploaded.")
+
+        isSalinityModified = config.getboolean("SensorTest", "is_salinity_modified")
+        if isSalinityModified:
+            salinity_option = config.get("SensorTest", "salinity_input_option")
+            if salinity_option == "Fixed Value":
+                fixed_salinity = config.getfloat("SensorTest", "fixed_salinity")
+                salinity = np.full(ensembles, fixed_salinity)
+                salinity *= 10
+            elif salinity_option == "File Upload":
+                salinity_file_path = config.get("SensorTest", "salinity_file_path")
+                df = pd.read_csv(salinity_file_path)
+                salinity = np.squeeze(df)
+                if len(salinity) != ensembles:
+                    print("""
+                          Error: Uploaded file ensembles and 
+                          actual ensembles mismatch
+                          """)
+                else:
+                    print("Salinity file uploaded.")
+
+        isTemperatureModified = config.getboolean(
+            "SensorTest", "is_temperature_modified"
+        )
+        if isTemperatureModified:
+            temperature_option = config.get("SensorTest", "temperature_input_option")
+            if temperature_option == "Fixed Value":
+                fixed_temperature = config.getfloat("SensorTest", "fixed_temperature")
+                temperature = np.full(ensembles, fixed_temperature)
+                temperature *= 10
+            elif temperature_option == "File Upload":
+                temperature_file_path = config.get(
+                    "SensorTest", "temperature_file_path"
+                )
+                df = pd.read_csv(temperature_file_path)
+                temperature = np.squeeze(df)
+                if len(temperature) != ensembles:
+                    print("""
+                          Error: Uploaded file ensembles and 
+                          actual ensembles mismatch
+                          """)
+                else:
+                    print("Temperature file uploaded.")
+
+        isRollCheck = config.getboolean("SensorTest", "roll_check")
+        if isRollCheck:
+            roll_cutoff = config.getint("SensorTest", "roll_cutoff")
+            mask = tilt_sensor_check(roll, mask, cutoff=roll_cutoff)
+
+        isPitchCheck = config.getboolean("SensorTest", "pitch_check")
+        if isPitchCheck:
+            pitch_cutoff = config.getint("SensorTest", "pitch_cutoff")
+            mask = tilt_sensor_check(pitch, mask, cutoff=pitch_cutoff)
+
+        isVelocityModified = config.getboolean("SensorTest", "velocity_modified")
+        if isVelocityModified:
+            velocity = sound_speed_correction(
+                velocity, sound, temperature, salinity, depth
+            )
 
     # QC Test
     isQCTest = config.getboolean("QCTest", "qc_test")
-
     if isQCTest:
-        ct = config.getint("QCTest", "correlation")
-        evt = config.getint("QCTest", "error_velocity")
-        et = config.getint("QCTest", "echo_intensity")
-        ft = config.getint("QCTest", "false_target")
-        is3Beam = config.getboolean("QCTest", "three_beam")
-        pgt = config.getint("QCTest", "percentage_good")
-
-        mask = pg_check(pgood, mask, pgt, threebeam=is3Beam)
-        mask = qc_check(correlation, mask, ct)
-        mask = qc_check(echo, mask, et)
-        mask = ev_check(velocity[3, :, :], mask, evt)
-        mask = false_target(echo, mask, ft, threebeam=True)
-        print("QC Test complete.")
+        isQCCheck = config.get("QCTest", "qc_check")
+        if isQCCheck:
+            ct = config.getint("QCTest", "correlation")
+            evt = config.getint("QCTest", "error_velocity")
+            et = config.getint("QCTest", "echo_intensity")
+            ft = config.getint("QCTest", "false_target")
+            is3beam = config.getboolean("QCTest", "three_beam")
+            if is3beam != None:
+                is3beam = int(is3beam)
+            beam_ignore = config.get("QCTest", "beam_ignore")
+            pgt = config.getint("QCTest", "percent_good")
+            orientation = config.get("QCTest", "orientation")
+            beam_ignore = config.getboolean(
+                "QCTest",
+            )
 
+            mask = pg_check(ds, mask, pgt, threebeam=is3beam)
+            mask = correlation_check(ds, mask, ct, is3beam, beam_ignore=beam_ignore)
+            mask = echo_check(ds, mask, et, is3beam, beam_ignore=beam_ignore)
+            mask = ev_check(ds, mask, evt)
+            mask = false_target(
+                ds, mask, ft, threebeam=is3beam, beam_ignore=beam_ignore
+            )
+
+            print("QC Check Complete.")
+
+        isBeamModified = config.getboolean("QCTest", "beam_modified")
+        if isBeamModified:
+            orientation = config.get("QCTest", "orientation")
+            print("Beam orientation changed.")
+
+    # Profile Test
     endpoints = None
     isProfileTest = config.getboolean("ProfileTest", "profile_test")
     if isProfileTest:
-        isTrimEnds = config.getboolean("ProfileTest", "trim_ends")
+        isTrimEnds = config.getboolean("ProfileTest", "trim_ends_check")
         if isTrimEnds:
-            start_index = config.getint("ProfileTest", "trim_ends_start_index")
-            end_index = config.getint("ProfileTest", "trim_ends_end_index")
-            # if start_index < 0 or start_index > ensembles:
-
+            start_index = config.getint("ProfileTest", "trim_start_ensemble")
+            end_index = config.getint("ProfileTest", "trim_end_ensemble")
             if start_index > 0:
                 mask[:, :start_index] = 1
 
@@ -99,31 +252,116 @@ def autoprocess(filepath):
 
             print("Trim Ends complete.")
 
-        isCutBins = config.getboolean("ProfileTest", "cut_bins")
+        isCutBins = config.getboolean("ProfileTest", "cutbins_sidelobe_check")
         if isCutBins:
-            add_cells = config.getint("ProfileTest", "cut_bins_add_cells")
-            mask = side_lobe_beam_angle(flobj, vlobj, mask, extra_cells=add_cells)
-
+            water_column_depth = config.getint("ProfileTest", "water_depth")
+            extra_cells = config.getint("ProfileTest", "extra_cells")
+            mask = side_lobe_beam_angle(
+                depth,
+                mask,
+                orientation=orientation,
+                water_column_depth=water_column_depth,
+                extra_cells=extra_cells,
+                cells=cells,
+                cell_size=cell_size,
+                bin1dist=bin1dist,
+                beam_angle=beam_angle,
+            )
             print("Cutbins complete.")
 
+        # Manual Cut Bins
+        # isManual_cutbins = config.getboolean("ProfileTest", "manual_cutbins")
+        # if isManual_cutbins:
+        #     raw_bins = config.get("ProfileTest", "manual_cut_bins")
+        #     bin_groups = raw_bins.split("]")
+        #
+        #     for group in bin_groups:
+        #         if group.strip():  # Ignore empty parts
+        #             # Clean and split the values
+        #             clean_group = group.replace("[", "").strip()
+        #             values = list(map(int, clean_group.split(",")))
+        #             min_cell, max_cell, min_ensemble, max_ensemble = values
+        #             mask = manual_cut_bins(
+        #                 mask, min_cell, max_cell, min_ensemble, max_ensemble
+        #             )
+        #
+        #     print("Manual cut bins applied.")
+
         isRegrid = config.getboolean("ProfileTest", "regrid")
         if isRegrid:
             print("File regridding started. This will take a few seconds ...")
-            regrid_option = config.get("ProfileTest", "regrid_option")
+
+            # regrid_option = config.get("ProfileTest", "regrid_option")
+            end_cell_option = config.get("ProfileTest", "end_cell_option")
+            interpolate = config.get("ProfileTest", "interpolate")
+            boundary = config.getint("ProfileTest", "boundary")
             z, velocity = regrid3d(
-                flobj,
-                vlobj,
+                depth,
                 velocity,
                 -32768,
                 trimends=endpoints,
+                orientation=orientation,
+                end_cell_option=end_cell_option,
+                method=interpolate,
+                boundary_limit=boundary,
+                cells=cells,
+                cell_size=cell_size,
+                bin1dist=bin1dist,
+                beams=beams,
+            )
+            z, echo = regrid3d(
+                depth,
+                echo,
+                -32768,
+                trimends=endpoints,
+                orientation=orientation,
+                method=interpolate,
+                boundary_limit=boundary,
+                cells=cells,
+                cell_size=cell_size,
+                bin1dist=bin1dist,
+                beams=beams,
             )
-            z, echo = regrid3d(flobj, vlobj, echo, -32768, trimends=endpoints)
             z, correlation = regrid3d(
-                flobj, vlobj, correlation, -32768, trimends=endpoints
+                depth,
+                correlation,
+                -32768,
+                trimends=endpoints,
+                orientation=orientation,
+                method=interpolate,
+                boundary_limit=boundary,
+                cells=cells,
+                cell_size=cell_size,
+                bin1dist=bin1dist,
+                beams=beams,
+            )
+            z, pgood = regrid3d(
+                depth,
+                pgood,
+                -32768,
+                trimends=endpoints,
+                orientation=orientation,
+                method=interpolate,
+                boundary_limit=boundary,
+                cells=cells,
+                cell_size=cell_size,
+                bin1dist=bin1dist,
+                beams=beams,
+            )
+            z, mask = regrid2d(
+                depth,
+                mask,
+                1,
+                trimends=endpoints,
+                orientation=orientation,
+                method=interpolate,
+                boundary_limit=boundary,
+                cells=cells,
+                cell_size=cell_size,
+                bin1dist=bin1dist,
             )
-            z, pgood = regrid3d(flobj, vlobj, pgood, -32768, trimends=endpoints)
-            z, mask = regrid2d(flobj, vlobj, mask, 1, trimends=endpoints)
             depth = z
+
             print("Regrid Complete.")
 
         print("Profile Test complete.")
@@ -134,14 +372,23 @@ def autoprocess(filepath):
             "VelocityTest", "magnetic_declination"
         )
         if isMagneticDeclination:
-            maglat = config.getfloat("VelocityTest", "latitude")
-            maglon = config.getfloat("VelocityTest", "longitude")
-            magdep = config.getfloat("VelocityTest", "depth")
-            magyear = config.getfloat("VelocityTest", "year")
-
-            velocity, mag = magnetic_declination(
-                velocity, maglat, maglon, magdep, magyear
-            )
+            magmethod = config.get("VelocityTest", "magnet_method")
+            maglat = config.getfloat("VelocityTest", "magnet_latitude")
+            maglon = config.getfloat("VelocityTest", "magnet_longitude")
+            magdep = config.getfloat("VelocityTest", "magnet_depth")
+            magyear = config.getfloat("VelocityTest", "magnet_year")
+            year = int(magyear)
+            #      mag = config.getfloat("VelocityTest", "mag")
+
+            if magmethod == "pygeomag":
+                mag = magdec(maglat, maglon, magdep, magyear)
+            elif magmethod.lower() == "api":
+                mag = wmm2020api(maglat, maglon, year)
+            elif magmethod.lower() == "manual":
+                mag = config.getint("VelocityTest", "magnet_user_input")
+            else:
+                mag = 0
+            velocity = velocity_modifier(velocity, mag)
             print(f"Magnetic Declination applied. The value is {mag[0]} degrees.")
 
         isCutOff = config.getboolean("VelocityTest", "cutoff")
@@ -156,46 +403,47 @@ def autoprocess(filepath):
 
         isDespike = config.getboolean("VelocityTest", "despike")
         if isDespike:
-            despike_kernal = config.getint("VelocityTest", "despike_kernal_size")
-            despike_cutoff = config.getint("VelocityTest", "despike_cutoff")
+            despike_kernel = config.getint("VelocityTest", "despike_kernel_size")
+            despike_cutoff = config.getfloat("VelocityTest", "despike_cutoff")
 
             mask = despike(
                 velocity[0, :, :],
                 mask,
-                kernal_size=despike_kernal,
+                kernel_size=despike_kernel,
                 cutoff=despike_cutoff,
             )
             mask = despike(
                 velocity[1, :, :],
                 mask,
-                kernal_size=despike_kernal,
+                kernel_size=despike_kernel,
                 cutoff=despike_cutoff,
             )
             print("Velocity data despiked.")
 
         isFlatline = config.getboolean("VelocityTest", "flatline")
         if isFlatline:
-            despike_kernal = config.getint("VelocityTest", "flatline_kernal_size")
+            despike_kernel = config.getint("VelocityTest", "flatline_kernel_size")
             despike_cutoff = config.getint("VelocityTest", "flatline_deviation")
 
             mask = flatline(
                 velocity[0, :, :],
                 mask,
-                kernal_size=despike_kernal,
+                kernel_size=despike_kernel,
                 cutoff=despike_cutoff,
             )
             mask = flatline(
                 velocity[1, :, :],
                 mask,
-                kernal_size=despike_kernal,
+                kernel_size=despike_kernel,
                 cutoff=despike_cutoff,
             )
             mask = flatline(
                 velocity[2, :, :],
                 mask,
-                kernal_size=despike_kernal,
+                kernel_size=despike_kernel,
                 cutoff=despike_cutoff,
             )
+
             print("Flatlines in velocity removed.")
 
         print("Velocity Test complete.")
@@ -239,39 +487,82 @@ def autoprocess(filepath):
         }
     )
 
-    date = pd.to_datetime(date_df)
+    date_raw = pd.to_datetime(date_df)
+    date_flead = pd.to_datetime(date_df)
+    date_vlead = pd.to_datetime(date_df)
+    date_final = pd.to_datetime(date_df)
 
     print("Time axis created.")
 
-    isWriteRawNC = config.get("DownloadOptions", "download_raw")
-    isWriteProcNC = config.get("DownloadOptions", "download_processed")
-    isAttributes = config.get("Optional", "attributes")
-
+    isAttributes = config.getboolean("DownloadOptions", "add_attributes_processed")
     if isAttributes:
-        attributes = [att for att in config["Optional"]]
-        attributes = dict(config["Optional"].items())
-        del attributes["attributes"]
+        attributes = [att for att in config["DownloadOptions"]]
+        attributes = dict(config["DownloadOptions"].items())
+        del attributes["add_attributes_processed"]
     else:
         attributes = None
 
+    isWriteRawNC = config.getboolean("DownloadOptions", "download_raw_netcdf")
+    isWritefleadNc = config.getboolean("DownloadOptions", "download_flead_netcdf")
+    isWriteVleadNC = config.getboolean("DownloadOptions", "download_vlead_netcdf")
+    isWriteProcNC = config.getboolean("DownloadOptions", "download_processed_netcdf")
+    filepath = config.get("FileSettings", "output_file_path")
+
+    print(isWriteRawNC)
     if isWriteRawNC:
-        filepath = config.get("FileSettings", "output_file_path")
-        filename = config.get("FileSettings", "output_file_name_raw")
+        filename = config.get("FileSettings", "output_file_name_raw_netcdf")
         output_file_path = os.path.join(filepath, filename)
-        if isAttributes:
-            wr.rawnc(full_input_file_path, output_file_path, attributes=attributes)
+        print(date_raw.shape)
+        wr.rawnc(
+            full_input_file_path,
+            output_file_path,
+            date_raw,
+            axis_option=axis_option,
+            attributes=attributes,
+        )
 
         print("Raw file written.")
 
+    if isWritefleadNc:
+        filename = config.get("FileSettings", "output_file_name_flead_netcdf")
+        output_file_path = os.path.join(filepath, filename)
+        wr.flead_nc(
+            full_input_file_path,
+            output_file_path,
+            date_flead,
+            axis_option=axis_option,
+            attributes=attributes,
+        )
+
+        print("Flead File written")
+
+    if isWriteVleadNC:
+        filename = config.get("FileSettings", "output_file_name_vlead_netcdf")
+        output_file_path = os.path.join(filepath, filename)
+        wr.vlead_nc(
+            full_input_file_path,
+            output_file_path,
+            date_vlead,
+            axis_option=axis_option,
+            attributes=attributes,
+        )
+
+        print("Vlead file written.")
+
+    depth1 = depth
+
     if isWriteProcNC:
-        filepath = config.get("FileSettings", "output_file_path")
-        filename = config.get("FileSettings", "output_file_name_processed")
-        full_file_path = os.path.join(filepath, filename)
+        filename = config.get("FileSettings", "output_file_name_processed_netcdf")
+        output_file_path = os.path.join(filepath, filename)
 
         wr.finalnc(
-            full_file_path,
-            depth,
-            date,
+            output_file_path,
+            depth1,
+            mask,
+            echo,
+            correlation,
+            pgood,
+            date_final,
             velocity,
             attributes=attributes,  # Pass edited attributes
         )