pyadps 0.1.1__py3-none-any.whl → 0.2.0b0__py3-none-any.whl

pyadps/utils/autoprocess.py

@@ -4,28 +4,22 @@ import os
 import numpy as np
 import pandas as pd
 import pyadps.utils.writenc as wr
-from netCDF4 import date2num
 from pyadps.utils import readrdi
-from pyadps.utils.profile_test import side_lobe_beam_angle, manual_cut_bins
-from pyadps.utils.profile_test import regrid2d, regrid3d
+from pyadps.utils.profile_test import side_lobe_beam_angle
+from pyadps.utils.regrid import regrid2d, regrid3d
 from pyadps.utils.signal_quality import (
     default_mask,
     ev_check,
     false_target,
     pg_check,
-    echo_check,
-    correlation_check,
+    qc_check,
 )
 from pyadps.utils.velocity_test import (
     despike,
     flatline,
+    magnetic_declination,
     velocity_cutoff,
-    magdec,
-    wmm2020api,
-    velocity_modifier,
 )
-from pyadps.utils.sensor_health import sound_speed_correction, tilt_sensor_check
-
 
 def main():
     # Get the config file
@@ -35,34 +29,16 @@ def main():
             autoprocess(filepath)
         else:
             print("File not found!")
-    except Exception as e:
-        import traceback
-
+    except:
         print("Error: Unable to process the data.")
-        traceback.print_exc()
-
 
-def autoprocess(config_file, binary_file_path=None):
-    # Load configuration
+def autoprocess(filepath):
     config = configparser.ConfigParser()
+    config.read(filepath)
+    input_file_name = config.get("FileSettings", "input_file_name")
+    input_file_path = config.get("FileSettings", "input_file_path")
 
-    # Decode and parse the config file
-    # Check if config_file is a file-like object or a file path
-    if hasattr(config_file, "read"):
-        # If it's a file-like object, read its content
-        config_content = config_file.read().decode("utf-8")
-    else:
-        # If it's a file path, open the file and read its content
-        with open(config_file, "r", encoding="utf-8") as file:
-            config_content = file.read()
-    config.read_string(config_content)
-
-    if not binary_file_path:
-        input_file_name = config.get("FileSettings", "input_file_name")
-        input_file_path = config.get("FileSettings", "input_file_path")
-        full_input_file_path = os.path.join(input_file_path, input_file_name)
-    else:
-        full_input_file_path = binary_file_path
+    full_input_file_path = os.path.join(input_file_path, input_file_name)
 
     print("File reading started. Please wait for a few seconds ...")
     ds = readrdi.ReadFile(full_input_file_path)
@@ -75,148 +51,44 @@ def autoprocess(config_file, binary_file_path=None):
     echo = ds.echo.data
     correlation = ds.correlation.data
     pgood = ds.percentgood.data
-    roll = ds.variableleader.roll.data
-    pitch = ds.variableleader.pitch.data
-    sound = ds.variableleader.speed_of_sound.data
-    depth = ds.variableleader.depth_of_transducer.data
-    temperature = (
-        ds.variableleader.temperature.data * ds.variableleader.temperature.scale
-    )
-    salinity = ds.variableleader.salinity.data * ds.variableleader.salinity.scale
-    orientation = ds.fixedleader.system_configuration()["Beam Direction"]
     ensembles = header.ensembles
     cells = flobj.field()["Cells"]
-    beams = flobj.field()["Beams"]
-    cell_size = flobj.field()["Depth Cell Len"]
-    bin1dist = flobj.field()["Bin 1 Dist"]
-    beam_angle = int(flobj.system_configuration()["Beam Angle"])
     fdata = flobj.fleader
     vdata = vlobj.vleader
-    #  depth = ds.variableleader.depth_of_transducer
 
-    # Initialize mask
-    mask = default_mask(ds)
-
-    # Debugging statement
+    mask = default_mask(flobj, velocity)
+    print("Default Mask created.")
     x = np.arange(0, ensembles, 1)
     y = np.arange(0, cells, 1)
-
-    axis_option = config.get("DownloadOptions", "axis_option")
-
-    # Sensor Test
-    isSensorTest = config.getboolean("SensorTest", "sensor_test")
-    if isSensorTest:
-        isDepthModified = config.getboolean("SensorTest", "is_depth_modified")
-        if isDepthModified:
-            depth_option = config.get("SensorTest", "depth_input_option")
-            if depth_option == "Fixed Value":
-                fixed_depth = config.getfloat("SensorTest", "fixed_depth")
-                depth = np.full(ensembles, fixed_depth)
-                depth *= 10
-            elif depth_option == "File Upload":
-                depth_file_path = config.get("SensorTest", "depth_file_path")
-                df = pd.read_csv(depth_file_path)
-                depth = np.squeeze(df)
-                if len(depth) != ensembles:
-                    print("""
-                          Error: Uploaded file ensembles and 
-                          actual ensembles mismatch
-                          """)
-                else:
-                    print("Depth file uploaded.")
-
-        isSalinityModified = config.getboolean("SensorTest", "is_salinity_modified")
-        if isSalinityModified:
-            salinity_option = config.get("SensorTest", "salinity_input_option")
-            if salinity_option == "Fixed Value":
-                fixed_salinity = config.getfloat("SensorTest", "fixed_salinity")
-                salinity = np.full(ensembles, fixed_salinity)
-                salinity *= 10
-            elif salinity_option == "File Upload":
-                salinity_file_path = config.get("SensorTest", "salinity_file_path")
-                df = pd.read_csv(salinity_file_path)
-                salinity = np.squeeze(df)
-                if len(salinity) != ensembles:
-                    print("""
-                          Error: Uploaded file ensembles and 
-                          actual ensembles mismatch
-                          """)
-                else:
-                    print("Salinity file uploaded.")
-
-        isTemperatureModified = config.getboolean(
-            "SensorTest", "is_temperature_modified"
-        )
-        if isTemperatureModified:
-            temperature_option = config.get("SensorTest", "temperature_input_option")
-            if temperature_option == "Fixed Value":
-                fixed_temperature = config.getfloat("SensorTest", "fixed_temperature")
-                temperature = np.full(ensembles, fixed_temperature)
-                temperature *= 10
-            elif temperature_option == "File Upload":
-                temperature_file_path = config.get(
-                    "SensorTest", "temperature_file_path"
-                )
-                df = pd.read_csv(temperature_file_path)
-                temperature = np.squeeze(df)
-                if len(temperature) != ensembles:
-                    print("""
-                          Error: Uploaded file ensembles and 
-                          actual ensembles mismatch
-                          """)
-                else:
-                    print("Temperature file uploaded.")
-
-        isRollCheck = config.getboolean("SensorTest", "roll_check")
-        if isRollCheck:
-            roll_cutoff = config.getint("SensorTest", "roll_cutoff")
-            mask = tilt_sensor_check(roll, mask, cutoff=roll_cutoff)
-
-        isPitchCheck = config.getboolean("SensorTest", "pitch_check")
-        if isPitchCheck:
-            pitch_cutoff = config.getint("SensorTest", "pitch_cutoff")
-            mask = tilt_sensor_check(pitch, mask, cutoff=pitch_cutoff)
-
-        isVelocityModified = config.getboolean("SensorTest", "velocity_modified")
-        if isVelocityModified:
-            velocity = sound_speed_correction(
-                velocity, sound, temperature, salinity, depth
-            )
+    depth = None
 
     # QC Test
     isQCTest = config.getboolean("QCTest", "qc_test")
+
     if isQCTest:
-        isQCCheck = config.get("QCTest", "qc_check")
-        if isQCCheck:
-            ct = config.getint("QCTest", "correlation")
-            evt = config.getint("QCTest", "error_velocity")
-            et = config.getint("QCTest", "echo_intensity")
-            ft = config.getint("QCTest", "false_target")
-            is3Beam = config.getboolean("QCTest", "three_beam")
-            pgt = config.getint("QCTest", "percent_good")
-            orientation = config.get("QCTest", "orientation")
-
-            mask = pg_check(ds, mask, pgt, threebeam=is3Beam)
-            mask = correlation_check(ds, mask, ct)
-            mask = echo_check(ds, mask, et)
-            mask = ev_check(ds, mask, evt)
-            mask = false_target(ds, mask, ft, threebeam=True)
-
-            print("QC Check Complete.")
-
-        isBeamModified = config.getboolean("QCTest", "beam_modified")
-        if isBeamModified:
-            orientation = config.get("QCTest", "orientation")
-            print("Beam orientation changed.")
-
-    # Profile Test
+        ct = config.getint("QCTest", "correlation")
+        evt = config.getint("QCTest", "error_velocity")
+        et = config.getint("QCTest", "echo_intensity")
+        ft = config.getint("QCTest", "false_target")
+        is3Beam = config.getboolean("QCTest", "three_beam")
+        pgt = config.getint("QCTest", "percentage_good")
+
+        mask = pg_check(pgood, mask, pgt, threebeam=is3Beam)
+        mask = qc_check(correlation, mask, ct)
+        mask = qc_check(echo, mask, et)
+        mask = ev_check(velocity[3, :, :], mask, evt)
+        mask = false_target(echo, mask, ft, threebeam=True)
+        print("QC Test complete.")
+
     endpoints = None
     isProfileTest = config.getboolean("ProfileTest", "profile_test")
     if isProfileTest:
-        isTrimEnds = config.getboolean("ProfileTest", "trim_ends_check")
+        isTrimEnds = config.getboolean("ProfileTest", "trim_ends")
         if isTrimEnds:
-            start_index = config.getint("ProfileTest", "trim_start_ensemble")
-            end_index = config.getint("ProfileTest", "trim_end_ensemble")
+            start_index = config.getint("ProfileTest", "trim_ends_start_index")
+            end_index = config.getint("ProfileTest", "trim_ends_end_index")
+            # if start_index < 0 or start_index > ensembles:
+
             if start_index > 0:
                 mask[:, :start_index] = 1
 
@@ -227,116 +99,31 @@ def autoprocess(config_file, binary_file_path=None):
 
             print("Trim Ends complete.")
 
-        isCutBins = config.getboolean("ProfileTest", "cutbins_sidelobe_check")
+        isCutBins = config.getboolean("ProfileTest", "cut_bins")
         if isCutBins:
-            water_column_depth = config.getint("ProfileTest", "water_depth")
-            extra_cells = config.getint("ProfileTest", "extra_cells")
-            mask = side_lobe_beam_angle(
-                depth,
-                mask,
-                orientation=orientation,
-                water_column_depth=water_column_depth,
-                extra_cells=extra_cells,
-                cells=cells,
-                cell_size=cell_size,
-                bin1dist=bin1dist,
-                beam_angle=beam_angle,
-            )
-            print("Cutbins complete.")
+            add_cells = config.getint("ProfileTest", "cut_bins_add_cells")
+            mask = side_lobe_beam_angle(flobj, vlobj, mask, extra_cells=add_cells)
 
-        # Manual Cut Bins
-        # isManual_cutbins = config.getboolean("ProfileTest", "manual_cutbins")
-        # if isManual_cutbins:
-        #     raw_bins = config.get("ProfileTest", "manual_cut_bins")
-        #     bin_groups = raw_bins.split("]")
-        #
-        #     for group in bin_groups:
-        #         if group.strip():  # Ignore empty parts
-        #             # Clean and split the values
-        #             clean_group = group.replace("[", "").strip()
-        #             values = list(map(int, clean_group.split(",")))
-        #             min_cell, max_cell, min_ensemble, max_ensemble = values
-        #             mask = manual_cut_bins(
-        #                 mask, min_cell, max_cell, min_ensemble, max_ensemble
-        #             )
-        #
-        #     print("Manual cut bins applied.")
+            print("Cutbins complete.")
 
         isRegrid = config.getboolean("ProfileTest", "regrid")
         if isRegrid:
             print("File regridding started. This will take a few seconds ...")
-
-            # regrid_option = config.get("ProfileTest", "regrid_option")
-            end_cell_option = config.get("ProfileTest", "end_cell_option")
-            interpolate = config.get("ProfileTest", "interpolate")
-            boundary = config.getint("ProfileTest", "boundary")
+            regrid_option = config.get("ProfileTest", "regrid_option")
             z, velocity = regrid3d(
-                depth,
+                flobj,
+                vlobj,
                 velocity,
                 -32768,
                 trimends=endpoints,
-                orientation=orientation,
-                end_cell_option=end_cell_option,
-                method=interpolate,
-                boundary_limit=boundary,
-                cells=cells,
-                cell_size=cell_size,
-                bin1dist=bin1dist,
-                beams=beams,
-            )
-            z, echo = regrid3d(
-                depth,
-                echo,
-                -32768,
-                trimends=endpoints,
-                orientation=orientation,
-                method=interpolate,
-                boundary_limit=boundary,
-                cells=cells,
-                cell_size=cell_size,
-                bin1dist=bin1dist,
-                beams=beams,
             )
+            z, echo = regrid3d(flobj, vlobj, echo, -32768, trimends=endpoints)
             z, correlation = regrid3d(
-                depth,
-                correlation,
-                -32768,
-                trimends=endpoints,
-                orientation=orientation,
-                method=interpolate,
-                boundary_limit=boundary,
-                cells=cells,
-                cell_size=cell_size,
-                bin1dist=bin1dist,
-                beams=beams,
-            )
-            z, pgood = regrid3d(
-                depth,
-                pgood,
-                -32768,
-                trimends=endpoints,
-                orientation=orientation,
-                method=interpolate,
-                boundary_limit=boundary,
-                cells=cells,
-                cell_size=cell_size,
-                bin1dist=bin1dist,
-                beams=beams,
-            )
-            z, mask = regrid2d(
-                depth,
-                mask,
-                1,
-                trimends=endpoints,
-                orientation=orientation,
-                method=interpolate,
-                boundary_limit=boundary,
-                cells=cells,
-                cell_size=cell_size,
-                bin1dist=bin1dist,
+                flobj, vlobj, correlation, -32768, trimends=endpoints
             )
+            z, pgood = regrid3d(flobj, vlobj, pgood, -32768, trimends=endpoints)
+            z, mask = regrid2d(flobj, vlobj, mask, 1, trimends=endpoints)
             depth = z
-
             print("Regrid Complete.")
 
         print("Profile Test complete.")
@@ -347,23 +134,14 @@ def autoprocess(config_file, binary_file_path=None):
             "VelocityTest", "magnetic_declination"
         )
         if isMagneticDeclination:
-            magmethod = config.get("VelocityTest", "magnet_method")
-            maglat = config.getfloat("VelocityTest", "magnet_latitude")
-            maglon = config.getfloat("VelocityTest", "magnet_longitude")
-            magdep = config.getfloat("VelocityTest", "magnet_depth")
-            magyear = config.getfloat("VelocityTest", "magnet_year")
-            year = int(magyear)
-            #      mag = config.getfloat("VelocityTest", "mag")
-
-            if magmethod == "pygeomag":
-                mag = magdec(maglat, maglon, magdep, magyear)
-            elif magmethod.lower() == "api":
-                mag = wmm2020api(maglat, maglon, year)
-            elif magmethod.lower() == "manual":
-                mag = config.getint("VelocityTest", "magnet_user_input")
-            else:
-                mag = 0
-            velocity = velocity_modifier(velocity, mag)
+            maglat = config.getfloat("VelocityTest", "latitude")
+            maglon = config.getfloat("VelocityTest", "longitude")
+            magdep = config.getfloat("VelocityTest", "depth")
+            magyear = config.getfloat("VelocityTest", "year")
+
+            velocity, mag = magnetic_declination(
+                velocity, maglat, maglon, magdep, magyear
+            )
             print(f"Magnetic Declination applied. The value is {mag[0]} degrees.")
 
         isCutOff = config.getboolean("VelocityTest", "cutoff")
@@ -378,47 +156,46 @@ def autoprocess(config_file, binary_file_path=None):
 
         isDespike = config.getboolean("VelocityTest", "despike")
         if isDespike:
-            despike_kernel = config.getint("VelocityTest", "despike_kernel_size")
-            despike_cutoff = config.getfloat("VelocityTest", "despike_cutoff")
+            despike_kernal = config.getint("VelocityTest", "despike_kernal_size")
+            despike_cutoff = config.getint("VelocityTest", "despike_cutoff")
 
             mask = despike(
                 velocity[0, :, :],
                 mask,
-                kernel_size=despike_kernel,
+                kernal_size=despike_kernal,
                 cutoff=despike_cutoff,
             )
             mask = despike(
                 velocity[1, :, :],
                 mask,
-                kernel_size=despike_kernel,
+                kernal_size=despike_kernal,
                 cutoff=despike_cutoff,
             )
             print("Velocity data despiked.")
 
         isFlatline = config.getboolean("VelocityTest", "flatline")
         if isFlatline:
-            despike_kernel = config.getint("VelocityTest", "flatline_kernel_size")
+            despike_kernal = config.getint("VelocityTest", "flatline_kernal_size")
             despike_cutoff = config.getint("VelocityTest", "flatline_deviation")
 
             mask = flatline(
                 velocity[0, :, :],
                 mask,
-                kernel_size=despike_kernel,
+                kernal_size=despike_kernal,
                 cutoff=despike_cutoff,
             )
             mask = flatline(
                 velocity[1, :, :],
                 mask,
-                kernel_size=despike_kernel,
+                kernal_size=despike_kernal,
                 cutoff=despike_cutoff,
             )
             mask = flatline(
                 velocity[2, :, :],
                 mask,
-                kernel_size=despike_kernel,
+                kernal_size=despike_kernal,
                 cutoff=despike_cutoff,
             )
-
             print("Flatlines in velocity removed.")
 
         print("Velocity Test complete.")
@@ -462,82 +239,39 @@ def autoprocess(config_file, binary_file_path=None):
         }
     )
 
-    date_raw = pd.to_datetime(date_df)
-    date_flead = pd.to_datetime(date_df)
-    date_vlead = pd.to_datetime(date_df)
-    date_final = pd.to_datetime(date_df)
+    date = pd.to_datetime(date_df)
 
     print("Time axis created.")
 
-    isAttributes = config.getboolean("DownloadOptions", "add_attributes_processed")
+    isWriteRawNC = config.get("DownloadOptions", "download_raw")
+    isWriteProcNC = config.get("DownloadOptions", "download_processed")
+    isAttributes = config.get("Optional", "attributes")
+
     if isAttributes:
-        attributes = [att for att in config["DownloadOptions"]]
-        attributes = dict(config["DownloadOptions"].items())
-        del attributes["add_attributes_processed"]
+        attributes = [att for att in config["Optional"]]
+        attributes = dict(config["Optional"].items())
+        del attributes["attributes"]
     else:
         attributes = None
 
-    isWriteRawNC = config.getboolean("DownloadOptions", "download_raw_netcdf")
-    isWritefleadNc = config.getboolean("DownloadOptions", "download_flead_netcdf")
-    isWriteVleadNC = config.getboolean("DownloadOptions", "download_vlead_netcdf")
-    isWriteProcNC = config.getboolean("DownloadOptions", "download_processed_netcdf")
-    filepath = config.get("FileSettings", "output_file_path")
-
-    print(isWriteRawNC)
     if isWriteRawNC:
-        filename = config.get("FileSettings", "output_file_name_raw_netcdf")
+        filepath = config.get("FileSettings", "output_file_path")
+        filename = config.get("FileSettings", "output_file_name_raw")
         output_file_path = os.path.join(filepath, filename)
-        print(date_raw.shape)
-        wr.rawnc(
-            full_input_file_path,
-            output_file_path,
-            date_raw,
-            axis_option=axis_option,
-            attributes=attributes,
-        )
+        if isAttributes:
+            wr.rawnc(full_input_file_path, output_file_path, attributes=attributes)
 
         print("Raw file written.")
 
-    if isWritefleadNc:
-        filename = config.get("FileSettings", "output_file_name_flead_netcdf")
-        output_file_path = os.path.join(filepath, filename)
-        wr.flead_nc(
-            full_input_file_path,
-            output_file_path,
-            date_flead,
-            axis_option=axis_option,
-            attributes=attributes,
-        )
-
-        print("Flead File written")
-
-    if isWriteVleadNC:
-        filename = config.get("FileSettings", "output_file_name_vlead_netcdf")
-        output_file_path = os.path.join(filepath, filename)
-        wr.vlead_nc(
-            full_input_file_path,
-            output_file_path,
-            date_vlead,
-            axis_option=axis_option,
-            attributes=attributes,
-        )
-
-        print("Vlead file written.")
-
-    depth1 = depth
-
     if isWriteProcNC:
-        filename = config.get("FileSettings", "output_file_name_processed_netcdf")
-        output_file_path = os.path.join(filepath, filename)
+        filepath = config.get("FileSettings", "output_file_path")
+        filename = config.get("FileSettings", "output_file_name_processed")
+        full_file_path = os.path.join(filepath, filename)
 
         wr.finalnc(
-            output_file_path,
-            depth1,
-            mask,
-            echo,
-            correlation,
-            pgood,
-            date_final,
+            full_file_path,
+            depth,
+            date,
             velocity,
             attributes=attributes,  # Pass edited attributes
         )