pyadps 0.2.1b0__py3-none-any.whl → 0.3.1b0__py3-none-any.whl

pyadps/utils/autoprocess.py

@@ -5,22 +5,25 @@ import numpy as np
 import pandas as pd
 import pyadps.utils.writenc as wr
 from pyadps.utils import readrdi
-from pyadps.utils.profile_test import side_lobe_beam_angle
-from pyadps.utils.regrid import regrid2d, regrid3d
+from pyadps.utils.profile_test import side_lobe_beam_angle, manual_cut_bins
+from pyadps.utils.profile_test import regrid2d, regrid3d
 from pyadps.utils.signal_quality import (
     default_mask,
     ev_check,
     false_target,
     pg_check,
-    qc_check,
+    echo_check,
+    correlation_check,
 )
 from pyadps.utils.velocity_test import (
     despike,
     flatline,
-    magnetic_declination,
     velocity_cutoff,
+    wmm2020api,
+    velocity_modifier,
 )
 
+
 def main():
     # Get the config file
     try:
@@ -29,16 +32,34 @@ def main():
             autoprocess(filepath)
         else:
             print("File not found!")
-    except:
+    except Exception as e:
+        import traceback
+
         print("Error: Unable to process the data.")
+        traceback.print_exc()
+
 
-def autoprocess(filepath):
+def autoprocess(config_file, binary_file_path=None):
+    # Load configuration
     config = configparser.ConfigParser()
-    config.read(filepath)
-    input_file_name = config.get("FileSettings", "input_file_name")
-    input_file_path = config.get("FileSettings", "input_file_path")
 
-    full_input_file_path = os.path.join(input_file_path, input_file_name)
+    # Decode and parse the config file
+    # Check if config_file is a file-like object or a file path
+    if hasattr(config_file, "read"):
+        # If it's a file-like object, read its content
+        config_content = config_file.read().decode("utf-8")
+    else:
+        # If it's a file path, open the file and read its content
+        with open(config_file, "r", encoding="utf-8") as file:
+            config_content = file.read()
+    config.read_string(config_content)
+
+    if not binary_file_path:
+        input_file_name = config.get("FileSettings", "input_file_name")
+        input_file_path = config.get("FileSettings", "input_file_path")
+        full_input_file_path = os.path.join(input_file_path, input_file_name)
+    else:
+        full_input_file_path = binary_file_path
 
     print("File reading started. Please wait for a few seconds ...")
     ds = readrdi.ReadFile(full_input_file_path)
@@ -55,13 +76,24 @@ def autoprocess(filepath):
     cells = flobj.field()["Cells"]
     fdata = flobj.fleader
     vdata = vlobj.vleader
+    #  depth = ds.variableleader.depth_of_transducer
 
-    mask = default_mask(flobj, velocity)
-    print("Default Mask created.")
+    # Initialize mask
+    mask = default_mask(ds)
+
+    # Debugging statement
     x = np.arange(0, ensembles, 1)
     y = np.arange(0, cells, 1)
     depth = None
 
+    axis_option = config.get("DownloadOptions", "axis_option")
+
+    # Sensor Test
+    isSensorTest = config.getboolean("SensorTest", "sensor_test")
+    isRollTest = config.getboolean("RollTest", "roll_test")
+    # if isSensorTest:
+    #     if isRollTest:
+
     # QC Test
     isQCTest = config.getboolean("QCTest", "qc_test")
 
@@ -72,14 +104,15 @@ def autoprocess(filepath):
         ft = config.getint("QCTest", "false_target")
         is3Beam = config.getboolean("QCTest", "three_beam")
         pgt = config.getint("QCTest", "percentage_good")
+        orientation = config.get("QCTest", "orientation")
 
-        mask = pg_check(pgood, mask, pgt, threebeam=is3Beam)
-        mask = qc_check(correlation, mask, ct)
-        mask = qc_check(echo, mask, et)
-        mask = ev_check(velocity[3, :, :], mask, evt)
-        mask = false_target(echo, mask, ft, threebeam=True)
-        print("QC Test complete.")
+        mask = pg_check(ds, mask, pgt, threebeam=is3Beam)
+        mask = correlation_check(ds, mask, ct)
+        mask = echo_check(ds, mask, et)
+        mask = ev_check(ds, mask, evt)
+        mask = false_target(ds, mask, ft, threebeam=True)
 
+    # Profile Test
     endpoints = None
     isProfileTest = config.getboolean("ProfileTest", "profile_test")
     if isProfileTest:
@@ -87,8 +120,6 @@ def autoprocess(filepath):
         if isTrimEnds:
             start_index = config.getint("ProfileTest", "trim_ends_start_index")
             end_index = config.getint("ProfileTest", "trim_ends_end_index")
-            # if start_index < 0 or start_index > ensembles:
-
             if start_index > 0:
                 mask[:, :start_index] = 1
 
@@ -101,29 +132,150 @@ def autoprocess(filepath):
 
         isCutBins = config.getboolean("ProfileTest", "cut_bins")
         if isCutBins:
+            water_column_depth = 0
             add_cells = config.getint("ProfileTest", "cut_bins_add_cells")
-            mask = side_lobe_beam_angle(flobj, vlobj, mask, extra_cells=add_cells)
+            if orientation == "down":
+                water_column_depth = config.get("ProfileTest", "water_column_depth")
+                water_column_depth = int(water_column_depth)
+                mask = side_lobe_beam_angle(
+                    ds,
+                    mask,
+                    orientation=orientation,
+                    water_column_depth=water_column_depth,
+                    extra_cells=add_cells,
+                )
+            else:
+                mask = side_lobe_beam_angle(
+                    ds,
+                    mask,
+                    orientation=orientation,
+                    water_column_depth=water_column_depth,
+                    extra_cells=add_cells,
+                )
 
             print("Cutbins complete.")
 
+        # Manual Cut Bins
+        isManual_cutbins = config.getboolean("ProfileTest", "manual_cutbins")
+        if isManual_cutbins:
+            raw_bins = config.get("ProfileTest", "manual_cut_bins")
+            bin_groups = raw_bins.split("]")
+
+            for group in bin_groups:
+                if group.strip():  # Ignore empty parts
+                    # Clean and split the values
+                    clean_group = group.replace("[", "").strip()
+                    values = list(map(int, clean_group.split(",")))
+                    min_cell, max_cell, min_ensemble, max_ensemble = values
+                    mask = manual_cut_bins(
+                        mask, min_cell, max_cell, min_ensemble, max_ensemble
+                    )
+
+            print("Manual cut bins applied.")
+
         isRegrid = config.getboolean("ProfileTest", "regrid")
         if isRegrid:
             print("File regridding started. This will take a few seconds ...")
+
             regrid_option = config.get("ProfileTest", "regrid_option")
-            z, velocity = regrid3d(
-                flobj,
-                vlobj,
-                velocity,
-                -32768,
-                trimends=endpoints,
-            )
-            z, echo = regrid3d(flobj, vlobj, echo, -32768, trimends=endpoints)
-            z, correlation = regrid3d(
-                flobj, vlobj, correlation, -32768, trimends=endpoints
-            )
-            z, pgood = regrid3d(flobj, vlobj, pgood, -32768, trimends=endpoints)
-            z, mask = regrid2d(flobj, vlobj, mask, 1, trimends=endpoints)
-            depth = z
+            interpolate = config.get("ProfileTest", "regrid_interpolation")
+            boundary = 0
+            if regrid_option == "Manual":
+                boundary = config.get("ProfileTest", "transducer_depth")
+                z, velocity = regrid3d(
+                    ds,
+                    velocity,
+                    -32768,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                z, echo = regrid3d(
+                    ds,
+                    echo,
+                    -32768,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                z, correlation = regrid3d(
+                    ds,
+                    correlation,
+                    -32768,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                z, pgood = regrid3d(
+                    ds,
+                    pgood,
+                    -32768,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                z, mask = regrid2d(
+                    ds,
+                    mask,
+                    1,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                depth = z
+            else:
+                z, velocity = regrid3d(
+                    ds,
+                    velocity,
+                    -32768,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                z, echo = regrid3d(
+                    ds,
+                    echo,
+                    -32768,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                z, correlation = regrid3d(
+                    ds,
+                    correlation,
+                    -32768,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                z, pgood = regrid3d(
+                    ds,
+                    pgood,
+                    -32768,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                z, mask = regrid2d(
+                    ds,
+                    mask,
+                    1,
+                    trimends=endpoints,
+                    orientation=orientation,
+                    method=interpolate,
+                    boundary_limit=boundary,
+                )
+                depth = z
+
             print("Regrid Complete.")
 
         print("Profile Test complete.")
@@ -138,12 +290,12 @@ def autoprocess(filepath):
             maglon = config.getfloat("VelocityTest", "longitude")
             magdep = config.getfloat("VelocityTest", "depth")
             magyear = config.getfloat("VelocityTest", "year")
+            year = int(magyear)
+            #      mag = config.getfloat("VelocityTest", "mag")
 
-            velocity, mag = magnetic_declination(
-                velocity, maglat, maglon, magdep, magyear
-            )
+            mag = wmm2020api(maglat, maglon, year)
+            velocity = velocity_modifier(velocity, mag)
             print(f"Magnetic Declination applied. The value is {mag[0]} degrees.")
-
         isCutOff = config.getboolean("VelocityTest", "cutoff")
         if isCutOff:
             maxu = config.getint("VelocityTest", "max_zonal_velocity")
@@ -196,6 +348,7 @@ def autoprocess(filepath):
                 kernal_size=despike_kernal,
                 cutoff=despike_cutoff,
             )
+
             print("Flatlines in velocity removed.")
 
         print("Velocity Test complete.")
@@ -239,14 +392,13 @@ def autoprocess(filepath):
         }
     )
 
-    date = pd.to_datetime(date_df)
+    date_raw = pd.to_datetime(date_df)
+    date_vlead = pd.to_datetime(date_df)
+    date_final = pd.to_datetime(date_df)
 
     print("Time axis created.")
 
-    isWriteRawNC = config.get("DownloadOptions", "download_raw")
-    isWriteProcNC = config.get("DownloadOptions", "download_processed")
     isAttributes = config.get("Optional", "attributes")
-
     if isAttributes:
         attributes = [att for att in config["Optional"]]
         attributes = dict(config["Optional"].items())
@@ -254,15 +406,44 @@ def autoprocess(filepath):
     else:
         attributes = None
 
+    isWriteRawNC = config.get("DownloadOptions", "download_raw")
+    isWriteVleadNC = config.get("DownloadOptions", "download_vlead")
+    isWriteProcNC = config.get("DownloadOptions", "download_processed")
+
     if isWriteRawNC:
         filepath = config.get("FileSettings", "output_file_path")
         filename = config.get("FileSettings", "output_file_name_raw")
         output_file_path = os.path.join(filepath, filename)
         if isAttributes:
-            wr.rawnc(full_input_file_path, output_file_path, attributes=attributes)
+            wr.rawnc(
+                full_input_file_path,
+                output_file_path,
+                date_raw,
+                axis_option,
+                attributes,
+                isAttributes,
+            )
 
         print("Raw file written.")
 
+    if isWriteVleadNC:
+        filepath = config.get("FileSettings", "output_file_path")
+        filename = config.get("FileSettings", "output_file_name_vlead")
+        output_file_path = os.path.join(filepath, filename)
+        if isAttributes:
+            wr.vlead_nc(
+                full_input_file_path,
+                output_file_path,
+                date_vlead,
+                axis_option,
+                attributes,
+                isAttributes,
+            )
+
+            print("Vlead file written.")
+
+    depth1 = depth
+
     if isWriteProcNC:
         filepath = config.get("FileSettings", "output_file_path")
         filename = config.get("FileSettings", "output_file_name_processed")
@@ -270,8 +451,9 @@ def autoprocess(filepath):
 
         wr.finalnc(
             full_file_path,
-            depth,
-            date,
+            depth1,
+            mask,
+            date_final,
             velocity,
             attributes=attributes,  # Pass edited attributes
         )

pyadps/utils/metadata/config.ini

@@ -4,8 +4,9 @@ input_file_path = /home/user/data/
 input_file_name = adcp_raw.000
 
 # Output file settings. Do not enter file extension.
-output_file_path = /home/user/output/
+output_file_path = /home/nio/Videos/output/
 output_file_name_raw = adcp_raw.nc
+output_file_name_vlead = adcp_vlead.nc
 output_file_name_processed = adcp_proc.nc
 
 # Choose between 'netcdf' or 'csv' for the raw output format
@@ -16,11 +17,18 @@ output_format_processed = csv
 
 [DownloadOptions]
 # Options to download raw and/or processed output files
+axis_option = ensemble
 download_raw = True
+download_vlead = True
 download_processed = True
 apply_mask = True
 download_mask = True
 
+[SensorTest]
+sensor_test = True
+transducer_depth = 200
+
+
 [QCTest]
 # Enable or Disable QC Test (True/False)
 qc_test = True
@@ -30,6 +38,8 @@ echo_intensity = 40
 false_target = 50
 three_beam = True
 percentage_good = 50
+orientation = up
+
 
 [ProfileTest]
 # Enable or Disable Profile Test (True/False)
@@ -39,17 +49,22 @@ trim_ends_start_index = 2
 trim_ends_end_index = 17086
 cut_bins = True
 cut_bins_add_cells = 2
+manual_cutbins = True
+manual_cut_bins = [2,10,0,8000] [40,42,0,8000] [12,14,0,8000]
 regrid = True
-regrid_option = Bin
+regrid_option = Cell
+regrid_interpolation = nearest
+transducer_depth = 200
+water_column_depth = 200
 
 [VelocityTest]
 # Enable or Disable Velocity Test (True/False)
 velocity_test = True
 magnetic_declination = True
 latitude = 0.0
-longitude = 0.1
+longitude = 83.0
 depth = 0
-year = 2024
+year = 2025
 cutoff = True
 max_zonal_velocity = 250
 max_meridional_velocity = 250
@@ -79,3 +94,6 @@ file_created_by = xxxx
 contact = abcd
 comments = No comments
 
+
+
+