pyadps 0.1.0b0__py3-none-any.whl → 0.2.0b0__py3-none-any.whl

pyadps/pages/07_Write_File.py

@@ -1,3 +1,4 @@
+import configparser
 import tempfile
 
 import numpy as np
@@ -6,8 +7,6 @@ import plotly.graph_objects as go
 import streamlit as st
 import utils.writenc as wr
 from plotly_resampler import FigureResampler
-import configparser
-
 
 if "flead" not in st.session_state:
     st.write(":red[Please Select Data!]")
@@ -89,6 +88,7 @@ else:
     st.session_state.final_depth = st.session_state.depth
 
 
+# Functions for plotting
 @st.cache_data
 def fillplot_plotly(
     x, y, data, maskdata, colorscale="balance", title="Data", mask=False
@@ -113,6 +113,7 @@ def fillplot_plotly(
         yaxis=dict(showline=True, mirror=True),
         title_text=title,
     )
+    fig.update_yaxes(autorange="reversed")
     st.plotly_chart(fig)
 
 
@@ -155,6 +156,7 @@ def call_plot(varname, beam, mask=False):
         )
 
 
+# Option to View Processed Data
 st.header("View Processed Data", divider="blue")
 var_option = st.selectbox(
     "Select a data type", ("Velocity", "Echo", "Correlation", "Percent Good")
@@ -170,6 +172,7 @@ if plot_button:
         call_plot(var_option, beam, mask=False)
 
 
+# Option to Write Processed Data
 st.header("Write Data", divider="blue")
 
 mask_data_radio = st.radio("Do you want to mask the final data?", ("Yes", "No"))
@@ -186,37 +189,57 @@ file_type_radio = st.radio("Select output file format:", ("NetCDF", "CSV"))
 
 if file_type_radio == "NetCDF":
     add_attr_button = st.checkbox("Add attributes to NetCDF file")
-    
+
     if add_attr_button:
         st.write("### Modify Attributes")
-        
+
         # Create two-column layout for attributes
         col1, col2 = st.columns(2)
-        
+
         with col1:
             # Display attributes in the first column
-            for key in ["Cruise_No.", "Ship_Name", "Project_No.", "Water_Depth_m", "Deployment_Depth_m","Deployment_Date","Recovery_Date"]:
+            for key in [
+                "Cruise_No.",
+                "Ship_Name",
+                "Project_No.",
+                "Water_Depth_m",
+                "Deployment_Depth_m",
+                "Deployment_Date",
+                "Recovery_Date",
+            ]:
                 if key in st.session_state.attributes:
-                   st.session_state.attributes[key] = st.text_input(key, value=st.session_state.attributes[key])
+                    st.session_state.attributes[key] = st.text_input(
+                        key, value=st.session_state.attributes[key]
+                    )
                 else:
-                   st.session_state.attributes[key] = st.text_input(key)
-                
+                    st.session_state.attributes[key] = st.text_input(key)
+
         with col2:
             # Display attributes in the second column
-            for key in ["Latitude", "Longitude","Platform_Type","Participants", "File_created_by", "Contact", "Comments"]:
+            for key in [
+                "Latitude",
+                "Longitude",
+                "Platform_Type",
+                "Participants",
+                "File_created_by",
+                "Contact",
+                "Comments",
+            ]:
                 if key in st.session_state.attributes:
-                   st.session_state.attributes[key] = st.text_input(key, value=st.session_state.attributes[key])
+                    st.session_state.attributes[key] = st.text_input(
+                        key, value=st.session_state.attributes[key]
+                    )
                 else:
-                   st.session_state.attributes[key] = st.text_input(key)     
-                   
-download_button = st.button("Generate Processed files")           
+                    st.session_state.attributes[key] = st.text_input(key)
+
+download_button = st.button("Generate Processed files")
 
 if download_button:
     st.session_state.processed_filename = file_write()
     st.write(":grey[Processed file created. Click the download button.]")
     st.write(st.session_state.processed_filename)
     depth = np.trunc(st.session_state.final_depth)
-    
+
     if file_type_radio == "NetCDF":
         if add_attr_button and st.session_state.attributes:
             # Generate file with attributes
@@ -225,7 +248,7 @@ if download_button:
                 depth,
                 st.session_state.date,
                 st.session_state.write_velocity,
-                attributes=st.session_state.attributes  # Pass edited attributes
+                attributes=st.session_state.attributes,  # Pass edited attributes
             )
         else:
             # Generate file without attributes
@@ -233,9 +256,9 @@ if download_button:
                 st.session_state.processed_filename,
                 depth,
                 st.session_state.date,
-                st.session_state.write_velocity
+                st.session_state.write_velocity,
             )
-    
+
     with open(st.session_state.processed_filename, "rb") as file:
         st.download_button(
             label="Download NetCDF File",
@@ -280,73 +303,129 @@ if download_button:
             file_name="vertical_velocity.csv",
             mime="text/csf",
         )
-        
-        
+
+
+# Option to Download Config file
+# ------------------------------
+
 # Header for the Config.ini File Generator
 st.header("Config.ini File Generator", divider="blue")
 
 # Radio button to decide whether to generate the config.ini file
-generate_config_radio = st.radio("Do you want to generate a config.ini file?", ("No", "Yes"))
+generate_config_radio = st.radio(
+    "Do you want to generate a config.ini file?", ("No", "Yes")
+)
+
 
 if generate_config_radio == "Yes":
     # Create a config parser object
     config = configparser.ConfigParser()
 
     # Main section
-    config["Main"] = {
-        "Input_FileName": st.session_state.fname
-    }
-
+    config["FileSettings"] = {}
+    config["DownloadOptions"] = {}
+    config["QCTest"] = {"qc_test": "False"}
+    config["ProfileTest"] = {"profile_test": "False"}
+    config["VelocityTest"] = {"velocity_test": "False"}
+    config["Optional"] = {"attributes": "False"}
+
+    config["FileSettings"]["input_file_path"] = ""
+    config["FileSettings"]["input_file_name"] = st.session_state.fname
+    config["FileSettings"]["output_file_path"] = ""
+    config["FileSettings"]["output_file_name_raw"] = ""
+    config["FileSettings"]["output_file_name_processed"] = ""
+    config["FileSettings"]["output_format_raw"] = str(file_type_radio).lower()
+    config["FileSettings"]["output_format_processed"] = str(file_type_radio).lower()
+
+    config["DownloadOptions"]["download_raw"] = "True"
+    config["DownloadOptions"]["download_processed"] = "True"
+    config["DownloadOptions"]["apply_mask"] = "True"
+    config["DownloadOptions"]["download_mask"] = "True"
+
+    # QC Test Options
     if st.session_state.isQCMask:
-        config["QC Test"] = {}
+        config["QCTest"]["qc_test"] = "True"
 
         # Add the contents of the current QC Mask thresholds
         if "newthresh" in st.session_state:
             for idx, row in st.session_state.newthresh.iterrows():
-                config["QC Test"][row["Threshold"].replace(" ", "_")] = row["Values"]
+                config["QCTest"][row["Threshold"].replace(" ", "_")] = row["Values"]
 
-
-    # Profile Test section
+    # Profile Test Options
     if st.session_state.isProfileMask:
-        config["Profile Test"] = {}
-
-        if st.session_state.update_mask:
+        config["ProfileTest"]["profile_test"] = "True"
 
-            config["Profile Test"]["Change_Range"] = str(st.session_state.ens_range)
-            config["Profile Test"]["Deployment_ensembles"] = str(st.session_state.start_ens)
-            config["Profile Test"]["Recovery_ensembles"] = str(st.session_state.end_ens)
+        if st.session_state.isTrimEnds:
+            config["ProfileTest"]["trim_ends"] = "True"
+            config["ProfileTest"]["trim_ends_start_index"] = str(
+                st.session_state.start_ens
+            )
+            config["ProfileTest"]["trim_ends_end_index"] = str(st.session_state.end_ens)
+        else:
+            config["ProfileTest"]["trim_ends"] = "False"
 
-        if st.session_state.update_mask_cutbin:
-            config["Profile Test"]["Beam"] = str(st.session_state.beam + 1)  # Adding 1 since beams are 1-based
-            config["Profile Test"]["cell_to_delete"] = str(st.session_state.extra_cells)
+        if st.session_state.isCutBins:
+            config["ProfileTest"]["cut_bins"] = "True"
+            config["ProfileTest"]["cut_bins_add_cells"] = str(
+                st.session_state.extra_cells
+            )
+        else:
+            config["ProfileTest"]["cut_bins"] = "False"
 
         if st.session_state.isGrid:
-            config["Profile Test"]["Regrid_Depth_cells"] = st.session_state.last_cell  # Bin or Surface
-
+            config["ProfileTest"]["regrid"] = "True"
+            config["ProfileTest"][
+                "Regrid_Option"
+            ] = st.session_state.end_bin_option
+        else:
+            config["ProfileTest"]["regrid"] = "False"
 
     # Velocity Test Section
     if st.session_state.isVelocityMask:
-        config["Velocity Test"] = {}
+        config["VelocityTest"]["velocity_test"] = "True"
 
         if st.session_state.isMagnet:
-            config["Velocity Test"]["Latitude"] = str(st.session_state.lat)
-            config["Velocity Test"]["Longitude"] = str(st.session_state.lon)
-            config["Velocity Test"]["Depth"] = str(st.session_state.magnetic_dec_depth)
-            config["Velocity Test"]["Year"] = str(st.session_state.year)
+            config["VelocityTest"]["magnetic_declination"] = str(True)
+            config["VelocityTest"]["latitude"] = str(st.session_state.lat)
+            config["VelocityTest"]["longitude"] = str(st.session_state.lon)
+            config["VelocityTest"]["depth"] = str(st.session_state.magnetic_dec_depth)
+            config["VelocityTest"]["year"] = str(st.session_state.year)
+        else:
+            config["VelocityTest"]["magnetic_declination"] = str(False)
 
         if st.session_state.isCutoff:
-            config["Velocity Test"]["Max_Zoank"] = str(st.session_state.maxuvel)
-            config["Velocity Test"]["Max_Meridional"] = str(st.session_state.maxvvel)
-            config["Velocity Test"]["Max_Vertical"] = str(st.session_state.maxwvel)
+            config["VelocityTest"]["cutoff"] = str(True)
+            config["VelocityTest"]["max_zonal_velocity"] = str(st.session_state.maxuvel)
+            config["VelocityTest"]["max_meridional_velocity"] = str(
+                st.session_state.maxvvel
+            )
+            config["VelocityTest"]["max_vertical_velocity"] = str(
+                st.session_state.maxwvel
+            )
+        else:
+            config["VelocityTest"]["cutoff"] = str(False)
 
         if st.session_state.isDespike:
-            config["Velocity Test"]["Despike_Kernal_Size"] = str(st.session_state.despike_kernal)
-            config["Velocity Test"]["Despike_Cutoff"] = str(st.session_state.despike_cutoff)
+            config["VelocityTest"]["despike"] = str(True)
+            config["VelocityTest"]["despike_Kernal_Size"] = str(
+                st.session_state.despike_kernal
+            )
+            config["VelocityTest"]["despike_Cutoff"] = str(
+                st.session_state.despike_cutoff
+            )
+        else:
+            config["VelocityTest"]["Despike"] = str(False)
 
         if st.session_state.isFlatline:
-            config["Velocity Test"]["Flatline_Kernal"] = str(st.session_state.flatline_kernal)
-            config["Velocity Test"]["Flatline_Deviation"] = str(st.session_state.flatline_cutoff)
-
+            config["VelocityTest"]["flatline"] = str(True)
+            config["VelocityTest"]["flatline_kernal_size"] = str(
+                st.session_state.flatline_kernal
+            )
+            config["VelocityTest"]["flatline_deviation"] = str(
+                st.session_state.flatline_cutoff
+            )
+        else:
+            config["VelocityTest"]["flatline"] = str(False)
 
     # Optional section (attributes)
     config["Optional"] = {}
@@ -365,3 +444,9 @@ if generate_config_radio == "Yes":
             data=file,
             file_name="config.ini",
         )
+
+    display_config_radio = st.radio(
+        "Do you want to display config.ini file?", ("No", "Yes")
+    )
+    if display_config_radio == "Yes":
+        st.write({section: dict(config[section]) for section in config.sections()})

pyadps/utils/__init__.py

@@ -9,4 +9,4 @@ from pyadps.utils.regrid import *
 from pyadps.utils.signal_quality import *
 from pyadps.utils.velocity_test import *
 from pyadps.utils.writenc import *
-
+# from pyadps.utils.autoprocess import *

pyadps/utils/autoprocess.py

@@ -0,0 +1,282 @@
+import configparser
+import os
+
+import numpy as np
+import pandas as pd
+import pyadps.utils.writenc as wr
+from pyadps.utils import readrdi
+from pyadps.utils.profile_test import side_lobe_beam_angle
+from pyadps.utils.regrid import regrid2d, regrid3d
+from pyadps.utils.signal_quality import (
+    default_mask,
+    ev_check,
+    false_target,
+    pg_check,
+    qc_check,
+)
+from pyadps.utils.velocity_test import (
+    despike,
+    flatline,
+    magnetic_declination,
+    velocity_cutoff,
+)
+
+def main():
+    # Get the config file
+    try:
+        filepath = input("Enter config file name: ")
+        if os.path.exists(filepath):
+            autoprocess(filepath)
+        else:
+            print("File not found!")
+    except:
+        print("Error: Unable to process the data.")
+
+def autoprocess(filepath):
+    config = configparser.ConfigParser()
+    config.read(filepath)
+    input_file_name = config.get("FileSettings", "input_file_name")
+    input_file_path = config.get("FileSettings", "input_file_path")
+
+    full_input_file_path = os.path.join(input_file_path, input_file_name)
+
+    print("File reading started. Please wait for a few seconds ...")
+    ds = readrdi.ReadFile(full_input_file_path)
+    print("File reading complete.")
+
+    header = ds.fileheader
+    flobj = ds.fixedleader
+    vlobj = ds.variableleader
+    velocity = ds.velocity.data
+    echo = ds.echo.data
+    correlation = ds.correlation.data
+    pgood = ds.percentgood.data
+    ensembles = header.ensembles
+    cells = flobj.field()["Cells"]
+    fdata = flobj.fleader
+    vdata = vlobj.vleader
+
+    mask = default_mask(flobj, velocity)
+    print("Default Mask created.")
+    x = np.arange(0, ensembles, 1)
+    y = np.arange(0, cells, 1)
+    depth = None
+
+    # QC Test
+    isQCTest = config.getboolean("QCTest", "qc_test")
+
+    if isQCTest:
+        ct = config.getint("QCTest", "correlation")
+        evt = config.getint("QCTest", "error_velocity")
+        et = config.getint("QCTest", "echo_intensity")
+        ft = config.getint("QCTest", "false_target")
+        is3Beam = config.getboolean("QCTest", "three_beam")
+        pgt = config.getint("QCTest", "percentage_good")
+
+        mask = pg_check(pgood, mask, pgt, threebeam=is3Beam)
+        mask = qc_check(correlation, mask, ct)
+        mask = qc_check(echo, mask, et)
+        mask = ev_check(velocity[3, :, :], mask, evt)
+        mask = false_target(echo, mask, ft, threebeam=True)
+        print("QC Test complete.")
+
+    endpoints = None
+    isProfileTest = config.getboolean("ProfileTest", "profile_test")
+    if isProfileTest:
+        isTrimEnds = config.getboolean("ProfileTest", "trim_ends")
+        if isTrimEnds:
+            start_index = config.getint("ProfileTest", "trim_ends_start_index")
+            end_index = config.getint("ProfileTest", "trim_ends_end_index")
+            # if start_index < 0 or start_index > ensembles:
+
+            if start_index > 0:
+                mask[:, :start_index] = 1
+
+            if end_index < x[-1]:
+                mask[:, end_index:] = 1
+
+            endpoints = np.array([start_index, end_index])
+
+            print("Trim Ends complete.")
+
+        isCutBins = config.getboolean("ProfileTest", "cut_bins")
+        if isCutBins:
+            add_cells = config.getint("ProfileTest", "cut_bins_add_cells")
+            mask = side_lobe_beam_angle(flobj, vlobj, mask, extra_cells=add_cells)
+
+            print("Cutbins complete.")
+
+        isRegrid = config.getboolean("ProfileTest", "regrid")
+        if isRegrid:
+            print("File regridding started. This will take a few seconds ...")
+            regrid_option = config.get("ProfileTest", "regrid_option")
+            z, velocity = regrid3d(
+                flobj,
+                vlobj,
+                velocity,
+                -32768,
+                trimends=endpoints,
+            )
+            z, echo = regrid3d(flobj, vlobj, echo, -32768, trimends=endpoints)
+            z, correlation = regrid3d(
+                flobj, vlobj, correlation, -32768, trimends=endpoints
+            )
+            z, pgood = regrid3d(flobj, vlobj, pgood, -32768, trimends=endpoints)
+            z, mask = regrid2d(flobj, vlobj, mask, 1, trimends=endpoints)
+            depth = z
+            print("Regrid Complete.")
+
+        print("Profile Test complete.")
+
+    isVelocityTest = config.getboolean("VelocityTest", "velocity_test")
+    if isVelocityTest:
+        isMagneticDeclination = config.getboolean(
+            "VelocityTest", "magnetic_declination"
+        )
+        if isMagneticDeclination:
+            maglat = config.getfloat("VelocityTest", "latitude")
+            maglon = config.getfloat("VelocityTest", "longitude")
+            magdep = config.getfloat("VelocityTest", "depth")
+            magyear = config.getfloat("VelocityTest", "year")
+
+            velocity, mag = magnetic_declination(
+                velocity, maglat, maglon, magdep, magyear
+            )
+            print(f"Magnetic Declination applied. The value is {mag[0]} degrees.")
+
+        isCutOff = config.getboolean("VelocityTest", "cutoff")
+        if isCutOff:
+            maxu = config.getint("VelocityTest", "max_zonal_velocity")
+            maxv = config.getint("VelocityTest", "max_meridional_velocity")
+            maxw = config.getint("VelocityTest", "max_vertical_velocity")
+            mask = velocity_cutoff(velocity[0, :, :], mask, cutoff=maxu)
+            mask = velocity_cutoff(velocity[1, :, :], mask, cutoff=maxv)
+            mask = velocity_cutoff(velocity[2, :, :], mask, cutoff=maxw)
+            print("Maximum velocity cutoff applied.")
+
+        isDespike = config.getboolean("VelocityTest", "despike")
+        if isDespike:
+            despike_kernal = config.getint("VelocityTest", "despike_kernal_size")
+            despike_cutoff = config.getint("VelocityTest", "despike_cutoff")
+
+            mask = despike(
+                velocity[0, :, :],
+                mask,
+                kernal_size=despike_kernal,
+                cutoff=despike_cutoff,
+            )
+            mask = despike(
+                velocity[1, :, :],
+                mask,
+                kernal_size=despike_kernal,
+                cutoff=despike_cutoff,
+            )
+            print("Velocity data despiked.")
+
+        isFlatline = config.getboolean("VelocityTest", "flatline")
+        if isFlatline:
+            despike_kernal = config.getint("VelocityTest", "flatline_kernal_size")
+            despike_cutoff = config.getint("VelocityTest", "flatline_deviation")
+
+            mask = flatline(
+                velocity[0, :, :],
+                mask,
+                kernal_size=despike_kernal,
+                cutoff=despike_cutoff,
+            )
+            mask = flatline(
+                velocity[1, :, :],
+                mask,
+                kernal_size=despike_kernal,
+                cutoff=despike_cutoff,
+            )
+            mask = flatline(
+                velocity[2, :, :],
+                mask,
+                kernal_size=despike_kernal,
+                cutoff=despike_cutoff,
+            )
+            print("Flatlines in velocity removed.")
+
+        print("Velocity Test complete.")
+
+    # Apply mask to velocity data
+    isApplyMask = config.get("DownloadOptions", "apply_mask")
+    if isApplyMask:
+        velocity[:, mask == 1] = -32768
+        print("Mask Applied.")
+
+    # Create Depth axis if regrid not applied
+    if depth is None:
+        mean_depth = np.mean(vlobj.vleader["Depth of Transducer"]) / 10
+        mean_depth = np.trunc(mean_depth)
+        cells = flobj.field()["Cells"]
+        cell_size = flobj.field()["Depth Cell Len"] / 100
+        bin1dist = flobj.field()["Bin 1 Dist"] / 100
+        max_depth = mean_depth - bin1dist
+        min_depth = max_depth - cells * cell_size
+        depth = np.arange(-1 * max_depth, -1 * min_depth, cell_size)
+
+        print("WARNING: File not regrided. Depth axis created based on mean depth.")
+
+    # Create Time axis
+    year = vlobj.vleader["RTC Year"]
+    month = vlobj.vleader["RTC Month"]
+    day = vlobj.vleader["RTC Day"]
+    hour = vlobj.vleader["RTC Hour"]
+    minute = vlobj.vleader["RTC Minute"]
+    second = vlobj.vleader["RTC Second"]
+
+    year = year + 2000
+    date_df = pd.DataFrame(
+        {
+            "year": year,
+            "month": month,
+            "day": day,
+            "hour": hour,
+            "minute": minute,
+            "second": second,
+        }
+    )
+
+    date = pd.to_datetime(date_df)
+
+    print("Time axis created.")
+
+    isWriteRawNC = config.get("DownloadOptions", "download_raw")
+    isWriteProcNC = config.get("DownloadOptions", "download_processed")
+    isAttributes = config.get("Optional", "attributes")
+
+    if isAttributes:
+        attributes = [att for att in config["Optional"]]
+        attributes = dict(config["Optional"].items())
+        del attributes["attributes"]
+    else:
+        attributes = None
+
+    if isWriteRawNC:
+        filepath = config.get("FileSettings", "output_file_path")
+        filename = config.get("FileSettings", "output_file_name_raw")
+        output_file_path = os.path.join(filepath, filename)
+        if isAttributes:
+            wr.rawnc(full_input_file_path, output_file_path, attributes=attributes)
+
+        print("Raw file written.")
+
+    if isWriteProcNC:
+        filepath = config.get("FileSettings", "output_file_path")
+        filename = config.get("FileSettings", "output_file_name_processed")
+        full_file_path = os.path.join(filepath, filename)
+
+        wr.finalnc(
+            full_file_path,
+            depth,
+            date,
+            velocity,
+            attributes=attributes,  # Pass edited attributes
+        )
+        print("Processed file written.")
+
+
+if __name__ == "__main__":
+    main()

pyadps/utils/metadata/config.ini

@@ -0,0 +1,81 @@
+[FileSettings]
+# Input file settings
+input_file_path = /home/user/data/ 
+input_file_name = adcp_raw.000
+
+# Output file settings. Do not enter file extension.
+output_file_path = /home/user/output/
+output_file_name_raw = adcp_raw.nc
+output_file_name_processed = adcp_proc.nc
+
+# Choose between 'netcdf' or 'csv' for the raw output format
+output_format_raw = netcdf
+
+# Choose between 'netcdf' or 'csv' for the processed output format
+output_format_processed = csv
+
+[DownloadOptions]
+# Options to download raw and/or processed output files
+download_raw = True
+download_processed = True
+apply_mask = True
+download_mask = True
+
+[QCTest]
+# Enable or Disable QC Test (True/False)
+qc_test = True
+correlation = 64
+error_velocity = 2000
+echo_intensity = 40
+false_target = 50
+three_beam = True
+percentage_good = 50
+
+[ProfileTest]
+# Enable or Disable Profile Test (True/False)
+profile_test = True
+trim_ends = True
+trim_ends_start_index = 2
+trim_ends_end_index = 17086
+cut_bins = True
+cut_bins_add_cells = 2
+regrid = True
+regrid_option = Bin
+
+[VelocityTest]
+# Enable or Disable Velocity Test (True/False)
+velocity_test = True
+magnetic_declination = True
+latitude = 0.0
+longitude = 0.1
+depth = 0
+year = 2024
+cutoff = True
+max_zonal_velocity = 250
+max_meridional_velocity = 250
+max_vertical_velocity = 15
+despike = True
+despike_kernal_size = 5
+despike_cutoff = 150
+flatline = True 
+flatline_kernal_size = 13
+flatline_deviation = 1
+
+[Optional]
+# Options to add attributes to netcdf file (True/False)
+attributes = True
+cruise_no. = Ship999
+ship_name = RV Vessel Name
+project_no. = GAP9999
+water_depth_m = 1000
+deployment_depth_m = 300
+deployment_date = 12/10/2023
+recovery_date = 8/10/2024
+latitude = 15.0
+longitude = 77.0
+platform_type = Moored
+participants = abcd, efgh, ijkl
+file_created_by = xxxx
+contact = abcd
+comments = No comments
+