pyadps 0.1.1__py3-none-any.whl → 0.2.1b0__py3-none-any.whl

pyadps/pages/03_Download_Raw_File.py

@@ -20,18 +20,14 @@ if "fname" not in st.session_state:
 if "rawfilename" not in st.session_state:
     st.session_state.rawfilename = "rawfile.nc"
 
-if "fleadfilename" not in st.session_state:
-    st.session_state.fleadfilename = "flead.nc"
-
 if "vleadfilename" not in st.session_state:
     st.session_state.vleadfilename = "vlead.nc"
 
 if "attributes" not in st.session_state:
     st.session_state.attributes = {}
-
-if "add_attributes_DRW" not in st.session_state:
-    st.session_state.add_attributes_DRW = "No"  # Default value
-
+    
+if "add_attributes" not in st.session_state:
+    st.session_state.add_attributes = "No"  # Default value
 
 ################ Functions #######################
 @st.cache_data()
@@ -42,7 +38,6 @@ def file_access(uploaded_file):
         f.write(uploaded_file.getvalue())
     return path
 
-
 @st.cache_data
 def read_file(filepath):
     ds = rd.ReadFile(st.session_state.fpath)
@@ -50,61 +45,25 @@ def read_file(filepath):
         ds.fixensemble()
     st.session_state.ds = ds
 
-
 @st.cache_data
 def file_write(path, axis_option, add_attributes=True):
     tempdirname = tempfile.TemporaryDirectory(delete=False)
     st.session_state.rawfilename = tempdirname.name + "/rawfile.nc"
-
+    
     if add_attributes:
-        wr.rawnc(
-            path,
-            st.session_state.rawfilename,
-            st.session_state.date1,
-            axis_option,
-            attributes=st.session_state.attributes,
-        )
+        wr.rawnc(path, st.session_state.rawfilename, st.session_state.date1, axis_option, attributes=st.session_state.attributes)
     else:
-        wr.rawnc(
-            path, st.session_state.rawfilename, st.session_state.date1, axis_option
-        )
-
-@st.cache_data
-def file_write_flead(path, axis_option, add_attributes=True):
-    tempvardirname = tempfile.TemporaryDirectory(delete=False)
-    st.session_state.fleadfilename = tempvardirname.name + "/flead.nc"
-
-    if add_attributes:
-        wr.flead_nc(
-            path,
-            st.session_state.fleadfilename,
-            st.session_state.date2,
-            axis_option,
-            attributes=st.session_state.attributes,
-        )
-    else:
-        wr.flead_nc(
-            path, st.session_state.fleadfilename, st.session_state.date2, axis_option
-        )
+        wr.rawnc(path, st.session_state.rawfilename, st.session_state.date1,axis_option)
 
 @st.cache_data
 def file_write_vlead(path, axis_option, add_attributes=True):
     tempvardirname = tempfile.TemporaryDirectory(delete=False)
     st.session_state.vleadfilename = tempvardirname.name + "/vlead.nc"
-
+    
     if add_attributes:
-        wr.vlead_nc(
-            path,
-            st.session_state.vleadfilename,
-            st.session_state.date3,
-            axis_option,
-            attributes=st.session_state.attributes,
-        )
+        wr.vlead_nc(path, st.session_state.vleadfilename, st.session_state.date2, axis_option, attributes=st.session_state.attributes)
     else:
-        wr.vlead_nc(
-            path, st.session_state.vleadfilename, st.session_state.date3, axis_option
-        )
-
+        wr.vlead_nc(path, st.session_state.vleadfilename, st.session_state.date2, axis_option)
 
 if "axis_option" not in st.session_state:
     st.session_state.axis_option = "ensemble"  # Default value
@@ -113,67 +72,51 @@ if "axis_option" not in st.session_state:
 st.header("NetCDF File", divider="blue")
 
 # Option to add attributes
-st.session_state.add_attributes_DRW = st.radio(
-    "Do you want to add attributes to the NetCDF file?", ["No", "Yes"], horizontal=True
-)
+add_attributes = st.radio("Do you want to add attributes to the NetCDF file?", ["No", "Yes"], horizontal=True)
 
-if st.session_state.add_attributes_DRW == "Yes":
+if add_attributes == "Yes":
     st.write("### Please fill in the attributes:")
-
+    
     # Two-column layout
     col1, col2 = st.columns(2)
-
+    
     with col1:
-        st.session_state.attributes["Cruise_No."] = st.text_input("Cruise No.")
-        st.session_state.attributes["Ship_Name"] = st.text_input("Ship Name")
-        st.session_state.attributes["Project_No."] = st.text_input("Project No.")
-        st.session_state.attributes["Water_Depth_m"] = st.text_input("Water Depth (m)")
-        st.session_state.attributes["Deployment_Depth_m"] = st.text_input(
-            "Deployment Depth (m)"
-        )
-        st.session_state.attributes["Deployment_Date"] = st.date_input(
-            "Deployment Date"
-        )
-        st.session_state.attributes["Recovery_Date"] = st.date_input("Recovery Date")
+        st.session_state.attributes['Cruise_No.'] = st.text_input("Cruise No.")
+        st.session_state.attributes['Ship_Name'] = st.text_input("Ship Name")
+        st.session_state.attributes['Project_No.'] = st.text_input("Project No.")
+        st.session_state.attributes['Water_Depth_m'] = st.text_input("Water Depth (m)")
+        st.session_state.attributes['Deployment_Depth_m'] = st.text_input("Deployment Depth (m)")
+        st.session_state.attributes['Deployment_Date'] = st.date_input("Deployment Date")
+        st.session_state.attributes['Recovery_Date'] = st.date_input("Recovery Date")
+
 
     with col2:
-        st.session_state.attributes["Latitude"] = st.text_input("Latitude")
-        st.session_state.attributes["Longitude"] = st.text_input("Longitude")
-        st.session_state.attributes["Platform_Type"] = st.text_input("Platform Type")
-        st.session_state.attributes["Participants"] = st.text_area("Participants")
-        st.session_state.attributes["File_created_by"] = st.text_input(
-            "File created by"
-        )
-        st.session_state.attributes["Contact"] = st.text_input("Contact")
-        st.session_state.attributes["Comments"] = st.text_area("Comments")
+        st.session_state.attributes['Latitude'] = st.text_input("Latitude")
+        st.session_state.attributes['Longitude'] = st.text_input("Longitude")
+        st.session_state.attributes['Platform_Type'] = st.text_input("Platform Type")
+        st.session_state.attributes['Participants'] = st.text_area("Participants")
+        st.session_state.attributes['File_created_by'] = st.text_input("File created by")
+        st.session_state.attributes['Contact'] = st.text_input("Contact")
+        st.session_state.attributes['Comments'] = st.text_area("Comments")
 
     st.write("Attributes will be added to the NetCDF file once you submit.")
 
 # Dropdown for axis_option
-st.session_state.axis_option_DRW = st.selectbox(
+axis_option = st.selectbox(
     "Select x-axis option:",
-    options=["time", "ensemble"],
-    index=0,  # Default to "time"
+    options=["ensemble", "time"],
+    index=0  # Default to "ensemble"
 )
 
 # Ensure it is passed correctly
-# st.session_state.axis_option = axis_option
+st.session_state.axis_option = axis_option
 
 # Buttons to generate files
-st.session_state.rawnc_download_DRW = st.button("Generate Raw NetCDF File")
-st.session_state.fleadnc_download_DRW = st.button(
-    "Generate Raw Fixed Leader NetCDF File"
-)
-st.session_state.vleadnc_download_DRW = st.button(
-    "Generate Raw Variable Leader NetCDF File"
-)
+download_button = st.button("Generate Raw NetCDF File")
+download_var_button = st.button("Generate Raw Variable Leader NetCDF File")
 
-if st.session_state.rawnc_download_DRW:
-    file_write(
-        st.session_state.fpath,
-        st.session_state.axis_option_DRW,
-        st.session_state.add_attributes_DRW == "Yes",
-    )
+if download_button:
+    file_write(st.session_state.fpath,   st.session_state.axis_option , add_attributes == "Yes")
     st.write(st.session_state.rawfilename)
     with open(st.session_state.rawfilename, "rb") as file:
         st.download_button(
@@ -182,26 +125,8 @@ if st.session_state.rawnc_download_DRW:
             file_name="rawfile.nc",
         )
 
-if st.session_state.fleadnc_download_DRW:
-    file_write_flead(
-        st.session_state.fpath,
-        st.session_state.axis_option,
-        st.session_state.add_attributes_DRW == "Yes",
-    )
-    st.write(st.session_state.fleadfilename)
-    with open(st.session_state.fleadfilename, "rb") as file:
-        st.download_button(
-            label="Download Fixed Leader",
-            data=file,
-            file_name="flead.nc",
-        )
-
-if st.session_state.vleadnc_download_DRW:
-    file_write_vlead(
-        st.session_state.fpath,
-        st.session_state.axis_option,
-        st.session_state.add_attributes_DRW == "Yes",
-    )
+if download_var_button:
+    file_write_vlead(st.session_state.fpath,  st.session_state.axis_option, add_attributes == "Yes")
     st.write(st.session_state.vleadfilename)
     with open(st.session_state.vleadfilename, "rb") as file:
         st.download_button(
@@ -210,7 +135,6 @@ if st.session_state.vleadnc_download_DRW:
             file_name="vlead.nc",
         )
 
-
 def download_csv_with_ensemble(data, filename):
     # Create ensemble numbers from 1 to the number of rows in the data
     ensembles = np.arange(1, len(next(iter(data.values()))) + 1)
@@ -220,62 +144,60 @@ def download_csv_with_ensemble(data, filename):
     df.insert(0, "RDI_Ensemble", ensembles)  # Add ensemble numbers as the first column
 
     # Export the DataFrame as a CSV
-    csv = df.to_csv(index=False).encode("utf-8")
+    csv = df.to_csv(index=False).encode('utf-8')
     return st.download_button(
         label=f"Download {filename} as CSV",
         data=csv,
         file_name=f"{filename}.csv",
-        mime="text/csv",
+        mime='text/csv',
     )
-
-
+    
 def download_csv(data, filename):
     # Convert data to DataFrame if it's not already one
     if isinstance(data, dict):
         df = pd.DataFrame.from_dict(data, orient="index").T
     else:
         df = pd.DataFrame(data)
-
+    
     # Export the DataFrame as a CSV
-    csv = df.to_csv(index=False).encode("utf-8")
+    csv = df.to_csv(index=False).encode('utf-8')
     return st.download_button(
         label=f"Download {filename} as CSV",
         data=csv,
         file_name=f"{filename}.csv",
-        mime="text/csv",
+        mime='text/csv',
     )
-
-
+    
+    
 def download_csv1(data, filename):
     # Convert data to DataFrame if it's not already one
     if isinstance(data, dict):
         df = pd.DataFrame.from_dict(data, orient="index").T
     else:
         df = pd.DataFrame(data)
-
+    
     # Create ensemble and depth arrays
     ensembles = np.arange(1, df.shape[0] + 1)
     depths = np.arange(1, df.shape[1] + 1)
-
+    
     # Add ensemble numbers as the first column
-    df.insert(0, "Ensemble", ensembles)
-
+    df.insert(0, 'Ensemble', ensembles)
+    
     # Transpose the DataFrame to switch rows and columns
     df = df.T
-
+    
     # Add depth values as the first row
-    df.insert(0, "Depth", [""] + list(depths))
-
+    df.insert(0, 'Depth', [''] + list(depths))
+    
     # Export the DataFrame as a CSV
-    csv = df.to_csv(index=False, header=False).encode("utf-8")
+    csv = df.to_csv(index=False, header=False).encode('utf-8')
     return st.download_button(
         label=f"Download {filename} as CSV",
         data=csv,
         file_name=f"{filename}.csv",
-        mime="text/csv",
+        mime='text/csv',
     )
 
-
 # Load data
 fdata = st.session_state.flead.fleader
 vdata = st.session_state.vlead.vleader
@@ -293,36 +215,25 @@ X, Y = np.meshgrid(x, y)
 st.header("Download Raw Data CSV File", divider="blue")
 
 # Selection for the data category
-st.session_state.rawcsv_option_DRW = st.selectbox(
+data_type = st.selectbox(
     "Select data type to download:",
-    [
-        "Velocity",
-        "Echo Intensity",
-        "Correlation",
-        "Percent Good",
-        "Variable Leader",
-        "Fixed Leader",
-    ],
+    ["Fixed Leader", "Variable Leader", "Velocity", "Echo Intensity", "Correlation", "Percent Good"]
 )
 
 # Show corresponding variable options based on selection
-if st.session_state.rawcsv_option_DRW == "Fixed Leader":
+if data_type == "Fixed Leader":
     # Combine all variables of Fixed Leader into one DataFrame
     f_combined_data = {var: fdata[var] for var in fdata.keys()}
     download_csv_with_ensemble(f_combined_data, "Fixed_Leader_All_Variables")
 
-elif st.session_state.rawcsv_option_DRW == "Variable Leader":
+elif data_type == "Variable Leader":
     # Combine all variables of Variable Leader into one DataFrame
     v_combined_data = {var: vdata[var] for var in vdata.keys()}
     download_csv(v_combined_data, "Variable_Leader_All_Variables")
 
 else:
-    st.session_state.rawcsv_beam_DRW = st.radio(
-        "Select beam to download", (1, 2, 3, 4), horizontal=True
-    )
+    beam_download = st.radio("Select beam to download", (1, 2, 3, 4), horizontal=True)
 
-    data_type = st.session_state.rawcsv_option_DRW
-    beam_download = st.session_state.rawcsv_beam_DRW
     if data_type == "Velocity":
         download_data = velocity[beam_download - 1, :, :]
     elif data_type == "Echo Intensity":
@@ -333,3 +244,4 @@ else:
         download_data = pgood[beam_download - 1, :, :]
 
     download_csv1(download_data, f"{data_type}_Beam_{beam_download}")
+

pyadps/pages/04_QC_Test.py

@@ -0,0 +1,334 @@
+import numpy as np
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import streamlit as st
+from plotly.subplots import make_subplots
+from plotly_resampler import FigureResampler
+from streamlit.runtime.state import session_state
+from utils.signal_quality import ev_check, false_target, pg_check, qc_check
+
+if "flead" not in st.session_state:
+    st.write(":red[Please Select Data!]")
+    st.stop()
+
+# `mask` holds the temporary changes in the page
+# `qcmask` holds the final changes in the page
+if "mask" not in st.session_state:
+    st.session_state.mask = np.copy(st.session_state.orig_mask)
+
+if not st.session_state.isQCMask:
+    st.write(":grey[Creating a new mask file ...]")
+    st.session_state.qc_mask = np.copy(st.session_state.orig_mask)
+    st.session_state.isSubmit = False
+else:
+    st.write(":grey[Working on a saved mask file ...]")
+    st.write(":orange[WARNING! QC test already completed. Reset to change settings.]")
+    reset_button1 = st.button("Reset Mask Data")
+    if reset_button1:
+        st.session_state.mask = np.copy(st.session_state.orig_mask)
+        st.session_state.qc_mask = np.copy(st.session_state.orig_mask)
+        st.write(":green[Mask data is reset to default]")
+
+if "isThresh" not in st.session_state:
+    st.session_state.isThresh = False
+
+# Load data
+flobj = st.session_state.flead
+vlobj = st.session_state.vlead
+velocity = st.session_state.velocity
+echo = st.session_state.echo
+correlation = st.session_state.correlation
+pgood = st.session_state.pgood
+ensembles = st.session_state.head.ensembles
+cells = flobj.field()["Cells"]
+fdata = flobj.fleader
+vdata = vlobj.vleader
+x = np.arange(0, ensembles, 1)
+y = np.arange(0, cells, 1)
+
+
+@st.cache_data
+def fillplot_plotly(data, colorscale="balance"):
+    fig = FigureResampler(go.Figure())
+    data1 = np.where(data == -32768, np.nan, data)
+    fig.add_trace(
+        go.Heatmap(z=data1[:, 0:-1], x=x, y=y, colorscale=colorscale, hoverongaps=False)
+    )
+    st.plotly_chart(fig)
+
+
+@st.cache_data
+def plot_noise(dep=0, rec=-1):
+    n = dep
+    m = rec
+    colorleft = [
+        "rgb(240, 255, 255)",
+        "rgb(115, 147, 179)",
+        "rgb(100, 149, 237)",
+        "rgb(15, 82, 186)",
+    ]
+    colorright = [
+        "rgb(250, 200, 152)",
+        "rgb(255, 165, 0)",
+        "rgb(255, 95, 31)",
+        "rgb(139, 64, 0)",
+    ]
+    fig = make_subplots(
+        rows=1,
+        cols=2,
+        subplot_titles=[
+            f"Deployment Ensemble ({x[n]+1})",
+            f"Recovery Ensemble ({x[m]+1})",
+        ],
+    )
+    for i in range(4):
+        fig.add_trace(
+            go.Scatter(
+                x=echo[i, :, n],
+                y=y,
+                name=f"Beam (D) {i+1}",
+                line=dict(color=colorleft[i]),
+            ),
+            row=1,
+            col=1,
+        )
+    for i in range(4):
+        fig.add_trace(
+            go.Scatter(
+                x=echo[i, :, m],
+                y=y,
+                name=f"Beam (R)  {i+1}",
+                line=dict(color=colorright[i]),
+            ),
+            row=1,
+            col=2,
+        )
+
+    fig.update_layout(height=600, width=800, title_text="Echo Intensity")
+    fig.update_xaxes(title="Echo (count)")
+    fig.update_yaxes(title="Cells")
+    st.plotly_chart(fig)
+    
+#########  NOISE FLOOR IDENTIFICATION ##############
+dn = rn = 1
+st.header("Noise Floor Identification", divider="blue")
+st.write(
+    """
+    If the ADCP has collected data from the air either 
+    before deployment or after recovery, this data can 
+    be used to estimate the echo intensity threshold. 
+    The plots below show the echo intensity from the first 
+    and last ensembles. The noise level is typically around 
+    30-40 counts throughout the entire profile.
+"""
+)
+dn = st.number_input("Deployment Ensemble", x[0] + 1, x[-1] + 1, x[0] + 1)
+# r = st.number_input("Recovery Ensemble", -1 * (x[-1] + 1), -1 * (x[0] + 1), -1)
+rn = st.number_input("Recovery Ensemble", x[0] + 1, x[-1] + 1, x[-1] + 1)
+dn = dn - 1
+rn = rn - 1
+
+plot_noise(dep=dn, rec=rn)
+
+
+################## QC Test ###################
+
+st.header("Quality Control Tests", divider="blue")
+st.write("")
+
+left, right = st.columns([1, 1])
+with left:
+    st.write(""" Teledyne RDI recommends these quality control tests, 
+                 some of which can be configured before deployment. 
+                 The pre-deployment values configured for the ADCP are listed 
+                 in the table below. The noise-floor identification graph above 
+                 can assist in determining the echo intensity threshold. 
+                 For more information about these tests, 
+                 refer to *Acoustic Doppler Current Profiler Principles of 
+                 Operation: A Practical Primer* by Teledyne RDI.""")
+    fdata = st.session_state.flead.field()
+    st.divider()
+    st.write(":blue-background[Additional Information:]")
+    st.write(f"Number of Pings per Ensemble: `{fdata["Pings"]}`")
+    st.write(f"Number of Beams: `{fdata["Beams"]}`")
+    st.divider()
+    st.write(":red-background[Thresholds used during deployment:]")
+    thresh = pd.DataFrame(
+        [
+            ["Correlation", fdata["Correlation Thresh"]],
+            ["Error Velocity", fdata["Error Velocity Thresh"]],
+            ["Echo Intensity", 0],
+            ["False Target", fdata["False Target Thresh"]],
+            ["Percentage Good", fdata["Percent Good Min"]],
+        ],
+        columns=["Threshold", "Values"],
+    )
+
+    st.write(thresh)
+
+with right:
+    with st.form(key="my_form"):
+        st.write("Would you like to apply new threshold?")
+
+        ct = st.number_input(
+            "Select Correlation Threshold",
+            0,
+            255,
+            fdata["Correlation Thresh"],
+        )
+
+        evt = st.number_input(
+            "Select Error Velocity Threshold",
+            0,
+            9999,
+            fdata["Error Velocity Thresh"],
+        )
+
+        et = st.number_input(
+            "Select Echo Intensity Threshold",
+            0,
+            255,
+            0,
+        )
+
+        ft = st.number_input(
+            "Select False Target Threshold",
+            0,
+            255,
+            fdata["False Target Thresh"],
+        )
+
+        option = st.selectbox(
+            "Would you like to use a three-beam solution?", (True, False)
+        )
+
+        pgt = st.number_input(
+            "Select Percent Good Threshold",
+            0,
+            100,
+            fdata["Percent Good Min"],
+        )
+        submit_button = st.form_submit_button(label="Submit")
+
+
+mask = st.session_state.mask
+with left:
+    if submit_button:
+        st.session_state.newthresh = pd.DataFrame(
+            [
+                ["Correlation", str(ct)],
+                ["Error Velocity", str(evt)],
+                ["Echo Intensity", str(et)],
+                ["False Target", str(ft)],
+                ["Three Beam", str(option)],
+                ["Percentage Good", str(pgt)],
+            ],
+            columns=["Threshold", "Values"],
+        )
+        st.session_state.isThresh = True
+        # st.write(st.session_state.newthresh)
+
+        mask = pg_check(pgood, mask, pgt, threebeam=option)
+        mask = qc_check(correlation, mask, ct)
+        mask = qc_check(echo, mask, et)
+        mask = ev_check(velocity[3, :, :], mask, evt)
+        mask = false_target(echo, mask, ft, threebeam=True)
+        st.session_state.mask = mask
+
+    if st.session_state.isThresh:
+        st.write(":green-background[Current Thresholds]")
+        st.write(st.session_state.newthresh)
+
+
+
+st.header("Mask File", divider="blue")
+st.write(
+"""
+Displayed the mask file. 
+Ensure to save any necessary changes or apply additional thresholds if needed.
+"""
+)
+
+
+if st.button("Display mask file"):
+    st.subheader("Default Mask File")
+    st.write(
+    """
+ADCP assigns missing values based on thresholds set before deployment.
+These values cannot be recovered and the default 
+"""
+)
+    fillplot_plotly(st.session_state.orig_mask, colorscale="greys")
+
+
+    st.subheader("Update Mask File")
+    st.write(
+    """
+Update, display and save the updated mask file after applying threshold.
+If thresholds are not saved, default mask file is used. 
+"""
+)
+# values, counts = np.unique(mask, return_counts=True)
+    fillplot_plotly(st.session_state.mask, colorscale="greys")
+
+############## SENSOR HEALTH ######################
+st.header("Sensor Health", divider="blue")
+st.write("The following details can be used to determine whether the additional sensors are functioning properly.")
+# ################## Pressure Sensor Check ###################
+# st.subheader("Pressure Sensor Check", divider="orange")
+#
+# st.subheader("Temperature Sensor Check", divider="orange")
+#
+# st.subheader("Tilt Sensor Check", divider="orange")
+################## Fix Orientation ###################
+st.subheader("Fix Orientation", divider="orange")
+
+
+if st.session_state.beam_direction == 'Up':
+    beamalt = 'Down' 
+else:
+    beamalt = 'Up'
+st.write(f"The current orientation of ADCP is `{st.session_state.beam_direction}`. Use the below option to correct the orientation.")
+
+beamdir_select = st.radio(f'Change orientation to {beamalt}', ['No', 'Yes'])
+if beamdir_select == 'Yes':
+    st.session_state.beam_direction = beamalt
+    st.write(f"The orientation changed to `{st.session_state.beam_direction}`")
+
+
+
+
+################## Save Button #############
+st.header("Save Data", divider="blue")
+col1, col2 = st.columns([1, 1])
+with col1:
+    save_mask_button = st.button(label="Save Mask Data")
+
+    if save_mask_button:
+        # st.session_state.mask = mask
+        st.session_state.qc_mask = np.copy(st.session_state.mask)
+        st.session_state.isQCMask = True
+        st.session_state.isProfileMask = False
+        st.session_state.isGridSave = False
+        st.session_state.isVelocityMask = False
+        st.write(":green[Mask file saved]")
+    else:
+        st.write(":red[Mask data not saved]")
+with col2:
+    reset_mask_button = st.button("Reset mask Data")
+    if reset_mask_button:
+        st.session_state.mask = np.copy(st.session_state.orig_mask)
+        st.session_state.isQCMask = False
+        st.session_state.isGrid = False
+        st.session_state.isProfileMask = False
+        st.session_state.isVelocityMask = False
+        st.write(":green[Mask data is reset to default]")
+
+
+
+
+
+
+
+