pyadps 0.3.3b0__py3-none-any.whl

pyadps/Home_Page.py

@@ -0,0 +1,42 @@
+import streamlit as st
+
+
+def main():
+    st.set_page_config(
+        page_title="ADCP Data Processing Software",
+        page_icon=":world_map:️",
+        layout="wide",
+        initial_sidebar_state="auto",
+        menu_items={
+            "Get Help": "http://github.com/p-amol/adps",
+            "Report a bug": "http://github.com/adps/issues",
+            "About": "# Python ADCP Data Processing Software (PyADPS)",
+        },
+    )
+
+    """
+    # **Python ADCP Data Processing Software (pyadps)**
+    `pyadps` is a software for processing Teledyne RDI Acoustic Doppler Current Profiler (ADCP) PD0 files. Currently the software can process the data from Workhorse ADCPs.
+
+    ## Features
+
+    * Access RDI ADCP binary files using Python 3
+    * Convert RDI binary files to netcdf
+    * Process ADCP data 
+
+    ## Contribute
+    Issue Tracker: http://github.com/adps/issues
+    Source Code: http://github.com/p-amol/adps
+
+    ## Support
+    If you are having issues, please let us know.
+    We have a mailing list located at: adps-python@google-groups.com
+    
+    ## License
+    The project is licensed under the MIT license.
+
+    """
+
+
+if __name__ == "__main__":
+    main()

pyadps/__init__.py

@@ -0,0 +1,8 @@
+#import utils.pyreadrdi
+#import utils.readrdi
+# pyadps/__init__.py
+
+# Importing main modules and functions to simplify access
+from pyadps.Home_Page import main as show_home_page
+from pyadps.pages import *
+from pyadps.utils import *

pyadps/__main__.py

@@ -0,0 +1,15 @@
+# pyadps/__main__.py
+
+import subprocess
+import os
+
+def main():
+    # Get the absolute path to Home_Page.py
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    home_page_path = os.path.join(current_dir, 'Home_Page.py')
+
+    # Run the Streamlit app
+    subprocess.run(["streamlit", "run", home_page_path])
+
+if __name__ == "__main__":
+    main()

pyadps/pages/01_Read_File.py

@@ -0,0 +1,458 @@
+import os
+import tempfile
+
+import numpy as np
+import pandas as pd
+import streamlit as st
+import utils.readrdi as rd
+from utils.signal_quality import default_mask
+from utils.readrdi import ReadFile
+
+# To make the page wider if the user presses the reload button.
+st.set_page_config(layout="wide")
+
+"""
+Streamlit page to load ADCP binary file and display File Header
+and Fixed Leader data
+"""
+
+if "fname" not in st.session_state:
+    st.session_state.fname = "No file selected"
+
+if "rawfilename" not in st.session_state:
+    st.session_state.rawfilename = "rawfile.nc"
+
+if "vleadfilename" not in st.session_state:
+    st.session_state.vleadfilename = "vlead.nc"
+
+
+################ Functions #######################
+@st.cache_data()
+def file_access(uploaded_file):
+    """
+    Function creates temporary directory to store the uploaded file.
+    The path of the file is returned
+
+    Args:
+        uploaded_file (string): Name of the uploaded file
+
+    Returns:
+        path (string): Path of the uploaded file
+    """
+    temp_dir = tempfile.mkdtemp()
+    path = os.path.join(temp_dir, uploaded_file.name)
+    with open(path, "wb") as f:
+        f.write(uploaded_file.getvalue())
+    return path
+
+
+def color_bool(val):
+    """
+    Takes a scalar and returns a string with
+    the css color property.
+    """
+    if isinstance(val, bool):
+        if val:
+            color = "green"
+        else:
+            color = "red"
+    else:
+        color = "orange"
+    return "color: %s" % color
+
+
+def color_bool2(val):
+    """
+    Takes a scalar and returns a string with
+    the css color property. The following colors
+    are assinged for the string
+        "True": green,
+        "False": red
+         Any other string: orange
+
+    Args:
+        val (string): Any string data
+
+    Returns:
+        The input string with css color property added
+    """
+    if val == "True" or val == "Data type is healthy":
+        color = "green"
+    elif val == "False":
+        color = "red"
+    # elif val in st.session_state.ds.warnings.values():
+    #     color = "orange"
+    else:
+        color = "orange"
+    return "color: %s" % color
+
+
+@st.cache_data
+def read_file(filepath):
+    ds = rd.ReadFile(st.session_state.fpath)
+    if not ds.isEnsembleEqual:
+        ds.fixensemble()
+    st.session_state.ds = ds
+    # return ds
+
+
+uploaded_file = st.file_uploader("Upload RDI ADCP Binary File", type="000")
+
+if uploaded_file is not None:
+    # st.cache_data.clear
+
+    # Get path
+    st.session_state.fpath = file_access(uploaded_file)
+    # Get data
+    read_file(st.session_state.fpath)
+    ds = st.session_state.ds
+    head = ds.fileheader
+    flead = ds.fixedleader
+    vlead = ds.variableleader
+    velocity = ds.velocity.data
+    correlation = ds.correlation.data
+    echo = ds.echo.data
+    pgood = ds.percentgood.data
+    beamdir = ds.fixedleader.system_configuration()["Beam Direction"]
+
+    st.session_state.fname = uploaded_file.name
+    st.session_state.head = ds.fileheader
+    st.session_state.flead = ds.fixedleader
+    st.session_state.vlead = ds.variableleader
+    st.session_state.velocity = ds.velocity.data
+    st.session_state.echo = ds.echo.data
+    st.session_state.correlation = ds.correlation.data
+    st.session_state.pgood = ds.percentgood.data
+    st.session_state.beam_direction = beamdir
+    st.session_state.sound_speed = ds.variableleader.speed_of_sound.data
+    st.session_state.depth = ds.variableleader.depth_of_transducer.data
+    st.session_state.temperature = (
+        ds.variableleader.temperature.data * ds.variableleader.temperature.scale
+    )
+    st.session_state.salinity = (
+        ds.variableleader.salinity.data * ds.variableleader.salinity.scale
+    )
+
+    # st.session_state.flead = flead
+    # st.session_state.vlead = vlead
+    # st.session_state.head = head
+    # st.session_state.velocity = velocity
+    # st.session_state.echo = echo
+    # st.session_state.correlation = correlation
+    # st.session_state.pgood = pgood
+    st.write("You selected `%s`" % st.session_state.fname)
+
+elif "flead" in st.session_state:
+    st.write("You selected `%s`" % st.session_state.fname)
+else:
+    # reset the cache and resources if the user press reload button.
+    st.cache_data.clear()
+    st.cache_resource.clear()
+    st.stop()
+
+########## TIME AXIS ##############
+
+# Time axis is extracted and stored as Pandas datetime
+year = st.session_state.vlead.vleader["RTC Year"]
+month = st.session_state.vlead.vleader["RTC Month"]
+day = st.session_state.vlead.vleader["RTC Day"]
+hour = st.session_state.vlead.vleader["RTC Hour"]
+minute = st.session_state.vlead.vleader["RTC Minute"]
+second = st.session_state.vlead.vleader["RTC Second"]
+
+# Recent ADCP binary files have Y2K compliant clock. The Century
+# is stored in`RTC Century`. As all files may not have this clock
+# we have added 2000 to the year.
+# CHECKS:
+# Are all our data Y2K compliant?
+# Should we give users the options to correct the data?
+
+year = year + 2000
+date_df = pd.DataFrame(
+    {
+        "year": year,
+        "month": month,
+        "day": day,
+        "hour": hour,
+        "minute": minute,
+        "second": second,
+    }
+)
+
+st.session_state.date = pd.to_datetime(date_df)
+st.session_state.date1 = pd.to_datetime(date_df)
+st.session_state.date2 = pd.to_datetime(date_df)
+st.session_state.ensemble_axis = np.arange(0, st.session_state.head.ensembles, 1)
+st.session_state.axis_option = "time"
+
+
+# ---------- Initialize all options -------------
+# ------------------------
+# Page: Download Raw File
+# ------------------------
+# Widgets
+st.session_state.add_attributes_DRW = "No"
+st.session_state.axis_option_DRW = "time"
+st.session_state.rawnc_download_DRW = False
+st.session_state.vleadnc_download_DRW = False
+st.session_state.rawcsv_option_DRW = "Velocity"
+st.session_state.rawcsv_beam_DRW = 1
+st.session_state.rawcsv_download_DRW = False
+
+# ------------------
+# Page: Sensor Test
+# ------------------
+st.session_state.isSensorTest = False
+st.session_state.isFirstSensorVisit = True
+
+# -- Tab 1: Depth Correction
+st.session_state.isDepthModified_ST = False
+# Widgets
+# Options: "Fixed Value", "File Upload"
+st.session_state.depthoption_ST = "Fixed Value"
+st.session_state.isFixedDepth_ST = False
+st.session_state.fixeddepth_ST = 0
+st.session_state.isUploadDepth_ST = False
+
+# -- Tab 2: Salinity Correction
+st.session_state.isSalinityModified_ST = False
+# Widgets
+st.session_state.salinityoption_ST = "Fixed Value"
+st.session_state.isFixedSalinity_ST = False
+st.session_state.fixedsalinity_ST = 35
+st.session_state.isUploadSalinity_ST = False
+
+# -- Tab 3: Temperature Correction
+st.session_state.isTemperatureModified_ST = False
+# Widgets
+st.session_state.temperatureoption_ST = "Fixed Value"
+st.session_state.isFixedTemperature_ST = False
+st.session_state.fixedtemperature_ST = 0
+st.session_state.isUploadTemperature_ST = False
+
+# -- Tab 7: Pitch, Roll, Velocity Correction
+st.session_state.isRollCheck_ST = False
+st.session_state.isPitchCheck_ST = False
+st.session_state.isVelocityModifiedSound_ST = False
+# Widgets
+st.session_state.roll_cutoff_ST = 359
+st.session_state.pitch_cutoff_ST = 359
+
+# ------------------
+# Page: QC Test
+# ------------------
+# Global Test
+st.session_state.isQCTest = False
+st.session_state.isFirstQCVisit = True
+
+# Tab 2: Apply QC
+st.session_state.isQCCheck_QCT = False
+# Widgets
+st.session_state.ct_QCT = 64
+st.session_state.et_QCT = 0
+st.session_state.evt_QCT = 2000
+st.session_state.ft_QCT = 50
+st.session_state.is3beam_QCT = True
+st.session_state.pgt_QCT = 0
+
+# Data Modifications
+st.session_state.isBeamModified_QCT = False
+# Widgets
+st.session_state.beam_direction_QCT = st.session_state.beam_direction
+
+# ------------------
+# Page: Profile Test
+# ------------------
+st.session_state.isProfileTest = False
+st.session_state.isFirstProfileVisit = True
+
+# Tab1: Trim Ends
+st.session_state.isTrimEndsCheck_PT = False
+# Widgets
+st.session_state.start_ens_PT = 0
+st.session_state.end_ens_PT = st.session_state.head.ensembles
+
+# Tab2: Cutbins - Sidelobe
+st.session_state.isCutBinSideLobeCheck_PT = False
+st.session_state.extra_cells_PT = 0
+st.session_state.water_depth_PT = 0
+
+# Tab3: Cutbins - Manual
+st.session_state.isCutBinManualCheck_PT = False
+
+# Tab4: Regrid
+st.session_state.isRegridCheck_PT = False
+st.session_state.end_cell_option_PT = "Cell"
+st.session_state.interpolate_PT = "nearest"
+st.session_state.manualdepth_PT = 0
+
+# ------------------
+# Page: Velocity Test
+# ------------------
+# Global Test
+st.session_state.isVelocityTest = False
+# Check if visiting the page first time
+st.session_state.isFirstVelocityVisit = True
+# Local Tests:
+# Tab1: Magnetic Declination
+st.session_state.isMagnetCheck_VT = False
+# OPTIONS: pygeomag, API, Manual
+st.session_state.magnet_method_VT = "pygeomag"
+st.session_state.magnet_lat_VT = 0
+st.session_state.magnet_lon_VT = 0
+st.session_state.magnet_year_VT = 2025
+st.session_state.magnet_depth_VT = 0
+st.session_state.magnet_user_input_VT = 0
+
+# Tab2: Velocity Cutoff
+st.session_state.isCutoffCheck_VT = False
+st.session_state.maxuvel_VT = 250
+st.session_state.maxvvel_VT = 250
+st.session_state.maxwvel_VT = 15
+
+# Tab3: Despike
+st.session_state.isDespikeCheck_VT = False
+st.session_state.despike_kernal_VT = 5
+st.session_state.despike_cutoff_VT = 3
+
+# Tab4: Flatline
+st.session_state.isFlatlineCheck_VT = False
+st.session_state.flatline_kernal_VT = 5
+st.session_state.flatline_cutoff_VT = 3
+
+# ------------------
+# Page: Write File
+# ------------------
+st.session_state.isWriteFile = True
+st.session_state.isAttributes = False
+st.session_state.mask_data_WF = "Yes"
+# FileTypes: NetCDF, CSV
+st.session_state.file_type_WF = "NetCDF"
+st.session_state.isProcessedNetcdfDownload_WF = True
+st.session_state.isProcessedCSVDownload_WF = False
+
+# MASK DATA
+# The velocity data has missing values due to the cutoff
+# criteria used before deployment. The `default_mask` uses
+# the velocity to create a mask. This mask  file is stored
+# in the session_state.
+#
+# WARNING: Never Change `st.session_state.orig_mask` in the code!
+#
+if "orig_mask" not in st.session_state:
+    ds = st.session_state.ds
+    st.session_state.orig_mask = default_mask(ds)
+
+# ----------------------
+# Page returning options
+# ----------------------
+# This checks if we have returned back to the page after saving the data
+st.session_state.isSensorPageReturn = False
+st.session_state.isQCPageReturn = False
+st.session_state.isProfilePageReturn = False
+st.session_state.isVelocityPageReturn = False
+
+########## FILE HEADER ###############
+st.header("File Header", divider="blue")
+st.write(
+    """
+        Header information is the first item sent by the ADCP. You may check the file size, total ensembles, and available data types. The function also checks if the total bytes and data types are uniform for all ensembles. 
+        """
+)
+
+left1, right1 = st.columns(2)
+with left1:
+    check_button = st.button("Check File Health")
+    if check_button:
+        cf = st.session_state.head.check_file()
+        if (
+            cf["File Size Match"]
+            and cf["Byte Uniformity"]
+            and cf["Data Type Uniformity"]
+        ):
+            st.write("Your file appears healthy! :sunglasses:")
+        else:
+            st.write("Your file appears corrupted! :worried:")
+
+        cf["File Size (MB)"] = "{:,.2f}".format(cf["File Size (MB)"])
+        st.write(f"Total no. of Ensembles: :green[{st.session_state.head.ensembles}]")
+        df = pd.DataFrame(cf.items(), columns=pd.array(["Check", "Details"]))
+        df = df.astype("str")
+        st.write(df.style.map(color_bool2, subset="Details"))
+        # st.write(df)
+with right1:
+    datatype_button = st.button("Display Data Types")
+    if datatype_button:
+        st.write(
+            pd.DataFrame(
+                st.session_state.head.data_types(),
+                columns=pd.array(["Available Data Types"]),
+            )
+        )
+
+if st.session_state.ds.isWarning:
+    st.write(
+        """ 
+            Warnings detected while reading. Data sets may still be available for processing.
+            Click `Display Warning` to display warnings for each data types.
+        """
+    )
+    warning_button = st.button("Display Warnings")
+    df2 = pd.DataFrame(
+        st.session_state.ds.warnings.items(),
+        columns=pd.array(["Data Type", "Warnings"]),
+    )
+    if warning_button:
+        st.write(df2.style.map(color_bool2, subset=["Warnings"]))
+
+############ FIXED LEADER #############
+
+st.header("Fixed Leader (Static Variables)", divider="blue")
+st.write(
+    """
+        Fixed Leader data refers to the non-dynamic WorkHorse ADCP data like the hardware information and the thresholds. Typically, values remain constant over time. They only change when you change certain commands, although there are occasional exceptions. You can confirm this using the :blue[**Fleader Uniformity Check**]. Click :blue[**Fixed Leader**] to display the values for the first ensemble.
+        """
+)
+
+
+flead_check_button = st.button("Fleader Uniformity Check")
+if flead_check_button:
+    st.write("The following variables are non-uniform:")
+    for keys, values in st.session_state.flead.is_uniform().items():
+        if not values:
+            st.markdown(f":blue[**- {keys}**]")
+    st.write("Displaying all static variables")
+    df = pd.DataFrame(st.session_state.flead.is_uniform(), index=[0]).T
+    st.write(df.style.map(color_bool))
+
+flead_button = st.button("Fixed Leader")
+if flead_button:
+    # Pandas array should have all elements with same data type.
+    # Except Sl. no., which is np.uint64, rest are np.int64.
+    # Convert all datatype to uint64
+    fl_dict = st.session_state.flead.field().items()
+    new_dict = {}
+    for key, value in fl_dict:
+        new_dict[key] = value.astype(np.uint64)
+
+    df = pd.DataFrame(
+        {
+            "Fields": new_dict.keys(),
+            "Values": new_dict.values(),
+        }
+    )
+    st.dataframe(df, use_container_width=True)
+
+left, centre, right = st.columns(3)
+with left:
+    st.dataframe(st.session_state.flead.system_configuration())
+
+with centre:
+    st.dataframe(st.session_state.flead.ez_sensor())
+    #     st.write(output)
+with right:
+    # st.write(st.session_state.flead.ex_coord_trans())
+    df = pd.DataFrame(st.session_state.flead.ex_coord_trans(), index=[0]).T
+    df = df.astype("str")
+    st.write((df.style.map(color_bool2)))
+    # st.dataframe(df)

pyadps/pages/02_View_Raw_Data.py

@@ -0,0 +1,164 @@
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import streamlit as st
+from plotly_resampler import FigureResampler
+
+if "flead" not in st.session_state:
+    st.write(":red[Please Select Data!]")
+    st.stop()
+
+# Load data
+fdata = st.session_state.flead.fleader
+vdata = st.session_state.vlead.vleader
+velocity = st.session_state.velocity
+echo = st.session_state.echo
+correlation = st.session_state.correlation
+pgood = st.session_state.pgood
+
+x = np.arange(0, st.session_state.head.ensembles, 1)
+y = np.arange(0, fdata["Cells"][0], 1)
+
+X, Y = np.meshgrid(x, y)
+
+
+@st.cache_data
+def fillplot_matplotlib(data):
+    fig, ax = plt.subplots()
+    cs = ax.contourf(X, Y, data)
+    fig.colorbar(cs)
+    st.pyplot(fig)
+
+
+@st.cache_data
+def fillplot_plotly(data, colorscale="balance", title="Data", xaxis="time"):
+    if xaxis == "time":
+        xdata = st.session_state.date
+    elif xaxis == "ensemble":
+        xdata = x
+    else:
+        xdata = x
+    fig = FigureResampler(go.Figure())
+    data1 = np.where(data == -32768, np.nan, data)
+    fig.add_trace(
+        go.Heatmap(
+            z=data1[:, 0:-1],
+            x=xdata,
+            y=y,
+            colorscale=colorscale,
+            hoverongaps=False,
+        )
+    )
+    fig.update_layout(
+        xaxis=dict(showline=True, mirror=True),
+        yaxis=dict(showline=True, mirror=True),
+        title_text=title,
+    )
+    st.plotly_chart(fig)
+
+
+@st.cache_data
+def lineplot(data, title, xaxis="time"):
+    if xaxis == "time":
+        df = pd.DataFrame({"date": st.session_state.date, title: data})
+        fig = px.line(df, x="date", y=title)
+    else:
+        df = pd.DataFrame({"ensemble": x, title: data})
+        fig = px.line(df, x="ensemble", y=title)
+
+    st.plotly_chart(fig)
+
+
+# Introduction
+st.header("View Raw Data", divider="orange")
+st.write(
+    """
+Displays all variables available in the raw file. No processing has been carried out. 
+Data might be missing because of the quality-check criteria used before deployment.\n 
+Either `time` or `ensemble` axis can be chosen as the abscissa (x-axis).
+The ordinate (y-axis) for the heatmap is `bins` as the depth correction is not applied. 
+"""
+)
+xbutton = st.radio("Select an x-axis to plot", ["time", "ensemble"], horizontal=True)
+
+
+tab1, tab2, tab3, tab4 = st.tabs(["Primary Data", "Variable Leader", "Fixed Leader", "Advanced"])
+
+with tab3:
+# Fixed Leader Plots
+    st.header("Fixed Leader", divider="blue")
+    fbutton = st.radio("Select a dynamic variable to plot:", fdata.keys(), horizontal=True)
+    lineplot(fdata[fbutton], fbutton, xaxis=str(xbutton))
+
+with tab2:
+# Variable Leader Plots
+    st.header("Variable Leader", divider="blue")
+    vbutton = st.radio("Select a dynamic variable to plot:", vdata.keys(), horizontal=True)
+    lineplot(vdata[vbutton], vbutton, xaxis=str(xbutton))
+
+with tab1:
+    st.header("Velocity, Echo Intensity, Correlation & Percent Good", divider="blue")
+
+
+    def call_plot(varname, beam, xaxis="time"):
+        if varname == "Velocity":
+            fillplot_plotly(velocity[beam - 1, :, :], title=varname, xaxis=xaxis)
+        elif varname == "Echo":
+            fillplot_plotly(echo[beam - 1, :, :], title=varname, xaxis=xaxis)
+        elif varname == "Correlation":
+            fillplot_plotly(correlation[beam - 1, :, :], title=varname, xaxis=xaxis)
+        elif varname == "Percent Good":
+            fillplot_plotly(pgood[beam - 1, :, :], title=varname, xaxis=xaxis)
+
+
+    var_option = st.selectbox(
+        "Select a data type", ("Velocity", "Echo", "Correlation", "Percent Good")
+    )
+    beam = st.radio("Select beam", (1, 2, 3, 4), horizontal=True)
+    call_plot(var_option, beam, xaxis=str(xbutton))
+
+
+with tab4:
+    st.header("Advanced Data", divider="blue")
+    adv_option = st.selectbox(
+        "Select a data type", ("Bit Result", 
+                            "ADC Channel", 
+                            "Error Status Word 1", 
+                            "Error Status Word 2",
+                            "Error Status Word 3",
+                            "Error Status Word 4"))
+    if adv_option == "Bit Result": 
+        bitdata = st.session_state.vlead.bitresult()
+        st.subheader("BIT Result", divider="orange")
+        st.write("""
+        This field contains the results of Workhorse ADCPs builtin test functions.
+        A zero indicates a successful BIT result.
+        """)
+        bitbutton = st.radio("Select a dynamic variable to plot:", bitdata.keys(), horizontal=True)
+        lineplot(bitdata[bitbutton], bitbutton, xaxis=str(xbutton))
+
+    elif adv_option == "ADC Channel":
+        adcdata = st.session_state.vlead.adc_channel()
+        st.subheader("ADC Channel", divider="orange")
+        adcbutton = st.radio("Select a dynamic variable to plot:", adcdata.keys(), horizontal=True)
+        lineplot(adcdata[adcbutton], adcbutton, xaxis=str(xbutton))
+    elif adv_option == "Error Status Word 1":
+        errordata1 = st.session_state.vlead.error_status_word(esw=1)
+        st.subheader("Error Status Word", divider="orange")
+        errorbutton = st.radio("Select a dynamic variable to plot:", errordata1.keys(), horizontal=True)
+        lineplot(errordata1[errorbutton], errorbutton, xaxis=str(xbutton))
+    elif adv_option == "Error Status Word 2":
+        errordata2 = st.session_state.vlead.error_status_word(esw=2)
+        errorbutton = st.radio("Select a dynamic variable to plot:", errordata2.keys(), horizontal=True)
+        lineplot(errordata2[errorbutton], errorbutton, xaxis=str(xbutton))
+    elif adv_option == "Error Status Word 3":
+        errordata3 = st.session_state.vlead.error_status_word(esw=3)
+        errorbutton = st.radio("Select a dynamic variable to plot:", errordata3.keys(), horizontal=True)
+        lineplot(errordata3[errorbutton], errorbutton, xaxis=str(xbutton))
+    elif adv_option == "Error Status Word 4":
+        errordata4 = st.session_state.vlead.error_status_word(esw=4)
+        errorbutton = st.radio("Select a dynamic variable to plot:", errordata4.keys(), horizontal=True)
+        lineplot(errordata4[errorbutton], errorbutton, xaxis=str(xbutton))
+