pyadps 0.2.0b0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyadps/Home_Page.py +11 -5
- pyadps/pages/01_Read_File.py +623 -211
- pyadps/pages/02_View_Raw_Data.py +97 -41
- pyadps/pages/03_Download_Raw_File.py +200 -67
- pyadps/pages/04_Sensor_Health.py +905 -0
- pyadps/pages/05_QC_Test.py +493 -0
- pyadps/pages/06_Profile_Test.py +971 -0
- pyadps/pages/07_Velocity_Test.py +600 -0
- pyadps/pages/08_Write_File.py +623 -0
- pyadps/pages/09_Add-Ons.py +168 -0
- pyadps/utils/__init__.py +5 -3
- pyadps/utils/autoprocess.py +371 -80
- pyadps/utils/logging_utils.py +269 -0
- pyadps/utils/metadata/config.ini +22 -4
- pyadps/utils/metadata/demo.000 +0 -0
- pyadps/utils/metadata/flmeta.json +420 -420
- pyadps/utils/metadata/vlmeta.json +611 -565
- pyadps/utils/multifile.py +292 -0
- pyadps/utils/plotgen.py +505 -3
- pyadps/utils/profile_test.py +720 -125
- pyadps/utils/pyreadrdi.py +164 -92
- pyadps/utils/readrdi.py +436 -186
- pyadps/utils/script.py +197 -147
- pyadps/utils/sensor_health.py +120 -0
- pyadps/utils/signal_quality.py +472 -68
- pyadps/utils/velocity_test.py +79 -31
- pyadps/utils/writenc.py +222 -39
- {pyadps-0.2.0b0.dist-info → pyadps-0.3.0.dist-info}/METADATA +63 -33
- pyadps-0.3.0.dist-info/RECORD +35 -0
- {pyadps-0.2.0b0.dist-info → pyadps-0.3.0.dist-info}/WHEEL +1 -1
- {pyadps-0.2.0b0.dist-info → pyadps-0.3.0.dist-info}/entry_points.txt +1 -0
- pyadps/pages/04_QC_Test.py +0 -334
- pyadps/pages/05_Profile_Test.py +0 -575
- pyadps/pages/06_Velocity_Test.py +0 -341
- pyadps/pages/07_Write_File.py +0 -452
- pyadps/utils/cutbin.py +0 -413
- pyadps/utils/regrid.py +0 -279
- pyadps-0.2.0b0.dist-info/RECORD +0 -31
- {pyadps-0.2.0b0.dist-info → pyadps-0.3.0.dist-info}/LICENSE +0 -0
pyadps/pages/01_Read_File.py
CHANGED
@@ -1,18 +1,28 @@
|
|
1
1
|
import os
|
2
2
|
import tempfile
|
3
|
+
import time
|
4
|
+
from typing import Any, Dict, Union, Tuple
|
3
5
|
|
6
|
+
import numpy as np
|
4
7
|
import pandas as pd
|
5
8
|
import streamlit as st
|
9
|
+
import plotly.express as px
|
6
10
|
import utils.readrdi as rd
|
7
|
-
import utils.writenc as wr
|
8
|
-
from streamlit.runtime.state import session_state
|
9
11
|
from utils.signal_quality import default_mask
|
10
12
|
|
13
|
+
|
14
|
+
# Set page configuration to wide layout
|
15
|
+
st.set_page_config(layout="wide")
|
16
|
+
|
17
|
+
|
11
18
|
"""
|
12
|
-
Streamlit
|
13
|
-
and Fixed Leader data
|
19
|
+
Streamlit application for ADCP (Acoustic Doppler Current Profiler) binary file processing.
|
20
|
+
This page loads ADCP binary files and displays File Header and Fixed Leader data. Once
|
21
|
+
the file is loaded, the data processing and visualization options will be available
|
22
|
+
in other tabs.
|
14
23
|
"""
|
15
24
|
|
25
|
+
# Initialize session state variables if they don't exist
|
16
26
|
if "fname" not in st.session_state:
|
17
27
|
st.session_state.fname = "No file selected"
|
18
28
|
|
@@ -23,264 +33,666 @@ if "vleadfilename" not in st.session_state:
|
|
23
33
|
st.session_state.vleadfilename = "vlead.nc"
|
24
34
|
|
25
35
|
|
26
|
-
################ Functions #######################
|
36
|
+
################ Helper Functions #######################
|
27
37
|
@st.cache_data()
|
28
|
-
def
|
38
|
+
def read_file(uploaded_file) -> None:
|
29
39
|
"""
|
30
|
-
|
31
|
-
|
40
|
+
Creates temporary directory to store the uploaded ADCP binary file.
|
41
|
+
Reads ADCP binary file. Stores the file path and data in session state.
|
32
42
|
|
33
43
|
Args:
|
34
|
-
uploaded_file
|
35
|
-
|
36
|
-
Returns:
|
37
|
-
path (string): Path of the uploaded file
|
44
|
+
uploaded_file: The uploaded file object from Streamlit
|
38
45
|
"""
|
46
|
+
# Create a temporary directory and store the file for reading
|
39
47
|
temp_dir = tempfile.mkdtemp()
|
40
48
|
path = os.path.join(temp_dir, uploaded_file.name)
|
41
49
|
with open(path, "wb") as f:
|
42
50
|
f.write(uploaded_file.getvalue())
|
43
|
-
return path
|
44
51
|
|
52
|
+
# Read the file
|
53
|
+
ds = rd.ReadFile(path)
|
54
|
+
# By default, the ensemble is fixed while reading
|
55
|
+
if not ds.isEnsembleEqual:
|
56
|
+
ds.fixensemble()
|
57
|
+
|
58
|
+
# Store file path and data in session state
|
59
|
+
st.session_state.fpath = path
|
60
|
+
st.session_state.ds = ds
|
45
61
|
|
46
|
-
|
62
|
+
|
63
|
+
def color_bool(val: bool) -> str:
|
47
64
|
"""
|
48
|
-
|
49
|
-
|
65
|
+
Returns CSS color formatting based on boolean value.
|
66
|
+
|
67
|
+
Args:
|
68
|
+
val: Boolean value to be colorized
|
69
|
+
|
70
|
+
Returns:
|
71
|
+
str: CSS color styling (green for True, red for False, orange for non-boolean)
|
50
72
|
"""
|
51
73
|
if isinstance(val, bool):
|
52
|
-
if val
|
53
|
-
color = "green"
|
54
|
-
else:
|
55
|
-
color = "red"
|
74
|
+
color = "green" if val else "red"
|
56
75
|
else:
|
57
76
|
color = "orange"
|
58
|
-
return "color:
|
77
|
+
return f"color: {color}"
|
59
78
|
|
60
79
|
|
61
|
-
def color_bool2(val):
|
80
|
+
def color_bool2(val: str) -> str:
|
62
81
|
"""
|
63
|
-
|
64
|
-
the css color property. The following colors
|
65
|
-
are assinged for the string
|
66
|
-
"True": green,
|
67
|
-
"False": red
|
68
|
-
Any other string: orange
|
82
|
+
Returns CSS color formatting based on string value.
|
69
83
|
|
70
84
|
Args:
|
71
|
-
val
|
85
|
+
val: String value to be colorized
|
72
86
|
|
73
87
|
Returns:
|
74
|
-
|
88
|
+
str: CSS color styling (green for "True"/"healthy", red for "False", orange otherwise)
|
75
89
|
"""
|
76
90
|
if val == "True" or val == "Data type is healthy":
|
77
91
|
color = "green"
|
78
92
|
elif val == "False":
|
79
93
|
color = "red"
|
80
|
-
# elif val in st.session_state.ds.warnings.values():
|
81
|
-
# color = "orange"
|
82
94
|
else:
|
83
95
|
color = "orange"
|
84
|
-
return "color:
|
96
|
+
return f"color: {color}"
|
97
|
+
|
98
|
+
|
99
|
+
def initialize_session_state() -> None:
|
100
|
+
"""Initialize all session state variables for different pages of the application."""
|
101
|
+
|
102
|
+
st.session_state.isTimeAxisModified = False
|
103
|
+
st.session_state.isSnapTimeAxis = False
|
104
|
+
st.session_state.time_snap_frequency = "h"
|
105
|
+
st.session_state.time_snap_tolerance = 5
|
106
|
+
st.session_state.time_target_minute = 0
|
107
|
+
st.session_state.isTimeGapFilled = False
|
108
|
+
|
109
|
+
# Download Raw File page settings
|
110
|
+
st.session_state.add_attributes_DRW = "No"
|
111
|
+
st.session_state.axis_option_DRW = "time"
|
112
|
+
st.session_state.rawnc_download_DRW = False
|
113
|
+
st.session_state.vleadnc_download_DRW = False
|
114
|
+
st.session_state.rawcsv_option_DRW = "Velocity"
|
115
|
+
st.session_state.rawcsv_beam_DRW = 1
|
116
|
+
st.session_state.rawcsv_download_DRW = False
|
117
|
+
|
118
|
+
# Sensor Test page settings
|
119
|
+
st.session_state.isSensorTest = False
|
120
|
+
st.session_state.isFirstSensorVisit = True
|
121
|
+
|
122
|
+
# Depth Correction settings
|
123
|
+
st.session_state.isDepthModified_ST = False
|
124
|
+
st.session_state.depthoption_ST = "Fixed Value"
|
125
|
+
st.session_state.isFixedDepth_ST = False
|
126
|
+
st.session_state.fixeddepth_ST = 0
|
127
|
+
st.session_state.isUploadDepth_ST = False
|
128
|
+
|
129
|
+
# Salinity Correction settings
|
130
|
+
st.session_state.isSalinityModified_ST = False
|
131
|
+
st.session_state.salinityoption_ST = "Fixed Value"
|
132
|
+
st.session_state.isFixedSalinity_ST = False
|
133
|
+
st.session_state.fixedsalinity_ST = 35
|
134
|
+
st.session_state.isUploadSalinity_ST = False
|
135
|
+
|
136
|
+
# Temperature Correction settings
|
137
|
+
st.session_state.isTemperatureModified_ST = False
|
138
|
+
st.session_state.temperatureoption_ST = "Fixed Value"
|
139
|
+
st.session_state.isFixedTemperature_ST = False
|
140
|
+
st.session_state.fixedtemperature_ST = 0
|
141
|
+
st.session_state.isUploadTemperature_ST = False
|
142
|
+
|
143
|
+
# Pitch, Roll, Velocity Correction settings
|
144
|
+
st.session_state.isRollCheck_ST = False
|
145
|
+
st.session_state.isPitchCheck_ST = False
|
146
|
+
st.session_state.isVelocityModifiedSound_ST = False
|
147
|
+
st.session_state.roll_cutoff_ST = 359
|
148
|
+
st.session_state.pitch_cutoff_ST = 359
|
149
|
+
|
150
|
+
# QC Test page settings
|
151
|
+
st.session_state.isQCTest = False
|
152
|
+
st.session_state.isFirstQCVisit = True
|
153
|
+
st.session_state.isQCCheck_QCT = False
|
154
|
+
st.session_state.ct_QCT = 64
|
155
|
+
st.session_state.et_QCT = 0
|
156
|
+
st.session_state.evt_QCT = 2000
|
157
|
+
st.session_state.ft_QCT = 50
|
158
|
+
st.session_state.is3beam_QCT = True
|
159
|
+
st.session_state.pgt_QCT = 0
|
160
|
+
st.session_state.isBeamModified_QCT = False
|
161
|
+
|
162
|
+
# Profile Test page settings
|
163
|
+
st.session_state.isProfileTest = False
|
164
|
+
st.session_state.isFirstProfileVisit = True
|
165
|
+
st.session_state.isTrimEndsCheck_PT = False
|
166
|
+
st.session_state.start_ens_PT = 0
|
167
|
+
st.session_state.end_ens_PT = st.session_state.head.ensembles
|
168
|
+
st.session_state.isCutBinSideLobeCheck_PT = False
|
169
|
+
st.session_state.extra_cells_PT = 0
|
170
|
+
st.session_state.water_depth_PT = 0
|
171
|
+
st.session_state.isCutBinManualCheck_PT = False
|
172
|
+
st.session_state.isRegridCheck_PT = False
|
173
|
+
st.session_state.end_cell_option_PT = "Cell"
|
174
|
+
st.session_state.interpolate_PT = "nearest"
|
175
|
+
st.session_state.manualdepth_PT = 0
|
176
|
+
|
177
|
+
# Velocity Test page settings
|
178
|
+
st.session_state.isVelocityTest = False
|
179
|
+
st.session_state.isFirstVelocityVisit = True
|
180
|
+
st.session_state.isMagnetCheck_VT = False
|
181
|
+
st.session_state.magnet_method_VT = "pygeomag"
|
182
|
+
st.session_state.magnet_lat_VT = 0
|
183
|
+
st.session_state.magnet_lon_VT = 0
|
184
|
+
st.session_state.magnet_year_VT = 2025
|
185
|
+
st.session_state.magnet_depth_VT = 0
|
186
|
+
st.session_state.magnet_user_input_VT = 0
|
187
|
+
st.session_state.isCutoffCheck_VT = False
|
188
|
+
st.session_state.maxuvel_VT = 250
|
189
|
+
st.session_state.maxvvel_VT = 250
|
190
|
+
st.session_state.maxwvel_VT = 15
|
191
|
+
st.session_state.isDespikeCheck_VT = False
|
192
|
+
st.session_state.despike_kernel_VT = 5
|
193
|
+
st.session_state.despike_cutoff_VT = 3
|
194
|
+
st.session_state.isFlatlineCheck_VT = False
|
195
|
+
st.session_state.flatline_kernel_VT = 5
|
196
|
+
st.session_state.flatline_cutoff_VT = 3
|
197
|
+
|
198
|
+
# Write File page settings
|
199
|
+
st.session_state.isWriteFile = True
|
200
|
+
st.session_state.isAttributes = False
|
201
|
+
st.session_state.mask_data_WF = "Yes"
|
202
|
+
st.session_state.file_type_WF = "NetCDF"
|
203
|
+
st.session_state.isProcessedNetcdfDownload_WF = True
|
204
|
+
st.session_state.isProcessedCSVDownload_WF = False
|
205
|
+
|
206
|
+
# Page return flags
|
207
|
+
st.session_state.isSensorPageReturn = False
|
208
|
+
st.session_state.isQCPageReturn = False
|
209
|
+
st.session_state.isProfilePageReturn = False
|
210
|
+
st.session_state.isVelocityPageReturn = False
|
211
|
+
|
212
|
+
|
213
|
+
def display_diagnostic_plots(ds):
|
214
|
+
"""Displays the interactive plots for diagnosing the time axis."""
|
215
|
+
st.subheader("1. Diagnose with a Plot")
|
216
|
+
|
217
|
+
plot_choice = st.selectbox(
|
218
|
+
"Select plot type:",
|
219
|
+
("Time Interval Between Ensembles", "Time Components vs. Ensembles"),
|
220
|
+
)
|
85
221
|
|
222
|
+
if plot_choice == "Time Interval Between Ensembles":
|
223
|
+
time_s = ds.time
|
224
|
+
if len(time_s) > 1:
|
225
|
+
time_diff_seconds = time_s.diff().dt.total_seconds().dropna()
|
226
|
+
plot_df = pd.DataFrame({"Time Difference (seconds)": time_diff_seconds})
|
227
|
+
plot_df.index.name = "Ensemble Number"
|
228
|
+
plot_df.reset_index(inplace=True)
|
229
|
+
|
230
|
+
fig = px.line(
|
231
|
+
plot_df,
|
232
|
+
x="Ensemble Number",
|
233
|
+
y="Time Difference (seconds)",
|
234
|
+
title="Time Interval Between Ensembles",
|
235
|
+
)
|
236
|
+
st.plotly_chart(fig, use_container_width=True)
|
237
|
+
st.info(
|
238
|
+
"**How to interpret this chart:** A **flat horizontal line** indicates a perfectly regular time interval. **Spikes** represent large gaps in the data. A **noisy or wandering line** indicates time drift."
|
239
|
+
)
|
240
|
+
else:
|
241
|
+
st.write("Not enough data points to plot intervals.")
|
242
|
+
|
243
|
+
elif plot_choice == "Time Components vs. Ensembles":
|
244
|
+
time_s = ds.time
|
245
|
+
if len(time_s) > 0:
|
246
|
+
selected_comp = st.radio(
|
247
|
+
"Select time component to plot:",
|
248
|
+
("Hour", "Minute", "Second"),
|
249
|
+
index=1,
|
250
|
+
horizontal=True,
|
251
|
+
key="time_comp_plot_radio",
|
252
|
+
)
|
86
253
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
254
|
+
plot_df = pd.DataFrame(
|
255
|
+
{
|
256
|
+
"Ensemble Number": np.arange(len(time_s)),
|
257
|
+
"Hour": time_s.dt.hour,
|
258
|
+
"Minute": time_s.dt.minute,
|
259
|
+
"Second": time_s.dt.second,
|
260
|
+
}
|
261
|
+
)
|
94
262
|
|
263
|
+
fig = px.line(
|
264
|
+
plot_df,
|
265
|
+
x="Ensemble Number",
|
266
|
+
y=selected_comp,
|
267
|
+
title=f"{selected_comp} Component vs. Ensembles",
|
268
|
+
)
|
269
|
+
st.plotly_chart(fig, use_container_width=True)
|
270
|
+
st.info(
|
271
|
+
"**How to interpret this chart:** For regular data, the **Hour** plot should be a repeating sawtooth wave. The **Minute** and **Second** plots should be flat lines at 0. Jumps indicate gaps, while slopes or noise indicate drift."
|
272
|
+
)
|
273
|
+
else:
|
274
|
+
st.write("No data to plot.")
|
95
275
|
|
96
|
-
uploaded_file = st.file_uploader("Upload RDI ADCP Binary File", type="000")
|
97
276
|
|
98
|
-
|
99
|
-
|
277
|
+
def display_diagnostic_tables(ds):
|
278
|
+
"""
|
279
|
+
Displays the frequency distribution plots by default.
|
280
|
+
Provides optional checkboxes to view the detailed data tables.
|
281
|
+
"""
|
282
|
+
st.subheader("2. Diagnose with Frequency Distributions")
|
283
|
+
st.markdown(
|
284
|
+
"Use the plots for a quick visual check of time component and interval frequencies. Check the boxes to see the detailed data tables."
|
285
|
+
)
|
100
286
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
287
|
+
st.markdown("##### Component Frequencies")
|
288
|
+
comp = st.selectbox(
|
289
|
+
"Select time component to analyze:",
|
290
|
+
("minute", "hour", "second"),
|
291
|
+
key="time_comp_select",
|
292
|
+
)
|
293
|
+
if comp and hasattr(ds, "get_time_component_frequency"):
|
294
|
+
freq_series = ds.get_time_component_frequency(comp)
|
295
|
+
if not freq_series.empty:
|
296
|
+
freq_df = freq_series.reset_index()
|
297
|
+
freq_df.columns = [comp.capitalize(), "Count"]
|
298
|
+
|
299
|
+
# The plot is now displayed by default
|
300
|
+
fig_comp = px.bar(
|
301
|
+
freq_df,
|
302
|
+
x=comp.capitalize(),
|
303
|
+
y="Count",
|
304
|
+
title=f"Distribution of '{comp.capitalize()}' values",
|
305
|
+
)
|
306
|
+
st.plotly_chart(fig_comp, use_container_width=True)
|
114
307
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
st.
|
122
|
-
st.
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
"
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
308
|
+
# The table is now optional
|
309
|
+
if st.checkbox(f"Show data table for '{comp.capitalize()}' frequency"):
|
310
|
+
st.dataframe(freq_df, use_container_width=True)
|
311
|
+
else:
|
312
|
+
st.write("No time data to analyze.")
|
313
|
+
|
314
|
+
st.markdown("---")
|
315
|
+
st.markdown("##### Interval Frequencies")
|
316
|
+
interval_freq = ds.get_time_interval_frequency()
|
317
|
+
if interval_freq:
|
318
|
+
df_interval = pd.DataFrame(
|
319
|
+
interval_freq.items(), columns=["Time Difference", "Frequency"]
|
320
|
+
)
|
321
|
+
|
322
|
+
# The plot is now displayed by default
|
323
|
+
fig_interval = px.bar(
|
324
|
+
df_interval,
|
325
|
+
x="Time Difference",
|
326
|
+
y="Frequency",
|
327
|
+
title="Distribution of Time Intervals",
|
328
|
+
)
|
329
|
+
st.plotly_chart(fig_interval, use_container_width=True)
|
330
|
+
|
331
|
+
# The table is now optional
|
332
|
+
if st.checkbox("Show data table for interval frequency"):
|
333
|
+
st.dataframe(df_interval, use_container_width=True)
|
334
|
+
|
335
|
+
else:
|
336
|
+
st.write("Not enough data to calculate intervals.")
|
337
|
+
|
338
|
+
|
339
|
+
def display_correction_tools(ds):
|
340
|
+
"""Displays the widgets for correcting the time axis."""
|
341
|
+
# --- Step 3: Correction for minor drifts (Snap) ---
|
342
|
+
st.subheader("3. Correct Minor Time Drifts (Snap)")
|
343
|
+
st.markdown(
|
344
|
+
"Use this if your data has small, inconsistent time drifts. This will **round** each timestamp to the nearest specified interval or minute."
|
345
|
+
)
|
346
|
+
|
347
|
+
col1, col2 = st.columns(2)
|
348
|
+
snap_freq = "h"
|
349
|
+
target_minute = None
|
350
|
+
|
351
|
+
with col1:
|
352
|
+
default_target_minute = 0
|
353
|
+
minute_freq = ds.get_time_component_frequency("minute")
|
354
|
+
if not minute_freq.empty:
|
355
|
+
default_target_minute = minute_freq.index[0]
|
356
|
+
target_minute = st.slider(
|
357
|
+
"Select minute to round to (0-59):",
|
358
|
+
min_value=0,
|
359
|
+
max_value=59,
|
360
|
+
value=default_target_minute,
|
361
|
+
step=1,
|
362
|
+
)
|
363
|
+
|
364
|
+
with col2:
|
365
|
+
st.markdown("**Set Correction Tolerance:**")
|
366
|
+
tolerance_min = st.number_input(
|
367
|
+
"Max allowed correction (minutes)",
|
368
|
+
min_value=1,
|
369
|
+
max_value=60,
|
370
|
+
value=5,
|
371
|
+
step=1,
|
372
|
+
help="The operation will be aborted if any timestamp needs to be changed by more than this amount.",
|
373
|
+
)
|
374
|
+
snap_tolerance = f"{tolerance_min}min"
|
375
|
+
|
376
|
+
if st.button("Snap Time Axis"):
|
377
|
+
if hasattr(ds, "snap_time_axis"):
|
378
|
+
with st.spinner("Snapping time axis..."):
|
379
|
+
success, message = ds.snap_time_axis(
|
380
|
+
freq=snap_freq,
|
381
|
+
tolerance=snap_tolerance,
|
382
|
+
target_minute=target_minute,
|
383
|
+
)
|
384
|
+
if success:
|
385
|
+
st.session_state.isTimeAxisModified = True
|
386
|
+
st.session_state.isSnapTimeAxis = True
|
387
|
+
st.session_state.time_snap_frequency = snap_freq
|
388
|
+
st.session_state.time_snap_tolerance = snap_tolerance
|
389
|
+
st.session_state.time_target_minute = target_minute
|
390
|
+
st.success(message)
|
391
|
+
time.sleep(2)
|
392
|
+
st.rerun()
|
393
|
+
else:
|
394
|
+
st.error(message)
|
395
|
+
else:
|
396
|
+
st.error("`snap_time_axis` method not found in `readrdi`.")
|
397
|
+
|
398
|
+
# --- Step 4: Fill Missing Time Gaps ---
|
399
|
+
st.subheader("4. Fill Missing Time Gaps")
|
400
|
+
st.markdown(
|
401
|
+
"Use this **after snapping** (if needed) to make the time axis perfectly uniform. It finds any large time gaps and fills them with masked (missing) data."
|
181
402
|
)
|
182
403
|
|
183
|
-
|
184
|
-
|
185
|
-
st.
|
186
|
-
|
187
|
-
|
188
|
-
st.session_state.
|
404
|
+
if st.button("Fill Time Axis Gaps"):
|
405
|
+
if hasattr(ds, "fill_time_axis"):
|
406
|
+
with st.spinner("Filling gaps in time axis..."):
|
407
|
+
success, message = ds.fill_time_axis()
|
408
|
+
if success:
|
409
|
+
st.session_state.isTimeAxisModified = True
|
410
|
+
st.session_state.isTimeGapFilled = True
|
411
|
+
st.success(message)
|
412
|
+
time.sleep(2)
|
413
|
+
st.rerun()
|
414
|
+
else:
|
415
|
+
st.warning(message)
|
416
|
+
else:
|
417
|
+
st.error("`fill_time_axis` method not found in `readrdi`.")
|
418
|
+
|
189
419
|
|
190
|
-
|
191
|
-
st.header("File Header", divider="blue")
|
192
|
-
st.write(
|
420
|
+
def process_date() -> None:
|
193
421
|
"""
|
194
|
-
|
422
|
+
Displays time axis info and, if irregular, provides diagnostic and correction tools.
|
423
|
+
"""
|
424
|
+
ds = st.session_state.ds
|
425
|
+
is_regular = ds.isTimeRegular
|
426
|
+
interval_info = ds.get_time_interval()
|
427
|
+
|
428
|
+
st.header("Time Axis Information", divider="blue")
|
429
|
+
|
430
|
+
if st.session_state.get("isTimeAxisModified", False):
|
431
|
+
st.warning(
|
432
|
+
"⚠️ **Warning:** The time axis has been modified from its original state."
|
433
|
+
)
|
434
|
+
|
435
|
+
st.write(
|
195
436
|
"""
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
check_button = st.button("Check File Health")
|
201
|
-
if check_button:
|
202
|
-
cf = st.session_state.head.check_file()
|
203
|
-
if (
|
204
|
-
cf["File Size Match"]
|
205
|
-
and cf["Byte Uniformity"]
|
206
|
-
and cf["Data Type Uniformity"]
|
207
|
-
):
|
208
|
-
st.write("Your file appears healthy! :sunglasses:")
|
209
|
-
else:
|
210
|
-
st.write("Your file appears corrupted! :worried:")
|
437
|
+
This section analyzes the time intervals between your ADCP ensembles to determine
|
438
|
+
if the data was recorded at a regular frequency (e.g., every minute, every hour).
|
439
|
+
"""
|
440
|
+
)
|
211
441
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
st.write(
|
222
|
-
|
223
|
-
|
224
|
-
|
442
|
+
st.write(f"**Total Ensembles:** {ds.ensembles}")
|
443
|
+
st.write(f"**Start Time:** {ds.time.iloc[0]}")
|
444
|
+
st.write(f"**End Time:** {ds.time.iloc[-1]}")
|
445
|
+
st.write(f"**Total Time:** {ds.time.iloc[-1] - ds.time.iloc[0]}")
|
446
|
+
|
447
|
+
st.subheader("Check Time Irregularities", divider="orange")
|
448
|
+
|
449
|
+
if is_regular:
|
450
|
+
st.success("**Equal Time Interval:** The time axis is regular and uniform.")
|
451
|
+
st.write(f"**Detected Time Interval:** :green[{interval_info}]")
|
452
|
+
else:
|
453
|
+
st.error("**Equal Time Interval:** The time axis is irregular.")
|
454
|
+
st.write(f"**Most Common Interval (Mode):** :red[{interval_info}]")
|
455
|
+
st.warning(
|
456
|
+
"Irregularities can be caused by minor time drifts during recording or by missing data (gaps). "
|
457
|
+
"Use the tools below to diagnose and correct these issues."
|
458
|
+
)
|
459
|
+
st.markdown("---")
|
460
|
+
|
461
|
+
if st.checkbox("🔬 **Diagnose & Correct Time Axis**"):
|
462
|
+
with st.container(border=True):
|
463
|
+
# Call the new helper functions
|
464
|
+
display_diagnostic_plots(ds)
|
465
|
+
display_diagnostic_tables(ds)
|
466
|
+
display_correction_tools(ds)
|
467
|
+
|
468
|
+
# Finalize session state variables
|
469
|
+
st.session_state.ensemble_axis = np.arange(0, ds.ensembles, 1)
|
470
|
+
st.session_state.axis_option = "time"
|
471
|
+
st.session_state.date = ds.time
|
472
|
+
st.session_state.date1 = ds.time
|
473
|
+
st.session_state.date2 = ds.time
|
474
|
+
st.session_state.date3 = ds.time
|
475
|
+
|
476
|
+
|
477
|
+
def display_file_header() -> None:
|
478
|
+
"""Display file header information and health checks."""
|
479
|
+
st.header("File Header", divider="blue")
|
480
|
+
st.write(
|
481
|
+
"""
|
482
|
+
Header information is the first item sent by the ADCP. You may check the file size,
|
483
|
+
total ensembles, and available data types. The function also checks if the total bytes
|
484
|
+
and data types are uniform for all ensembles.
|
485
|
+
"""
|
486
|
+
)
|
487
|
+
|
488
|
+
# Create two columns for buttons
|
489
|
+
left1, right1 = st.columns(2)
|
490
|
+
|
491
|
+
with left1:
|
492
|
+
# File health check button
|
493
|
+
if st.button("Check File Health"):
|
494
|
+
cf = st.session_state.head.check_file()
|
495
|
+
|
496
|
+
# Check if file is healthy
|
497
|
+
if (
|
498
|
+
cf["File Size Match"]
|
499
|
+
and cf["Byte Uniformity"]
|
500
|
+
and cf["Data Type Uniformity"]
|
501
|
+
):
|
502
|
+
st.write("Your file appears healthy! :sunglasses:")
|
503
|
+
else:
|
504
|
+
st.write("Your file appears corrupted! :worried:")
|
505
|
+
|
506
|
+
# Format file size
|
507
|
+
cf["File Size (MB)"] = "{:,.2f}".format(cf["File Size (MB)"])
|
508
|
+
|
509
|
+
# Display ensemble count
|
510
|
+
st.write(
|
511
|
+
f"Total no. of Ensembles: :green[{st.session_state.head.ensembles}]"
|
512
|
+
)
|
513
|
+
|
514
|
+
# Display health check results with coloring
|
515
|
+
df = pd.DataFrame(cf.items(), columns=pd.array(["Check", "Details"]))
|
516
|
+
df = df.astype("str")
|
517
|
+
st.write(df.style.map(color_bool2, subset="Details"))
|
518
|
+
|
519
|
+
with right1:
|
520
|
+
# Display data types button
|
521
|
+
if st.button("Display Data Types"):
|
522
|
+
st.write(
|
523
|
+
pd.DataFrame(
|
524
|
+
st.session_state.head.data_types(),
|
525
|
+
columns=pd.array(["Available Data Types"]),
|
526
|
+
)
|
225
527
|
)
|
528
|
+
|
529
|
+
# Display warnings if any
|
530
|
+
if st.session_state.ds.isWarning:
|
531
|
+
st.write(
|
532
|
+
"""
|
533
|
+
Warnings detected while reading. Data sets may still be available for processing.
|
534
|
+
Click `Display Warning` to display warnings for each data type.
|
535
|
+
"""
|
226
536
|
)
|
227
537
|
|
228
|
-
if st.
|
538
|
+
if st.button("Display Warnings"):
|
539
|
+
df2 = pd.DataFrame(
|
540
|
+
st.session_state.ds.warnings.items(),
|
541
|
+
columns=pd.array(["Data Type", "Warnings"]),
|
542
|
+
)
|
543
|
+
st.write(df2.style.map(color_bool2, subset=["Warnings"]))
|
544
|
+
|
545
|
+
|
546
|
+
def display_fixed_leader() -> None:
|
547
|
+
"""Display fixed leader (static variables) information."""
|
548
|
+
st.header("Fixed Leader (Static Variables)", divider="blue")
|
229
549
|
st.write(
|
230
|
-
"""
|
231
|
-
|
232
|
-
|
550
|
+
"""
|
551
|
+
Fixed Leader data refers to the non-dynamic WorkHorse ADCP data like hardware information
|
552
|
+
and thresholds. Typically, values remain constant over time. They only change when you
|
553
|
+
change certain commands, although there are occasional exceptions. You can confirm this
|
554
|
+
using the :blue[**Fleader Uniformity Check**]. Click :blue[**Fixed Leader**] to display
|
555
|
+
the values for the first ensemble.
|
233
556
|
"""
|
234
557
|
)
|
235
|
-
warning_button = st.button("Display Warnings")
|
236
|
-
df2 = pd.DataFrame(
|
237
|
-
st.session_state.ds.warnings.items(),
|
238
|
-
columns=pd.array(["Data Type", "Warnings"]),
|
239
|
-
)
|
240
|
-
if warning_button:
|
241
|
-
st.write(df2.style.map(color_bool2, subset=["Warnings"]))
|
242
558
|
|
243
|
-
|
559
|
+
# Uniformity check button
|
560
|
+
if st.button("Fleader Uniformity Check"):
|
561
|
+
# Get uniformity check results
|
562
|
+
uniformity_results = st.session_state.flead.is_uniform()
|
563
|
+
|
564
|
+
# Display non-uniform variables
|
565
|
+
st.write("The following variables are non-uniform:")
|
566
|
+
for key, is_uniform in uniformity_results.items():
|
567
|
+
if not is_uniform:
|
568
|
+
st.markdown(f":blue[**- {key}**]")
|
569
|
+
|
570
|
+
# Display all static variables with color coding
|
571
|
+
st.write("Displaying all static variables")
|
572
|
+
df = pd.DataFrame(uniformity_results, index=[0]).T
|
573
|
+
st.write(df.style.map(color_bool))
|
574
|
+
|
575
|
+
# Display fixed leader data button
|
576
|
+
if st.button("Fixed Leader"):
|
577
|
+
# Convert all values to uint64 to ensure consistent datatypes
|
578
|
+
fl_dict = st.session_state.flead.field().items()
|
579
|
+
new_dict = {}
|
580
|
+
for key, value in fl_dict:
|
581
|
+
new_dict[key] = value.astype(np.uint64)
|
582
|
+
|
583
|
+
# Create and display dataframe
|
584
|
+
df = pd.DataFrame(
|
585
|
+
{
|
586
|
+
"Fields": new_dict.keys(),
|
587
|
+
"Values": new_dict.values(),
|
588
|
+
}
|
589
|
+
)
|
590
|
+
st.dataframe(df, use_container_width=True)
|
591
|
+
|
592
|
+
# Create three columns for additional information
|
593
|
+
left, center, right = st.columns(3)
|
594
|
+
|
595
|
+
with left:
|
596
|
+
# Display system configuration
|
597
|
+
st.dataframe(st.session_state.flead.system_configuration())
|
598
|
+
|
599
|
+
with center:
|
600
|
+
# Display EZ sensor information
|
601
|
+
st.dataframe(st.session_state.flead.ez_sensor())
|
602
|
+
|
603
|
+
with right:
|
604
|
+
# Display coordinate transformation with color coding
|
605
|
+
df = pd.DataFrame(st.session_state.flead.ex_coord_trans(), index=[0]).T
|
606
|
+
df = df.astype("str")
|
607
|
+
st.write(df.style.map(color_bool2))
|
608
|
+
|
244
609
|
|
245
|
-
|
246
|
-
st.write(
|
610
|
+
def process_uploaded_file(uploaded_file) -> None:
|
247
611
|
"""
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
st.
|
260
|
-
|
261
|
-
st.
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
612
|
+
Process the uploaded ADCP binary file and store relevant data in session state.
|
613
|
+
|
614
|
+
Args:
|
615
|
+
uploaded_file: The uploaded file object from Streamlit
|
616
|
+
"""
|
617
|
+
ds = st.session_state.ds
|
618
|
+
|
619
|
+
# Store data in session state
|
620
|
+
st.session_state.fname = uploaded_file.name
|
621
|
+
st.session_state.head = ds.fileheader
|
622
|
+
st.session_state.flead = ds.fixedleader
|
623
|
+
st.session_state.vlead = ds.variableleader
|
624
|
+
st.session_state.velocity = ds.velocity.data
|
625
|
+
st.session_state.echo = ds.echo.data
|
626
|
+
st.session_state.correlation = ds.correlation.data
|
627
|
+
st.session_state.pgood = ds.percentgood.data
|
628
|
+
st.session_state.beam_direction = ds.fixedleader.system_configuration()[
|
629
|
+
"Beam Direction"
|
630
|
+
]
|
631
|
+
|
632
|
+
# Store sensor data with scaling applied where needed
|
633
|
+
st.session_state.sound_speed = ds.variableleader.speed_of_sound.data
|
634
|
+
st.session_state.depth = ds.variableleader.depth_of_transducer.data
|
635
|
+
st.session_state.temperature = (
|
636
|
+
ds.variableleader.temperature.data * ds.variableleader.temperature.scale
|
637
|
+
)
|
638
|
+
st.session_state.salinity = (
|
639
|
+
ds.variableleader.salinity.data * ds.variableleader.salinity.scale
|
270
640
|
)
|
271
|
-
st.dataframe(df, use_container_width=True)
|
272
|
-
|
273
|
-
left, centre, right = st.columns(3)
|
274
|
-
with left:
|
275
|
-
st.dataframe(st.session_state.flead.system_configuration())
|
276
|
-
|
277
|
-
with centre:
|
278
|
-
st.dataframe(st.session_state.flead.ez_sensor())
|
279
|
-
# st.write(output)
|
280
|
-
with right:
|
281
|
-
# st.write(st.session_state.flead.ex_coord_trans())
|
282
|
-
df = pd.DataFrame(st.session_state.flead.ex_coord_trans(), index=[0]).T
|
283
|
-
df = df.astype("str")
|
284
|
-
st.write((df.style.map(color_bool2)))
|
285
|
-
# st.dataframe(df)
|
286
641
|
|
642
|
+
# Create mask for velocity data if not already created
|
643
|
+
if "orig_mask" not in st.session_state:
|
644
|
+
st.session_state.orig_mask = default_mask(ds)
|
645
|
+
|
646
|
+
# Display confirmation message
|
647
|
+
st.write(f"You selected `{st.session_state.fname}`")
|
648
|
+
|
649
|
+
|
650
|
+
def main():
|
651
|
+
"""Main application function that handles the workflow."""
|
652
|
+
# Title and file upload
|
653
|
+
st.title("ADCP Data Analysis Tool")
|
654
|
+
|
655
|
+
# File uploader for ADCP binary files
|
656
|
+
uploaded_file = st.file_uploader("Upload RDI ADCP Binary File", type="000")
|
657
|
+
|
658
|
+
if uploaded_file is not None and uploaded_file.name != st.session_state.get(
|
659
|
+
"processed_filename", None
|
660
|
+
):
|
661
|
+
# Read the file
|
662
|
+
read_file(uploaded_file)
|
663
|
+
|
664
|
+
# Process the uploaded file
|
665
|
+
process_uploaded_file(uploaded_file)
|
666
|
+
|
667
|
+
# Initialize all session state variables
|
668
|
+
initialize_session_state()
|
669
|
+
|
670
|
+
# Process time data
|
671
|
+
process_date()
|
672
|
+
|
673
|
+
# Display file header section
|
674
|
+
display_file_header()
|
675
|
+
|
676
|
+
# Display fixed leader section
|
677
|
+
display_fixed_leader()
|
678
|
+
|
679
|
+
elif "flead" in st.session_state:
|
680
|
+
# If file was previously uploaded and processed
|
681
|
+
st.write(f"You selected `{st.session_state.fname}`")
|
682
|
+
st.info("Refresh to upload a new file.")
|
683
|
+
|
684
|
+
process_date()
|
685
|
+
# Display file header and fixed leader information
|
686
|
+
display_file_header()
|
687
|
+
display_fixed_leader()
|
688
|
+
|
689
|
+
else:
|
690
|
+
st.info("Please upload a file to begin.")
|
691
|
+
# No file loaded, clear cache and stop
|
692
|
+
st.cache_data.clear()
|
693
|
+
st.cache_resource.clear()
|
694
|
+
st.stop()
|
695
|
+
|
696
|
+
|
697
|
+
if __name__ == "__main__":
|
698
|
+
main()
|