oceanstream 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanstream/__init__.py +0 -0
- oceanstream/cli/__init__.py +3 -0
- oceanstream/cli/main.py +267 -0
- oceanstream/denoise/__init__.py +2 -0
- oceanstream/denoise/apply_masks.py +49 -0
- oceanstream/denoise/applying_masks_handler.py +173 -0
- oceanstream/denoise/background_noise_remover.py +97 -0
- oceanstream/denoise/create_masks.py +87 -0
- oceanstream/denoise/noise_masks.py +398 -0
- oceanstream/denoise/types.py +73 -0
- oceanstream/echodata/__init__.py +5 -0
- oceanstream/echodata/ensure_time_continuity.py +75 -0
- oceanstream/echodata/mvbs_computation.py +77 -0
- oceanstream/echodata/raw_handler.py +577 -0
- oceanstream/echodata/sv_computation.py +118 -0
- oceanstream/echodata/sv_dataset_extension.py +126 -0
- oceanstream/echodata/sv_interpolation.py +260 -0
- oceanstream/echodata/target_strength_computation.py +109 -0
- oceanstream/exports/__init__.py +2 -0
- oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
- oceanstream/exports/csv/csv_export_from_raw.py +92 -0
- oceanstream/exports/csv/csv_export_nasc.py +135 -0
- oceanstream/exports/frequency_differencing_handler.py +210 -0
- oceanstream/exports/nasc.py +23 -0
- oceanstream/exports/nasc_computation.py +62 -0
- oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
- oceanstream/exports/shoals/shoal_process.py +224 -0
- oceanstream/exports/shoals/shoals_handler.py +35 -0
- oceanstream/plot/__init__.py +2 -0
- oceanstream/plot/echogram.py +542 -0
- oceanstream/process/__init__.py +21 -0
- oceanstream/process/combine_zarr.py +90 -0
- oceanstream/process/file_processor.py +151 -0
- oceanstream/process/folder_processor.py +246 -0
- oceanstream/process/process.py +77 -0
- oceanstream/process/processed_data_io.py +140 -0
- oceanstream/settings/__init__.py +1 -0
- oceanstream/settings/config.py +17 -0
- oceanstream/utils.py +101 -0
- oceanstream-0.0.0.dev0.dist-info/LICENSE +21 -0
- oceanstream-0.0.0.dev0.dist-info/METADATA +214 -0
- oceanstream-0.0.0.dev0.dist-info/RECORD +46 -0
- oceanstream-0.0.0.dev0.dist-info/WHEEL +5 -0
- oceanstream-0.0.0.dev0.dist-info/dependency_links.txt +1 -0
- oceanstream-0.0.0.dev0.dist-info/entry_points.txt +2 -0
- oceanstream-0.0.0.dev0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import os
|
|
3
|
+
import xarray as xr
|
|
4
|
+
import numpy as np
|
|
5
|
+
from matplotlib.colors import LinearSegmentedColormap, Colormap
|
|
6
|
+
import matplotlib.pyplot as plt
|
|
7
|
+
import matplotlib.dates as mdates
|
|
8
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Optional, Union
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def plot_individual_channel(ds_Sv, channel, output_path, file_base_name,
|
|
14
|
+
cmap='viridis',
|
|
15
|
+
regions2d=None,
|
|
16
|
+
region_ids=None, region_class=None):
|
|
17
|
+
"""Plot and save echogram for a single channel with optional regions and enhancements."""
|
|
18
|
+
import pandas as pd
|
|
19
|
+
plot_params = {
|
|
20
|
+
"vmin": -70,
|
|
21
|
+
"vmax": -30,
|
|
22
|
+
"cmap": cmap
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
full_channel_name = ds_Sv.channel.values[channel]
|
|
26
|
+
channel_name = "_".join(full_channel_name.split()[:3])
|
|
27
|
+
|
|
28
|
+
plt.figure(figsize=(20, 10))
|
|
29
|
+
colors = ['black', 'yellow', 'red', 'orange']
|
|
30
|
+
|
|
31
|
+
idx = 0
|
|
32
|
+
labels_added = set()
|
|
33
|
+
|
|
34
|
+
echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
|
|
35
|
+
ax = plt.subplot(1, 1, 1)
|
|
36
|
+
|
|
37
|
+
echogram_data = np.flipud(ds_Sv.Sv.isel(channel=channel).T.values)
|
|
38
|
+
plt.pcolormesh(ds_Sv.ping_time, ds_Sv.range_sample, echogram_data, **plot_params)
|
|
39
|
+
cbar = plt.colorbar(ax=ax, orientation="vertical")
|
|
40
|
+
cbar.set_label("Volume backscattering strength (dB re 1m$^{-1}$)")
|
|
41
|
+
|
|
42
|
+
if region_ids:
|
|
43
|
+
idx = _plot_region_ids(colors, ds_Sv, idx, labels_added, region_ids, regions2d)
|
|
44
|
+
|
|
45
|
+
if region_class:
|
|
46
|
+
_plot_region_classes(channel_name, colors, ds_Sv, idx, labels_added, region_class, regions2d)
|
|
47
|
+
|
|
48
|
+
if regions2d:
|
|
49
|
+
plt.legend()
|
|
50
|
+
|
|
51
|
+
ax.invert_yaxis()
|
|
52
|
+
plt.grid(True, linestyle='--', alpha=0.5)
|
|
53
|
+
|
|
54
|
+
# Format the x-axis to show time (HH:mm)
|
|
55
|
+
ax.xaxis_date()
|
|
56
|
+
ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
|
|
57
|
+
|
|
58
|
+
# Add horizontal lines at specific depth intervals
|
|
59
|
+
depth_intervals = np.arange(0, ds_Sv.range_sample.max(), 10) # Example interval of 10 units
|
|
60
|
+
for depth in depth_intervals:
|
|
61
|
+
ax.axhline(depth, color='gray', linestyle='--', linewidth=0.5)
|
|
62
|
+
|
|
63
|
+
# Convert the first ping_time to a Python datetime object
|
|
64
|
+
date_str = pd.to_datetime(ds_Sv.ping_time.values[0]).strftime('%Y-%m-%d')
|
|
65
|
+
|
|
66
|
+
# Add date label on the right side
|
|
67
|
+
plt.xlabel('Time (HH:MM)', fontsize=14)
|
|
68
|
+
plt.ylabel('Depth', fontsize=14)
|
|
69
|
+
plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
|
|
70
|
+
plt.text(1.02, 0.5, date_str, transform=ax.transAxes, rotation=90, va='center', fontsize=12, fontweight='bold')
|
|
71
|
+
|
|
72
|
+
# Add a secondary y-axis for fathoms (1 fathom = 1.8288 meters)
|
|
73
|
+
secax = ax.secondary_yaxis('right', functions=(lambda x: x / 1.8288, lambda x: x * 1.8288))
|
|
74
|
+
secax.set_ylabel('Depth (fathoms)', fontsize=14)
|
|
75
|
+
|
|
76
|
+
plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
|
|
77
|
+
plt.close()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def plot_individual_channel_shaders(ds_Sv, channel, output_path, file_base_name, cmap='ocean_r'):
|
|
81
|
+
"""Plot and save echogram for a single channel with optional regions and enhancements."""
|
|
82
|
+
import holoviews as hv
|
|
83
|
+
from holoviews.operation.datashader import rasterize
|
|
84
|
+
from datashader import reductions as ds
|
|
85
|
+
hv.extension('bokeh', 'matplotlib')
|
|
86
|
+
|
|
87
|
+
full_channel_name = ds_Sv.channel.values[channel]
|
|
88
|
+
channel_name = "_".join(full_channel_name.split()[:3])
|
|
89
|
+
|
|
90
|
+
echogram_output_path_html = os.path.join(output_path, f"{file_base_name}_{channel_name}.html")
|
|
91
|
+
echogram_output_path_png = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
|
|
92
|
+
|
|
93
|
+
filtered_ds = ds_Sv['Sv']
|
|
94
|
+
if 'beam' in filtered_ds.dims:
|
|
95
|
+
filtered_ds = filtered_ds.isel(beam=0).drop('beam')
|
|
96
|
+
|
|
97
|
+
if 'channel' in filtered_ds.dims:
|
|
98
|
+
filtered_ds = filtered_ds.assign_coords({'frequency': ds_Sv.frequency_nominal})
|
|
99
|
+
filtered_ds = filtered_ds.swap_dims({'channel': 'frequency'})
|
|
100
|
+
if filtered_ds.frequency.size == 1:
|
|
101
|
+
filtered_ds = filtered_ds.isel(frequency=0)
|
|
102
|
+
|
|
103
|
+
# Handle NaNs by dropping NaNs directly from ping_time and range_sample
|
|
104
|
+
filtered_ds = filtered_ds.dropna(dim='ping_time', how='all')
|
|
105
|
+
filtered_ds = filtered_ds.dropna(dim='range_sample', how='all')
|
|
106
|
+
|
|
107
|
+
# Extract data for visualization
|
|
108
|
+
sv_data = filtered_ds.isel(frequency=channel)
|
|
109
|
+
ping_time = sv_data['ping_time']
|
|
110
|
+
range_sample = sv_data['range_sample']
|
|
111
|
+
|
|
112
|
+
sv_values = sv_data.values
|
|
113
|
+
ds_array = xr.DataArray(sv_values, coords=[ping_time, range_sample], dims=['ping_time', 'range_sample'])
|
|
114
|
+
hv_quadmesh = hv.QuadMesh(ds_array, kdims=['ping_time', 'range_sample'], vdims=['Sv'])
|
|
115
|
+
rasterized_quadmesh = rasterize(hv_quadmesh, aggregator=ds.mean('Sv'))
|
|
116
|
+
|
|
117
|
+
rasterized_quadmesh = rasterized_quadmesh.opts(
|
|
118
|
+
cmap='viridis',
|
|
119
|
+
colorbar=True,
|
|
120
|
+
responsive=True,
|
|
121
|
+
min_height=600,
|
|
122
|
+
clim=(np.nanmin(sv_values), np.nanmax(sv_values)),
|
|
123
|
+
tools=['hover', 'box_select'],
|
|
124
|
+
active_tools=['wheel_zoom'],
|
|
125
|
+
invert_yaxis=True,
|
|
126
|
+
hooks=[lambda plot, element: plot.handles['colorbar'].title('Sv')]
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# Save the plot as HTML
|
|
130
|
+
hv.save(rasterized_quadmesh, echogram_output_path_html, fmt='html')
|
|
131
|
+
|
|
132
|
+
# Save the plot as PNG
|
|
133
|
+
# hv.save(rasterized_quadmesh, echogram_output_path_png, fmt='png')
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def plot_individual_channel_enhanced(ds_Sv, channel, output_path, file_base_name,
|
|
137
|
+
cmap=None,
|
|
138
|
+
regions2d=None,
|
|
139
|
+
region_ids=None, region_class=None,
|
|
140
|
+
threshold=[-80, -50]):
|
|
141
|
+
"""Plot and save echogram for a single channel with optional regions and enhancements."""
|
|
142
|
+
full_channel_name = ds_Sv.channel.values[channel]
|
|
143
|
+
channel_name = "_".join(full_channel_name.split()[:3])
|
|
144
|
+
|
|
145
|
+
num_pings = ds_Sv.dims['ping_time']
|
|
146
|
+
num_samples = ds_Sv.dims['range_sample']
|
|
147
|
+
aspect_ratio = num_pings / num_samples
|
|
148
|
+
|
|
149
|
+
# Set base dimensions
|
|
150
|
+
base_width = 10 # Base width in inches
|
|
151
|
+
base_height = 10 # Base height in inches
|
|
152
|
+
|
|
153
|
+
# Adjust width and height based on aspect ratio, ensuring practical limits
|
|
154
|
+
if aspect_ratio > 1:
|
|
155
|
+
fig_width = base_width * aspect_ratio
|
|
156
|
+
fig_height = base_height
|
|
157
|
+
else:
|
|
158
|
+
fig_width = base_width
|
|
159
|
+
fig_height = base_height / aspect_ratio
|
|
160
|
+
|
|
161
|
+
fig_width = max(min(fig_width, 20), 10) # Width between 10 and 20 inches
|
|
162
|
+
fig_height = max(min(fig_height, 10), 5) # Height between 5 and 10 inches
|
|
163
|
+
|
|
164
|
+
plt.figure(figsize=(fig_width, fig_height))
|
|
165
|
+
|
|
166
|
+
echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
|
|
167
|
+
|
|
168
|
+
# Apply the same preprocessing steps from _plot_echogram
|
|
169
|
+
ds = ds_Sv
|
|
170
|
+
filter_var = 'channel'
|
|
171
|
+
filter_val = channel
|
|
172
|
+
if 'backscatter_i' in ds.variables:
|
|
173
|
+
filtered_ds = np.abs(ds.backscatter_r + 1j * ds.backscatter_i)
|
|
174
|
+
else:
|
|
175
|
+
filtered_ds = ds['Sv']
|
|
176
|
+
if 'beam' in filtered_ds.dims:
|
|
177
|
+
filtered_ds = filtered_ds.isel(beam=0).drop('beam')
|
|
178
|
+
|
|
179
|
+
if 'channel' in filtered_ds.dims:
|
|
180
|
+
filtered_ds = filtered_ds.assign_coords({'frequency': ds.frequency_nominal})
|
|
181
|
+
filtered_ds = filtered_ds.swap_dims({'channel': 'frequency'})
|
|
182
|
+
if filtered_ds.frequency.size == 1:
|
|
183
|
+
filtered_ds = filtered_ds.isel(frequency=0)
|
|
184
|
+
|
|
185
|
+
# Update axis labels
|
|
186
|
+
filtered_ds['ping_time'].attrs = {
|
|
187
|
+
'long_name': filtered_ds['ping_time'].attrs.get('long_name', 'Ping time'),
|
|
188
|
+
'units': filtered_ds['ping_time'].attrs.get('units', '')
|
|
189
|
+
}
|
|
190
|
+
filtered_ds['range_sample'].attrs = {
|
|
191
|
+
'long_name': filtered_ds['range_sample'].attrs.get('long_name', 'Depth'),
|
|
192
|
+
'units': filtered_ds['range_sample'].attrs.get('units', '')
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
# Handle NaNs by dropping NaNs directly from ping_time and range_sample
|
|
196
|
+
filtered_ds = filtered_ds.dropna(dim='ping_time', how='all')
|
|
197
|
+
filtered_ds = filtered_ds.dropna(dim='range_sample', how='all')
|
|
198
|
+
|
|
199
|
+
# Set the colormap
|
|
200
|
+
if cmap is None:
|
|
201
|
+
simrad_color_table = [
|
|
202
|
+
(1, 1, 1), (0.6235, 0.6235, 0.6235), (0.3725, 0.3725, 0.3725),
|
|
203
|
+
(0, 0, 1), (0, 0, 0.5), (0, 0.7490, 0), (0, 0.5, 0),
|
|
204
|
+
(1, 1, 0), (1, 0.5, 0), (1, 0, 0.7490), (1, 0, 0),
|
|
205
|
+
(0.6509, 0.3255, 0.2353), (0.4705, 0.2353, 0.1568)
|
|
206
|
+
]
|
|
207
|
+
cmap = LinearSegmentedColormap.from_list('Simrad', simrad_color_table)
|
|
208
|
+
cmap.set_bad(color='grey')
|
|
209
|
+
|
|
210
|
+
# Custom tick formatter for datetime64 values as floats
|
|
211
|
+
def format_datetime(x, pos=None):
|
|
212
|
+
try:
|
|
213
|
+
dt = x.astype('datetime64[ms]').astype('object')
|
|
214
|
+
tick_label = dt.strftime("%H:%M:%S")
|
|
215
|
+
except:
|
|
216
|
+
tick_label = ''
|
|
217
|
+
return tick_label
|
|
218
|
+
|
|
219
|
+
# Create the echogram using xarray's plot method
|
|
220
|
+
echogram_data = filtered_ds.isel(frequency=channel).T
|
|
221
|
+
echogram_data.plot(
|
|
222
|
+
x='ping_time',
|
|
223
|
+
y='range_sample',
|
|
224
|
+
yincrease=False,
|
|
225
|
+
vmin=threshold[0],
|
|
226
|
+
vmax=threshold[1],
|
|
227
|
+
cmap=cmap,
|
|
228
|
+
cbar_kwargs={'label': 'Volume backscattering strength (Sv re 1 m-1)'}
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
plt.gca().xaxis.set_major_formatter(plt.FuncFormatter(format_datetime))
|
|
232
|
+
plt.grid(True, linestyle='--', alpha=0.5)
|
|
233
|
+
plt.xlabel('Ping time', fontsize=14)
|
|
234
|
+
plt.ylabel('Depth', fontsize=14)
|
|
235
|
+
plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
|
|
236
|
+
plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
|
|
237
|
+
plt.close()
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_name,
|
|
241
|
+
cmap='viridis'):
|
|
242
|
+
full_channel_name = ds_Sv.channel.values[channel]
|
|
243
|
+
channel_name = "_".join(full_channel_name.split()[:3])
|
|
244
|
+
|
|
245
|
+
plt.figure(figsize=(30, 18))
|
|
246
|
+
echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
|
|
247
|
+
|
|
248
|
+
filtered_ds = ds_Sv['Sv']
|
|
249
|
+
if 'beam' in filtered_ds.dims:
|
|
250
|
+
filtered_ds = filtered_ds.isel(beam=0).drop('beam')
|
|
251
|
+
|
|
252
|
+
if 'channel' in filtered_ds.dims:
|
|
253
|
+
filtered_ds = filtered_ds.assign_coords({'frequency': ds_Sv.frequency_nominal})
|
|
254
|
+
filtered_ds = filtered_ds.swap_dims({'channel': 'frequency'})
|
|
255
|
+
if filtered_ds.frequency.size == 1:
|
|
256
|
+
filtered_ds = filtered_ds.isel(frequency=0)
|
|
257
|
+
|
|
258
|
+
# Update axis labels
|
|
259
|
+
filtered_ds['ping_time'].attrs = {
|
|
260
|
+
'long_name': filtered_ds['ping_time'].attrs.get('long_name', 'Ping time'),
|
|
261
|
+
'units': filtered_ds['ping_time'].attrs.get('units', '')
|
|
262
|
+
}
|
|
263
|
+
filtered_ds['range_sample'].attrs = {
|
|
264
|
+
'long_name': filtered_ds['range_sample'].attrs.get('long_name', 'Depth'),
|
|
265
|
+
'units': filtered_ds['range_sample'].attrs.get('units', '')
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
# Handle NaNs by dropping NaNs directly from ping_time and range_sample
|
|
269
|
+
filtered_ds = filtered_ds.dropna(dim='ping_time', how='all')
|
|
270
|
+
filtered_ds = filtered_ds.dropna(dim='range_sample', how='all')
|
|
271
|
+
|
|
272
|
+
# Create the echogram using xarray's plot method
|
|
273
|
+
filtered_ds.isel(frequency=channel).T.plot(
|
|
274
|
+
x='ping_time',
|
|
275
|
+
y='range_sample',
|
|
276
|
+
yincrease=False,
|
|
277
|
+
vmin=-80,
|
|
278
|
+
vmax=-50,
|
|
279
|
+
cmap='ocean_r',
|
|
280
|
+
cbar_kwargs={},
|
|
281
|
+
add_colorbar=False
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
plt.title('')
|
|
285
|
+
plt.axis('off')
|
|
286
|
+
plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
|
|
287
|
+
plt.close()
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name,
|
|
291
|
+
cmap='viridis',
|
|
292
|
+
regions2d=None,
|
|
293
|
+
region_ids=None, region_class=None):
|
|
294
|
+
"""Plot and save echogram for a single channel with optional regions and enhancements."""
|
|
295
|
+
full_channel_name = ds_Sv.channel.values[channel]
|
|
296
|
+
channel_name = "_".join(full_channel_name.split()[:3])
|
|
297
|
+
|
|
298
|
+
plt.figure(figsize=(30, 18))
|
|
299
|
+
echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
|
|
300
|
+
|
|
301
|
+
# Apply the same preprocessing steps from _plot_echogram
|
|
302
|
+
ds = ds_Sv
|
|
303
|
+
|
|
304
|
+
filtered_ds = ds['Sv']
|
|
305
|
+
if 'beam' in filtered_ds.dims:
|
|
306
|
+
filtered_ds = filtered_ds.isel(beam=0).drop('beam')
|
|
307
|
+
|
|
308
|
+
if 'channel' in filtered_ds.dims:
|
|
309
|
+
filtered_ds = filtered_ds.assign_coords({'frequency': ds.frequency_nominal})
|
|
310
|
+
filtered_ds = filtered_ds.swap_dims({'channel': 'frequency'})
|
|
311
|
+
if filtered_ds.frequency.size == 1:
|
|
312
|
+
filtered_ds = filtered_ds.isel(frequency=0)
|
|
313
|
+
|
|
314
|
+
# Update axis labels
|
|
315
|
+
filtered_ds['ping_time'].attrs = {
|
|
316
|
+
'long_name': filtered_ds['ping_time'].attrs.get('long_name', 'Ping time'),
|
|
317
|
+
'units': filtered_ds['ping_time'].attrs.get('units', '')
|
|
318
|
+
}
|
|
319
|
+
filtered_ds['range_sample'].attrs = {
|
|
320
|
+
'long_name': filtered_ds['range_sample'].attrs.get('long_name', 'Depth'),
|
|
321
|
+
'units': filtered_ds['range_sample'].attrs.get('units', '')
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
# Handle NaNs by dropping NaNs directly from ping_time and range_sample
|
|
325
|
+
filtered_ds = filtered_ds.dropna(dim='ping_time', how='all')
|
|
326
|
+
filtered_ds = filtered_ds.dropna(dim='range_sample', how='all')
|
|
327
|
+
|
|
328
|
+
# Create the echogram using xarray's plot method
|
|
329
|
+
filtered_ds.isel(frequency=channel).T.plot(
|
|
330
|
+
x='ping_time',
|
|
331
|
+
y='range_sample',
|
|
332
|
+
yincrease=False,
|
|
333
|
+
vmin=-80,
|
|
334
|
+
vmax=-50,
|
|
335
|
+
cmap='ocean_r',
|
|
336
|
+
cbar_kwargs={'label': 'Volume backscattering strength (Sv re 1 m-1)'}
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
plt.grid(True, linestyle='--', alpha=0.5)
|
|
340
|
+
plt.xlabel('Ping time', fontsize=14)
|
|
341
|
+
plt.ylabel('Depth', fontsize=14)
|
|
342
|
+
plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
|
|
343
|
+
plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
|
|
344
|
+
plt.close()
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=None, client=None):
|
|
348
|
+
"""Plot the echogram data and the regions."""
|
|
349
|
+
from dask.distributed import wait
|
|
350
|
+
|
|
351
|
+
if not plt.isinteractive():
|
|
352
|
+
plt.switch_backend('Agg')
|
|
353
|
+
|
|
354
|
+
futures = []
|
|
355
|
+
|
|
356
|
+
for channel in range(ds_Sv.dims['channel']):
|
|
357
|
+
future = client.submit(plot_individual_channel_simplified, ds_Sv, channel, output_path, file_base_name, cmap)
|
|
358
|
+
futures.append(future)
|
|
359
|
+
|
|
360
|
+
wait(futures)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def plot_sv_data(ds_Sv, file_base_name=None, output_path=None, cmap=None, regions2d=None, region_ids=None,
|
|
364
|
+
region_class=None):
|
|
365
|
+
"""Plot the echogram data and the regions."""
|
|
366
|
+
if not plt.isinteractive():
|
|
367
|
+
plt.switch_backend('Agg')
|
|
368
|
+
|
|
369
|
+
for channel in range(ds_Sv.dims['channel']):
|
|
370
|
+
plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, cmap, regions2d, region_ids,
|
|
371
|
+
region_class)
|
|
372
|
+
# plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_name, cmap)
|
|
373
|
+
# plot_individual_channel_shaders(ds_Sv=ds_Sv, channel=channel, output_path=output_path,
|
|
374
|
+
# file_base_name=file_base_name, cmap='ocean_r')
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
async def plot_sv_data_with_progress(ds_Sv, file_base_name=None, output_path=None, progress=None, plot_task=None,
|
|
378
|
+
cmap='viridis',
|
|
379
|
+
regions2d=None, region_ids=None, region_class=None):
|
|
380
|
+
loop = asyncio.get_running_loop()
|
|
381
|
+
with ThreadPoolExecutor() as pool:
|
|
382
|
+
total_steps = 100
|
|
383
|
+
progress_step = 95 / total_steps
|
|
384
|
+
|
|
385
|
+
plot_task_future = loop.run_in_executor(pool, plot_sv_data, ds_Sv, file_base_name, output_path, cmap,
|
|
386
|
+
regions2d,
|
|
387
|
+
region_ids,
|
|
388
|
+
region_class)
|
|
389
|
+
|
|
390
|
+
for step in range(total_steps):
|
|
391
|
+
if plot_task_future.done():
|
|
392
|
+
break
|
|
393
|
+
await asyncio.sleep(0.1)
|
|
394
|
+
progress.update(plot_task, advance=progress_step)
|
|
395
|
+
|
|
396
|
+
await plot_task_future
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def plot_all_channels(
|
|
400
|
+
dataset1: xr.Dataset,
|
|
401
|
+
dataset2: Optional[xr.Dataset] = None,
|
|
402
|
+
variable_name: str = "Sv",
|
|
403
|
+
name: str = "",
|
|
404
|
+
save_path: Optional[Union[str, Path]] = "",
|
|
405
|
+
**kwargs,
|
|
406
|
+
):
|
|
407
|
+
"""
|
|
408
|
+
Plots echograms for all channels from one or two xarray Datasets.
|
|
409
|
+
|
|
410
|
+
This function iterates over channels in the specified variable of the given dataset(s) and creates echogram plots.
|
|
411
|
+
Each channel's data is plotted in a separate figure. When two datasets are provided, their respective echograms
|
|
412
|
+
for each channel are plotted side by side for comparison.
|
|
413
|
+
|
|
414
|
+
Parameters:
|
|
415
|
+
- dataset1 (xr.Dataset): The first xarray Dataset to plot.
|
|
416
|
+
- dataset2 (xr.Dataset, optional): The second xarray Dataset to plot alongside the first. Defaults to None.
|
|
417
|
+
- variable_name (str, optional): The name of the variable to plot from the dataset. Defaults to "Sv".
|
|
418
|
+
- name (str, optional): Base name for the output plot files. Defaults to empty string"".
|
|
419
|
+
- save_path ((str, Path) optional): Path where to save the images default is current working dir.
|
|
420
|
+
- **kwargs: Arbitrary keyword arguments. Commonly used for plot customization like `vmin`, `vmax`, and `cmap`.
|
|
421
|
+
|
|
422
|
+
Saves:
|
|
423
|
+
- PNG files for each channel's echogram, named using the variable name, the `name` parameter and channel name.
|
|
424
|
+
|
|
425
|
+
Example:
|
|
426
|
+
>> plot_all_channels(dataset1, dataset2, variable_name="Sv", name="echogram", vmin=-70, vmax=-30, cmap='inferno')
|
|
427
|
+
This will create and save echogram plots comparing dataset1 and dataset2 for each channel, using specified plot settings.
|
|
428
|
+
|
|
429
|
+
Note:
|
|
430
|
+
- If only one dataset is provided, echograms for that dataset alone will be plotted.
|
|
431
|
+
- The function handles plotting parameters such as color range (`vmin` and `vmax`) and colormap (`cmap`) via kwargs.
|
|
432
|
+
"""
|
|
433
|
+
for ch in dataset1[variable_name].channel.values:
|
|
434
|
+
plt.figure(figsize=(20, 10))
|
|
435
|
+
|
|
436
|
+
# Configure plotting parameters
|
|
437
|
+
plot_params = {
|
|
438
|
+
"vmin": kwargs.get("vmin", -80),
|
|
439
|
+
"vmax": kwargs.get("vmax", -50),
|
|
440
|
+
"cmap": kwargs.get("cmap", "ocean_r"),
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
if dataset2:
|
|
444
|
+
# First subplot for dataset1
|
|
445
|
+
ax1 = plt.subplot(1, 2, 1)
|
|
446
|
+
mappable1 = ax1.pcolormesh(
|
|
447
|
+
np.rot90(dataset1[variable_name].sel(channel=ch).values), **plot_params
|
|
448
|
+
)
|
|
449
|
+
plt.title(f"Original Data {ch}")
|
|
450
|
+
|
|
451
|
+
# Second subplot for dataset2
|
|
452
|
+
ax2 = plt.subplot(1, 2, 2)
|
|
453
|
+
ax2.pcolormesh(np.rot90(dataset2[variable_name].sel(channel=ch).values), **plot_params)
|
|
454
|
+
plt.title(f"Downsampled Data {ch}")
|
|
455
|
+
|
|
456
|
+
# Create a common colorbar
|
|
457
|
+
plt.colorbar(mappable1, ax=[ax1, ax2], orientation="vertical")
|
|
458
|
+
|
|
459
|
+
else:
|
|
460
|
+
ax = plt.subplot(1, 1, 1)
|
|
461
|
+
plt.pcolormesh(np.rot90(dataset1[variable_name].sel(channel=ch).values), **plot_params)
|
|
462
|
+
plt.title(f"{variable_name} Data {ch}")
|
|
463
|
+
|
|
464
|
+
# Create a colorbar
|
|
465
|
+
plt.colorbar(ax=ax, orientation="vertical")
|
|
466
|
+
|
|
467
|
+
# Save the figure
|
|
468
|
+
if save_path:
|
|
469
|
+
used_path = Path(save_path)
|
|
470
|
+
used_path = used_path / f"{name}_{variable_name}_channel_{ch}.png"
|
|
471
|
+
else:
|
|
472
|
+
used_path = f"{name}_{variable_name}_channel_{ch}.png"
|
|
473
|
+
plt.savefig(used_path)
|
|
474
|
+
plt.close()
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def plot_regions_only(regions2d=None, output_path=None, region_ids=None, region_classes=None, evr_base_name=None):
|
|
478
|
+
"""Plot the echogram data and the regions."""
|
|
479
|
+
plt.figure(figsize=(20, 6))
|
|
480
|
+
colors = ['red', 'blue', 'green', 'orange', 'purple', 'brown', 'pink', 'gray', 'olive', 'cyan']
|
|
481
|
+
|
|
482
|
+
idx = 0
|
|
483
|
+
labels_added = set()
|
|
484
|
+
|
|
485
|
+
regions_output_path = os.path.join(output_path, f"{evr_base_name}_regions.png")
|
|
486
|
+
|
|
487
|
+
if region_ids:
|
|
488
|
+
for region_id in region_ids:
|
|
489
|
+
regions_closed = regions2d.data[regions2d.data['region_id'] == region_id]
|
|
490
|
+
color = colors[idx % len(colors)]
|
|
491
|
+
label = f"Region ID: {region_id}"
|
|
492
|
+
for _, point in regions_closed.iterrows():
|
|
493
|
+
plt.plot(point["time"], point["depth"], fillstyle='full', markersize=1, color=color,
|
|
494
|
+
label=label if label not in labels_added else "")
|
|
495
|
+
labels_added.add(label)
|
|
496
|
+
idx += 1
|
|
497
|
+
|
|
498
|
+
if region_classes:
|
|
499
|
+
for region_class in region_classes:
|
|
500
|
+
regions_closed = regions2d.close_region(region_class=region_class)
|
|
501
|
+
color = colors[idx % len(colors)]
|
|
502
|
+
label = f"Region Class: {region_class}"
|
|
503
|
+
for _, point in regions_closed.iterrows():
|
|
504
|
+
plt.plot(point["time"], point["depth"], fillstyle='full', markersize=2, color=color,
|
|
505
|
+
label=label if label not in labels_added else "")
|
|
506
|
+
labels_added.add(label)
|
|
507
|
+
idx += 1
|
|
508
|
+
|
|
509
|
+
plt.legend()
|
|
510
|
+
plt.xlabel('Time')
|
|
511
|
+
plt.ylabel('Depth')
|
|
512
|
+
|
|
513
|
+
plt.title('Regions with Labels')
|
|
514
|
+
plt.savefig(regions_output_path)
|
|
515
|
+
plt.show()
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _plot_region_classes(channel_name, colors, ds_Sv, idx, labels_added, region_class, regions2d):
|
|
519
|
+
for region in region_class:
|
|
520
|
+
regions_closed = regions2d.close_region(region_class=region)
|
|
521
|
+
color = colors[idx % len(colors)]
|
|
522
|
+
label = f"Region Class: {region}"
|
|
523
|
+
for _, point in regions_closed.iterrows():
|
|
524
|
+
depth = ds_Sv.range_sample.max().values - point["depth"]
|
|
525
|
+
plt.plot(point["time"], depth, fillstyle='full', markersize=5, color=color,
|
|
526
|
+
label=label if label not in labels_added else "")
|
|
527
|
+
labels_added.add(label)
|
|
528
|
+
idx += 1
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _plot_region_ids(colors, ds_Sv, idx, labels_added, region_ids, regions2d):
|
|
532
|
+
for region_id in region_ids:
|
|
533
|
+
regions_closed = regions2d.close_region(region_id=region_id)
|
|
534
|
+
color = colors[idx % len(colors)]
|
|
535
|
+
label = f"Region ID: {region_id}"
|
|
536
|
+
for _, point in regions_closed.iterrows():
|
|
537
|
+
depth = ds_Sv.range_sample.max().values - point["depth"]
|
|
538
|
+
plt.plot(point["time"], depth, fillstyle='full', markersize=5, color=color,
|
|
539
|
+
label=label if label not in labels_added else "")
|
|
540
|
+
labels_added.add(label)
|
|
541
|
+
idx += 1
|
|
542
|
+
return idx
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from .process import compute_sv, process_file_with_progress, read_file_with_progress
|
|
2
|
+
from .processed_data_io import read_processed, write_processed
|
|
3
|
+
from .combine_zarr import combine_zarr_files, read_zarr_files
|
|
4
|
+
from .file_processor import process_raw_file_with_progress, compute_Sv_to_zarr, convert_raw_file, compute_single_file
|
|
5
|
+
from .folder_processor import convert_raw_files, process_zarr_files
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"compute_sv",
|
|
9
|
+
"process_file_with_progress",
|
|
10
|
+
"process_raw_file_with_progress",
|
|
11
|
+
"convert_raw_file",
|
|
12
|
+
"process_zarr_files",
|
|
13
|
+
"compute_Sv_to_zarr",
|
|
14
|
+
"convert_raw_files",
|
|
15
|
+
"compute_single_file",
|
|
16
|
+
"read_processed",
|
|
17
|
+
"write_processed",
|
|
18
|
+
"read_file_with_progress",
|
|
19
|
+
"combine_zarr_files",
|
|
20
|
+
"read_zarr_files"
|
|
21
|
+
]
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import echopype as ep
|
|
3
|
+
import re
|
|
4
|
+
import time
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
from rich import print
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import psutil
|
|
9
|
+
from oceanstream.echodata import check_reversed_time, fix_time_reversions
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
logging.basicConfig(level="DEBUG", format='%(asctime)s - %(levelname)s - %(message)s')
|
|
13
|
+
|
|
14
|
+
def configure_logging(log_level=logging.ERROR):
|
|
15
|
+
logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def read_zarr_files(input_folder):
|
|
19
|
+
input_path = Path(input_folder)
|
|
20
|
+
if not input_path.is_dir():
|
|
21
|
+
logging.error("Input folder does not exist: %s", input_folder)
|
|
22
|
+
return
|
|
23
|
+
|
|
24
|
+
zarr_files = list(input_path.glob("*.zarr"))
|
|
25
|
+
if not zarr_files:
|
|
26
|
+
logging.error("No .zarr files found in directory: %s", input_folder)
|
|
27
|
+
return
|
|
28
|
+
|
|
29
|
+
file_info = []
|
|
30
|
+
for file in zarr_files:
|
|
31
|
+
creation_time = from_filename(file.stem)
|
|
32
|
+
file_info.append((file, creation_time))
|
|
33
|
+
|
|
34
|
+
filtered_file_info = [item for item in file_info if item[1] is not None]
|
|
35
|
+
filtered_file_info.sort(key=lambda x: x[1])
|
|
36
|
+
|
|
37
|
+
sorted_files = [file for file, _ in filtered_file_info]
|
|
38
|
+
|
|
39
|
+
return sorted_files
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def fix_time(ed):
|
|
43
|
+
if check_reversed_time(ed, "Sonar/Beam_group1", "ping_time"):
|
|
44
|
+
ed = fix_time_reversions(ed, {"Sonar/Beam_group1": "ping_time"})
|
|
45
|
+
if check_reversed_time(ed, "Environment", "time1"):
|
|
46
|
+
ed = fix_time_reversions(ed, {"Environment": "time1"})
|
|
47
|
+
return ed
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None, log_level=logging.DEBUG):
|
|
51
|
+
start_time = time.time()
|
|
52
|
+
|
|
53
|
+
logging.debug("Starting to combine Zarr files from folder: %s", input_folder)
|
|
54
|
+
|
|
55
|
+
output_path = Path(zarr_output_file)
|
|
56
|
+
sorted_files = read_zarr_files(input_folder)
|
|
57
|
+
logging.debug("Found %d .zarr files in directory: %s", len(sorted_files), input_folder)
|
|
58
|
+
|
|
59
|
+
if not sorted_files:
|
|
60
|
+
logging.error("No valid .zarr files with creation time found in directory: %s", input_folder)
|
|
61
|
+
return
|
|
62
|
+
|
|
63
|
+
ed_list = []
|
|
64
|
+
|
|
65
|
+
for zarr_file in sorted_files:
|
|
66
|
+
ed = ep.open_converted(zarr_file, chunks=chunks)
|
|
67
|
+
ed = fix_time(ed)
|
|
68
|
+
|
|
69
|
+
ed_list.append(ed)
|
|
70
|
+
|
|
71
|
+
combined_ed = ep.combine_echodata(ed_list)
|
|
72
|
+
combined_ed.to_zarr(output_path)
|
|
73
|
+
|
|
74
|
+
end_time = time.time()
|
|
75
|
+
total_time = end_time - start_time
|
|
76
|
+
|
|
77
|
+
print(f"\nTotal time taken: {total_time:.2f} seconds")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def from_filename(file_name):
|
|
81
|
+
"""Extract creation time from the file name if it follows a specific pattern."""
|
|
82
|
+
pattern = r'(\d{4}[A-Z])?-D(\d{8})-T(\d{6})'
|
|
83
|
+
match = re.search(pattern, file_name)
|
|
84
|
+
if match:
|
|
85
|
+
date_str = match.group(2)
|
|
86
|
+
time_str = match.group(3)
|
|
87
|
+
creation_time = datetime.strptime(date_str + time_str, '%Y%m%d%H%M%S')
|
|
88
|
+
return creation_time
|
|
89
|
+
|
|
90
|
+
return None
|