oceanstream 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. oceanstream/__init__.py +0 -0
  2. oceanstream/cli/__init__.py +3 -0
  3. oceanstream/cli/main.py +267 -0
  4. oceanstream/denoise/__init__.py +2 -0
  5. oceanstream/denoise/apply_masks.py +49 -0
  6. oceanstream/denoise/applying_masks_handler.py +173 -0
  7. oceanstream/denoise/background_noise_remover.py +97 -0
  8. oceanstream/denoise/create_masks.py +87 -0
  9. oceanstream/denoise/noise_masks.py +398 -0
  10. oceanstream/denoise/types.py +73 -0
  11. oceanstream/echodata/__init__.py +5 -0
  12. oceanstream/echodata/ensure_time_continuity.py +75 -0
  13. oceanstream/echodata/mvbs_computation.py +77 -0
  14. oceanstream/echodata/raw_handler.py +577 -0
  15. oceanstream/echodata/sv_computation.py +118 -0
  16. oceanstream/echodata/sv_dataset_extension.py +126 -0
  17. oceanstream/echodata/sv_interpolation.py +260 -0
  18. oceanstream/echodata/target_strength_computation.py +109 -0
  19. oceanstream/exports/__init__.py +2 -0
  20. oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
  21. oceanstream/exports/csv/csv_export_from_raw.py +92 -0
  22. oceanstream/exports/csv/csv_export_nasc.py +135 -0
  23. oceanstream/exports/frequency_differencing_handler.py +210 -0
  24. oceanstream/exports/nasc.py +23 -0
  25. oceanstream/exports/nasc_computation.py +62 -0
  26. oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
  27. oceanstream/exports/shoals/shoal_process.py +224 -0
  28. oceanstream/exports/shoals/shoals_handler.py +35 -0
  29. oceanstream/plot/__init__.py +2 -0
  30. oceanstream/plot/echogram.py +542 -0
  31. oceanstream/process/__init__.py +21 -0
  32. oceanstream/process/combine_zarr.py +90 -0
  33. oceanstream/process/file_processor.py +151 -0
  34. oceanstream/process/folder_processor.py +246 -0
  35. oceanstream/process/process.py +77 -0
  36. oceanstream/process/processed_data_io.py +140 -0
  37. oceanstream/settings/__init__.py +1 -0
  38. oceanstream/settings/config.py +17 -0
  39. oceanstream/utils.py +101 -0
  40. oceanstream-0.0.0.dev0.dist-info/LICENSE +21 -0
  41. oceanstream-0.0.0.dev0.dist-info/METADATA +214 -0
  42. oceanstream-0.0.0.dev0.dist-info/RECORD +46 -0
  43. oceanstream-0.0.0.dev0.dist-info/WHEEL +5 -0
  44. oceanstream-0.0.0.dev0.dist-info/dependency_links.txt +1 -0
  45. oceanstream-0.0.0.dev0.dist-info/entry_points.txt +2 -0
  46. oceanstream-0.0.0.dev0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,542 @@
1
+ import asyncio
2
+ import os
3
+ import xarray as xr
4
+ import numpy as np
5
+ from matplotlib.colors import LinearSegmentedColormap, Colormap
6
+ import matplotlib.pyplot as plt
7
+ import matplotlib.dates as mdates
8
+ from concurrent.futures import ThreadPoolExecutor
9
+ from pathlib import Path
10
+ from typing import Optional, Union
11
+
12
+
13
+ def plot_individual_channel(ds_Sv, channel, output_path, file_base_name,
14
+ cmap='viridis',
15
+ regions2d=None,
16
+ region_ids=None, region_class=None):
17
+ """Plot and save echogram for a single channel with optional regions and enhancements."""
18
+ import pandas as pd
19
+ plot_params = {
20
+ "vmin": -70,
21
+ "vmax": -30,
22
+ "cmap": cmap
23
+ }
24
+
25
+ full_channel_name = ds_Sv.channel.values[channel]
26
+ channel_name = "_".join(full_channel_name.split()[:3])
27
+
28
+ plt.figure(figsize=(20, 10))
29
+ colors = ['black', 'yellow', 'red', 'orange']
30
+
31
+ idx = 0
32
+ labels_added = set()
33
+
34
+ echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
35
+ ax = plt.subplot(1, 1, 1)
36
+
37
+ echogram_data = np.flipud(ds_Sv.Sv.isel(channel=channel).T.values)
38
+ plt.pcolormesh(ds_Sv.ping_time, ds_Sv.range_sample, echogram_data, **plot_params)
39
+ cbar = plt.colorbar(ax=ax, orientation="vertical")
40
+ cbar.set_label("Volume backscattering strength (dB re 1m$^{-1}$)")
41
+
42
+ if region_ids:
43
+ idx = _plot_region_ids(colors, ds_Sv, idx, labels_added, region_ids, regions2d)
44
+
45
+ if region_class:
46
+ _plot_region_classes(channel_name, colors, ds_Sv, idx, labels_added, region_class, regions2d)
47
+
48
+ if regions2d:
49
+ plt.legend()
50
+
51
+ ax.invert_yaxis()
52
+ plt.grid(True, linestyle='--', alpha=0.5)
53
+
54
+ # Format the x-axis to show time (HH:mm)
55
+ ax.xaxis_date()
56
+ ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
57
+
58
+ # Add horizontal lines at specific depth intervals
59
+ depth_intervals = np.arange(0, ds_Sv.range_sample.max(), 10) # Example interval of 10 units
60
+ for depth in depth_intervals:
61
+ ax.axhline(depth, color='gray', linestyle='--', linewidth=0.5)
62
+
63
+ # Convert the first ping_time to a Python datetime object
64
+ date_str = pd.to_datetime(ds_Sv.ping_time.values[0]).strftime('%Y-%m-%d')
65
+
66
+ # Add date label on the right side
67
+ plt.xlabel('Time (HH:MM)', fontsize=14)
68
+ plt.ylabel('Depth', fontsize=14)
69
+ plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
70
+ plt.text(1.02, 0.5, date_str, transform=ax.transAxes, rotation=90, va='center', fontsize=12, fontweight='bold')
71
+
72
+ # Add a secondary y-axis for fathoms (1 fathom = 1.8288 meters)
73
+ secax = ax.secondary_yaxis('right', functions=(lambda x: x / 1.8288, lambda x: x * 1.8288))
74
+ secax.set_ylabel('Depth (fathoms)', fontsize=14)
75
+
76
+ plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
77
+ plt.close()
78
+
79
+
80
+ def plot_individual_channel_shaders(ds_Sv, channel, output_path, file_base_name, cmap='ocean_r'):
81
+ """Plot and save echogram for a single channel with optional regions and enhancements."""
82
+ import holoviews as hv
83
+ from holoviews.operation.datashader import rasterize
84
+ from datashader import reductions as ds
85
+ hv.extension('bokeh', 'matplotlib')
86
+
87
+ full_channel_name = ds_Sv.channel.values[channel]
88
+ channel_name = "_".join(full_channel_name.split()[:3])
89
+
90
+ echogram_output_path_html = os.path.join(output_path, f"{file_base_name}_{channel_name}.html")
91
+ echogram_output_path_png = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
92
+
93
+ filtered_ds = ds_Sv['Sv']
94
+ if 'beam' in filtered_ds.dims:
95
+ filtered_ds = filtered_ds.isel(beam=0).drop('beam')
96
+
97
+ if 'channel' in filtered_ds.dims:
98
+ filtered_ds = filtered_ds.assign_coords({'frequency': ds_Sv.frequency_nominal})
99
+ filtered_ds = filtered_ds.swap_dims({'channel': 'frequency'})
100
+ if filtered_ds.frequency.size == 1:
101
+ filtered_ds = filtered_ds.isel(frequency=0)
102
+
103
+ # Handle NaNs by dropping NaNs directly from ping_time and range_sample
104
+ filtered_ds = filtered_ds.dropna(dim='ping_time', how='all')
105
+ filtered_ds = filtered_ds.dropna(dim='range_sample', how='all')
106
+
107
+ # Extract data for visualization
108
+ sv_data = filtered_ds.isel(frequency=channel)
109
+ ping_time = sv_data['ping_time']
110
+ range_sample = sv_data['range_sample']
111
+
112
+ sv_values = sv_data.values
113
+ ds_array = xr.DataArray(sv_values, coords=[ping_time, range_sample], dims=['ping_time', 'range_sample'])
114
+ hv_quadmesh = hv.QuadMesh(ds_array, kdims=['ping_time', 'range_sample'], vdims=['Sv'])
115
+ rasterized_quadmesh = rasterize(hv_quadmesh, aggregator=ds.mean('Sv'))
116
+
117
+ rasterized_quadmesh = rasterized_quadmesh.opts(
118
+ cmap='viridis',
119
+ colorbar=True,
120
+ responsive=True,
121
+ min_height=600,
122
+ clim=(np.nanmin(sv_values), np.nanmax(sv_values)),
123
+ tools=['hover', 'box_select'],
124
+ active_tools=['wheel_zoom'],
125
+ invert_yaxis=True,
126
+ hooks=[lambda plot, element: plot.handles['colorbar'].title('Sv')]
127
+ )
128
+
129
+ # Save the plot as HTML
130
+ hv.save(rasterized_quadmesh, echogram_output_path_html, fmt='html')
131
+
132
+ # Save the plot as PNG
133
+ # hv.save(rasterized_quadmesh, echogram_output_path_png, fmt='png')
134
+
135
+
136
+ def plot_individual_channel_enhanced(ds_Sv, channel, output_path, file_base_name,
137
+ cmap=None,
138
+ regions2d=None,
139
+ region_ids=None, region_class=None,
140
+ threshold=[-80, -50]):
141
+ """Plot and save echogram for a single channel with optional regions and enhancements."""
142
+ full_channel_name = ds_Sv.channel.values[channel]
143
+ channel_name = "_".join(full_channel_name.split()[:3])
144
+
145
+ num_pings = ds_Sv.dims['ping_time']
146
+ num_samples = ds_Sv.dims['range_sample']
147
+ aspect_ratio = num_pings / num_samples
148
+
149
+ # Set base dimensions
150
+ base_width = 10 # Base width in inches
151
+ base_height = 10 # Base height in inches
152
+
153
+ # Adjust width and height based on aspect ratio, ensuring practical limits
154
+ if aspect_ratio > 1:
155
+ fig_width = base_width * aspect_ratio
156
+ fig_height = base_height
157
+ else:
158
+ fig_width = base_width
159
+ fig_height = base_height / aspect_ratio
160
+
161
+ fig_width = max(min(fig_width, 20), 10) # Width between 10 and 20 inches
162
+ fig_height = max(min(fig_height, 10), 5) # Height between 5 and 10 inches
163
+
164
+ plt.figure(figsize=(fig_width, fig_height))
165
+
166
+ echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
167
+
168
+ # Apply the same preprocessing steps from _plot_echogram
169
+ ds = ds_Sv
170
+ filter_var = 'channel'
171
+ filter_val = channel
172
+ if 'backscatter_i' in ds.variables:
173
+ filtered_ds = np.abs(ds.backscatter_r + 1j * ds.backscatter_i)
174
+ else:
175
+ filtered_ds = ds['Sv']
176
+ if 'beam' in filtered_ds.dims:
177
+ filtered_ds = filtered_ds.isel(beam=0).drop('beam')
178
+
179
+ if 'channel' in filtered_ds.dims:
180
+ filtered_ds = filtered_ds.assign_coords({'frequency': ds.frequency_nominal})
181
+ filtered_ds = filtered_ds.swap_dims({'channel': 'frequency'})
182
+ if filtered_ds.frequency.size == 1:
183
+ filtered_ds = filtered_ds.isel(frequency=0)
184
+
185
+ # Update axis labels
186
+ filtered_ds['ping_time'].attrs = {
187
+ 'long_name': filtered_ds['ping_time'].attrs.get('long_name', 'Ping time'),
188
+ 'units': filtered_ds['ping_time'].attrs.get('units', '')
189
+ }
190
+ filtered_ds['range_sample'].attrs = {
191
+ 'long_name': filtered_ds['range_sample'].attrs.get('long_name', 'Depth'),
192
+ 'units': filtered_ds['range_sample'].attrs.get('units', '')
193
+ }
194
+
195
+ # Handle NaNs by dropping NaNs directly from ping_time and range_sample
196
+ filtered_ds = filtered_ds.dropna(dim='ping_time', how='all')
197
+ filtered_ds = filtered_ds.dropna(dim='range_sample', how='all')
198
+
199
+ # Set the colormap
200
+ if cmap is None:
201
+ simrad_color_table = [
202
+ (1, 1, 1), (0.6235, 0.6235, 0.6235), (0.3725, 0.3725, 0.3725),
203
+ (0, 0, 1), (0, 0, 0.5), (0, 0.7490, 0), (0, 0.5, 0),
204
+ (1, 1, 0), (1, 0.5, 0), (1, 0, 0.7490), (1, 0, 0),
205
+ (0.6509, 0.3255, 0.2353), (0.4705, 0.2353, 0.1568)
206
+ ]
207
+ cmap = LinearSegmentedColormap.from_list('Simrad', simrad_color_table)
208
+ cmap.set_bad(color='grey')
209
+
210
+ # Custom tick formatter for datetime64 values as floats
211
+ def format_datetime(x, pos=None):
212
+ try:
213
+ dt = x.astype('datetime64[ms]').astype('object')
214
+ tick_label = dt.strftime("%H:%M:%S")
215
+ except:
216
+ tick_label = ''
217
+ return tick_label
218
+
219
+ # Create the echogram using xarray's plot method
220
+ echogram_data = filtered_ds.isel(frequency=channel).T
221
+ echogram_data.plot(
222
+ x='ping_time',
223
+ y='range_sample',
224
+ yincrease=False,
225
+ vmin=threshold[0],
226
+ vmax=threshold[1],
227
+ cmap=cmap,
228
+ cbar_kwargs={'label': 'Volume backscattering strength (Sv re 1 m-1)'}
229
+ )
230
+
231
+ plt.gca().xaxis.set_major_formatter(plt.FuncFormatter(format_datetime))
232
+ plt.grid(True, linestyle='--', alpha=0.5)
233
+ plt.xlabel('Ping time', fontsize=14)
234
+ plt.ylabel('Depth', fontsize=14)
235
+ plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
236
+ plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
237
+ plt.close()
238
+
239
+
240
+ def plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_name,
241
+ cmap='viridis'):
242
+ full_channel_name = ds_Sv.channel.values[channel]
243
+ channel_name = "_".join(full_channel_name.split()[:3])
244
+
245
+ plt.figure(figsize=(30, 18))
246
+ echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
247
+
248
+ filtered_ds = ds_Sv['Sv']
249
+ if 'beam' in filtered_ds.dims:
250
+ filtered_ds = filtered_ds.isel(beam=0).drop('beam')
251
+
252
+ if 'channel' in filtered_ds.dims:
253
+ filtered_ds = filtered_ds.assign_coords({'frequency': ds_Sv.frequency_nominal})
254
+ filtered_ds = filtered_ds.swap_dims({'channel': 'frequency'})
255
+ if filtered_ds.frequency.size == 1:
256
+ filtered_ds = filtered_ds.isel(frequency=0)
257
+
258
+ # Update axis labels
259
+ filtered_ds['ping_time'].attrs = {
260
+ 'long_name': filtered_ds['ping_time'].attrs.get('long_name', 'Ping time'),
261
+ 'units': filtered_ds['ping_time'].attrs.get('units', '')
262
+ }
263
+ filtered_ds['range_sample'].attrs = {
264
+ 'long_name': filtered_ds['range_sample'].attrs.get('long_name', 'Depth'),
265
+ 'units': filtered_ds['range_sample'].attrs.get('units', '')
266
+ }
267
+
268
+ # Handle NaNs by dropping NaNs directly from ping_time and range_sample
269
+ filtered_ds = filtered_ds.dropna(dim='ping_time', how='all')
270
+ filtered_ds = filtered_ds.dropna(dim='range_sample', how='all')
271
+
272
+ # Create the echogram using xarray's plot method
273
+ filtered_ds.isel(frequency=channel).T.plot(
274
+ x='ping_time',
275
+ y='range_sample',
276
+ yincrease=False,
277
+ vmin=-80,
278
+ vmax=-50,
279
+ cmap='ocean_r',
280
+ cbar_kwargs={},
281
+ add_colorbar=False
282
+ )
283
+
284
+ plt.title('')
285
+ plt.axis('off')
286
+ plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
287
+ plt.close()
288
+
289
+
290
+ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name,
291
+ cmap='viridis',
292
+ regions2d=None,
293
+ region_ids=None, region_class=None):
294
+ """Plot and save echogram for a single channel with optional regions and enhancements."""
295
+ full_channel_name = ds_Sv.channel.values[channel]
296
+ channel_name = "_".join(full_channel_name.split()[:3])
297
+
298
+ plt.figure(figsize=(30, 18))
299
+ echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
300
+
301
+ # Apply the same preprocessing steps from _plot_echogram
302
+ ds = ds_Sv
303
+
304
+ filtered_ds = ds['Sv']
305
+ if 'beam' in filtered_ds.dims:
306
+ filtered_ds = filtered_ds.isel(beam=0).drop('beam')
307
+
308
+ if 'channel' in filtered_ds.dims:
309
+ filtered_ds = filtered_ds.assign_coords({'frequency': ds.frequency_nominal})
310
+ filtered_ds = filtered_ds.swap_dims({'channel': 'frequency'})
311
+ if filtered_ds.frequency.size == 1:
312
+ filtered_ds = filtered_ds.isel(frequency=0)
313
+
314
+ # Update axis labels
315
+ filtered_ds['ping_time'].attrs = {
316
+ 'long_name': filtered_ds['ping_time'].attrs.get('long_name', 'Ping time'),
317
+ 'units': filtered_ds['ping_time'].attrs.get('units', '')
318
+ }
319
+ filtered_ds['range_sample'].attrs = {
320
+ 'long_name': filtered_ds['range_sample'].attrs.get('long_name', 'Depth'),
321
+ 'units': filtered_ds['range_sample'].attrs.get('units', '')
322
+ }
323
+
324
+ # Handle NaNs by dropping NaNs directly from ping_time and range_sample
325
+ filtered_ds = filtered_ds.dropna(dim='ping_time', how='all')
326
+ filtered_ds = filtered_ds.dropna(dim='range_sample', how='all')
327
+
328
+ # Create the echogram using xarray's plot method
329
+ filtered_ds.isel(frequency=channel).T.plot(
330
+ x='ping_time',
331
+ y='range_sample',
332
+ yincrease=False,
333
+ vmin=-80,
334
+ vmax=-50,
335
+ cmap='ocean_r',
336
+ cbar_kwargs={'label': 'Volume backscattering strength (Sv re 1 m-1)'}
337
+ )
338
+
339
+ plt.grid(True, linestyle='--', alpha=0.5)
340
+ plt.xlabel('Ping time', fontsize=14)
341
+ plt.ylabel('Depth', fontsize=14)
342
+ plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
343
+ plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
344
+ plt.close()
345
+
346
+
347
+ def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=None, client=None):
348
+ """Plot the echogram data and the regions."""
349
+ from dask.distributed import wait
350
+
351
+ if not plt.isinteractive():
352
+ plt.switch_backend('Agg')
353
+
354
+ futures = []
355
+
356
+ for channel in range(ds_Sv.dims['channel']):
357
+ future = client.submit(plot_individual_channel_simplified, ds_Sv, channel, output_path, file_base_name, cmap)
358
+ futures.append(future)
359
+
360
+ wait(futures)
361
+
362
+
363
+ def plot_sv_data(ds_Sv, file_base_name=None, output_path=None, cmap=None, regions2d=None, region_ids=None,
364
+ region_class=None):
365
+ """Plot the echogram data and the regions."""
366
+ if not plt.isinteractive():
367
+ plt.switch_backend('Agg')
368
+
369
+ for channel in range(ds_Sv.dims['channel']):
370
+ plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, cmap, regions2d, region_ids,
371
+ region_class)
372
+ # plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_name, cmap)
373
+ # plot_individual_channel_shaders(ds_Sv=ds_Sv, channel=channel, output_path=output_path,
374
+ # file_base_name=file_base_name, cmap='ocean_r')
375
+
376
+
377
+ async def plot_sv_data_with_progress(ds_Sv, file_base_name=None, output_path=None, progress=None, plot_task=None,
378
+ cmap='viridis',
379
+ regions2d=None, region_ids=None, region_class=None):
380
+ loop = asyncio.get_running_loop()
381
+ with ThreadPoolExecutor() as pool:
382
+ total_steps = 100
383
+ progress_step = 95 / total_steps
384
+
385
+ plot_task_future = loop.run_in_executor(pool, plot_sv_data, ds_Sv, file_base_name, output_path, cmap,
386
+ regions2d,
387
+ region_ids,
388
+ region_class)
389
+
390
+ for step in range(total_steps):
391
+ if plot_task_future.done():
392
+ break
393
+ await asyncio.sleep(0.1)
394
+ progress.update(plot_task, advance=progress_step)
395
+
396
+ await plot_task_future
397
+
398
+
399
+ def plot_all_channels(
400
+ dataset1: xr.Dataset,
401
+ dataset2: Optional[xr.Dataset] = None,
402
+ variable_name: str = "Sv",
403
+ name: str = "",
404
+ save_path: Optional[Union[str, Path]] = "",
405
+ **kwargs,
406
+ ):
407
+ """
408
+ Plots echograms for all channels from one or two xarray Datasets.
409
+
410
+ This function iterates over channels in the specified variable of the given dataset(s) and creates echogram plots.
411
+ Each channel's data is plotted in a separate figure. When two datasets are provided, their respective echograms
412
+ for each channel are plotted side by side for comparison.
413
+
414
+ Parameters:
415
+ - dataset1 (xr.Dataset): The first xarray Dataset to plot.
416
+ - dataset2 (xr.Dataset, optional): The second xarray Dataset to plot alongside the first. Defaults to None.
417
+ - variable_name (str, optional): The name of the variable to plot from the dataset. Defaults to "Sv".
418
+ - name (str, optional): Base name for the output plot files. Defaults to empty string"".
419
+ - save_path ((str, Path) optional): Path where to save the images default is current working dir.
420
+ - **kwargs: Arbitrary keyword arguments. Commonly used for plot customization like `vmin`, `vmax`, and `cmap`.
421
+
422
+ Saves:
423
+ - PNG files for each channel's echogram, named using the variable name, the `name` parameter and channel name.
424
+
425
+ Example:
426
+ >> plot_all_channels(dataset1, dataset2, variable_name="Sv", name="echogram", vmin=-70, vmax=-30, cmap='inferno')
427
+ This will create and save echogram plots comparing dataset1 and dataset2 for each channel, using specified plot settings.
428
+
429
+ Note:
430
+ - If only one dataset is provided, echograms for that dataset alone will be plotted.
431
+ - The function handles plotting parameters such as color range (`vmin` and `vmax`) and colormap (`cmap`) via kwargs.
432
+ """
433
+ for ch in dataset1[variable_name].channel.values:
434
+ plt.figure(figsize=(20, 10))
435
+
436
+ # Configure plotting parameters
437
+ plot_params = {
438
+ "vmin": kwargs.get("vmin", -80),
439
+ "vmax": kwargs.get("vmax", -50),
440
+ "cmap": kwargs.get("cmap", "ocean_r"),
441
+ }
442
+
443
+ if dataset2:
444
+ # First subplot for dataset1
445
+ ax1 = plt.subplot(1, 2, 1)
446
+ mappable1 = ax1.pcolormesh(
447
+ np.rot90(dataset1[variable_name].sel(channel=ch).values), **plot_params
448
+ )
449
+ plt.title(f"Original Data {ch}")
450
+
451
+ # Second subplot for dataset2
452
+ ax2 = plt.subplot(1, 2, 2)
453
+ ax2.pcolormesh(np.rot90(dataset2[variable_name].sel(channel=ch).values), **plot_params)
454
+ plt.title(f"Downsampled Data {ch}")
455
+
456
+ # Create a common colorbar
457
+ plt.colorbar(mappable1, ax=[ax1, ax2], orientation="vertical")
458
+
459
+ else:
460
+ ax = plt.subplot(1, 1, 1)
461
+ plt.pcolormesh(np.rot90(dataset1[variable_name].sel(channel=ch).values), **plot_params)
462
+ plt.title(f"{variable_name} Data {ch}")
463
+
464
+ # Create a colorbar
465
+ plt.colorbar(ax=ax, orientation="vertical")
466
+
467
+ # Save the figure
468
+ if save_path:
469
+ used_path = Path(save_path)
470
+ used_path = used_path / f"{name}_{variable_name}_channel_{ch}.png"
471
+ else:
472
+ used_path = f"{name}_{variable_name}_channel_{ch}.png"
473
+ plt.savefig(used_path)
474
+ plt.close()
475
+
476
+
477
+ def plot_regions_only(regions2d=None, output_path=None, region_ids=None, region_classes=None, evr_base_name=None):
478
+ """Plot the echogram data and the regions."""
479
+ plt.figure(figsize=(20, 6))
480
+ colors = ['red', 'blue', 'green', 'orange', 'purple', 'brown', 'pink', 'gray', 'olive', 'cyan']
481
+
482
+ idx = 0
483
+ labels_added = set()
484
+
485
+ regions_output_path = os.path.join(output_path, f"{evr_base_name}_regions.png")
486
+
487
+ if region_ids:
488
+ for region_id in region_ids:
489
+ regions_closed = regions2d.data[regions2d.data['region_id'] == region_id]
490
+ color = colors[idx % len(colors)]
491
+ label = f"Region ID: {region_id}"
492
+ for _, point in regions_closed.iterrows():
493
+ plt.plot(point["time"], point["depth"], fillstyle='full', markersize=1, color=color,
494
+ label=label if label not in labels_added else "")
495
+ labels_added.add(label)
496
+ idx += 1
497
+
498
+ if region_classes:
499
+ for region_class in region_classes:
500
+ regions_closed = regions2d.close_region(region_class=region_class)
501
+ color = colors[idx % len(colors)]
502
+ label = f"Region Class: {region_class}"
503
+ for _, point in regions_closed.iterrows():
504
+ plt.plot(point["time"], point["depth"], fillstyle='full', markersize=2, color=color,
505
+ label=label if label not in labels_added else "")
506
+ labels_added.add(label)
507
+ idx += 1
508
+
509
+ plt.legend()
510
+ plt.xlabel('Time')
511
+ plt.ylabel('Depth')
512
+
513
+ plt.title('Regions with Labels')
514
+ plt.savefig(regions_output_path)
515
+ plt.show()
516
+
517
+
518
+ def _plot_region_classes(channel_name, colors, ds_Sv, idx, labels_added, region_class, regions2d):
519
+ for region in region_class:
520
+ regions_closed = regions2d.close_region(region_class=region)
521
+ color = colors[idx % len(colors)]
522
+ label = f"Region Class: {region}"
523
+ for _, point in regions_closed.iterrows():
524
+ depth = ds_Sv.range_sample.max().values - point["depth"]
525
+ plt.plot(point["time"], depth, fillstyle='full', markersize=5, color=color,
526
+ label=label if label not in labels_added else "")
527
+ labels_added.add(label)
528
+ idx += 1
529
+
530
+
531
+ def _plot_region_ids(colors, ds_Sv, idx, labels_added, region_ids, regions2d):
532
+ for region_id in region_ids:
533
+ regions_closed = regions2d.close_region(region_id=region_id)
534
+ color = colors[idx % len(colors)]
535
+ label = f"Region ID: {region_id}"
536
+ for _, point in regions_closed.iterrows():
537
+ depth = ds_Sv.range_sample.max().values - point["depth"]
538
+ plt.plot(point["time"], depth, fillstyle='full', markersize=5, color=color,
539
+ label=label if label not in labels_added else "")
540
+ labels_added.add(label)
541
+ idx += 1
542
+ return idx
@@ -0,0 +1,21 @@
1
+ from .process import compute_sv, process_file_with_progress, read_file_with_progress
2
+ from .processed_data_io import read_processed, write_processed
3
+ from .combine_zarr import combine_zarr_files, read_zarr_files
4
+ from .file_processor import process_raw_file_with_progress, compute_Sv_to_zarr, convert_raw_file, compute_single_file
5
+ from .folder_processor import convert_raw_files, process_zarr_files
6
+
7
+ __all__ = [
8
+ "compute_sv",
9
+ "process_file_with_progress",
10
+ "process_raw_file_with_progress",
11
+ "convert_raw_file",
12
+ "process_zarr_files",
13
+ "compute_Sv_to_zarr",
14
+ "convert_raw_files",
15
+ "compute_single_file",
16
+ "read_processed",
17
+ "write_processed",
18
+ "read_file_with_progress",
19
+ "combine_zarr_files",
20
+ "read_zarr_files"
21
+ ]
@@ -0,0 +1,90 @@
1
+ import logging
2
+ import echopype as ep
3
+ import re
4
+ import time
5
+ from datetime import datetime
6
+ from rich import print
7
+ from pathlib import Path
8
+ import psutil
9
+ from oceanstream.echodata import check_reversed_time, fix_time_reversions
10
+
11
+
12
+ logging.basicConfig(level="DEBUG", format='%(asctime)s - %(levelname)s - %(message)s')
13
+
14
+ def configure_logging(log_level=logging.ERROR):
15
+ logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s')
16
+
17
+
18
+ def read_zarr_files(input_folder):
19
+ input_path = Path(input_folder)
20
+ if not input_path.is_dir():
21
+ logging.error("Input folder does not exist: %s", input_folder)
22
+ return
23
+
24
+ zarr_files = list(input_path.glob("*.zarr"))
25
+ if not zarr_files:
26
+ logging.error("No .zarr files found in directory: %s", input_folder)
27
+ return
28
+
29
+ file_info = []
30
+ for file in zarr_files:
31
+ creation_time = from_filename(file.stem)
32
+ file_info.append((file, creation_time))
33
+
34
+ filtered_file_info = [item for item in file_info if item[1] is not None]
35
+ filtered_file_info.sort(key=lambda x: x[1])
36
+
37
+ sorted_files = [file for file, _ in filtered_file_info]
38
+
39
+ return sorted_files
40
+
41
+
42
+ def fix_time(ed):
43
+ if check_reversed_time(ed, "Sonar/Beam_group1", "ping_time"):
44
+ ed = fix_time_reversions(ed, {"Sonar/Beam_group1": "ping_time"})
45
+ if check_reversed_time(ed, "Environment", "time1"):
46
+ ed = fix_time_reversions(ed, {"Environment": "time1"})
47
+ return ed
48
+
49
+
50
+ def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None, log_level=logging.DEBUG):
51
+ start_time = time.time()
52
+
53
+ logging.debug("Starting to combine Zarr files from folder: %s", input_folder)
54
+
55
+ output_path = Path(zarr_output_file)
56
+ sorted_files = read_zarr_files(input_folder)
57
+ logging.debug("Found %d .zarr files in directory: %s", len(sorted_files), input_folder)
58
+
59
+ if not sorted_files:
60
+ logging.error("No valid .zarr files with creation time found in directory: %s", input_folder)
61
+ return
62
+
63
+ ed_list = []
64
+
65
+ for zarr_file in sorted_files:
66
+ ed = ep.open_converted(zarr_file, chunks=chunks)
67
+ ed = fix_time(ed)
68
+
69
+ ed_list.append(ed)
70
+
71
+ combined_ed = ep.combine_echodata(ed_list)
72
+ combined_ed.to_zarr(output_path)
73
+
74
+ end_time = time.time()
75
+ total_time = end_time - start_time
76
+
77
+ print(f"\nTotal time taken: {total_time:.2f} seconds")
78
+
79
+
80
+ def from_filename(file_name):
81
+ """Extract creation time from the file name if it follows a specific pattern."""
82
+ pattern = r'(\d{4}[A-Z])?-D(\d{8})-T(\d{6})'
83
+ match = re.search(pattern, file_name)
84
+ if match:
85
+ date_str = match.group(2)
86
+ time_str = match.group(3)
87
+ creation_time = datetime.strptime(date_str + time_str, '%Y%m%d%H%M%S')
88
+ return creation_time
89
+
90
+ return None