oceanstream 0.0.6.dev0__tar.gz → 0.0.10.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/PKG-INFO +3 -2
  2. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/cli/main.py +29 -18
  3. oceanstream-0.0.10.dev0/oceanstream/cli/requirements.txt +2 -0
  4. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/core.py +18 -6
  5. oceanstream-0.0.10.dev0/oceanstream/denoise/requirements.txt +0 -0
  6. oceanstream-0.0.10.dev0/oceanstream/echodata/requirements.txt +1 -0
  7. oceanstream-0.0.10.dev0/oceanstream/exports/requirements.txt +4 -0
  8. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/plot/echogram.py +45 -14
  9. oceanstream-0.0.10.dev0/oceanstream/plot/requirements.txt +2 -0
  10. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/combine_zarr.py +2 -3
  11. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/file_processor.py +58 -16
  12. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/folder_processor.py +51 -25
  13. oceanstream-0.0.10.dev0/oceanstream/process/requirements.txt +0 -0
  14. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/settings/defaults.config.json +2 -2
  15. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/PKG-INFO +3 -2
  16. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/SOURCES.txt +6 -0
  17. oceanstream-0.0.10.dev0/pyproject.toml +8 -0
  18. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/setup.cfg +3 -2
  19. oceanstream-0.0.6.dev0/pyproject.toml +0 -55
  20. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/.flake8 +0 -0
  21. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/.gitattributes +0 -0
  22. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/.gitignore +0 -0
  23. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/CODE_OF_CONDUCT.md +0 -0
  24. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/LICENSE +0 -0
  25. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/MANIFEST.in +0 -0
  26. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/README.md +0 -0
  27. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/notebooks/README.md +0 -0
  28. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/__init__.py +0 -0
  29. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/cli/__init__.py +0 -0
  30. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/__init__.py +0 -0
  31. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/apply_masks.py +0 -0
  32. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/applying_masks_handler.py +0 -0
  33. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/background_noise_remover.py +0 -0
  34. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/create_masks.py +0 -0
  35. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/noise_masks.py +0 -0
  36. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/types.py +0 -0
  37. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/__init__.py +0 -0
  38. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/ensure_time_continuity.py +0 -0
  39. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/mvbs_computation.py +0 -0
  40. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/raw_handler.py +0 -0
  41. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/sv_computation.py +0 -0
  42. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/sv_dataset_extension.py +0 -0
  43. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/sv_interpolation.py +0 -0
  44. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/target_strength_computation.py +0 -0
  45. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/__init__.py +0 -0
  46. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/csv/csv_export_from_Sv.py +0 -0
  47. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/csv/csv_export_from_raw.py +0 -0
  48. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/csv/csv_export_nasc.py +0 -0
  49. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/frequency_differencing_handler.py +0 -0
  50. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/nasc.py +0 -0
  51. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/nasc_computation.py +0 -0
  52. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/shoals/shoal_detection_handler.py +0 -0
  53. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/shoals/shoal_process.py +0 -0
  54. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/shoals/shoals_handler.py +0 -0
  55. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/plot/__init__.py +0 -0
  56. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/__init__.py +0 -0
  57. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/process.py +0 -0
  58. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/processed_data_io.py +0 -0
  59. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/settings/__init__.py +0 -0
  60. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/settings/config.py +0 -0
  61. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/utils.py +0 -0
  62. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/dependency_links.txt +0 -0
  63. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/entry_points.txt +0 -0
  64. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/requires.txt +0 -0
  65. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/top_level.txt +0 -0
  66. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/requirements-dev.txt +0 -0
  67. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/requirements-local.txt +0 -0
  68. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/requirements.txt +0 -0
  69. {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oceanstream
3
- Version: 0.0.6.dev0
3
+ Version: 0.0.10.dev0
4
4
  Summary: OceanStream: process raw sonar data at scale
5
5
  Home-page: https://oceanstream.io
6
6
  Author: Pineview Labs
@@ -9,10 +9,11 @@ Maintainer: Andrei Rusu
9
9
  Maintainer-email: hello@pineview.io
10
10
  License: MIT
11
11
  Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.10
12
13
  Classifier: Programming Language :: Python :: 3.11
13
14
  Classifier: License :: OSI Approved :: MIT License
14
15
  Classifier: Operating System :: OS Independent
15
- Requires-Python: >=3.11
16
+ Requires-Python: >=3.10
16
17
  Description-Content-Type: text/markdown
17
18
  Provides-Extra: cli
18
19
  Provides-Extra: echodata
@@ -11,7 +11,7 @@ from pathlib import Path
11
11
  from rich import print
12
12
  from rich.traceback import install, Traceback
13
13
  from oceanstream.settings import load_config
14
- from dask.distributed import LocalCluster, Client
14
+ from dask.distributed import LocalCluster, Client, Variable
15
15
  from rich.console import Console
16
16
 
17
17
 
@@ -113,16 +113,21 @@ def convert(
113
113
  filePath = Path(source)
114
114
  configData = initialize(settings, filePath, log_level=log_level)
115
115
 
116
- if filePath.is_file():
117
- from oceanstream.process import convert_raw_file
118
- convert_raw_file(filePath, configData)
119
- print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
120
- elif filePath.is_dir():
121
- from oceanstream.process import convert_raw_files
122
- convert_raw_files(configData, workers_count=workers_count)
123
- else:
124
- print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
125
- sys.exit(1)
116
+ try:
117
+ if filePath.is_file():
118
+ from oceanstream.process import convert_raw_file
119
+ convert_raw_file(filePath, configData)
120
+ print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
121
+ elif filePath.is_dir():
122
+ from oceanstream.process import convert_raw_files
123
+ convert_raw_files(configData, workers_count=workers_count)
124
+ else:
125
+ print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
126
+ except KeyboardInterrupt:
127
+ logging.info("KeyboardInterrupt received, terminating processes...")
128
+ except Exception as e:
129
+ logging.exception("Error processing folder %s", configData['raw_path'])
130
+ print(Traceback())
126
131
 
127
132
 
128
133
  @app.command()
@@ -171,7 +176,7 @@ def combine(
171
176
  logging.info("KeyboardInterrupt received, terminating processes...")
172
177
  except Exception as e:
173
178
  logging.exception("Error processing folder %s", configData['raw_path'])
174
- traceback.print_exc()
179
+ print(Traceback())
175
180
  finally:
176
181
  status.stop()
177
182
  client.close()
@@ -190,6 +195,7 @@ def compute_sv(
190
195
  sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
191
196
  show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
192
197
  plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
198
+ use_dask: bool = typer.Option(False, help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
193
199
  depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
194
200
  waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
195
201
  show_choices=["CW", "BB"]),
@@ -208,11 +214,14 @@ def compute_sv(
208
214
  file_path = Path(source)
209
215
  config_data = initialize(settings_dict, file_path, log_level=log_level)
210
216
 
217
+ client = None
211
218
  console = Console()
219
+ single_file = file_path.is_dir() and source.endswith(".zarr")
212
220
  with console.status("Processing...", spinner="dots") as status:
213
221
  status.start()
214
- cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
215
- client = Client(cluster)
222
+ if use_dask or not single_file:
223
+ cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
224
+ client = Client(cluster)
216
225
 
217
226
  try:
218
227
  if file_path.is_dir() and source.endswith(".zarr"):
@@ -230,13 +239,14 @@ def compute_sv(
230
239
  f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
231
240
  f"http://localhost:8787/status for progress")
232
241
  from oceanstream.process import process_zarr_files
233
-
242
+ processed_count_var = Variable('processed_count', client)
234
243
  process_zarr_files(config_data,
235
244
  workers_count=workers_count,
236
245
  status=status,
237
246
  chunks=config_data.get('base_chunk_sizes'),
238
247
  plot_echogram=plot_echogram,
239
248
  waveform_mode=waveform_mode,
249
+ processed_count_var=processed_count_var,
240
250
  depth_offset=depth_offset)
241
251
  else:
242
252
  print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
@@ -245,10 +255,11 @@ def compute_sv(
245
255
  logging.info("KeyboardInterrupt received, terminating processes...")
246
256
  except Exception as e:
247
257
  logging.exception("Error while processing %s", config_data['raw_path'])
248
- traceback.print_exc()
258
+ print(Traceback())
249
259
  finally:
250
- client.close()
251
- cluster.close()
260
+ if use_dask:
261
+ client.close()
262
+ cluster.close()
252
263
  status.stop()
253
264
 
254
265
 
@@ -0,0 +1,2 @@
1
+ typer
2
+ rich
@@ -14,6 +14,9 @@ logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(messa
14
14
 
15
15
  def initialize(settings, file_path, log_level=None):
16
16
  logging.debug(f"Initializing with settings: {settings}, file path: {file_path}, log level: {log_level}")
17
+ if "config" not in settings:
18
+ settings["config"] = ""
19
+
17
20
  config_data = load_config(settings["config"])
18
21
  config_data["raw_path"] = file_path
19
22
 
@@ -27,6 +30,9 @@ def initialize(settings, file_path, log_level=None):
27
30
  if settings["output_folder"] is not None:
28
31
  config_data["output_folder"] = settings["output_folder"]
29
32
 
33
+ if settings['cloud_storage'] is not None:
34
+ config_data['cloud_storage'] = settings['cloud_storage']
35
+
30
36
  return config_data
31
37
 
32
38
 
@@ -103,8 +109,7 @@ def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
103
109
 
104
110
  file_name = f"{Path(dir_path).stem}-combined.zarr"
105
111
  zarr_output_file = os.path.join(config_data['output_folder'], file_name)
106
- logging.info(
107
- f"Combining Zarr files to {zarr_output_file}; navigate to http://localhost:8787/status for progress")
112
+ logging.info(f"Combining Zarr files to {zarr_output_file}")
108
113
 
109
114
  combine_zarr_files(dir_path, zarr_output_file=zarr_output_file, chunks=chunks)
110
115
  logging.info("Zarr files have been combined successfully.")
@@ -113,18 +118,24 @@ def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
113
118
 
114
119
 
115
120
  def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONAR_MODEL, plot_echogram=False,
116
- depth_offset=0.0, waveform_mode="CW", config=None, log_level="WARNING", chunks=None):
117
- logging.debug("Starting compute_sv function")
121
+ depth_offset=0.0, waveform_mode="CW", log_level="WARNING", chunks=None, config=None,
122
+ processed_count_var=None):
118
123
  settings_dict = {
119
- "config": config,
120
124
  "sonar_model": sonar_model,
121
125
  "output_folder": output or DEFAULT_OUTPUT_FOLDER
122
126
  }
127
+
128
+ if config is not None:
129
+ settings_dict.update(config)
130
+ # settings_dict["config"] = ''
131
+
123
132
  file_path = Path(source)
124
133
  config_data = initialize(settings_dict, file_path, log_level=log_level)
125
134
 
126
135
  if chunks:
127
136
  config_data['chunks'] = chunks
137
+ else:
138
+ config_data['chunks'] = config_data.get('base_chunk_sizes', None)
128
139
 
129
140
  if file_path.is_dir() and source.endswith(".zarr"):
130
141
  logging.debug(f"Computing Sv for Zarr root file: {file_path}")
@@ -136,7 +147,8 @@ def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONA
136
147
  logging.debug(f"Processing Zarr files in directory: {file_path}")
137
148
  from oceanstream.process import process_zarr_files
138
149
 
139
- process_zarr_files(config_data, workers_count=workers_count, chunks=chunks, plot_echogram=plot_echogram,
150
+ process_zarr_files(config_data, workers_count=workers_count, chunks=chunks,
151
+ processed_count_var=processed_count_var, plot_echogram=plot_echogram,
140
152
  waveform_mode=waveform_mode, depth_offset=depth_offset)
141
153
  else:
142
154
  logging.error(f"The provided path '{source}' is not a valid Zarr root.")
@@ -0,0 +1,4 @@
1
+ haversine
2
+ pandas
3
+ scipy
4
+ numpy
@@ -1,7 +1,10 @@
1
1
  import asyncio
2
2
  import os
3
+ import tempfile
3
4
  import xarray as xr
4
5
  import numpy as np
6
+ import logging
7
+
5
8
  from matplotlib.colors import LinearSegmentedColormap, Colormap
6
9
  import matplotlib.pyplot as plt
7
10
  import matplotlib.dates as mdates
@@ -287,18 +290,13 @@ def plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_na
287
290
  plt.close()
288
291
 
289
292
 
290
- def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name,
291
- cmap='viridis',
292
- regions2d=None,
293
- region_ids=None, region_class=None):
293
+ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, echogram_path=None,
294
+ config_data=None, cmap='ocean_r'):
294
295
  """Plot and save echogram for a single channel with optional regions and enhancements."""
295
296
  full_channel_name = ds_Sv.channel.values[channel]
296
297
  channel_name = "_".join(full_channel_name.split()[:3])
297
298
 
298
299
  plt.figure(figsize=(30, 18))
299
- echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
300
-
301
- # Apply the same preprocessing steps from _plot_echogram
302
300
  ds = ds_Sv
303
301
 
304
302
  filtered_ds = ds['Sv']
@@ -332,7 +330,7 @@ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_na
332
330
  yincrease=False,
333
331
  vmin=-80,
334
332
  vmax=-50,
335
- cmap='ocean_r',
333
+ cmap=cmap,
336
334
  cbar_kwargs={'label': 'Volume backscattering strength (Sv re 1 m-1)'}
337
335
  )
338
336
 
@@ -340,8 +338,20 @@ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_na
340
338
  plt.xlabel('Ping time', fontsize=14)
341
339
  plt.ylabel('Depth', fontsize=14)
342
340
  plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
343
- plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
344
- plt.close()
341
+
342
+ echogram_file_name = f"{file_base_name}_{channel_name}.png"
343
+
344
+ if config_data and 'cloud_storage' in config_data:
345
+ with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as temp_file:
346
+ plt.savefig(temp_file.name, dpi=300, bbox_inches='tight')
347
+ plt.close()
348
+ echogram_output_path = os.path.join(echogram_path, echogram_file_name)
349
+ upload_to_cloud_storage(temp_file.name, echogram_output_path, config_data['cloud_storage'])
350
+ os.remove(temp_file.name)
351
+ else:
352
+ echogram_output_path = os.path.join(output_path, echogram_file_name)
353
+ plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
354
+ plt.close()
345
355
 
346
356
 
347
357
  def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=None, client=None):
@@ -360,15 +370,15 @@ def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=Non
360
370
  wait(futures)
361
371
 
362
372
 
363
- def plot_sv_data(ds_Sv, file_base_name=None, output_path=None, cmap=None, regions2d=None, region_ids=None,
364
- region_class=None):
373
+ def plot_sv_data(ds_Sv, file_base_name=None, output_path=None, echogram_path=None, config_data=None, cmap=None):
365
374
  """Plot the echogram data and the regions."""
366
375
  if not plt.isinteractive():
367
376
  plt.switch_backend('Agg')
368
377
 
369
378
  for channel in range(ds_Sv.dims['channel']):
370
- plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, cmap, regions2d, region_ids,
371
- region_class)
379
+ plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, echogram_path=echogram_path,
380
+ config_data=config_data,
381
+ cmap='ocean_r')
372
382
  # plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_name, cmap)
373
383
  # plot_individual_channel_shaders(ds_Sv=ds_Sv, channel=channel, output_path=output_path,
374
384
  # file_base_name=file_base_name, cmap='ocean_r')
@@ -540,3 +550,24 @@ def _plot_region_ids(colors, ds_Sv, idx, labels_added, region_ids, regions2d):
540
550
  labels_added.add(label)
541
551
  idx += 1
542
552
  return idx
553
+
554
+
555
+ def upload_to_cloud_storage(local_path, remote_path, cloud_storage_config):
556
+ storage_type = cloud_storage_config['storage_type']
557
+ container_name = cloud_storage_config['container_name']
558
+ storage_options = cloud_storage_config['storage_options']
559
+
560
+ if storage_type == 'azure':
561
+ upload_to_azure_blob(local_path, remote_path, container_name, storage_options)
562
+ else:
563
+ raise ValueError(f"Unsupported storage type: {storage_type}")
564
+
565
+
566
+ def upload_to_azure_blob(local_path, remote_path, container_name, storage_options):
567
+ from azure.storage.blob import BlobServiceClient
568
+ blob_service_client = BlobServiceClient.from_connection_string(storage_options['connection_string'])
569
+ blob_client = blob_service_client.get_blob_client(container=container_name, blob=remote_path)
570
+
571
+ with open(local_path, "rb") as data:
572
+ blob_client.upload_blob(data, overwrite=True)
573
+ logging.info(f"Uploaded {local_path} to Azure Blob Storage as {remote_path}")
@@ -0,0 +1,2 @@
1
+ matplotlib
2
+ cmocean
@@ -5,7 +5,6 @@ import time
5
5
  from datetime import datetime
6
6
  from rich import print
7
7
  from pathlib import Path
8
- import psutil
9
8
  from oceanstream.echodata import check_reversed_time, fix_time_reversions
10
9
 
11
10
 
@@ -21,7 +20,7 @@ def read_zarr_files(input_folder):
21
20
  logging.error("Input folder does not exist: %s", input_folder)
22
21
  return
23
22
 
24
- zarr_files = list(input_path.glob("*.zarr"))
23
+ zarr_files = list(input_path.rglob("*.zarr"))
25
24
  if not zarr_files:
26
25
  logging.error("No .zarr files found in directory: %s", input_folder)
27
26
  return
@@ -47,7 +46,7 @@ def fix_time(ed):
47
46
  return ed
48
47
 
49
48
 
50
- def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None, log_level=logging.DEBUG):
49
+ def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None):
51
50
  start_time = time.time()
52
51
 
53
52
  logging.debug("Starting to combine Zarr files from folder: %s", input_folder)
@@ -15,7 +15,6 @@ from oceanstream.plot import plot_sv_data_with_progress, plot_sv_data
15
15
  from oceanstream.echodata import get_campaign_metadata
16
16
  from .process import compute_sv, process_file_with_progress, read_file_with_progress
17
17
 
18
-
19
18
  install(show_locals=True, width=120)
20
19
 
21
20
 
@@ -23,13 +22,14 @@ def get_chunk_sizes(var_dims, chunk_sizes):
23
22
  return {dim: chunk_sizes[dim] for dim in var_dims if dim in chunk_sizes}
24
23
 
25
24
 
26
- def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False, **kwargs):
25
+ def compute_Sv_to_zarr(echodata, config_data, base_path=None, chunks=None, plot_echogram=False, **kwargs):
27
26
  """
28
27
  Compute Sv from echodata and save to zarr file.
29
28
 
30
29
  Args:
31
30
  echodata:
32
31
  config_data:
32
+ base_path:
33
33
  chunks:
34
34
  plot_echogram:
35
35
  **kwargs:
@@ -42,11 +42,21 @@ def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False,
42
42
  encode_mode = waveform_mode == "CW" and "power" or "complex"
43
43
  Sv = compute_sv(echodata, encode_mode=encode_mode, **kwargs)
44
44
 
45
- parent_folder = os.path.join(Path(config_data["output_folder"]), file_path.stem)
46
- if not os.path.exists(parent_folder):
47
- os.makedirs(parent_folder)
45
+ if base_path:
46
+ relative_path = file_path.relative_to(base_path)
47
+
48
+ if relative_path.parent != ".":
49
+ zarr_path = Path(relative_path.parent) / file_path.stem
50
+ else:
51
+ zarr_path = relative_path.stem
52
+ else:
53
+ zarr_path = file_path.stem
48
54
 
49
- output_path = os.path.join(parent_folder, f"{file_path.stem}_Sv.zarr")
55
+ output_path = Path(config_data["output_folder"]) / zarr_path
56
+ output_path.mkdir(parents=True, exist_ok=True)
57
+
58
+ echogram_path = zarr_path
59
+ zarr_file_name = f"{file_path.stem}_Sv.zarr"
50
60
 
51
61
  if chunks is not None:
52
62
  for var in Sv.data_vars:
@@ -56,14 +66,15 @@ def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False,
56
66
  if 'chunks' in Sv[var].encoding:
57
67
  del Sv[var].encoding['chunks']
58
68
 
59
- print("Removing background noise...")
60
69
  ds_processed = Sv
70
+
61
71
  # ds_processed = apply_background_noise_removal(Sv, config=config_data)
62
- ds_processed.to_zarr(output_path)
72
+ write_zarr_file(zarr_path, zarr_file_name, ds_processed, config_data, output_path)
63
73
 
64
74
  if plot_echogram:
65
75
  try:
66
- plot_sv_data(ds_processed, file_base_name=file_path.stem, output_path=parent_folder)
76
+ plot_sv_data(ds_processed, file_base_name=file_path.stem, output_path=output_path,
77
+ echogram_path=echogram_path, config_data=config_data)
67
78
  except Exception as e:
68
79
  logging.exception(f"Error plotting echogram for {file_path}:")
69
80
  raise e
@@ -71,13 +82,22 @@ def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False,
71
82
  return output_path
72
83
 
73
84
 
85
+ def write_zarr_file(zarr_path, zarr_file_name, ds_processed, config_data=None, output_path=None):
86
+ if 'cloud_storage' in config_data:
87
+ store = get_chunk_store(config_data['cloud_storage'], Path(zarr_path) / zarr_file_name)
88
+ else:
89
+ store = os.path.join(output_path, zarr_file_name)
90
+
91
+ ds_processed.to_zarr(store, mode='w')
92
+
93
+
74
94
  async def process_raw_file_with_progress(config_data, plot_echogram, waveform_mode="CW", depth_offset=0):
75
95
  try:
76
96
  with Progress(
77
- TextColumn("[progress.description]{task.description}"),
78
- BarColumn(),
79
- TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
80
- TimeElapsedColumn()
97
+ TextColumn("[progress.description]{task.description}"),
98
+ BarColumn(),
99
+ TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
100
+ TimeElapsedColumn()
81
101
  ) as progress:
82
102
  print(f"[green] Processing file: {config_data['raw_path']}[/green]")
83
103
  read_task = progress.add_task("[cyan]Reading raw file data...", total=100)
@@ -117,14 +137,25 @@ async def process_raw_file_with_progress(config_data, plot_echogram, waveform_mo
117
137
  logging.exception(f"Error processing file {config_data['raw_path']}: {e}")
118
138
 
119
139
 
120
- def convert_raw_file(file_path, config_data, progress_queue=None):
140
+ def convert_raw_file(file_path, config_data, base_path=None, progress_queue=None):
121
141
  logging.debug("Starting processing of file: %s", file_path)
122
142
  from oceanstream.echodata import read_file
123
143
 
124
144
  try:
125
- file_config_data = {**config_data, 'raw_path': Path(file_path)}
145
+ file_path_obj = Path(file_path)
146
+ file_config_data = {**config_data, 'raw_path': file_path_obj}
147
+
148
+ if base_path:
149
+ relative_path = file_path_obj.relative_to(base_path)
150
+ relative_path = relative_path.parent
151
+ else:
152
+ relative_path = file_path_obj.name
153
+
154
+ output_path = Path(config_data["output_folder"]) / relative_path
155
+ output_path.mkdir(parents=True, exist_ok=True)
156
+
126
157
  echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
127
- echodata.to_zarr(save_path=config_data["output_folder"], overwrite=True, parallel=False)
158
+ echodata.to_zarr(save_path=output_path, overwrite=True, parallel=False)
128
159
 
129
160
  if progress_queue:
130
161
  progress_queue.put(file_path)
@@ -133,6 +164,17 @@ def convert_raw_file(file_path, config_data, progress_queue=None):
133
164
  print(Traceback())
134
165
 
135
166
 
167
+ def get_chunk_store(storage_config, path):
168
+ if storage_config['storage_type'] == 'azure':
169
+ from adlfs import AzureBlobFileSystem
170
+ azfs = AzureBlobFileSystem(**storage_config['storage_options'])
171
+
172
+ return azfs.get_mapper(f"{storage_config['container_name']}/{path}")
173
+
174
+ else:
175
+ raise ValueError(f"Unsupported storage type: {storage_config['storage_type']}")
176
+
177
+
136
178
  def compute_single_file(config_data, **kwargs):
137
179
  file_path = config_data["raw_path"]
138
180
  start_time = time.time()
@@ -7,6 +7,7 @@ import sys
7
7
  import traceback
8
8
  import warnings
9
9
 
10
+ from dask import delayed, compute
10
11
  from pathlib import Path
11
12
  from datetime import datetime
12
13
  from rich import print
@@ -21,8 +22,10 @@ from .process import compute_sv
21
22
  from .processed_data_io import write_processed
22
23
  from .file_processor import convert_raw_file, compute_single_file
23
24
 
24
- logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
25
+ # logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
25
26
  warnings.filterwarnings("ignore", module="echopype")
27
+ warnings.filterwarnings("ignore", category=UserWarning)
28
+ warnings.filterwarnings("ignore", category=FutureWarning)
26
29
  pool = None
27
30
 
28
31
 
@@ -123,6 +126,15 @@ def signal_handler(sig, frame):
123
126
  sys.exit(0)
124
127
 
125
128
 
129
+ def find_raw_files(base_dir):
130
+ raw_files = []
131
+ for root, dirs, files in os.walk(base_dir):
132
+ for file in files:
133
+ if file.endswith('.raw'):
134
+ raw_files.append(os.path.join(root, file))
135
+ return raw_files
136
+
137
+
126
138
  def convert_raw_files(config_data, workers_count=os.cpu_count()):
127
139
  global pool
128
140
 
@@ -132,8 +144,7 @@ def convert_raw_files(config_data, workers_count=os.cpu_count()):
132
144
  configure_logging(log_level)
133
145
 
134
146
  print(f"Starting to convert folder: {dir_path} to Zarr using {workers_count} parallel processes...")
135
- raw_files = [os.path.join(dir_path, f) for f in os.listdir(dir_path) if
136
- os.path.isfile(os.path.join(dir_path, f)) and f.endswith('.raw')]
147
+ raw_files = find_raw_files(dir_path)
137
148
  file_info = []
138
149
 
139
150
  if not raw_files:
@@ -166,7 +177,8 @@ def convert_raw_files(config_data, workers_count=os.cpu_count()):
166
177
  pool = Pool(processes=workers_count)
167
178
 
168
179
  # Partial function with config_data, progress_queue and other arguments
169
- process_func = partial(convert_raw_file, config_data=config_data, progress_queue=progress_queue)
180
+ process_func = partial(convert_raw_file, config_data=config_data, progress_queue=progress_queue,
181
+ base_path=dir_path)
170
182
 
171
183
  # Run the progress updater in a separate process
172
184
  progress_updater = Process(target=update_progress, args=(progress_queue, len(sorted_files), log_level))
@@ -192,14 +204,25 @@ def convert_raw_files(config_data, workers_count=os.cpu_count()):
192
204
  logging.exception("Error processing folder %s: %s", config_data['raw_path'], e)
193
205
 
194
206
 
195
- def process_single_zarr_file(file_path, config_data, chunks=None, plot_echogram=False, waveform_mode="CW",
196
- depth_offset=0):
207
+ def process_single_zarr_file(file_path, config_data, base_path=None, chunks=None, plot_echogram=False, waveform_mode="CW",
208
+ depth_offset=0, total_files=1, processed_count_var=None):
197
209
  file_config_data = {**config_data, 'raw_path': Path(file_path)}
198
- compute_single_file(file_config_data, chunks=chunks, plot_echogram=plot_echogram,
210
+
211
+ processed_count = None
212
+
213
+ if processed_count_var:
214
+ processed_count = processed_count_var.get() + 1
215
+ processed_count_var.set(processed_count)
216
+
217
+ compute_single_file(file_config_data, base_path=base_path, chunks=chunks, plot_echogram=plot_echogram,
199
218
  waveform_mode=waveform_mode, depth_offset=depth_offset)
200
219
 
220
+ if processed_count:
221
+ print(f"Processed file: {file_path}. {processed_count}/{total_files}")
222
+
201
223
 
202
- def process_zarr_files(config_data, workers_count=os.cpu_count(), status=None, chunks=None, plot_echogram=False,
224
+ def process_zarr_files(config_data, workers_count=os.cpu_count(), status=None, chunks=None, processed_count_var=None,
225
+ plot_echogram=False,
203
226
  waveform_mode="CW", depth_offset=0):
204
227
  dir_path = config_data['raw_path']
205
228
  zarr_files = read_zarr_files(dir_path)
@@ -208,24 +231,26 @@ def process_zarr_files(config_data, workers_count=os.cpu_count(), status=None, c
208
231
  logging.error("No valid .zarr files with creation time found in directory: %s", dir_path)
209
232
  return
210
233
 
211
- logging.info(f"Found {len(zarr_files)} Zarr files in directory: {dir_path}")
212
-
213
- # def wrapped_process_single_zarr_file(file_path):
214
- # print(f"Computing Sv for {file_path}...\n")
215
- # return process_single_zarr_file(file_path, config_data,
216
- # chunks=chunks, plot_echogram=plot_echogram, waveform_mode=waveform_mode,
217
- # depth_offset=depth_offset)
218
- #
219
- # # Use Dask bag to parallelize the processing
220
- # bag = db.from_sequence(zarr_files, npartitions=workers_count)
221
- # bag = bag.map(wrapped_process_single_zarr_file)
222
- #
223
- # # Compute the results
224
- # results = bag.compute()
234
+ if status:
235
+ status.update(f"Found {len(zarr_files)} Zarr files in directory: {dir_path}\n")
236
+
237
+ tasks = []
238
+ total_files = len(zarr_files)
239
+
240
+ if processed_count_var:
241
+ processed_count_var.set(0)
242
+
225
243
  for file_path in zarr_files:
226
- status.update(f"Computing Sv for {file_path}...\n")
227
- process_single_zarr_file(file_path, config_data, chunks=chunks, plot_echogram=plot_echogram,
228
- waveform_mode=waveform_mode, depth_offset=depth_offset)
244
+ if status:
245
+ status.update(f"Computing Sv for {file_path}...\n")
246
+ task = delayed(process_single_zarr_file)(file_path, config_data, chunks=chunks, base_path=dir_path,
247
+ plot_echogram=plot_echogram,
248
+ waveform_mode=waveform_mode, depth_offset=depth_offset,
249
+ total_files=total_files, processed_count_var=processed_count_var)
250
+ tasks.append(task)
251
+
252
+ # Execute all tasks in parallel
253
+ compute(*tasks)
229
254
 
230
255
  logging.info("✅ All files have been processed")
231
256
 
@@ -243,4 +268,5 @@ def from_filename(file_name):
243
268
  return None
244
269
 
245
270
 
271
+
246
272
  signal.signal(signal.SIGINT, signal_handler)
@@ -111,8 +111,8 @@
111
111
  },
112
112
  "profile": false,
113
113
  "base_chunk_sizes": {
114
- "ping_time": 50,
115
- "range_sample": 50
114
+ "ping_time": 100,
115
+ "range_sample": 100
116
116
  },
117
117
  "sonar_model": null,
118
118
  "output_folder": "output",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oceanstream
3
- Version: 0.0.6.dev0
3
+ Version: 0.0.10.dev0
4
4
  Summary: OceanStream: process raw sonar data at scale
5
5
  Home-page: https://oceanstream.io
6
6
  Author: Pineview Labs
@@ -9,10 +9,11 @@ Maintainer: Andrei Rusu
9
9
  Maintainer-email: hello@pineview.io
10
10
  License: MIT
11
11
  Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.10
12
13
  Classifier: Programming Language :: Python :: 3.11
13
14
  Classifier: License :: OSI Approved :: MIT License
14
15
  Classifier: Operating System :: OS Independent
15
- Requires-Python: >=3.11
16
+ Requires-Python: >=3.10
16
17
  Description-Content-Type: text/markdown
17
18
  Provides-Extra: cli
18
19
  Provides-Extra: echodata
@@ -23,17 +23,20 @@ oceanstream.egg-info/requires.txt
23
23
  oceanstream.egg-info/top_level.txt
24
24
  oceanstream/cli/__init__.py
25
25
  oceanstream/cli/main.py
26
+ oceanstream/cli/requirements.txt
26
27
  oceanstream/denoise/__init__.py
27
28
  oceanstream/denoise/apply_masks.py
28
29
  oceanstream/denoise/applying_masks_handler.py
29
30
  oceanstream/denoise/background_noise_remover.py
30
31
  oceanstream/denoise/create_masks.py
31
32
  oceanstream/denoise/noise_masks.py
33
+ oceanstream/denoise/requirements.txt
32
34
  oceanstream/denoise/types.py
33
35
  oceanstream/echodata/__init__.py
34
36
  oceanstream/echodata/ensure_time_continuity.py
35
37
  oceanstream/echodata/mvbs_computation.py
36
38
  oceanstream/echodata/raw_handler.py
39
+ oceanstream/echodata/requirements.txt
37
40
  oceanstream/echodata/sv_computation.py
38
41
  oceanstream/echodata/sv_dataset_extension.py
39
42
  oceanstream/echodata/sv_interpolation.py
@@ -42,6 +45,7 @@ oceanstream/exports/__init__.py
42
45
  oceanstream/exports/frequency_differencing_handler.py
43
46
  oceanstream/exports/nasc.py
44
47
  oceanstream/exports/nasc_computation.py
48
+ oceanstream/exports/requirements.txt
45
49
  oceanstream/exports/csv/csv_export_from_Sv.py
46
50
  oceanstream/exports/csv/csv_export_from_raw.py
47
51
  oceanstream/exports/csv/csv_export_nasc.py
@@ -50,12 +54,14 @@ oceanstream/exports/shoals/shoal_process.py
50
54
  oceanstream/exports/shoals/shoals_handler.py
51
55
  oceanstream/plot/__init__.py
52
56
  oceanstream/plot/echogram.py
57
+ oceanstream/plot/requirements.txt
53
58
  oceanstream/process/__init__.py
54
59
  oceanstream/process/combine_zarr.py
55
60
  oceanstream/process/file_processor.py
56
61
  oceanstream/process/folder_processor.py
57
62
  oceanstream/process/process.py
58
63
  oceanstream/process/processed_data_io.py
64
+ oceanstream/process/requirements.txt
59
65
  oceanstream/settings/__init__.py
60
66
  oceanstream/settings/config.py
61
67
  oceanstream/settings/defaults.config.json
@@ -0,0 +1,8 @@
1
+ [build-system]
2
+ requires = [
3
+ "setuptools >= 48",
4
+ "setuptools_scm[toml] >= 4, <6",
5
+ "setuptools_scm_git_archive",
6
+ "wheel >= 0.29.0"
7
+ ]
8
+ build-backend = "setuptools.build_meta"
@@ -1,6 +1,6 @@
1
1
  [metadata]
2
2
  name = oceanstream
3
- version = 0.0.6-dev
3
+ version = 0.0.10-dev
4
4
  description = OceanStream: process raw sonar data at scale
5
5
  long_description = file: README.md
6
6
  long_description_content_type = text/markdown
@@ -13,6 +13,7 @@ license_files = LICENSE
13
13
  url = https://oceanstream.io
14
14
  classifiers =
15
15
  Programming Language :: Python :: 3
16
+ Programming Language :: Python :: 3.10
16
17
  Programming Language :: Python :: 3.11
17
18
  License :: OSI Approved :: MIT License
18
19
  Operating System :: OS Independent
@@ -20,7 +21,7 @@ classifiers =
20
21
  [options]
21
22
  packages = find:
22
23
  platforms = any
23
- python_requires = >=3.11
24
+ python_requires = >=3.10
24
25
  include_package_data = True
25
26
  setup_requires =
26
27
  setuptools_scm
@@ -1,55 +0,0 @@
1
- [build-system]
2
- requires = [
3
- "setuptools >= 48",
4
- "setuptools_scm[toml] >= 4, <6",
5
- "setuptools_scm_git_archive",
6
- "wheel >= 0.29.0"
7
- ]
8
- build-backend = "setuptools.build_meta"
9
-
10
- #[project]
11
- #name = "oceanstream"
12
- #version = "0.0.5-dev"
13
- #description = "OceanStream: process raw sonar data at scale"
14
- #readme = "README.md"
15
- #requires-python = ">=3.11"
16
- #authors = [
17
- # { name="Pineview Labs", email="hello@pineview.io" }
18
- #]
19
- #license = { file="LICENSE" }
20
- #classifiers = [
21
- # "Programming Language :: Python :: 3",
22
- # "Programming Language :: Python :: 3.10",
23
- # "Programming Language :: Python :: 3.11",
24
- # "License :: OSI Approved :: MIT License",
25
- # "Operating System :: OS Independent"
26
- #]
27
- #
28
- #dependencies = [
29
- # "git+https://github.com/OceanStreamIO/echopype-dev.git@oceanstream#egg=echopype"
30
- #]
31
- #
32
- #[project.optional-dependencies]
33
- #cli = [
34
- # "typer",
35
- # "rich"
36
- #]
37
- #plot = [
38
- # "matplotlib"
39
- #]
40
- #echodata = [
41
- # "pydantic"
42
- #]
43
- #
44
- #denoise = []
45
- #exports = []
46
- #process = []
47
- #complete = [
48
- # "typer",
49
- # "rich",
50
- # "matplotlib",
51
- # "pydantic"
52
- #]
53
- #
54
- #[project.scripts]
55
- #oceanstream = "oceanstream.cli.main:main"