oceanstream 0.0.8.dev0__tar.gz → 0.0.12.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/PKG-INFO +3 -2
  2. oceanstream-0.0.12.dev0/oceanstream/__init__.py +5 -0
  3. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/cli/main.py +29 -20
  4. oceanstream-0.0.12.dev0/oceanstream/cli/requirements.txt +1 -0
  5. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/core.py +44 -16
  6. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_dataset_extension.py +0 -2
  7. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/plot/echogram.py +45 -14
  8. oceanstream-0.0.12.dev0/oceanstream/process/aws/__init__.py +9 -0
  9. oceanstream-0.0.12.dev0/oceanstream/process/aws/s3.py +93 -0
  10. oceanstream-0.0.12.dev0/oceanstream/process/azure/__init__.py +8 -0
  11. oceanstream-0.0.12.dev0/oceanstream/process/azure/azure_processor.py +66 -0
  12. oceanstream-0.0.12.dev0/oceanstream/process/azure/blob_storage.py +54 -0
  13. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/combine_zarr.py +3 -4
  14. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/file_processor.py +76 -21
  15. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/folder_processor.py +51 -25
  16. oceanstream-0.0.12.dev0/oceanstream/process/requirements.txt +0 -0
  17. oceanstream-0.0.12.dev0/oceanstream/process/utils.py +159 -0
  18. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/settings/defaults.config.json +2 -2
  19. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/PKG-INFO +3 -2
  20. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/SOURCES.txt +6 -0
  21. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/requires.txt +3 -4
  22. oceanstream-0.0.12.dev0/pyproject.toml +8 -0
  23. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/requirements.txt +3 -0
  24. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/setup.cfg +3 -4
  25. oceanstream-0.0.8.dev0/oceanstream/__init__.py +0 -3
  26. oceanstream-0.0.8.dev0/oceanstream/cli/requirements.txt +0 -2
  27. oceanstream-0.0.8.dev0/oceanstream/echodata/requirements.txt +0 -1
  28. oceanstream-0.0.8.dev0/pyproject.toml +0 -55
  29. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/.flake8 +0 -0
  30. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/.gitattributes +0 -0
  31. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/.gitignore +0 -0
  32. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/CODE_OF_CONDUCT.md +0 -0
  33. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/LICENSE +0 -0
  34. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/MANIFEST.in +0 -0
  35. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/README.md +0 -0
  36. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/notebooks/README.md +0 -0
  37. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/cli/__init__.py +0 -0
  38. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/__init__.py +0 -0
  39. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/apply_masks.py +0 -0
  40. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/applying_masks_handler.py +0 -0
  41. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/background_noise_remover.py +0 -0
  42. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/create_masks.py +0 -0
  43. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/noise_masks.py +0 -0
  44. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/requirements.txt +0 -0
  45. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/types.py +0 -0
  46. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/__init__.py +0 -0
  47. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/ensure_time_continuity.py +0 -0
  48. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/mvbs_computation.py +0 -0
  49. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/raw_handler.py +0 -0
  50. {oceanstream-0.0.8.dev0/oceanstream/process → oceanstream-0.0.12.dev0/oceanstream/echodata}/requirements.txt +0 -0
  51. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_computation.py +0 -0
  52. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_interpolation.py +0 -0
  53. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/target_strength_computation.py +0 -0
  54. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/__init__.py +0 -0
  55. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/csv/csv_export_from_Sv.py +0 -0
  56. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/csv/csv_export_from_raw.py +0 -0
  57. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/csv/csv_export_nasc.py +0 -0
  58. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/frequency_differencing_handler.py +0 -0
  59. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/nasc.py +0 -0
  60. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/nasc_computation.py +0 -0
  61. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/requirements.txt +0 -0
  62. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/shoals/shoal_detection_handler.py +0 -0
  63. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/shoals/shoal_process.py +0 -0
  64. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/shoals/shoals_handler.py +0 -0
  65. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/plot/__init__.py +0 -0
  66. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/plot/requirements.txt +0 -0
  67. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/__init__.py +0 -0
  68. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/process.py +0 -0
  69. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/processed_data_io.py +0 -0
  70. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/settings/__init__.py +0 -0
  71. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/settings/config.py +0 -0
  72. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/utils.py +0 -0
  73. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/dependency_links.txt +0 -0
  74. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/entry_points.txt +0 -0
  75. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/top_level.txt +0 -0
  76. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/requirements-dev.txt +0 -0
  77. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/requirements-local.txt +0 -0
  78. {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oceanstream
3
- Version: 0.0.8.dev0
3
+ Version: 0.0.12.dev0
4
4
  Summary: OceanStream: process raw sonar data at scale
5
5
  Home-page: https://oceanstream.io
6
6
  Author: Pineview Labs
@@ -9,10 +9,11 @@ Maintainer: Andrei Rusu
9
9
  Maintainer-email: hello@pineview.io
10
10
  License: MIT
11
11
  Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.10
12
13
  Classifier: Programming Language :: Python :: 3.11
13
14
  Classifier: License :: OSI Approved :: MIT License
14
15
  Classifier: Operating System :: OS Independent
15
- Requires-Python: >=3.11
16
+ Requires-Python: >=3.10
16
17
  Description-Content-Type: text/markdown
17
18
  Provides-Extra: cli
18
19
  Provides-Extra: echodata
@@ -0,0 +1,5 @@
1
+ from .core import initialize, compute_sv, combine, convert, process_raw_file
2
+
3
+ __all__ = [
4
+ "compute_sv", "combine", "convert", "process_raw_file", "initialize"
5
+ ]
@@ -1,5 +1,3 @@
1
- import traceback
2
-
3
1
  import typer
4
2
  import asyncio
5
3
  import os
@@ -11,7 +9,7 @@ from pathlib import Path
11
9
  from rich import print
12
10
  from rich.traceback import install, Traceback
13
11
  from oceanstream.settings import load_config
14
- from dask.distributed import LocalCluster, Client
12
+ from dask.distributed import LocalCluster, Client, Variable
15
13
  from rich.console import Console
16
14
 
17
15
 
@@ -113,16 +111,21 @@ def convert(
113
111
  filePath = Path(source)
114
112
  configData = initialize(settings, filePath, log_level=log_level)
115
113
 
116
- if filePath.is_file():
117
- from oceanstream.process import convert_raw_file
118
- convert_raw_file(filePath, configData)
119
- print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
120
- elif filePath.is_dir():
121
- from oceanstream.process import convert_raw_files
122
- convert_raw_files(configData, workers_count=workers_count)
123
- else:
124
- print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
125
- sys.exit(1)
114
+ try:
115
+ if filePath.is_file():
116
+ from oceanstream.process import convert_raw_file
117
+ convert_raw_file(filePath, configData)
118
+ print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
119
+ elif filePath.is_dir():
120
+ from oceanstream.process import convert_raw_files
121
+ convert_raw_files(configData, workers_count=workers_count)
122
+ else:
123
+ print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
124
+ except KeyboardInterrupt:
125
+ logging.info("KeyboardInterrupt received, terminating processes...")
126
+ except Exception as e:
127
+ logging.exception("Error processing folder %s", configData['raw_path'])
128
+ print(Traceback())
126
129
 
127
130
 
128
131
  @app.command()
@@ -171,7 +174,7 @@ def combine(
171
174
  logging.info("KeyboardInterrupt received, terminating processes...")
172
175
  except Exception as e:
173
176
  logging.exception("Error processing folder %s", configData['raw_path'])
174
- traceback.print_exc()
177
+ print(Traceback())
175
178
  finally:
176
179
  status.stop()
177
180
  client.close()
@@ -190,6 +193,7 @@ def compute_sv(
190
193
  sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
191
194
  show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
192
195
  plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
196
+ use_dask: bool = typer.Option(False, help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
193
197
  depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
194
198
  waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
195
199
  show_choices=["CW", "BB"]),
@@ -208,11 +212,14 @@ def compute_sv(
208
212
  file_path = Path(source)
209
213
  config_data = initialize(settings_dict, file_path, log_level=log_level)
210
214
 
215
+ client = None
211
216
  console = Console()
217
+ single_file = file_path.is_dir() and source.endswith(".zarr")
212
218
  with console.status("Processing...", spinner="dots") as status:
213
219
  status.start()
214
- cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
215
- client = Client(cluster)
220
+ if use_dask or not single_file:
221
+ cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
222
+ client = Client(cluster)
216
223
 
217
224
  try:
218
225
  if file_path.is_dir() and source.endswith(".zarr"):
@@ -230,13 +237,14 @@ def compute_sv(
230
237
  f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
231
238
  f"http://localhost:8787/status for progress")
232
239
  from oceanstream.process import process_zarr_files
233
-
240
+ processed_count_var = Variable('processed_count', client)
234
241
  process_zarr_files(config_data,
235
242
  workers_count=workers_count,
236
243
  status=status,
237
244
  chunks=config_data.get('base_chunk_sizes'),
238
245
  plot_echogram=plot_echogram,
239
246
  waveform_mode=waveform_mode,
247
+ processed_count_var=processed_count_var,
240
248
  depth_offset=depth_offset)
241
249
  else:
242
250
  print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
@@ -245,10 +253,11 @@ def compute_sv(
245
253
  logging.info("KeyboardInterrupt received, terminating processes...")
246
254
  except Exception as e:
247
255
  logging.exception("Error while processing %s", config_data['raw_path'])
248
- traceback.print_exc()
256
+ print(Traceback())
249
257
  finally:
250
- client.close()
251
- cluster.close()
258
+ if use_dask:
259
+ client.close()
260
+ cluster.close()
252
261
  status.stop()
253
262
 
254
263
 
@@ -5,6 +5,7 @@ import sys
5
5
 
6
6
  from pathlib import Path
7
7
  from oceanstream.settings import load_config
8
+ from oceanstream.process import convert_raw_file, convert_raw_files, compute_single_file
8
9
 
9
10
  DEFAULT_OUTPUT_FOLDER = "output"
10
11
  DEFAULT_SONAR_MODEL = "EK60"
@@ -12,10 +13,20 @@ DEFAULT_SONAR_MODEL = "EK60"
12
13
  logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
13
14
 
14
15
 
15
- def initialize(settings, file_path, log_level=None):
16
+ def initialize(settings, file_path=None, log_level=None, chunks=None):
16
17
  logging.debug(f"Initializing with settings: {settings}, file path: {file_path}, log level: {log_level}")
18
+ if "config" not in settings:
19
+ settings["config"] = ""
20
+
17
21
  config_data = load_config(settings["config"])
18
- config_data["raw_path"] = file_path
22
+
23
+ if chunks:
24
+ config_data['chunks'] = chunks
25
+ else:
26
+ config_data['chunks'] = config_data.get('base_chunk_sizes', None)
27
+
28
+ if file_path is not None:
29
+ config_data["raw_path"] = file_path
19
30
 
20
31
  if log_level is not None:
21
32
  logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s', force=True)
@@ -24,9 +35,21 @@ def initialize(settings, file_path, log_level=None):
24
35
  if 'sonar_model' in settings and settings["sonar_model"] is not None:
25
36
  config_data["sonar_model"] = settings["sonar_model"]
26
37
 
27
- if settings["output_folder"] is not None:
38
+ if settings.get('plot_echogram', None) is not None:
39
+ config_data["plot_echogram"] = settings["plot_echogram"]
40
+
41
+ if settings.get('waveform_mode', None) is not None:
42
+ config_data["waveform_mode"] = settings["waveform_mode"]
43
+
44
+ if settings.get('depth_offset', None) is not None:
45
+ config_data["depth_offset"] = settings["depth_offset"]
46
+
47
+ if settings.get("output_folder", None) is not None:
28
48
  config_data["output_folder"] = settings["output_folder"]
29
49
 
50
+ if settings.get('cloud_storage', None) is not None:
51
+ config_data['cloud_storage'] = settings['cloud_storage']
52
+
30
53
  return config_data
31
54
 
32
55
 
@@ -57,27 +80,29 @@ def process_raw_file(source, output=None, sonar_model=None, plot_echogram=False,
57
80
  sys.exit(1)
58
81
 
59
82
 
60
- def convert(source, output=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
83
+ def convert(source, output=None, base_path=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
61
84
  logging.debug("Starting convert function")
62
85
  settings = {
63
- "config": config,
64
86
  "output_folder": output or DEFAULT_OUTPUT_FOLDER
65
87
  }
66
88
 
89
+ if config is not None:
90
+ settings.update(config)
91
+
67
92
  file_path = Path(source)
68
93
  config_data = initialize(settings, file_path, log_level=log_level)
69
94
 
70
95
  if chunks:
71
96
  config_data['chunks'] = chunks
97
+ else:
98
+ config_data['chunks'] = config_data.get('base_chunk_sizes', None)
72
99
 
73
100
  if file_path.is_file():
74
101
  logging.debug(f"Converting raw file: {file_path}")
75
- from oceanstream.process import convert_raw_file
76
- convert_raw_file(file_path, config_data)
102
+ convert_raw_file(file_path, config_data, base_path=base_path)
77
103
  logging.info(f"Converted raw file {source} to Zarr and wrote output to: {config_data['output_folder']}")
78
104
  elif file_path.is_dir():
79
105
  logging.debug(f"Converting raw files in directory: {file_path}")
80
- from oceanstream.process import convert_raw_files
81
106
  convert_raw_files(config_data, workers_count=workers_count)
82
107
  else:
83
108
  logging.error(f"The provided path '{source}' is not a valid file/folder.")
@@ -103,8 +128,7 @@ def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
103
128
 
104
129
  file_name = f"{Path(dir_path).stem}-combined.zarr"
105
130
  zarr_output_file = os.path.join(config_data['output_folder'], file_name)
106
- logging.info(
107
- f"Combining Zarr files to {zarr_output_file}; navigate to http://localhost:8787/status for progress")
131
+ logging.info(f"Combining Zarr files to {zarr_output_file}")
108
132
 
109
133
  combine_zarr_files(dir_path, zarr_output_file=zarr_output_file, chunks=chunks)
110
134
  logging.info("Zarr files have been combined successfully.")
@@ -113,30 +137,34 @@ def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
113
137
 
114
138
 
115
139
  def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONAR_MODEL, plot_echogram=False,
116
- depth_offset=0.0, waveform_mode="CW", config=None, log_level="WARNING", chunks=None):
117
- logging.debug("Starting compute_sv function")
140
+ depth_offset=0.0, waveform_mode="CW", log_level="WARNING", chunks=None, config=None,
141
+ processed_count_var=None):
118
142
  settings_dict = {
119
- "config": config,
120
143
  "sonar_model": sonar_model,
121
144
  "output_folder": output or DEFAULT_OUTPUT_FOLDER
122
145
  }
146
+
147
+ if config is not None:
148
+ settings_dict.update(config)
149
+
123
150
  file_path = Path(source)
124
151
  config_data = initialize(settings_dict, file_path, log_level=log_level)
125
152
 
126
153
  if chunks:
127
154
  config_data['chunks'] = chunks
155
+ else:
156
+ config_data['chunks'] = config_data.get('base_chunk_sizes', None)
128
157
 
129
158
  if file_path.is_dir() and source.endswith(".zarr"):
130
159
  logging.debug(f"Computing Sv for Zarr root file: {file_path}")
131
- from oceanstream.process import compute_single_file
132
-
133
160
  compute_single_file(config_data, chunks=chunks, plot_echogram=plot_echogram, waveform_mode=waveform_mode,
134
161
  depth_offset=depth_offset)
135
162
  elif file_path.is_dir():
136
163
  logging.debug(f"Processing Zarr files in directory: {file_path}")
137
164
  from oceanstream.process import process_zarr_files
138
165
 
139
- process_zarr_files(config_data, workers_count=workers_count, chunks=chunks, plot_echogram=plot_echogram,
166
+ process_zarr_files(config_data, workers_count=workers_count, chunks=chunks,
167
+ processed_count_var=processed_count_var, plot_echogram=plot_echogram,
140
168
  waveform_mode=waveform_mode, depth_offset=depth_offset)
141
169
  else:
142
170
  logging.error(f"The provided path '{source}' is not a valid Zarr root.")
@@ -96,8 +96,6 @@ def add_depth(Sv: Dataset, depth_offset: float = 0, tilt: float = 0, downward: b
96
96
  first_channel = Sv["channel"].values[0]
97
97
  first_ping_time = Sv["ping_time"].values[0]
98
98
 
99
- original_echo_range = Sv["echo_range"].sel(channel=first_channel, ping_time=first_ping_time).values
100
-
101
99
  # Slice the echo_range to get the desired range of values
102
100
  selected_echo_range = Sv["echo_range"].sel(channel=first_channel, ping_time=first_ping_time)
103
101
  selected_echo_range = selected_echo_range.values.tolist()
@@ -1,7 +1,10 @@
1
1
  import asyncio
2
2
  import os
3
+ import tempfile
3
4
  import xarray as xr
4
5
  import numpy as np
6
+ import logging
7
+
5
8
  from matplotlib.colors import LinearSegmentedColormap, Colormap
6
9
  import matplotlib.pyplot as plt
7
10
  import matplotlib.dates as mdates
@@ -287,18 +290,13 @@ def plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_na
287
290
  plt.close()
288
291
 
289
292
 
290
- def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name,
291
- cmap='viridis',
292
- regions2d=None,
293
- region_ids=None, region_class=None):
293
+ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, echogram_path=None,
294
+ config_data=None, cmap='ocean_r'):
294
295
  """Plot and save echogram for a single channel with optional regions and enhancements."""
295
296
  full_channel_name = ds_Sv.channel.values[channel]
296
297
  channel_name = "_".join(full_channel_name.split()[:3])
297
298
 
298
299
  plt.figure(figsize=(30, 18))
299
- echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
300
-
301
- # Apply the same preprocessing steps from _plot_echogram
302
300
  ds = ds_Sv
303
301
 
304
302
  filtered_ds = ds['Sv']
@@ -332,7 +330,7 @@ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_na
332
330
  yincrease=False,
333
331
  vmin=-80,
334
332
  vmax=-50,
335
- cmap='ocean_r',
333
+ cmap=cmap,
336
334
  cbar_kwargs={'label': 'Volume backscattering strength (Sv re 1 m-1)'}
337
335
  )
338
336
 
@@ -340,8 +338,20 @@ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_na
340
338
  plt.xlabel('Ping time', fontsize=14)
341
339
  plt.ylabel('Depth', fontsize=14)
342
340
  plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
343
- plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
344
- plt.close()
341
+
342
+ echogram_file_name = f"{file_base_name}_{channel_name}.png"
343
+
344
+ if config_data and 'cloud_storage' in config_data:
345
+ with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as temp_file:
346
+ plt.savefig(temp_file.name, dpi=300, bbox_inches='tight')
347
+ plt.close()
348
+ echogram_output_path = os.path.join(echogram_path, echogram_file_name)
349
+ upload_to_cloud_storage(temp_file.name, echogram_output_path, config_data['cloud_storage'])
350
+ os.remove(temp_file.name)
351
+ else:
352
+ echogram_output_path = os.path.join(output_path, echogram_file_name)
353
+ plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
354
+ plt.close()
345
355
 
346
356
 
347
357
  def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=None, client=None):
@@ -360,15 +370,15 @@ def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=Non
360
370
  wait(futures)
361
371
 
362
372
 
363
- def plot_sv_data(ds_Sv, file_base_name=None, output_path=None, cmap=None, regions2d=None, region_ids=None,
364
- region_class=None):
373
+ def plot_sv_data(ds_Sv, file_base_name=None, output_path=None, echogram_path=None, config_data=None, cmap=None):
365
374
  """Plot the echogram data and the regions."""
366
375
  if not plt.isinteractive():
367
376
  plt.switch_backend('Agg')
368
377
 
369
378
  for channel in range(ds_Sv.dims['channel']):
370
- plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, cmap, regions2d, region_ids,
371
- region_class)
379
+ plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, echogram_path=echogram_path,
380
+ config_data=config_data,
381
+ cmap='ocean_r')
372
382
  # plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_name, cmap)
373
383
  # plot_individual_channel_shaders(ds_Sv=ds_Sv, channel=channel, output_path=output_path,
374
384
  # file_base_name=file_base_name, cmap='ocean_r')
@@ -540,3 +550,24 @@ def _plot_region_ids(colors, ds_Sv, idx, labels_added, region_ids, regions2d):
540
550
  labels_added.add(label)
541
551
  idx += 1
542
552
  return idx
553
+
554
+
555
+ def upload_to_cloud_storage(local_path, remote_path, cloud_storage_config):
556
+ storage_type = cloud_storage_config['storage_type']
557
+ container_name = cloud_storage_config['container_name']
558
+ storage_options = cloud_storage_config['storage_options']
559
+
560
+ if storage_type == 'azure':
561
+ upload_to_azure_blob(local_path, remote_path, container_name, storage_options)
562
+ else:
563
+ raise ValueError(f"Unsupported storage type: {storage_type}")
564
+
565
+
566
+ def upload_to_azure_blob(local_path, remote_path, container_name, storage_options):
567
+ from azure.storage.blob import BlobServiceClient
568
+ blob_service_client = BlobServiceClient.from_connection_string(storage_options['connection_string'])
569
+ blob_client = blob_service_client.get_blob_client(container=container_name, blob=remote_path)
570
+
571
+ with open(local_path, "rb") as data:
572
+ blob_client.upload_blob(data, overwrite=True)
573
+ logging.info(f"Uploaded {local_path} to Azure Blob Storage as {remote_path}")
@@ -0,0 +1,9 @@
1
+ from .s3 import process_survey_data_with_progress, list_raw_files_from_bucket, convert_survey_data_from_bucket, \
2
+ download_file_from_bucket
3
+
4
+ __all__ = [
5
+ "process_survey_data_with_progress",
6
+ "list_raw_files_from_bucket",
7
+ "convert_survey_data_from_bucket",
8
+ "download_file_from_bucket"
9
+ ]
@@ -0,0 +1,93 @@
1
+ import os
2
+ import tempfile
3
+ import botocore
4
+ import boto3
5
+ from asyncio import CancelledError
6
+ from rich import print
7
+ from rich.traceback import install, Traceback
8
+ from tqdm.auto import tqdm
9
+ from dask import delayed
10
+ from botocore.config import Config
11
+
12
+
13
+ install(show_locals=False, width=120)
14
+
15
+
16
+ def process_survey_data_with_progress(files_by_day, bucket_name, client, config_data, process_func):
17
+ try:
18
+ total_files = sum(len(files) for files in files_by_day.values())
19
+ progress_bar = tqdm(total=total_files, desc="Processing Files", unit="file", ncols=100)
20
+
21
+ def update_progress(*args):
22
+ progress_bar.update()
23
+
24
+ temp_dir = tempfile.mkdtemp()
25
+ futures, temp_dir = convert_survey_data_from_bucket(files_by_day, temp_dir, bucket_name, client, config_data,
26
+ process_func)
27
+
28
+ for future in futures:
29
+ future.add_done_callback(update_progress)
30
+
31
+ client.gather(futures) # Ensure all tasks complete
32
+
33
+ progress_bar.close()
34
+ os.rmdir(temp_dir)
35
+ except KeyboardInterrupt:
36
+ print("Closing down.")
37
+ except CancelledError:
38
+ print("Closing down.")
39
+ except Exception as e:
40
+ print(f"[bold red]An error occurred:[/bold red] {e}")
41
+ print(f"{Traceback()}\n")
42
+
43
+
44
+ def convert_survey_data_from_bucket(files_by_day, temp_dir, bucket_name, dask_client, config_data, process_func):
45
+ """Process survey data from S3."""
46
+ tasks = []
47
+
48
+ for day, files in files_by_day.items():
49
+ for file in files:
50
+ task = delayed(_process_raw_file)(file, day, temp_dir, bucket_name, config_data, process_func)
51
+ tasks.append(task)
52
+
53
+ # Execute all tasks in parallel
54
+ futures = dask_client.compute(tasks)
55
+
56
+ return futures, temp_dir
57
+
58
+
59
+ def list_raw_files_from_bucket(bucket_name, prefix):
60
+ """List files in the S3 bucket along with their metadata."""
61
+ s3_client = boto3.client('s3', config=Config(signature_version=botocore.UNSIGNED))
62
+ response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
63
+
64
+ files = [
65
+ {
66
+ 'Key': item['Key'],
67
+ 'Size': item['Size'],
68
+ 'LastModified': item['LastModified']
69
+ }
70
+ for item in response.get('Contents', []) if item['Key'].endswith('.raw')
71
+ ]
72
+
73
+ return files
74
+
75
+
76
+ def download_file_from_bucket(bucket_name, s3_key, local_dir):
77
+ """Download a file from S3."""
78
+ s3_client = boto3.client('s3', config=Config(signature_version=botocore.UNSIGNED))
79
+ local_path = os.path.join(local_dir, os.path.basename(s3_key))
80
+ s3_client.download_file(bucket_name, s3_key, local_path)
81
+
82
+ return local_path
83
+
84
+
85
+ def _process_raw_file(file, day, temp_dir, bucket_name, config_data, process_func):
86
+ """Process a single file."""
87
+ day_dir = os.path.join(temp_dir, day)
88
+ os.makedirs(day_dir, exist_ok=True)
89
+ local_path = download_file_from_bucket(bucket_name, file['Key'], day_dir)
90
+
91
+ process_func(local_path, config=config_data, base_path=temp_dir)
92
+ os.remove(local_path)
93
+
@@ -0,0 +1,8 @@
1
+ from .blob_storage import list_zarr_files
2
+ from .azure_processor import process_survey_data_with_progress, process_survey_data
3
+
4
+ __all__ = [
5
+ "list_zarr_files",
6
+ "process_survey_data",
7
+ "process_survey_data_with_progress"
8
+ ]
@@ -0,0 +1,66 @@
1
+ import re
2
+
3
+ from asyncio import CancelledError
4
+ from rich import print
5
+ from rich.traceback import install, Traceback
6
+ from tqdm.auto import tqdm
7
+ from dask import delayed
8
+ from .blob_storage import open_zarr_store
9
+
10
+ install(show_locals=False, width=120)
11
+
12
+
13
+ def process_survey_data_with_progress(files_by_day, azfs, container_name, client, config_data, process_func):
14
+ try:
15
+ total_files = sum(len(files) for files in files_by_day.values())
16
+ progress_bar = tqdm(total=total_files, desc="Processing Files", unit="file", ncols=100)
17
+
18
+ def update_progress(*args):
19
+ progress_bar.update()
20
+
21
+ futures = process_survey_data(files_by_day, azfs, container_name, client, config_data, process_func)
22
+
23
+ for future in futures:
24
+ future.add_done_callback(update_progress)
25
+
26
+ client.gather(futures) # Ensure all tasks complete
27
+
28
+ progress_bar.close()
29
+ except KeyboardInterrupt:
30
+ print("Closing down.")
31
+ except CancelledError:
32
+ print("Closing down.")
33
+ except Exception as e:
34
+ print(f"[bold red]An error occurred:[/bold red] {e}")
35
+ print(f"{Traceback()}\n")
36
+
37
+
38
+ def process_survey_data(files_by_day, azfs, container_name, dask_client, config_data, process_func):
39
+ """Process survey data from S3."""
40
+
41
+ tasks = []
42
+
43
+ for day, files in files_by_day.items():
44
+ for file in files:
45
+ task = delayed(_process_zarr_file)(file['Key'], azfs, container_name, config_data, process_func)
46
+ tasks.append(task)
47
+
48
+ # Execute all tasks in parallel
49
+ futures = dask_client.compute(tasks)
50
+
51
+ return futures
52
+
53
+
54
+ def _process_zarr_file(file, azfs, container_name, config_data, process_func=None):
55
+ """Process a single Zarr file."""
56
+ base_path = file.replace('.zarr', '')
57
+ pattern = rf"^{re.escape(container_name)}/"
58
+ base_path = re.sub(pattern, '', base_path)
59
+
60
+ echodata = open_zarr_store(azfs, file, chunks=config_data['chunks'])
61
+
62
+ process_func(echodata, config_data, base_path=base_path,
63
+ chunks=config_data.get('chunks'),
64
+ plot_echogram=config_data.get('plot_echogram', False),
65
+ waveform_mode=config_data.get('waveform_mode', "CW"),
66
+ depth_offset=config_data.get('depth_offset', 0.0))
@@ -0,0 +1,54 @@
1
+ import echopype as ep
2
+
3
+
4
+ def list_zarr_files(azfs, path):
5
+ """List all Zarr files in the Azure Blob Storage container along with their metadata."""
6
+ zarr_files = []
7
+ for blob in azfs.ls(path, detail=True):
8
+ if blob['type'] == 'directory' and not blob['name'].endswith('.zarr'):
9
+ subdir_files = list_zarr_files(azfs, blob['name'])
10
+ zarr_files.extend(subdir_files)
11
+ elif blob['name'].endswith('.zarr'):
12
+ zarr_files.append({
13
+ 'Key': blob['name'],
14
+ 'Size': blob['size'] if blob['size'] else 0,
15
+ 'LastModified': blob['last_modified'] if 'last_modified' in blob else 0
16
+ })
17
+
18
+ return zarr_files
19
+
20
+
21
+ def open_zarr_store(azfs, store_name, chunks=None):
22
+ """Open a Zarr store from Azure Blob Storage."""
23
+ mapper = azfs.get_mapper(store_name)
24
+
25
+ return ep.open_converted(mapper, chunks=chunks)
26
+
27
+
28
+ def _list_zarr_files_extended(azfs, path):
29
+ """List all Zarr files in the Azure Blob Storage container along with their metadata."""
30
+ zarr_files = []
31
+ for blob in azfs.ls(path, detail=True):
32
+ if blob['type'] == 'directory' and not blob['name'].endswith('.zarr'):
33
+ subdir_files = list_zarr_files(azfs, blob['name'])
34
+ zarr_files.extend(subdir_files)
35
+ else:
36
+ # Calculate the total size and most recent modification date for the .zarr folder
37
+ total_size = 0
38
+ last_modified = None
39
+ for sub_blob in azfs.ls(blob['name'], detail=True):
40
+ if sub_blob['type'] == 'file':
41
+ total_size += sub_blob['size']
42
+ if last_modified is None or sub_blob['last_modified'] > last_modified:
43
+ last_modified = sub_blob['last_modified']
44
+
45
+ zarr_files.append({
46
+ 'Key': blob['name'],
47
+ 'Size': total_size,
48
+ 'LastModified': last_modified
49
+ })
50
+
51
+ return zarr_files
52
+
53
+
54
+
@@ -5,7 +5,6 @@ import time
5
5
  from datetime import datetime
6
6
  from rich import print
7
7
  from pathlib import Path
8
- import psutil
9
8
  from oceanstream.echodata import check_reversed_time, fix_time_reversions
10
9
 
11
10
 
@@ -21,7 +20,7 @@ def read_zarr_files(input_folder):
21
20
  logging.error("Input folder does not exist: %s", input_folder)
22
21
  return
23
22
 
24
- zarr_files = list(input_path.glob("*.zarr"))
23
+ zarr_files = list(input_path.rglob("*.zarr"))
25
24
  if not zarr_files:
26
25
  logging.error("No .zarr files found in directory: %s", input_folder)
27
26
  return
@@ -47,7 +46,7 @@ def fix_time(ed):
47
46
  return ed
48
47
 
49
48
 
50
- def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None, log_level=logging.DEBUG):
49
+ def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None):
51
50
  start_time = time.time()
52
51
 
53
52
  logging.debug("Starting to combine Zarr files from folder: %s", input_folder)
@@ -87,4 +86,4 @@ def from_filename(file_name):
87
86
  creation_time = datetime.strptime(date_str + time_str, '%Y%m%d%H%M%S')
88
87
  return creation_time
89
88
 
90
- return None
89
+ return None