oceanstream 0.0.8.dev0__tar.gz → 0.0.12.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/PKG-INFO +3 -2
- oceanstream-0.0.12.dev0/oceanstream/__init__.py +5 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/cli/main.py +29 -20
- oceanstream-0.0.12.dev0/oceanstream/cli/requirements.txt +1 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/core.py +44 -16
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_dataset_extension.py +0 -2
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/plot/echogram.py +45 -14
- oceanstream-0.0.12.dev0/oceanstream/process/aws/__init__.py +9 -0
- oceanstream-0.0.12.dev0/oceanstream/process/aws/s3.py +93 -0
- oceanstream-0.0.12.dev0/oceanstream/process/azure/__init__.py +8 -0
- oceanstream-0.0.12.dev0/oceanstream/process/azure/azure_processor.py +66 -0
- oceanstream-0.0.12.dev0/oceanstream/process/azure/blob_storage.py +54 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/combine_zarr.py +3 -4
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/file_processor.py +76 -21
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/folder_processor.py +51 -25
- oceanstream-0.0.12.dev0/oceanstream/process/requirements.txt +0 -0
- oceanstream-0.0.12.dev0/oceanstream/process/utils.py +159 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/settings/defaults.config.json +2 -2
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/PKG-INFO +3 -2
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/SOURCES.txt +6 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/requires.txt +3 -4
- oceanstream-0.0.12.dev0/pyproject.toml +8 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/requirements.txt +3 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/setup.cfg +3 -4
- oceanstream-0.0.8.dev0/oceanstream/__init__.py +0 -3
- oceanstream-0.0.8.dev0/oceanstream/cli/requirements.txt +0 -2
- oceanstream-0.0.8.dev0/oceanstream/echodata/requirements.txt +0 -1
- oceanstream-0.0.8.dev0/pyproject.toml +0 -55
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/.flake8 +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/.gitattributes +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/.gitignore +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/CODE_OF_CONDUCT.md +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/LICENSE +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/MANIFEST.in +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/README.md +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/notebooks/README.md +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/cli/__init__.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/__init__.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/apply_masks.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/applying_masks_handler.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/background_noise_remover.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/create_masks.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/noise_masks.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/requirements.txt +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/types.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/__init__.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/ensure_time_continuity.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/mvbs_computation.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/raw_handler.py +0 -0
- {oceanstream-0.0.8.dev0/oceanstream/process → oceanstream-0.0.12.dev0/oceanstream/echodata}/requirements.txt +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_computation.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_interpolation.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/target_strength_computation.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/__init__.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/csv/csv_export_from_Sv.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/csv/csv_export_from_raw.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/csv/csv_export_nasc.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/frequency_differencing_handler.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/nasc.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/nasc_computation.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/requirements.txt +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/shoals/shoal_detection_handler.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/shoals/shoal_process.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/shoals/shoals_handler.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/plot/__init__.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/plot/requirements.txt +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/__init__.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/process.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/processed_data_io.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/settings/__init__.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/settings/config.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/utils.py +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/dependency_links.txt +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/entry_points.txt +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/top_level.txt +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/requirements-dev.txt +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/requirements-local.txt +0 -0
- {oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: oceanstream
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.12.dev0
|
|
4
4
|
Summary: OceanStream: process raw sonar data at scale
|
|
5
5
|
Home-page: https://oceanstream.io
|
|
6
6
|
Author: Pineview Labs
|
|
@@ -9,10 +9,11 @@ Maintainer: Andrei Rusu
|
|
|
9
9
|
Maintainer-email: hello@pineview.io
|
|
10
10
|
License: MIT
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
14
|
Classifier: License :: OSI Approved :: MIT License
|
|
14
15
|
Classifier: Operating System :: OS Independent
|
|
15
|
-
Requires-Python: >=3.
|
|
16
|
+
Requires-Python: >=3.10
|
|
16
17
|
Description-Content-Type: text/markdown
|
|
17
18
|
Provides-Extra: cli
|
|
18
19
|
Provides-Extra: echodata
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import traceback
|
|
2
|
-
|
|
3
1
|
import typer
|
|
4
2
|
import asyncio
|
|
5
3
|
import os
|
|
@@ -11,7 +9,7 @@ from pathlib import Path
|
|
|
11
9
|
from rich import print
|
|
12
10
|
from rich.traceback import install, Traceback
|
|
13
11
|
from oceanstream.settings import load_config
|
|
14
|
-
from dask.distributed import LocalCluster, Client
|
|
12
|
+
from dask.distributed import LocalCluster, Client, Variable
|
|
15
13
|
from rich.console import Console
|
|
16
14
|
|
|
17
15
|
|
|
@@ -113,16 +111,21 @@ def convert(
|
|
|
113
111
|
filePath = Path(source)
|
|
114
112
|
configData = initialize(settings, filePath, log_level=log_level)
|
|
115
113
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
114
|
+
try:
|
|
115
|
+
if filePath.is_file():
|
|
116
|
+
from oceanstream.process import convert_raw_file
|
|
117
|
+
convert_raw_file(filePath, configData)
|
|
118
|
+
print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
|
|
119
|
+
elif filePath.is_dir():
|
|
120
|
+
from oceanstream.process import convert_raw_files
|
|
121
|
+
convert_raw_files(configData, workers_count=workers_count)
|
|
122
|
+
else:
|
|
123
|
+
print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
|
|
124
|
+
except KeyboardInterrupt:
|
|
125
|
+
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
126
|
+
except Exception as e:
|
|
127
|
+
logging.exception("Error processing folder %s", configData['raw_path'])
|
|
128
|
+
print(Traceback())
|
|
126
129
|
|
|
127
130
|
|
|
128
131
|
@app.command()
|
|
@@ -171,7 +174,7 @@ def combine(
|
|
|
171
174
|
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
172
175
|
except Exception as e:
|
|
173
176
|
logging.exception("Error processing folder %s", configData['raw_path'])
|
|
174
|
-
|
|
177
|
+
print(Traceback())
|
|
175
178
|
finally:
|
|
176
179
|
status.stop()
|
|
177
180
|
client.close()
|
|
@@ -190,6 +193,7 @@ def compute_sv(
|
|
|
190
193
|
sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
|
|
191
194
|
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
192
195
|
plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
|
|
196
|
+
use_dask: bool = typer.Option(False, help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
|
|
193
197
|
depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
|
|
194
198
|
waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
|
|
195
199
|
show_choices=["CW", "BB"]),
|
|
@@ -208,11 +212,14 @@ def compute_sv(
|
|
|
208
212
|
file_path = Path(source)
|
|
209
213
|
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
210
214
|
|
|
215
|
+
client = None
|
|
211
216
|
console = Console()
|
|
217
|
+
single_file = file_path.is_dir() and source.endswith(".zarr")
|
|
212
218
|
with console.status("Processing...", spinner="dots") as status:
|
|
213
219
|
status.start()
|
|
214
|
-
|
|
215
|
-
|
|
220
|
+
if use_dask or not single_file:
|
|
221
|
+
cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
|
|
222
|
+
client = Client(cluster)
|
|
216
223
|
|
|
217
224
|
try:
|
|
218
225
|
if file_path.is_dir() and source.endswith(".zarr"):
|
|
@@ -230,13 +237,14 @@ def compute_sv(
|
|
|
230
237
|
f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
|
|
231
238
|
f"http://localhost:8787/status for progress")
|
|
232
239
|
from oceanstream.process import process_zarr_files
|
|
233
|
-
|
|
240
|
+
processed_count_var = Variable('processed_count', client)
|
|
234
241
|
process_zarr_files(config_data,
|
|
235
242
|
workers_count=workers_count,
|
|
236
243
|
status=status,
|
|
237
244
|
chunks=config_data.get('base_chunk_sizes'),
|
|
238
245
|
plot_echogram=plot_echogram,
|
|
239
246
|
waveform_mode=waveform_mode,
|
|
247
|
+
processed_count_var=processed_count_var,
|
|
240
248
|
depth_offset=depth_offset)
|
|
241
249
|
else:
|
|
242
250
|
print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
|
|
@@ -245,10 +253,11 @@ def compute_sv(
|
|
|
245
253
|
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
246
254
|
except Exception as e:
|
|
247
255
|
logging.exception("Error while processing %s", config_data['raw_path'])
|
|
248
|
-
|
|
256
|
+
print(Traceback())
|
|
249
257
|
finally:
|
|
250
|
-
|
|
251
|
-
|
|
258
|
+
if use_dask:
|
|
259
|
+
client.close()
|
|
260
|
+
cluster.close()
|
|
252
261
|
status.stop()
|
|
253
262
|
|
|
254
263
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
typer
|
|
@@ -5,6 +5,7 @@ import sys
|
|
|
5
5
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from oceanstream.settings import load_config
|
|
8
|
+
from oceanstream.process import convert_raw_file, convert_raw_files, compute_single_file
|
|
8
9
|
|
|
9
10
|
DEFAULT_OUTPUT_FOLDER = "output"
|
|
10
11
|
DEFAULT_SONAR_MODEL = "EK60"
|
|
@@ -12,10 +13,20 @@ DEFAULT_SONAR_MODEL = "EK60"
|
|
|
12
13
|
logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
def initialize(settings, file_path, log_level=None):
|
|
16
|
+
def initialize(settings, file_path=None, log_level=None, chunks=None):
|
|
16
17
|
logging.debug(f"Initializing with settings: {settings}, file path: {file_path}, log level: {log_level}")
|
|
18
|
+
if "config" not in settings:
|
|
19
|
+
settings["config"] = ""
|
|
20
|
+
|
|
17
21
|
config_data = load_config(settings["config"])
|
|
18
|
-
|
|
22
|
+
|
|
23
|
+
if chunks:
|
|
24
|
+
config_data['chunks'] = chunks
|
|
25
|
+
else:
|
|
26
|
+
config_data['chunks'] = config_data.get('base_chunk_sizes', None)
|
|
27
|
+
|
|
28
|
+
if file_path is not None:
|
|
29
|
+
config_data["raw_path"] = file_path
|
|
19
30
|
|
|
20
31
|
if log_level is not None:
|
|
21
32
|
logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s', force=True)
|
|
@@ -24,9 +35,21 @@ def initialize(settings, file_path, log_level=None):
|
|
|
24
35
|
if 'sonar_model' in settings and settings["sonar_model"] is not None:
|
|
25
36
|
config_data["sonar_model"] = settings["sonar_model"]
|
|
26
37
|
|
|
27
|
-
if settings
|
|
38
|
+
if settings.get('plot_echogram', None) is not None:
|
|
39
|
+
config_data["plot_echogram"] = settings["plot_echogram"]
|
|
40
|
+
|
|
41
|
+
if settings.get('waveform_mode', None) is not None:
|
|
42
|
+
config_data["waveform_mode"] = settings["waveform_mode"]
|
|
43
|
+
|
|
44
|
+
if settings.get('depth_offset', None) is not None:
|
|
45
|
+
config_data["depth_offset"] = settings["depth_offset"]
|
|
46
|
+
|
|
47
|
+
if settings.get("output_folder", None) is not None:
|
|
28
48
|
config_data["output_folder"] = settings["output_folder"]
|
|
29
49
|
|
|
50
|
+
if settings.get('cloud_storage', None) is not None:
|
|
51
|
+
config_data['cloud_storage'] = settings['cloud_storage']
|
|
52
|
+
|
|
30
53
|
return config_data
|
|
31
54
|
|
|
32
55
|
|
|
@@ -57,27 +80,29 @@ def process_raw_file(source, output=None, sonar_model=None, plot_echogram=False,
|
|
|
57
80
|
sys.exit(1)
|
|
58
81
|
|
|
59
82
|
|
|
60
|
-
def convert(source, output=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
|
|
83
|
+
def convert(source, output=None, base_path=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
|
|
61
84
|
logging.debug("Starting convert function")
|
|
62
85
|
settings = {
|
|
63
|
-
"config": config,
|
|
64
86
|
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
65
87
|
}
|
|
66
88
|
|
|
89
|
+
if config is not None:
|
|
90
|
+
settings.update(config)
|
|
91
|
+
|
|
67
92
|
file_path = Path(source)
|
|
68
93
|
config_data = initialize(settings, file_path, log_level=log_level)
|
|
69
94
|
|
|
70
95
|
if chunks:
|
|
71
96
|
config_data['chunks'] = chunks
|
|
97
|
+
else:
|
|
98
|
+
config_data['chunks'] = config_data.get('base_chunk_sizes', None)
|
|
72
99
|
|
|
73
100
|
if file_path.is_file():
|
|
74
101
|
logging.debug(f"Converting raw file: {file_path}")
|
|
75
|
-
|
|
76
|
-
convert_raw_file(file_path, config_data)
|
|
102
|
+
convert_raw_file(file_path, config_data, base_path=base_path)
|
|
77
103
|
logging.info(f"Converted raw file {source} to Zarr and wrote output to: {config_data['output_folder']}")
|
|
78
104
|
elif file_path.is_dir():
|
|
79
105
|
logging.debug(f"Converting raw files in directory: {file_path}")
|
|
80
|
-
from oceanstream.process import convert_raw_files
|
|
81
106
|
convert_raw_files(config_data, workers_count=workers_count)
|
|
82
107
|
else:
|
|
83
108
|
logging.error(f"The provided path '{source}' is not a valid file/folder.")
|
|
@@ -103,8 +128,7 @@ def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
|
|
|
103
128
|
|
|
104
129
|
file_name = f"{Path(dir_path).stem}-combined.zarr"
|
|
105
130
|
zarr_output_file = os.path.join(config_data['output_folder'], file_name)
|
|
106
|
-
logging.info(
|
|
107
|
-
f"Combining Zarr files to {zarr_output_file}; navigate to http://localhost:8787/status for progress")
|
|
131
|
+
logging.info(f"Combining Zarr files to {zarr_output_file}")
|
|
108
132
|
|
|
109
133
|
combine_zarr_files(dir_path, zarr_output_file=zarr_output_file, chunks=chunks)
|
|
110
134
|
logging.info("Zarr files have been combined successfully.")
|
|
@@ -113,30 +137,34 @@ def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
|
|
|
113
137
|
|
|
114
138
|
|
|
115
139
|
def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONAR_MODEL, plot_echogram=False,
|
|
116
|
-
depth_offset=0.0, waveform_mode="CW",
|
|
117
|
-
|
|
140
|
+
depth_offset=0.0, waveform_mode="CW", log_level="WARNING", chunks=None, config=None,
|
|
141
|
+
processed_count_var=None):
|
|
118
142
|
settings_dict = {
|
|
119
|
-
"config": config,
|
|
120
143
|
"sonar_model": sonar_model,
|
|
121
144
|
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
122
145
|
}
|
|
146
|
+
|
|
147
|
+
if config is not None:
|
|
148
|
+
settings_dict.update(config)
|
|
149
|
+
|
|
123
150
|
file_path = Path(source)
|
|
124
151
|
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
125
152
|
|
|
126
153
|
if chunks:
|
|
127
154
|
config_data['chunks'] = chunks
|
|
155
|
+
else:
|
|
156
|
+
config_data['chunks'] = config_data.get('base_chunk_sizes', None)
|
|
128
157
|
|
|
129
158
|
if file_path.is_dir() and source.endswith(".zarr"):
|
|
130
159
|
logging.debug(f"Computing Sv for Zarr root file: {file_path}")
|
|
131
|
-
from oceanstream.process import compute_single_file
|
|
132
|
-
|
|
133
160
|
compute_single_file(config_data, chunks=chunks, plot_echogram=plot_echogram, waveform_mode=waveform_mode,
|
|
134
161
|
depth_offset=depth_offset)
|
|
135
162
|
elif file_path.is_dir():
|
|
136
163
|
logging.debug(f"Processing Zarr files in directory: {file_path}")
|
|
137
164
|
from oceanstream.process import process_zarr_files
|
|
138
165
|
|
|
139
|
-
process_zarr_files(config_data, workers_count=workers_count, chunks=chunks,
|
|
166
|
+
process_zarr_files(config_data, workers_count=workers_count, chunks=chunks,
|
|
167
|
+
processed_count_var=processed_count_var, plot_echogram=plot_echogram,
|
|
140
168
|
waveform_mode=waveform_mode, depth_offset=depth_offset)
|
|
141
169
|
else:
|
|
142
170
|
logging.error(f"The provided path '{source}' is not a valid Zarr root.")
|
{oceanstream-0.0.8.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_dataset_extension.py
RENAMED
|
@@ -96,8 +96,6 @@ def add_depth(Sv: Dataset, depth_offset: float = 0, tilt: float = 0, downward: b
|
|
|
96
96
|
first_channel = Sv["channel"].values[0]
|
|
97
97
|
first_ping_time = Sv["ping_time"].values[0]
|
|
98
98
|
|
|
99
|
-
original_echo_range = Sv["echo_range"].sel(channel=first_channel, ping_time=first_ping_time).values
|
|
100
|
-
|
|
101
99
|
# Slice the echo_range to get the desired range of values
|
|
102
100
|
selected_echo_range = Sv["echo_range"].sel(channel=first_channel, ping_time=first_ping_time)
|
|
103
101
|
selected_echo_range = selected_echo_range.values.tolist()
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import os
|
|
3
|
+
import tempfile
|
|
3
4
|
import xarray as xr
|
|
4
5
|
import numpy as np
|
|
6
|
+
import logging
|
|
7
|
+
|
|
5
8
|
from matplotlib.colors import LinearSegmentedColormap, Colormap
|
|
6
9
|
import matplotlib.pyplot as plt
|
|
7
10
|
import matplotlib.dates as mdates
|
|
@@ -287,18 +290,13 @@ def plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_na
|
|
|
287
290
|
plt.close()
|
|
288
291
|
|
|
289
292
|
|
|
290
|
-
def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name,
|
|
291
|
-
cmap='
|
|
292
|
-
regions2d=None,
|
|
293
|
-
region_ids=None, region_class=None):
|
|
293
|
+
def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, echogram_path=None,
|
|
294
|
+
config_data=None, cmap='ocean_r'):
|
|
294
295
|
"""Plot and save echogram for a single channel with optional regions and enhancements."""
|
|
295
296
|
full_channel_name = ds_Sv.channel.values[channel]
|
|
296
297
|
channel_name = "_".join(full_channel_name.split()[:3])
|
|
297
298
|
|
|
298
299
|
plt.figure(figsize=(30, 18))
|
|
299
|
-
echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
|
|
300
|
-
|
|
301
|
-
# Apply the same preprocessing steps from _plot_echogram
|
|
302
300
|
ds = ds_Sv
|
|
303
301
|
|
|
304
302
|
filtered_ds = ds['Sv']
|
|
@@ -332,7 +330,7 @@ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_na
|
|
|
332
330
|
yincrease=False,
|
|
333
331
|
vmin=-80,
|
|
334
332
|
vmax=-50,
|
|
335
|
-
cmap=
|
|
333
|
+
cmap=cmap,
|
|
336
334
|
cbar_kwargs={'label': 'Volume backscattering strength (Sv re 1 m-1)'}
|
|
337
335
|
)
|
|
338
336
|
|
|
@@ -340,8 +338,20 @@ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_na
|
|
|
340
338
|
plt.xlabel('Ping time', fontsize=14)
|
|
341
339
|
plt.ylabel('Depth', fontsize=14)
|
|
342
340
|
plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
|
|
343
|
-
|
|
344
|
-
|
|
341
|
+
|
|
342
|
+
echogram_file_name = f"{file_base_name}_{channel_name}.png"
|
|
343
|
+
|
|
344
|
+
if config_data and 'cloud_storage' in config_data:
|
|
345
|
+
with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as temp_file:
|
|
346
|
+
plt.savefig(temp_file.name, dpi=300, bbox_inches='tight')
|
|
347
|
+
plt.close()
|
|
348
|
+
echogram_output_path = os.path.join(echogram_path, echogram_file_name)
|
|
349
|
+
upload_to_cloud_storage(temp_file.name, echogram_output_path, config_data['cloud_storage'])
|
|
350
|
+
os.remove(temp_file.name)
|
|
351
|
+
else:
|
|
352
|
+
echogram_output_path = os.path.join(output_path, echogram_file_name)
|
|
353
|
+
plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
|
|
354
|
+
plt.close()
|
|
345
355
|
|
|
346
356
|
|
|
347
357
|
def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=None, client=None):
|
|
@@ -360,15 +370,15 @@ def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=Non
|
|
|
360
370
|
wait(futures)
|
|
361
371
|
|
|
362
372
|
|
|
363
|
-
def plot_sv_data(ds_Sv, file_base_name=None, output_path=None,
|
|
364
|
-
region_class=None):
|
|
373
|
+
def plot_sv_data(ds_Sv, file_base_name=None, output_path=None, echogram_path=None, config_data=None, cmap=None):
|
|
365
374
|
"""Plot the echogram data and the regions."""
|
|
366
375
|
if not plt.isinteractive():
|
|
367
376
|
plt.switch_backend('Agg')
|
|
368
377
|
|
|
369
378
|
for channel in range(ds_Sv.dims['channel']):
|
|
370
|
-
plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name,
|
|
371
|
-
|
|
379
|
+
plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, echogram_path=echogram_path,
|
|
380
|
+
config_data=config_data,
|
|
381
|
+
cmap='ocean_r')
|
|
372
382
|
# plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_name, cmap)
|
|
373
383
|
# plot_individual_channel_shaders(ds_Sv=ds_Sv, channel=channel, output_path=output_path,
|
|
374
384
|
# file_base_name=file_base_name, cmap='ocean_r')
|
|
@@ -540,3 +550,24 @@ def _plot_region_ids(colors, ds_Sv, idx, labels_added, region_ids, regions2d):
|
|
|
540
550
|
labels_added.add(label)
|
|
541
551
|
idx += 1
|
|
542
552
|
return idx
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def upload_to_cloud_storage(local_path, remote_path, cloud_storage_config):
|
|
556
|
+
storage_type = cloud_storage_config['storage_type']
|
|
557
|
+
container_name = cloud_storage_config['container_name']
|
|
558
|
+
storage_options = cloud_storage_config['storage_options']
|
|
559
|
+
|
|
560
|
+
if storage_type == 'azure':
|
|
561
|
+
upload_to_azure_blob(local_path, remote_path, container_name, storage_options)
|
|
562
|
+
else:
|
|
563
|
+
raise ValueError(f"Unsupported storage type: {storage_type}")
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def upload_to_azure_blob(local_path, remote_path, container_name, storage_options):
|
|
567
|
+
from azure.storage.blob import BlobServiceClient
|
|
568
|
+
blob_service_client = BlobServiceClient.from_connection_string(storage_options['connection_string'])
|
|
569
|
+
blob_client = blob_service_client.get_blob_client(container=container_name, blob=remote_path)
|
|
570
|
+
|
|
571
|
+
with open(local_path, "rb") as data:
|
|
572
|
+
blob_client.upload_blob(data, overwrite=True)
|
|
573
|
+
logging.info(f"Uploaded {local_path} to Azure Blob Storage as {remote_path}")
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from .s3 import process_survey_data_with_progress, list_raw_files_from_bucket, convert_survey_data_from_bucket, \
|
|
2
|
+
download_file_from_bucket
|
|
3
|
+
|
|
4
|
+
__all__ = [
|
|
5
|
+
"process_survey_data_with_progress",
|
|
6
|
+
"list_raw_files_from_bucket",
|
|
7
|
+
"convert_survey_data_from_bucket",
|
|
8
|
+
"download_file_from_bucket"
|
|
9
|
+
]
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tempfile
|
|
3
|
+
import botocore
|
|
4
|
+
import boto3
|
|
5
|
+
from asyncio import CancelledError
|
|
6
|
+
from rich import print
|
|
7
|
+
from rich.traceback import install, Traceback
|
|
8
|
+
from tqdm.auto import tqdm
|
|
9
|
+
from dask import delayed
|
|
10
|
+
from botocore.config import Config
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
install(show_locals=False, width=120)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def process_survey_data_with_progress(files_by_day, bucket_name, client, config_data, process_func):
|
|
17
|
+
try:
|
|
18
|
+
total_files = sum(len(files) for files in files_by_day.values())
|
|
19
|
+
progress_bar = tqdm(total=total_files, desc="Processing Files", unit="file", ncols=100)
|
|
20
|
+
|
|
21
|
+
def update_progress(*args):
|
|
22
|
+
progress_bar.update()
|
|
23
|
+
|
|
24
|
+
temp_dir = tempfile.mkdtemp()
|
|
25
|
+
futures, temp_dir = convert_survey_data_from_bucket(files_by_day, temp_dir, bucket_name, client, config_data,
|
|
26
|
+
process_func)
|
|
27
|
+
|
|
28
|
+
for future in futures:
|
|
29
|
+
future.add_done_callback(update_progress)
|
|
30
|
+
|
|
31
|
+
client.gather(futures) # Ensure all tasks complete
|
|
32
|
+
|
|
33
|
+
progress_bar.close()
|
|
34
|
+
os.rmdir(temp_dir)
|
|
35
|
+
except KeyboardInterrupt:
|
|
36
|
+
print("Closing down.")
|
|
37
|
+
except CancelledError:
|
|
38
|
+
print("Closing down.")
|
|
39
|
+
except Exception as e:
|
|
40
|
+
print(f"[bold red]An error occurred:[/bold red] {e}")
|
|
41
|
+
print(f"{Traceback()}\n")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def convert_survey_data_from_bucket(files_by_day, temp_dir, bucket_name, dask_client, config_data, process_func):
|
|
45
|
+
"""Process survey data from S3."""
|
|
46
|
+
tasks = []
|
|
47
|
+
|
|
48
|
+
for day, files in files_by_day.items():
|
|
49
|
+
for file in files:
|
|
50
|
+
task = delayed(_process_raw_file)(file, day, temp_dir, bucket_name, config_data, process_func)
|
|
51
|
+
tasks.append(task)
|
|
52
|
+
|
|
53
|
+
# Execute all tasks in parallel
|
|
54
|
+
futures = dask_client.compute(tasks)
|
|
55
|
+
|
|
56
|
+
return futures, temp_dir
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def list_raw_files_from_bucket(bucket_name, prefix):
|
|
60
|
+
"""List files in the S3 bucket along with their metadata."""
|
|
61
|
+
s3_client = boto3.client('s3', config=Config(signature_version=botocore.UNSIGNED))
|
|
62
|
+
response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
|
|
63
|
+
|
|
64
|
+
files = [
|
|
65
|
+
{
|
|
66
|
+
'Key': item['Key'],
|
|
67
|
+
'Size': item['Size'],
|
|
68
|
+
'LastModified': item['LastModified']
|
|
69
|
+
}
|
|
70
|
+
for item in response.get('Contents', []) if item['Key'].endswith('.raw')
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
return files
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def download_file_from_bucket(bucket_name, s3_key, local_dir):
|
|
77
|
+
"""Download a file from S3."""
|
|
78
|
+
s3_client = boto3.client('s3', config=Config(signature_version=botocore.UNSIGNED))
|
|
79
|
+
local_path = os.path.join(local_dir, os.path.basename(s3_key))
|
|
80
|
+
s3_client.download_file(bucket_name, s3_key, local_path)
|
|
81
|
+
|
|
82
|
+
return local_path
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _process_raw_file(file, day, temp_dir, bucket_name, config_data, process_func):
|
|
86
|
+
"""Process a single file."""
|
|
87
|
+
day_dir = os.path.join(temp_dir, day)
|
|
88
|
+
os.makedirs(day_dir, exist_ok=True)
|
|
89
|
+
local_path = download_file_from_bucket(bucket_name, file['Key'], day_dir)
|
|
90
|
+
|
|
91
|
+
process_func(local_path, config=config_data, base_path=temp_dir)
|
|
92
|
+
os.remove(local_path)
|
|
93
|
+
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
from asyncio import CancelledError
|
|
4
|
+
from rich import print
|
|
5
|
+
from rich.traceback import install, Traceback
|
|
6
|
+
from tqdm.auto import tqdm
|
|
7
|
+
from dask import delayed
|
|
8
|
+
from .blob_storage import open_zarr_store
|
|
9
|
+
|
|
10
|
+
install(show_locals=False, width=120)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def process_survey_data_with_progress(files_by_day, azfs, container_name, client, config_data, process_func):
|
|
14
|
+
try:
|
|
15
|
+
total_files = sum(len(files) for files in files_by_day.values())
|
|
16
|
+
progress_bar = tqdm(total=total_files, desc="Processing Files", unit="file", ncols=100)
|
|
17
|
+
|
|
18
|
+
def update_progress(*args):
|
|
19
|
+
progress_bar.update()
|
|
20
|
+
|
|
21
|
+
futures = process_survey_data(files_by_day, azfs, container_name, client, config_data, process_func)
|
|
22
|
+
|
|
23
|
+
for future in futures:
|
|
24
|
+
future.add_done_callback(update_progress)
|
|
25
|
+
|
|
26
|
+
client.gather(futures) # Ensure all tasks complete
|
|
27
|
+
|
|
28
|
+
progress_bar.close()
|
|
29
|
+
except KeyboardInterrupt:
|
|
30
|
+
print("Closing down.")
|
|
31
|
+
except CancelledError:
|
|
32
|
+
print("Closing down.")
|
|
33
|
+
except Exception as e:
|
|
34
|
+
print(f"[bold red]An error occurred:[/bold red] {e}")
|
|
35
|
+
print(f"{Traceback()}\n")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def process_survey_data(files_by_day, azfs, container_name, dask_client, config_data, process_func):
|
|
39
|
+
"""Process survey data from S3."""
|
|
40
|
+
|
|
41
|
+
tasks = []
|
|
42
|
+
|
|
43
|
+
for day, files in files_by_day.items():
|
|
44
|
+
for file in files:
|
|
45
|
+
task = delayed(_process_zarr_file)(file['Key'], azfs, container_name, config_data, process_func)
|
|
46
|
+
tasks.append(task)
|
|
47
|
+
|
|
48
|
+
# Execute all tasks in parallel
|
|
49
|
+
futures = dask_client.compute(tasks)
|
|
50
|
+
|
|
51
|
+
return futures
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _process_zarr_file(file, azfs, container_name, config_data, process_func=None):
|
|
55
|
+
"""Process a single Zarr file."""
|
|
56
|
+
base_path = file.replace('.zarr', '')
|
|
57
|
+
pattern = rf"^{re.escape(container_name)}/"
|
|
58
|
+
base_path = re.sub(pattern, '', base_path)
|
|
59
|
+
|
|
60
|
+
echodata = open_zarr_store(azfs, file, chunks=config_data['chunks'])
|
|
61
|
+
|
|
62
|
+
process_func(echodata, config_data, base_path=base_path,
|
|
63
|
+
chunks=config_data.get('chunks'),
|
|
64
|
+
plot_echogram=config_data.get('plot_echogram', False),
|
|
65
|
+
waveform_mode=config_data.get('waveform_mode', "CW"),
|
|
66
|
+
depth_offset=config_data.get('depth_offset', 0.0))
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import echopype as ep
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def list_zarr_files(azfs, path):
|
|
5
|
+
"""List all Zarr files in the Azure Blob Storage container along with their metadata."""
|
|
6
|
+
zarr_files = []
|
|
7
|
+
for blob in azfs.ls(path, detail=True):
|
|
8
|
+
if blob['type'] == 'directory' and not blob['name'].endswith('.zarr'):
|
|
9
|
+
subdir_files = list_zarr_files(azfs, blob['name'])
|
|
10
|
+
zarr_files.extend(subdir_files)
|
|
11
|
+
elif blob['name'].endswith('.zarr'):
|
|
12
|
+
zarr_files.append({
|
|
13
|
+
'Key': blob['name'],
|
|
14
|
+
'Size': blob['size'] if blob['size'] else 0,
|
|
15
|
+
'LastModified': blob['last_modified'] if 'last_modified' in blob else 0
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
return zarr_files
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def open_zarr_store(azfs, store_name, chunks=None):
|
|
22
|
+
"""Open a Zarr store from Azure Blob Storage."""
|
|
23
|
+
mapper = azfs.get_mapper(store_name)
|
|
24
|
+
|
|
25
|
+
return ep.open_converted(mapper, chunks=chunks)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _list_zarr_files_extended(azfs, path):
|
|
29
|
+
"""List all Zarr files in the Azure Blob Storage container along with their metadata."""
|
|
30
|
+
zarr_files = []
|
|
31
|
+
for blob in azfs.ls(path, detail=True):
|
|
32
|
+
if blob['type'] == 'directory' and not blob['name'].endswith('.zarr'):
|
|
33
|
+
subdir_files = list_zarr_files(azfs, blob['name'])
|
|
34
|
+
zarr_files.extend(subdir_files)
|
|
35
|
+
else:
|
|
36
|
+
# Calculate the total size and most recent modification date for the .zarr folder
|
|
37
|
+
total_size = 0
|
|
38
|
+
last_modified = None
|
|
39
|
+
for sub_blob in azfs.ls(blob['name'], detail=True):
|
|
40
|
+
if sub_blob['type'] == 'file':
|
|
41
|
+
total_size += sub_blob['size']
|
|
42
|
+
if last_modified is None or sub_blob['last_modified'] > last_modified:
|
|
43
|
+
last_modified = sub_blob['last_modified']
|
|
44
|
+
|
|
45
|
+
zarr_files.append({
|
|
46
|
+
'Key': blob['name'],
|
|
47
|
+
'Size': total_size,
|
|
48
|
+
'LastModified': last_modified
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
return zarr_files
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
|
|
@@ -5,7 +5,6 @@ import time
|
|
|
5
5
|
from datetime import datetime
|
|
6
6
|
from rich import print
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
import psutil
|
|
9
8
|
from oceanstream.echodata import check_reversed_time, fix_time_reversions
|
|
10
9
|
|
|
11
10
|
|
|
@@ -21,7 +20,7 @@ def read_zarr_files(input_folder):
|
|
|
21
20
|
logging.error("Input folder does not exist: %s", input_folder)
|
|
22
21
|
return
|
|
23
22
|
|
|
24
|
-
zarr_files = list(input_path.
|
|
23
|
+
zarr_files = list(input_path.rglob("*.zarr"))
|
|
25
24
|
if not zarr_files:
|
|
26
25
|
logging.error("No .zarr files found in directory: %s", input_folder)
|
|
27
26
|
return
|
|
@@ -47,7 +46,7 @@ def fix_time(ed):
|
|
|
47
46
|
return ed
|
|
48
47
|
|
|
49
48
|
|
|
50
|
-
def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None
|
|
49
|
+
def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None):
|
|
51
50
|
start_time = time.time()
|
|
52
51
|
|
|
53
52
|
logging.debug("Starting to combine Zarr files from folder: %s", input_folder)
|
|
@@ -87,4 +86,4 @@ def from_filename(file_name):
|
|
|
87
86
|
creation_time = datetime.strptime(date_str + time_str, '%Y%m%d%H%M%S')
|
|
88
87
|
return creation_time
|
|
89
88
|
|
|
90
|
-
return None
|
|
89
|
+
return None
|