oceanstream 0.0.10.dev0__tar.gz → 0.0.14.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/PKG-INFO +2 -1
- oceanstream-0.0.14.dev0/oceanstream/__init__.py +5 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/cli/main.py +113 -28
- oceanstream-0.0.14.dev0/oceanstream/cli/requirements.txt +1 -0
- oceanstream-0.0.14.dev0/oceanstream/convert/__init__.py +5 -0
- oceanstream-0.0.14.dev0/oceanstream/convert/convert_raw_file.py +58 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/core.py +47 -31
- oceanstream-0.0.14.dev0/oceanstream/echodata/dataset.py +586 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/sv_computation.py +8 -1
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/sv_dataset_extension.py +18 -19
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/sv_interpolation.py +9 -10
- oceanstream-0.0.14.dev0/oceanstream/exports/__init__.py +6 -0
- oceanstream-0.0.14.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +183 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/plot/echogram.py +16 -10
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/__init__.py +5 -3
- oceanstream-0.0.14.dev0/oceanstream/process/aws/__init__.py +9 -0
- oceanstream-0.0.14.dev0/oceanstream/process/aws/s3.py +94 -0
- oceanstream-0.0.14.dev0/oceanstream/process/azure/__init__.py +8 -0
- oceanstream-0.0.14.dev0/oceanstream/process/azure/azure_processor.py +66 -0
- oceanstream-0.0.14.dev0/oceanstream/process/azure/blob_storage.py +85 -0
- oceanstream-0.0.14.dev0/oceanstream/process/cloud.py +21 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/combine_zarr.py +1 -1
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/file_processor.py +128 -84
- oceanstream-0.0.14.dev0/oceanstream/process/file_processor_class.py +217 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/folder_processor.py +175 -52
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/process.py +5 -1
- oceanstream-0.0.14.dev0/oceanstream/process/requirements.txt +0 -0
- oceanstream-0.0.14.dev0/oceanstream/process/utils.py +226 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/PKG-INFO +2 -1
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/SOURCES.txt +11 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/requires.txt +8 -4
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/requirements.txt +6 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/setup.cfg +2 -3
- oceanstream-0.0.10.dev0/oceanstream/__init__.py +0 -3
- oceanstream-0.0.10.dev0/oceanstream/cli/requirements.txt +0 -2
- oceanstream-0.0.10.dev0/oceanstream/echodata/requirements.txt +0 -1
- oceanstream-0.0.10.dev0/oceanstream/exports/__init__.py +0 -2
- oceanstream-0.0.10.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +0 -89
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/.flake8 +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/.gitattributes +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/.gitignore +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/CODE_OF_CONDUCT.md +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/LICENSE +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/MANIFEST.in +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/README.md +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/notebooks/README.md +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/cli/__init__.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/__init__.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/apply_masks.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/applying_masks_handler.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/background_noise_remover.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/create_masks.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/noise_masks.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/requirements.txt +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/types.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/__init__.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/ensure_time_continuity.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/mvbs_computation.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/raw_handler.py +0 -0
- {oceanstream-0.0.10.dev0/oceanstream/process → oceanstream-0.0.14.dev0/oceanstream/echodata}/requirements.txt +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/target_strength_computation.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/csv/csv_export_from_raw.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/csv/csv_export_nasc.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/frequency_differencing_handler.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/nasc.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/nasc_computation.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/requirements.txt +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/shoals/shoal_detection_handler.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/shoals/shoal_process.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/shoals/shoals_handler.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/plot/__init__.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/plot/requirements.txt +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/processed_data_io.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/settings/__init__.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/settings/config.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/settings/defaults.config.json +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/utils.py +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/dependency_links.txt +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/entry_points.txt +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/top_level.txt +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/pyproject.toml +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/requirements-dev.txt +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/requirements-local.txt +0 -0
- {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: oceanstream
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.14.dev0
|
|
4
4
|
Summary: OceanStream: process raw sonar data at scale
|
|
5
5
|
Home-page: https://oceanstream.io
|
|
6
6
|
Author: Pineview Labs
|
|
@@ -20,6 +20,7 @@ Provides-Extra: echodata
|
|
|
20
20
|
Provides-Extra: plot
|
|
21
21
|
Provides-Extra: denoise
|
|
22
22
|
Provides-Extra: process
|
|
23
|
+
Provides-Extra: convert
|
|
23
24
|
Provides-Extra: exports
|
|
24
25
|
Provides-Extra: complete
|
|
25
26
|
License-File: LICENSE
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import traceback
|
|
2
|
+
from asyncio import CancelledError
|
|
2
3
|
|
|
3
4
|
import typer
|
|
4
5
|
import asyncio
|
|
@@ -6,6 +7,7 @@ import os
|
|
|
6
7
|
import logging
|
|
7
8
|
import sys
|
|
8
9
|
import warnings
|
|
10
|
+
import dask
|
|
9
11
|
|
|
10
12
|
from pathlib import Path
|
|
11
13
|
from rich import print
|
|
@@ -13,7 +15,7 @@ from rich.traceback import install, Traceback
|
|
|
13
15
|
from oceanstream.settings import load_config
|
|
14
16
|
from dask.distributed import LocalCluster, Client, Variable
|
|
15
17
|
from rich.console import Console
|
|
16
|
-
|
|
18
|
+
from oceanstream.process import compute_and_export_single_file, process_zarr_files
|
|
17
19
|
|
|
18
20
|
install(show_locals=False, width=120)
|
|
19
21
|
|
|
@@ -32,6 +34,12 @@ BANNER = """
|
|
|
32
34
|
|
|
33
35
|
logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
|
|
34
36
|
|
|
37
|
+
dask.config.set({
|
|
38
|
+
'distributed.comm.timeouts.connect': '60s', # Increase the connection timeout
|
|
39
|
+
'distributed.comm.timeouts.tcp': '120s', # Increase the TCP timeout
|
|
40
|
+
'distributed.comm.retry.count': 0
|
|
41
|
+
})
|
|
42
|
+
|
|
35
43
|
|
|
36
44
|
def initialize(settings, file_path, log_level=None):
|
|
37
45
|
config_data = load_config(settings["config"])
|
|
@@ -97,6 +105,8 @@ def convert(
|
|
|
97
105
|
output: str = typer.Option(None,
|
|
98
106
|
help="Destination path for saving Zarr converted data. Defaults to a predefined "
|
|
99
107
|
"directory if not specified."),
|
|
108
|
+
sonar_model: str = typer.Option(None, help="Sonar model used to collect the data",
|
|
109
|
+
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
100
110
|
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
|
|
101
111
|
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
102
112
|
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
@@ -107,6 +117,7 @@ def convert(
|
|
|
107
117
|
"""
|
|
108
118
|
settings = {
|
|
109
119
|
"config": config,
|
|
120
|
+
"sonar_model": sonar_model,
|
|
110
121
|
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
111
122
|
}
|
|
112
123
|
|
|
@@ -115,9 +126,10 @@ def convert(
|
|
|
115
126
|
|
|
116
127
|
try:
|
|
117
128
|
if filePath.is_file():
|
|
118
|
-
from oceanstream.
|
|
129
|
+
from oceanstream.convert import convert_raw_file
|
|
130
|
+
print(f"[blue]Converting raw file {source} to Zarr...[/blue]")
|
|
119
131
|
convert_raw_file(filePath, configData)
|
|
120
|
-
print(
|
|
132
|
+
print("✅ The file has been converted successfully.")
|
|
121
133
|
elif filePath.is_dir():
|
|
122
134
|
from oceanstream.process import convert_raw_files
|
|
123
135
|
convert_raw_files(configData, workers_count=workers_count)
|
|
@@ -126,8 +138,12 @@ def convert(
|
|
|
126
138
|
except KeyboardInterrupt:
|
|
127
139
|
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
128
140
|
except Exception as e:
|
|
129
|
-
|
|
130
|
-
|
|
141
|
+
if filePath.is_file():
|
|
142
|
+
logging.error("Error processing file %s", filePath)
|
|
143
|
+
else:
|
|
144
|
+
logging.exception("Error processing folder %s", configData['raw_path'])
|
|
145
|
+
|
|
146
|
+
logging.error(traceback.format_exc())
|
|
131
147
|
|
|
132
148
|
|
|
133
149
|
@app.command()
|
|
@@ -195,10 +211,12 @@ def compute_sv(
|
|
|
195
211
|
sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
|
|
196
212
|
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
197
213
|
plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
|
|
198
|
-
use_dask: bool = typer.Option(False,
|
|
214
|
+
use_dask: bool = typer.Option(False,
|
|
215
|
+
help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
|
|
199
216
|
depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
|
|
200
217
|
waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
|
|
201
218
|
show_choices=["CW", "BB"]),
|
|
219
|
+
encode_mode: str = typer.Option("power", help="Encode mode, can be either power or complex", show_choices=["power", "complex"]),
|
|
202
220
|
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
203
221
|
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
204
222
|
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
@@ -211,6 +229,88 @@ def compute_sv(
|
|
|
211
229
|
"sonar_model": sonar_model,
|
|
212
230
|
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
213
231
|
}
|
|
232
|
+
dask.config.set({'distributed.comm.retry.count': 1})
|
|
233
|
+
|
|
234
|
+
file_path = Path(source)
|
|
235
|
+
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
236
|
+
single_file = file_path.is_dir() and source.endswith(".zarr")
|
|
237
|
+
|
|
238
|
+
client = None
|
|
239
|
+
cluster = None
|
|
240
|
+
|
|
241
|
+
if use_dask or not single_file:
|
|
242
|
+
cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
|
|
243
|
+
client = Client(cluster)
|
|
244
|
+
|
|
245
|
+
try:
|
|
246
|
+
if file_path.is_dir() and source.endswith(".zarr"):
|
|
247
|
+
console = Console()
|
|
248
|
+
with console.status("Processing...", spinner="dots") as status:
|
|
249
|
+
status.start()
|
|
250
|
+
status.update(
|
|
251
|
+
f"[blue] Computing Sv for {file_path}...[/blue]" + use_dask * "– navigate to "
|
|
252
|
+
"http://localhost:8787/status for "
|
|
253
|
+
"progress")
|
|
254
|
+
|
|
255
|
+
chunks = None
|
|
256
|
+
if use_dask:
|
|
257
|
+
chunks = config_data.get('base_chunk_sizes')
|
|
258
|
+
|
|
259
|
+
compute_and_export_single_file(config_data,
|
|
260
|
+
chunks=chunks,
|
|
261
|
+
plot_echogram=plot_echogram,
|
|
262
|
+
waveform_mode=waveform_mode,
|
|
263
|
+
encode_mode=encode_mode,
|
|
264
|
+
depth_offset=depth_offset)
|
|
265
|
+
|
|
266
|
+
status.stop()
|
|
267
|
+
print("✅ The file have been processed successfully.")
|
|
268
|
+
elif file_path.is_dir():
|
|
269
|
+
print(f"Dashboard available at {client.dashboard_link}")
|
|
270
|
+
process_zarr_files(config_data,
|
|
271
|
+
client,
|
|
272
|
+
workers_count=workers_count,
|
|
273
|
+
chunks=config_data.get('base_chunk_sizes'),
|
|
274
|
+
plot_echogram=plot_echogram,
|
|
275
|
+
waveform_mode=waveform_mode,
|
|
276
|
+
depth_offset=depth_offset)
|
|
277
|
+
else:
|
|
278
|
+
print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
|
|
279
|
+
sys.exit(1)
|
|
280
|
+
except KeyboardInterrupt:
|
|
281
|
+
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
282
|
+
except Exception as e:
|
|
283
|
+
logging.exception("Error while processing %s", config_data['raw_path'])
|
|
284
|
+
print(Traceback())
|
|
285
|
+
finally:
|
|
286
|
+
if client is not None:
|
|
287
|
+
client.close()
|
|
288
|
+
|
|
289
|
+
if cluster is not None:
|
|
290
|
+
cluster.close()
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
@app.command()
|
|
294
|
+
def export_location(
|
|
295
|
+
source: str = typer.Option(..., help="Path to a Zarr root file or a directory containing Zarr files"),
|
|
296
|
+
output: str = typer.Option(None,
|
|
297
|
+
help="Destination path for saving the exported data. Defaults to a predefined "
|
|
298
|
+
"directory if not specified."),
|
|
299
|
+
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for parallel processing"),
|
|
300
|
+
use_dask: bool = typer.Option(False,
|
|
301
|
+
help="Start a Local Dask cluster for parallel processing (always enabled for "
|
|
302
|
+
"multiple files)"),
|
|
303
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
304
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
305
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
306
|
+
):
|
|
307
|
+
"""
|
|
308
|
+
Given a Zarr dataset containing Sv data, exports the GPS location data to a JSON file.
|
|
309
|
+
"""
|
|
310
|
+
settings_dict = {
|
|
311
|
+
"config": config,
|
|
312
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
313
|
+
}
|
|
214
314
|
file_path = Path(source)
|
|
215
315
|
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
216
316
|
|
|
@@ -227,27 +327,17 @@ def compute_sv(
|
|
|
227
327
|
if file_path.is_dir() and source.endswith(".zarr"):
|
|
228
328
|
status.update(
|
|
229
329
|
f"[blue] Computing Sv for {file_path}...[/blue] – navigate to http://localhost:8787/status for progress")
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
compute_single_file(config_data,
|
|
233
|
-
chunks=config_data.get('base_chunk_sizes'),
|
|
234
|
-
plot_echogram=plot_echogram,
|
|
235
|
-
waveform_mode=waveform_mode,
|
|
236
|
-
depth_offset=depth_offset)
|
|
330
|
+
# TODO: Implement export_location_json
|
|
237
331
|
elif file_path.is_dir():
|
|
238
332
|
status.update(
|
|
239
333
|
f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
|
|
240
334
|
f"http://localhost:8787/status for progress")
|
|
241
|
-
from oceanstream.process import
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
plot_echogram=plot_echogram,
|
|
248
|
-
waveform_mode=waveform_mode,
|
|
249
|
-
processed_count_var=processed_count_var,
|
|
250
|
-
depth_offset=depth_offset)
|
|
335
|
+
from oceanstream.process import export_location_from_zarr_files
|
|
336
|
+
|
|
337
|
+
export_location_from_zarr_files(config_data,
|
|
338
|
+
workers_count=workers_count,
|
|
339
|
+
client=client,
|
|
340
|
+
chunks=config_data.get('base_chunk_sizes'))
|
|
251
341
|
else:
|
|
252
342
|
print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
|
|
253
343
|
sys.exit(1)
|
|
@@ -263,11 +353,6 @@ def compute_sv(
|
|
|
263
353
|
status.stop()
|
|
264
354
|
|
|
265
355
|
|
|
266
|
-
@app.command()
|
|
267
|
-
def export():
|
|
268
|
-
typer.echo("Export data...")
|
|
269
|
-
|
|
270
|
-
|
|
271
356
|
def main():
|
|
272
357
|
print(BANNER)
|
|
273
358
|
warnings.filterwarnings("ignore", category=UserWarning)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
typer
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
import traceback
|
|
7
|
+
import shutil
|
|
8
|
+
from asyncio import CancelledError
|
|
9
|
+
import echopype as ep
|
|
10
|
+
|
|
11
|
+
import xarray as xr
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from oceanstream.echodata import read_file
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def convert_raw_file(file_path, config_data, base_path=None, progress_counter=None, counter_lock=None):
|
|
17
|
+
logging.debug("Starting processing of file: %s", file_path)
|
|
18
|
+
|
|
19
|
+
file_path_obj = Path(file_path)
|
|
20
|
+
file_config_data = {**config_data, 'raw_path': file_path_obj}
|
|
21
|
+
|
|
22
|
+
if base_path:
|
|
23
|
+
relative_path = file_path_obj.relative_to(base_path)
|
|
24
|
+
relative_path = relative_path.parent
|
|
25
|
+
else:
|
|
26
|
+
relative_path = None
|
|
27
|
+
|
|
28
|
+
echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
|
|
29
|
+
file_name = file_path_obj.stem + ".zarr"
|
|
30
|
+
|
|
31
|
+
if 'cloud_storage' in config_data:
|
|
32
|
+
from oceanstream.process.cloud import get_chunk_store
|
|
33
|
+
|
|
34
|
+
if relative_path:
|
|
35
|
+
file_location = Path(relative_path) / file_name
|
|
36
|
+
else:
|
|
37
|
+
file_location = file_name
|
|
38
|
+
store = get_chunk_store(config_data['cloud_storage'], file_location)
|
|
39
|
+
echodata.to_zarr(save_path=store, overwrite=True, parallel=False)
|
|
40
|
+
|
|
41
|
+
output_dest = config_data['cloud_storage']['container_name'] + "/" + file_location
|
|
42
|
+
else:
|
|
43
|
+
if relative_path:
|
|
44
|
+
output_path = Path(config_data["output_folder"]) / relative_path
|
|
45
|
+
output_path.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
else:
|
|
47
|
+
output_path = Path(config_data["output_folder"])
|
|
48
|
+
|
|
49
|
+
output_dest = output_path / file_name
|
|
50
|
+
echodata.to_zarr(save_path=output_dest, overwrite=True, parallel=False)
|
|
51
|
+
|
|
52
|
+
if counter_lock is not None:
|
|
53
|
+
with counter_lock:
|
|
54
|
+
progress_counter.value += 1
|
|
55
|
+
|
|
56
|
+
logging.debug("Finished processing of file: %s", file_path)
|
|
57
|
+
|
|
58
|
+
return output_dest
|
|
@@ -5,6 +5,7 @@ import sys
|
|
|
5
5
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from oceanstream.settings import load_config
|
|
8
|
+
from oceanstream.process import convert_raw_files, compute_single_file
|
|
8
9
|
|
|
9
10
|
DEFAULT_OUTPUT_FOLDER = "output"
|
|
10
11
|
DEFAULT_SONAR_MODEL = "EK60"
|
|
@@ -12,13 +13,20 @@ DEFAULT_SONAR_MODEL = "EK60"
|
|
|
12
13
|
logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
def initialize(settings, file_path, log_level=None):
|
|
16
|
+
def initialize(settings, file_path=None, log_level=None, chunks=None):
|
|
16
17
|
logging.debug(f"Initializing with settings: {settings}, file path: {file_path}, log level: {log_level}")
|
|
17
18
|
if "config" not in settings:
|
|
18
19
|
settings["config"] = ""
|
|
19
20
|
|
|
20
21
|
config_data = load_config(settings["config"])
|
|
21
|
-
|
|
22
|
+
|
|
23
|
+
if chunks:
|
|
24
|
+
config_data['chunks'] = chunks
|
|
25
|
+
else:
|
|
26
|
+
config_data['chunks'] = config_data.get('base_chunk_sizes', None)
|
|
27
|
+
|
|
28
|
+
if file_path is not None:
|
|
29
|
+
config_data["raw_path"] = file_path
|
|
22
30
|
|
|
23
31
|
if log_level is not None:
|
|
24
32
|
logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s', force=True)
|
|
@@ -27,10 +35,19 @@ def initialize(settings, file_path, log_level=None):
|
|
|
27
35
|
if 'sonar_model' in settings and settings["sonar_model"] is not None:
|
|
28
36
|
config_data["sonar_model"] = settings["sonar_model"]
|
|
29
37
|
|
|
30
|
-
if settings
|
|
38
|
+
if settings.get('plot_echogram', None) is not None:
|
|
39
|
+
config_data["plot_echogram"] = settings["plot_echogram"]
|
|
40
|
+
|
|
41
|
+
if settings.get('waveform_mode', None) is not None:
|
|
42
|
+
config_data["waveform_mode"] = settings["waveform_mode"]
|
|
43
|
+
|
|
44
|
+
if settings.get('depth_offset', None) is not None:
|
|
45
|
+
config_data["depth_offset"] = settings["depth_offset"]
|
|
46
|
+
|
|
47
|
+
if settings.get("output_folder", None) is not None:
|
|
31
48
|
config_data["output_folder"] = settings["output_folder"]
|
|
32
49
|
|
|
33
|
-
if settings
|
|
50
|
+
if settings.get('cloud_storage', None) is not None:
|
|
34
51
|
config_data['cloud_storage'] = settings['cloud_storage']
|
|
35
52
|
|
|
36
53
|
return config_data
|
|
@@ -63,30 +80,32 @@ def process_raw_file(source, output=None, sonar_model=None, plot_echogram=False,
|
|
|
63
80
|
sys.exit(1)
|
|
64
81
|
|
|
65
82
|
|
|
66
|
-
def convert(source, output=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
83
|
+
# def convert(source, output=None, base_path=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
|
|
84
|
+
# logging.debug("Starting convert function")
|
|
85
|
+
# settings = {
|
|
86
|
+
# "output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
87
|
+
# }
|
|
88
|
+
#
|
|
89
|
+
# if config is not None:
|
|
90
|
+
# settings.update(config)
|
|
91
|
+
#
|
|
92
|
+
# file_path = Path(source)
|
|
93
|
+
# config_data = initialize(settings, file_path, log_level=log_level)
|
|
94
|
+
#
|
|
95
|
+
# if chunks:
|
|
96
|
+
# config_data['chunks'] = chunks
|
|
97
|
+
# else:
|
|
98
|
+
# config_data['chunks'] = config_data.get('base_chunk_sizes', None)
|
|
99
|
+
#
|
|
100
|
+
# if file_path.is_file():
|
|
101
|
+
# logging.debug(f"Converting raw file: {file_path}")
|
|
102
|
+
# convert_raw_file(file_path, config_data, base_path=base_path)
|
|
103
|
+
# logging.info(f"Converted raw file {source} to Zarr and wrote output to: {config_data['output_folder']}")
|
|
104
|
+
# elif file_path.is_dir():
|
|
105
|
+
# logging.debug(f"Converting raw files in directory: {file_path}")
|
|
106
|
+
# convert_raw_files(config_data, workers_count=workers_count)
|
|
107
|
+
# else:
|
|
108
|
+
# logging.error(f"The provided path '{source}' is not a valid file/folder.")
|
|
90
109
|
|
|
91
110
|
|
|
92
111
|
def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
|
|
@@ -127,7 +146,6 @@ def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONA
|
|
|
127
146
|
|
|
128
147
|
if config is not None:
|
|
129
148
|
settings_dict.update(config)
|
|
130
|
-
# settings_dict["config"] = ''
|
|
131
149
|
|
|
132
150
|
file_path = Path(source)
|
|
133
151
|
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
@@ -139,8 +157,6 @@ def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONA
|
|
|
139
157
|
|
|
140
158
|
if file_path.is_dir() and source.endswith(".zarr"):
|
|
141
159
|
logging.debug(f"Computing Sv for Zarr root file: {file_path}")
|
|
142
|
-
from oceanstream.process import compute_single_file
|
|
143
|
-
|
|
144
160
|
compute_single_file(config_data, chunks=chunks, plot_echogram=plot_echogram, waveform_mode=waveform_mode,
|
|
145
161
|
depth_offset=depth_offset)
|
|
146
162
|
elif file_path.is_dir():
|