oceanstream 0.0.12.dev0__tar.gz → 0.0.15.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/PKG-INFO +2 -1
- oceanstream-0.0.15.dev0/oceanstream/__init__.py +5 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/cli/main.py +115 -28
- oceanstream-0.0.15.dev0/oceanstream/convert/__init__.py +5 -0
- oceanstream-0.0.15.dev0/oceanstream/convert/convert_raw_file.py +53 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/core.py +27 -27
- oceanstream-0.0.15.dev0/oceanstream/echodata/dataset.py +586 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/echodata/sv_computation.py +8 -1
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/echodata/sv_dataset_extension.py +18 -17
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/echodata/sv_interpolation.py +9 -10
- oceanstream-0.0.15.dev0/oceanstream/exports/__init__.py +6 -0
- oceanstream-0.0.15.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +183 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/plot/echogram.py +16 -10
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/__init__.py +5 -3
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/aws/s3.py +3 -2
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/azure/__init__.py +1 -1
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/azure/blob_storage.py +34 -3
- oceanstream-0.0.15.dev0/oceanstream/process/cloud.py +21 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/file_processor.py +110 -79
- oceanstream-0.0.15.dev0/oceanstream/process/file_processor_class.py +217 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/folder_processor.py +175 -52
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/process.py +5 -1
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/utils.py +68 -1
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream.egg-info/PKG-INFO +2 -1
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream.egg-info/SOURCES.txt +5 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream.egg-info/requires.txt +5 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/requirements.txt +3 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/setup.cfg +2 -1
- oceanstream-0.0.12.dev0/oceanstream/__init__.py +0 -5
- oceanstream-0.0.12.dev0/oceanstream/exports/__init__.py +0 -2
- oceanstream-0.0.12.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +0 -89
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/.flake8 +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/.gitattributes +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/.gitignore +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/CODE_OF_CONDUCT.md +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/LICENSE +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/MANIFEST.in +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/README.md +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/notebooks/README.md +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/cli/__init__.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/cli/requirements.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/denoise/__init__.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/denoise/apply_masks.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/denoise/applying_masks_handler.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/denoise/background_noise_remover.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/denoise/create_masks.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/denoise/noise_masks.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/denoise/requirements.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/denoise/types.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/echodata/__init__.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/echodata/ensure_time_continuity.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/echodata/mvbs_computation.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/echodata/raw_handler.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/echodata/requirements.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/echodata/target_strength_computation.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/exports/csv/csv_export_from_raw.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/exports/csv/csv_export_nasc.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/exports/frequency_differencing_handler.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/exports/nasc.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/exports/nasc_computation.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/exports/requirements.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/exports/shoals/shoal_detection_handler.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/exports/shoals/shoal_process.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/exports/shoals/shoals_handler.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/plot/__init__.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/plot/requirements.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/aws/__init__.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/azure/azure_processor.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/combine_zarr.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/processed_data_io.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/process/requirements.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/settings/__init__.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/settings/config.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/settings/defaults.config.json +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream/utils.py +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream.egg-info/dependency_links.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream.egg-info/entry_points.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/oceanstream.egg-info/top_level.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/pyproject.toml +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/requirements-dev.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/requirements-local.txt +0 -0
- {oceanstream-0.0.12.dev0 → oceanstream-0.0.15.dev0}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: oceanstream
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.15.dev0
|
|
4
4
|
Summary: OceanStream: process raw sonar data at scale
|
|
5
5
|
Home-page: https://oceanstream.io
|
|
6
6
|
Author: Pineview Labs
|
|
@@ -20,6 +20,7 @@ Provides-Extra: echodata
|
|
|
20
20
|
Provides-Extra: plot
|
|
21
21
|
Provides-Extra: denoise
|
|
22
22
|
Provides-Extra: process
|
|
23
|
+
Provides-Extra: convert
|
|
23
24
|
Provides-Extra: exports
|
|
24
25
|
Provides-Extra: complete
|
|
25
26
|
License-File: LICENSE
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import traceback
|
|
2
|
+
from asyncio import CancelledError
|
|
3
|
+
|
|
1
4
|
import typer
|
|
2
5
|
import asyncio
|
|
3
6
|
import os
|
|
4
7
|
import logging
|
|
5
8
|
import sys
|
|
6
9
|
import warnings
|
|
10
|
+
import dask
|
|
7
11
|
|
|
8
12
|
from pathlib import Path
|
|
9
13
|
from rich import print
|
|
@@ -11,7 +15,7 @@ from rich.traceback import install, Traceback
|
|
|
11
15
|
from oceanstream.settings import load_config
|
|
12
16
|
from dask.distributed import LocalCluster, Client, Variable
|
|
13
17
|
from rich.console import Console
|
|
14
|
-
|
|
18
|
+
from oceanstream.process import compute_and_export_single_file, process_zarr_files
|
|
15
19
|
|
|
16
20
|
install(show_locals=False, width=120)
|
|
17
21
|
|
|
@@ -30,6 +34,12 @@ BANNER = """
|
|
|
30
34
|
|
|
31
35
|
logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
|
|
32
36
|
|
|
37
|
+
dask.config.set({
|
|
38
|
+
'distributed.comm.timeouts.connect': '60s', # Increase the connection timeout
|
|
39
|
+
'distributed.comm.timeouts.tcp': '120s', # Increase the TCP timeout
|
|
40
|
+
'distributed.comm.retry.count': 0
|
|
41
|
+
})
|
|
42
|
+
|
|
33
43
|
|
|
34
44
|
def initialize(settings, file_path, log_level=None):
|
|
35
45
|
config_data = load_config(settings["config"])
|
|
@@ -95,6 +105,8 @@ def convert(
|
|
|
95
105
|
output: str = typer.Option(None,
|
|
96
106
|
help="Destination path for saving Zarr converted data. Defaults to a predefined "
|
|
97
107
|
"directory if not specified."),
|
|
108
|
+
sonar_model: str = typer.Option(None, help="Sonar model used to collect the data",
|
|
109
|
+
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
98
110
|
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
|
|
99
111
|
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
100
112
|
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
@@ -105,6 +117,7 @@ def convert(
|
|
|
105
117
|
"""
|
|
106
118
|
settings = {
|
|
107
119
|
"config": config,
|
|
120
|
+
"sonar_model": sonar_model,
|
|
108
121
|
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
109
122
|
}
|
|
110
123
|
|
|
@@ -113,9 +126,10 @@ def convert(
|
|
|
113
126
|
|
|
114
127
|
try:
|
|
115
128
|
if filePath.is_file():
|
|
116
|
-
from oceanstream.
|
|
129
|
+
from oceanstream.convert import convert_raw_file
|
|
130
|
+
print(f"[blue]Converting raw file {source} to Zarr...[/blue]")
|
|
117
131
|
convert_raw_file(filePath, configData)
|
|
118
|
-
print(
|
|
132
|
+
print("✅ The file has been converted successfully.")
|
|
119
133
|
elif filePath.is_dir():
|
|
120
134
|
from oceanstream.process import convert_raw_files
|
|
121
135
|
convert_raw_files(configData, workers_count=workers_count)
|
|
@@ -124,8 +138,12 @@ def convert(
|
|
|
124
138
|
except KeyboardInterrupt:
|
|
125
139
|
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
126
140
|
except Exception as e:
|
|
127
|
-
|
|
128
|
-
|
|
141
|
+
if filePath.is_file():
|
|
142
|
+
logging.error("Error processing file %s", filePath)
|
|
143
|
+
else:
|
|
144
|
+
logging.exception("Error processing folder %s", configData['raw_path'])
|
|
145
|
+
|
|
146
|
+
logging.error(traceback.format_exc())
|
|
129
147
|
|
|
130
148
|
|
|
131
149
|
@app.command()
|
|
@@ -193,10 +211,12 @@ def compute_sv(
|
|
|
193
211
|
sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
|
|
194
212
|
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
195
213
|
plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
|
|
196
|
-
use_dask: bool = typer.Option(False,
|
|
214
|
+
use_dask: bool = typer.Option(False,
|
|
215
|
+
help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
|
|
197
216
|
depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
|
|
198
217
|
waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
|
|
199
218
|
show_choices=["CW", "BB"]),
|
|
219
|
+
encode_mode: str = typer.Option("power", help="Encode mode, can be either power or complex", show_choices=["power", "complex"]),
|
|
200
220
|
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
201
221
|
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
202
222
|
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
@@ -209,6 +229,88 @@ def compute_sv(
|
|
|
209
229
|
"sonar_model": sonar_model,
|
|
210
230
|
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
211
231
|
}
|
|
232
|
+
dask.config.set({'distributed.comm.retry.count': 1})
|
|
233
|
+
|
|
234
|
+
file_path = Path(source)
|
|
235
|
+
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
236
|
+
single_file = file_path.is_dir() and source.endswith(".zarr")
|
|
237
|
+
|
|
238
|
+
client = None
|
|
239
|
+
cluster = None
|
|
240
|
+
|
|
241
|
+
if use_dask or not single_file:
|
|
242
|
+
cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
|
|
243
|
+
client = Client(cluster)
|
|
244
|
+
|
|
245
|
+
try:
|
|
246
|
+
if file_path.is_dir() and source.endswith(".zarr"):
|
|
247
|
+
console = Console()
|
|
248
|
+
with console.status("Processing...", spinner="dots") as status:
|
|
249
|
+
status.start()
|
|
250
|
+
status.update(
|
|
251
|
+
f"[blue] Computing Sv for {file_path}...[/blue]" + use_dask * "– navigate to "
|
|
252
|
+
"http://localhost:8787/status for "
|
|
253
|
+
"progress")
|
|
254
|
+
|
|
255
|
+
chunks = None
|
|
256
|
+
if use_dask:
|
|
257
|
+
chunks = config_data.get('base_chunk_sizes')
|
|
258
|
+
|
|
259
|
+
compute_and_export_single_file(config_data,
|
|
260
|
+
chunks=chunks,
|
|
261
|
+
plot_echogram=plot_echogram,
|
|
262
|
+
waveform_mode=waveform_mode,
|
|
263
|
+
encode_mode=encode_mode,
|
|
264
|
+
depth_offset=depth_offset)
|
|
265
|
+
|
|
266
|
+
status.stop()
|
|
267
|
+
print("✅ The file have been processed successfully.")
|
|
268
|
+
elif file_path.is_dir():
|
|
269
|
+
print(f"Dashboard available at {client.dashboard_link}")
|
|
270
|
+
process_zarr_files(config_data,
|
|
271
|
+
client,
|
|
272
|
+
workers_count=workers_count,
|
|
273
|
+
chunks=config_data.get('base_chunk_sizes'),
|
|
274
|
+
plot_echogram=plot_echogram,
|
|
275
|
+
waveform_mode=waveform_mode,
|
|
276
|
+
depth_offset=depth_offset)
|
|
277
|
+
else:
|
|
278
|
+
print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
|
|
279
|
+
sys.exit(1)
|
|
280
|
+
except KeyboardInterrupt:
|
|
281
|
+
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
282
|
+
except Exception as e:
|
|
283
|
+
logging.exception("Error while processing %s", config_data['raw_path'])
|
|
284
|
+
print(Traceback())
|
|
285
|
+
finally:
|
|
286
|
+
if client is not None:
|
|
287
|
+
client.close()
|
|
288
|
+
|
|
289
|
+
if cluster is not None:
|
|
290
|
+
cluster.close()
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
@app.command()
|
|
294
|
+
def export_location(
|
|
295
|
+
source: str = typer.Option(..., help="Path to a Zarr root file or a directory containing Zarr files"),
|
|
296
|
+
output: str = typer.Option(None,
|
|
297
|
+
help="Destination path for saving the exported data. Defaults to a predefined "
|
|
298
|
+
"directory if not specified."),
|
|
299
|
+
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for parallel processing"),
|
|
300
|
+
use_dask: bool = typer.Option(False,
|
|
301
|
+
help="Start a Local Dask cluster for parallel processing (always enabled for "
|
|
302
|
+
"multiple files)"),
|
|
303
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
304
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
305
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
306
|
+
):
|
|
307
|
+
"""
|
|
308
|
+
Given a Zarr dataset containing Sv data, exports the GPS location data to a JSON file.
|
|
309
|
+
"""
|
|
310
|
+
settings_dict = {
|
|
311
|
+
"config": config,
|
|
312
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
313
|
+
}
|
|
212
314
|
file_path = Path(source)
|
|
213
315
|
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
214
316
|
|
|
@@ -225,27 +327,17 @@ def compute_sv(
|
|
|
225
327
|
if file_path.is_dir() and source.endswith(".zarr"):
|
|
226
328
|
status.update(
|
|
227
329
|
f"[blue] Computing Sv for {file_path}...[/blue] – navigate to http://localhost:8787/status for progress")
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
compute_single_file(config_data,
|
|
231
|
-
chunks=config_data.get('base_chunk_sizes'),
|
|
232
|
-
plot_echogram=plot_echogram,
|
|
233
|
-
waveform_mode=waveform_mode,
|
|
234
|
-
depth_offset=depth_offset)
|
|
330
|
+
# TODO: Implement export_location_json
|
|
235
331
|
elif file_path.is_dir():
|
|
236
332
|
status.update(
|
|
237
333
|
f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
|
|
238
334
|
f"http://localhost:8787/status for progress")
|
|
239
|
-
from oceanstream.process import
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
plot_echogram=plot_echogram,
|
|
246
|
-
waveform_mode=waveform_mode,
|
|
247
|
-
processed_count_var=processed_count_var,
|
|
248
|
-
depth_offset=depth_offset)
|
|
335
|
+
from oceanstream.process import export_location_from_zarr_files
|
|
336
|
+
|
|
337
|
+
export_location_from_zarr_files(config_data,
|
|
338
|
+
workers_count=workers_count,
|
|
339
|
+
client=client,
|
|
340
|
+
chunks=config_data.get('base_chunk_sizes'))
|
|
249
341
|
else:
|
|
250
342
|
print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
|
|
251
343
|
sys.exit(1)
|
|
@@ -261,11 +353,6 @@ def compute_sv(
|
|
|
261
353
|
status.stop()
|
|
262
354
|
|
|
263
355
|
|
|
264
|
-
@app.command()
|
|
265
|
-
def export():
|
|
266
|
-
typer.echo("Export data...")
|
|
267
|
-
|
|
268
|
-
|
|
269
356
|
def main():
|
|
270
357
|
print(BANNER)
|
|
271
358
|
warnings.filterwarnings("ignore", category=UserWarning)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from oceanstream.echodata import read_file
|
|
6
|
+
|
|
7
|
+
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
|
|
8
|
+
logger = logging.getLogger('oceanstream')
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def convert_raw_file(file_path, config_data, base_path=None, progress_counter=None, counter_lock=None):
|
|
12
|
+
logger.info("Starting processing of file: %s", file_path)
|
|
13
|
+
|
|
14
|
+
file_path_obj = Path(file_path)
|
|
15
|
+
file_config_data = {**config_data, 'raw_path': file_path_obj}
|
|
16
|
+
|
|
17
|
+
if base_path:
|
|
18
|
+
relative_path = file_path_obj.relative_to(base_path)
|
|
19
|
+
relative_path = relative_path.parent
|
|
20
|
+
else:
|
|
21
|
+
relative_path = None
|
|
22
|
+
|
|
23
|
+
echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
|
|
24
|
+
file_name = file_path_obj.stem + ".zarr"
|
|
25
|
+
|
|
26
|
+
if 'cloud_storage' in config_data:
|
|
27
|
+
from oceanstream.process.cloud import get_chunk_store
|
|
28
|
+
|
|
29
|
+
if relative_path:
|
|
30
|
+
file_location = Path(relative_path) / file_name
|
|
31
|
+
else:
|
|
32
|
+
file_location = file_name
|
|
33
|
+
store = get_chunk_store(config_data['cloud_storage'], file_location)
|
|
34
|
+
echodata.to_zarr(save_path=store, overwrite=True, parallel=False)
|
|
35
|
+
|
|
36
|
+
output_dest = config_data['cloud_storage']['container_name'] + "/" + file_location
|
|
37
|
+
else:
|
|
38
|
+
if relative_path:
|
|
39
|
+
output_path = Path(config_data["output_folder"]) / relative_path
|
|
40
|
+
output_path.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
else:
|
|
42
|
+
output_path = Path(config_data["output_folder"])
|
|
43
|
+
|
|
44
|
+
output_dest = output_path / file_name
|
|
45
|
+
echodata.to_zarr(save_path=output_dest, overwrite=True, parallel=False)
|
|
46
|
+
|
|
47
|
+
if counter_lock is not None:
|
|
48
|
+
with counter_lock:
|
|
49
|
+
progress_counter.value += 1
|
|
50
|
+
|
|
51
|
+
logger.info("Finished converting file: %s", file_path)
|
|
52
|
+
|
|
53
|
+
return output_dest
|
|
@@ -5,7 +5,7 @@ import sys
|
|
|
5
5
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from oceanstream.settings import load_config
|
|
8
|
-
from oceanstream.process import
|
|
8
|
+
from oceanstream.process import convert_raw_files, compute_single_file
|
|
9
9
|
|
|
10
10
|
DEFAULT_OUTPUT_FOLDER = "output"
|
|
11
11
|
DEFAULT_SONAR_MODEL = "EK60"
|
|
@@ -80,32 +80,32 @@ def process_raw_file(source, output=None, sonar_model=None, plot_echogram=False,
|
|
|
80
80
|
sys.exit(1)
|
|
81
81
|
|
|
82
82
|
|
|
83
|
-
def convert(source, output=None, base_path=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
83
|
+
# def convert(source, output=None, base_path=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
|
|
84
|
+
# logging.debug("Starting convert function")
|
|
85
|
+
# settings = {
|
|
86
|
+
# "output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
87
|
+
# }
|
|
88
|
+
#
|
|
89
|
+
# if config is not None:
|
|
90
|
+
# settings.update(config)
|
|
91
|
+
#
|
|
92
|
+
# file_path = Path(source)
|
|
93
|
+
# config_data = initialize(settings, file_path, log_level=log_level)
|
|
94
|
+
#
|
|
95
|
+
# if chunks:
|
|
96
|
+
# config_data['chunks'] = chunks
|
|
97
|
+
# else:
|
|
98
|
+
# config_data['chunks'] = config_data.get('base_chunk_sizes', None)
|
|
99
|
+
#
|
|
100
|
+
# if file_path.is_file():
|
|
101
|
+
# logging.debug(f"Converting raw file: {file_path}")
|
|
102
|
+
# convert_raw_file(file_path, config_data, base_path=base_path)
|
|
103
|
+
# logging.info(f"Converted raw file {source} to Zarr and wrote output to: {config_data['output_folder']}")
|
|
104
|
+
# elif file_path.is_dir():
|
|
105
|
+
# logging.debug(f"Converting raw files in directory: {file_path}")
|
|
106
|
+
# convert_raw_files(config_data, workers_count=workers_count)
|
|
107
|
+
# else:
|
|
108
|
+
# logging.error(f"The provided path '{source}' is not a valid file/folder.")
|
|
109
109
|
|
|
110
110
|
|
|
111
111
|
def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
|