oceanstream 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanstream/__init__.py +0 -0
- oceanstream/cli/__init__.py +3 -0
- oceanstream/cli/main.py +267 -0
- oceanstream/denoise/__init__.py +2 -0
- oceanstream/denoise/apply_masks.py +49 -0
- oceanstream/denoise/applying_masks_handler.py +173 -0
- oceanstream/denoise/background_noise_remover.py +97 -0
- oceanstream/denoise/create_masks.py +87 -0
- oceanstream/denoise/noise_masks.py +398 -0
- oceanstream/denoise/types.py +73 -0
- oceanstream/echodata/__init__.py +5 -0
- oceanstream/echodata/ensure_time_continuity.py +75 -0
- oceanstream/echodata/mvbs_computation.py +77 -0
- oceanstream/echodata/raw_handler.py +577 -0
- oceanstream/echodata/sv_computation.py +118 -0
- oceanstream/echodata/sv_dataset_extension.py +126 -0
- oceanstream/echodata/sv_interpolation.py +260 -0
- oceanstream/echodata/target_strength_computation.py +109 -0
- oceanstream/exports/__init__.py +2 -0
- oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
- oceanstream/exports/csv/csv_export_from_raw.py +92 -0
- oceanstream/exports/csv/csv_export_nasc.py +135 -0
- oceanstream/exports/frequency_differencing_handler.py +210 -0
- oceanstream/exports/nasc.py +23 -0
- oceanstream/exports/nasc_computation.py +62 -0
- oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
- oceanstream/exports/shoals/shoal_process.py +224 -0
- oceanstream/exports/shoals/shoals_handler.py +35 -0
- oceanstream/plot/__init__.py +2 -0
- oceanstream/plot/echogram.py +542 -0
- oceanstream/process/__init__.py +21 -0
- oceanstream/process/combine_zarr.py +90 -0
- oceanstream/process/file_processor.py +151 -0
- oceanstream/process/folder_processor.py +246 -0
- oceanstream/process/process.py +77 -0
- oceanstream/process/processed_data_io.py +140 -0
- oceanstream/settings/__init__.py +1 -0
- oceanstream/settings/config.py +17 -0
- oceanstream/utils.py +101 -0
- oceanstream-0.0.0.dev0.dist-info/LICENSE +21 -0
- oceanstream-0.0.0.dev0.dist-info/METADATA +214 -0
- oceanstream-0.0.0.dev0.dist-info/RECORD +46 -0
- oceanstream-0.0.0.dev0.dist-info/WHEEL +5 -0
- oceanstream-0.0.0.dev0.dist-info/dependency_links.txt +1 -0
- oceanstream-0.0.0.dev0.dist-info/entry_points.txt +2 -0
- oceanstream-0.0.0.dev0.dist-info/top_level.txt +1 -0
oceanstream/__init__.py
ADDED
|
File without changes
|
oceanstream/cli/main.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import traceback
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
import asyncio
|
|
5
|
+
import os
|
|
6
|
+
import logging
|
|
7
|
+
import sys
|
|
8
|
+
import warnings
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from rich import print
|
|
12
|
+
from rich.traceback import install, Traceback
|
|
13
|
+
from oceanstream.settings import load_config
|
|
14
|
+
from dask.distributed import LocalCluster, Client
|
|
15
|
+
from rich.console import Console
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
install(show_locals=False, width=120)
|
|
19
|
+
|
|
20
|
+
DEFAULT_OUTPUT_FOLDER = "output"
|
|
21
|
+
DEFAULT_SONAR_MODEL = "EK60"
|
|
22
|
+
|
|
23
|
+
BANNER = """
|
|
24
|
+
_
|
|
25
|
+
| |
|
|
26
|
+
___ ___ ___ __ _ _ __ ___| |_ _ __ ___ __ _ _ __ ___
|
|
27
|
+
/ _ \ / __/ _ \/ _` | '_ \ / __| __| '__/ _ \/ _` | '_ ` _ \
|
|
28
|
+
| (_) | (_| __/ (_| | | | | \__ \ |_| | | __/ (_| | | | | | |
|
|
29
|
+
\___/ \___\___|\__,_|_| |_| |___/\__|_| \___|\__,_|_| |_| |_|
|
|
30
|
+
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def initialize(settings, file_path, log_level=None):
|
|
37
|
+
config_data = load_config(settings["config"])
|
|
38
|
+
config_data["raw_path"] = file_path
|
|
39
|
+
|
|
40
|
+
if log_level is not None:
|
|
41
|
+
logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s', force=True)
|
|
42
|
+
config_data["log_level"] = log_level
|
|
43
|
+
|
|
44
|
+
if 'sonar_model' in settings and settings["sonar_model"] is not None:
|
|
45
|
+
config_data["sonar_model"] = settings["sonar_model"]
|
|
46
|
+
|
|
47
|
+
if settings["output_folder"] is not None:
|
|
48
|
+
config_data["output_folder"] = settings["output_folder"]
|
|
49
|
+
|
|
50
|
+
return config_data
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
app = typer.Typer(help="OceanStream CLI")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@app.command()
|
|
57
|
+
def process(
|
|
58
|
+
source: str = typer.Option(..., help="Path to a raw data file"),
|
|
59
|
+
output: str = typer.Option(None,
|
|
60
|
+
help="Destination path for saving processed data. Defaults to a predefined "
|
|
61
|
+
"directory if not specified."),
|
|
62
|
+
sonar_model: str = typer.Option(None, help="Sonar model used to collect the data",
|
|
63
|
+
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
64
|
+
plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
|
|
65
|
+
depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
|
|
66
|
+
waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
|
|
67
|
+
show_choices=["CW", "BB"]),
|
|
68
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
69
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
70
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
71
|
+
):
|
|
72
|
+
"""
|
|
73
|
+
Process the input raw file and save the result to the specified output folder.
|
|
74
|
+
"""
|
|
75
|
+
settings = {
|
|
76
|
+
"config": config,
|
|
77
|
+
"sonar_model": sonar_model,
|
|
78
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
filePath = Path(source)
|
|
82
|
+
configData = initialize(settings, filePath, log_level=log_level)
|
|
83
|
+
|
|
84
|
+
if filePath.is_file():
|
|
85
|
+
from oceanstream.process import process_raw_file_with_progress
|
|
86
|
+
asyncio.run(
|
|
87
|
+
process_raw_file_with_progress(configData, plot_echogram, waveform_mode=waveform_mode,
|
|
88
|
+
depth_offset=depth_offset))
|
|
89
|
+
else:
|
|
90
|
+
print(f"[red]❌ The provided path '{source}' is not a valid raw file.[/red]")
|
|
91
|
+
sys.exit(1)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@app.command()
|
|
95
|
+
def convert(
|
|
96
|
+
source: str = typer.Option(..., help="Path to a raw data file/folder"),
|
|
97
|
+
output: str = typer.Option(None,
|
|
98
|
+
help="Destination path for saving Zarr converted data. Defaults to a predefined "
|
|
99
|
+
"directory if not specified."),
|
|
100
|
+
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
|
|
101
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
102
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
103
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
104
|
+
):
|
|
105
|
+
"""
|
|
106
|
+
Convert the input raw data file(s) to Zarr and save the result to the specified output folder.
|
|
107
|
+
"""
|
|
108
|
+
settings = {
|
|
109
|
+
"config": config,
|
|
110
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
filePath = Path(source)
|
|
114
|
+
configData = initialize(settings, filePath, log_level=log_level)
|
|
115
|
+
|
|
116
|
+
if filePath.is_file():
|
|
117
|
+
from oceanstream.process import convert_raw_file
|
|
118
|
+
convert_raw_file(filePath, configData)
|
|
119
|
+
print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
|
|
120
|
+
elif filePath.is_dir():
|
|
121
|
+
from oceanstream.process import convert_raw_files
|
|
122
|
+
convert_raw_files(configData, workers_count=workers_count)
|
|
123
|
+
else:
|
|
124
|
+
print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
|
|
125
|
+
sys.exit(1)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@app.command()
|
|
129
|
+
def combine(
|
|
130
|
+
source: str = typer.Option(..., help="Path to a source data folder where zarr files are located"),
|
|
131
|
+
output: str = typer.Option(None,
|
|
132
|
+
help="Destination path for saving combined zarr file. Defaults to a predefined "
|
|
133
|
+
"directory if not specified."),
|
|
134
|
+
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
|
|
135
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
136
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
137
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
138
|
+
):
|
|
139
|
+
"""
|
|
140
|
+
Process the input zarr data files and save the combined zarr file to the specified output folder.
|
|
141
|
+
"""
|
|
142
|
+
settings = {
|
|
143
|
+
"config": config,
|
|
144
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
dir_path = Path(source)
|
|
148
|
+
configData = initialize(settings, dir_path, log_level=log_level)
|
|
149
|
+
|
|
150
|
+
if dir_path.is_dir():
|
|
151
|
+
console = Console()
|
|
152
|
+
with console.status("Processing...", spinner="dots") as status:
|
|
153
|
+
status.start()
|
|
154
|
+
|
|
155
|
+
cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
|
|
156
|
+
client = Client(cluster)
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
from oceanstream.process import combine_zarr_files
|
|
160
|
+
|
|
161
|
+
file_name = f"{Path(dir_path).stem}-combined.zarr"
|
|
162
|
+
zarr_output_file = os.path.join(configData['output_folder'], file_name)
|
|
163
|
+
status.update(
|
|
164
|
+
f"Combining zarr files to {zarr_output_file}; navigate to http://localhost:8787/status for progress")
|
|
165
|
+
|
|
166
|
+
combine_zarr_files(dir_path,
|
|
167
|
+
zarr_output_file=zarr_output_file,
|
|
168
|
+
chunks=configData.get('base_chunk_sizes'))
|
|
169
|
+
print("\nZarr files have been combined successfully.")
|
|
170
|
+
except KeyboardInterrupt:
|
|
171
|
+
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
172
|
+
except Exception as e:
|
|
173
|
+
logging.exception("Error processing folder %s", configData['raw_path'])
|
|
174
|
+
traceback.print_exc()
|
|
175
|
+
finally:
|
|
176
|
+
status.stop()
|
|
177
|
+
client.close()
|
|
178
|
+
cluster.close()
|
|
179
|
+
else:
|
|
180
|
+
print(f"[red]❌ The provided path '{source}' is not a valid folder.[/red]")
|
|
181
|
+
sys.exit(1)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@app.command()
|
|
185
|
+
def compute_sv(
|
|
186
|
+
source: str = typer.Option(..., help="Path to a Zarr root file or a directory containing Zarr files"),
|
|
187
|
+
output: str = typer.Option(None,
|
|
188
|
+
help="Destination path for saving Sv data. Defaults to a predefined directory if not specified."),
|
|
189
|
+
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for Sv computation"),
|
|
190
|
+
sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
|
|
191
|
+
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
192
|
+
plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
|
|
193
|
+
depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
|
|
194
|
+
waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
|
|
195
|
+
show_choices=["CW", "BB"]),
|
|
196
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
197
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
198
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
199
|
+
):
|
|
200
|
+
"""
|
|
201
|
+
Compute the volume backscattering strength (Sv) from the Zarr file.
|
|
202
|
+
"""
|
|
203
|
+
settings_dict = {
|
|
204
|
+
"config": config,
|
|
205
|
+
"sonar_model": sonar_model,
|
|
206
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
207
|
+
}
|
|
208
|
+
file_path = Path(source)
|
|
209
|
+
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
210
|
+
|
|
211
|
+
console = Console()
|
|
212
|
+
with console.status("Processing...", spinner="dots") as status:
|
|
213
|
+
status.start()
|
|
214
|
+
cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
|
|
215
|
+
client = Client(cluster)
|
|
216
|
+
|
|
217
|
+
try:
|
|
218
|
+
if file_path.is_dir() and source.endswith(".zarr"):
|
|
219
|
+
status.update(
|
|
220
|
+
f"[blue] Computing Sv for {file_path}...[/blue] – navigate to http://localhost:8787/status for progress")
|
|
221
|
+
from oceanstream.process import compute_single_file
|
|
222
|
+
|
|
223
|
+
compute_single_file(config_data,
|
|
224
|
+
chunks=config_data.get('base_chunk_sizes'),
|
|
225
|
+
plot_echogram=plot_echogram,
|
|
226
|
+
waveform_mode=waveform_mode,
|
|
227
|
+
depth_offset=depth_offset)
|
|
228
|
+
elif file_path.is_dir():
|
|
229
|
+
status.update(
|
|
230
|
+
f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
|
|
231
|
+
f"http://localhost:8787/status for progress")
|
|
232
|
+
from oceanstream.process import process_zarr_files
|
|
233
|
+
|
|
234
|
+
process_zarr_files(config_data,
|
|
235
|
+
workers_count=workers_count,
|
|
236
|
+
status=status,
|
|
237
|
+
chunks=config_data.get('base_chunk_sizes'),
|
|
238
|
+
plot_echogram=plot_echogram,
|
|
239
|
+
waveform_mode=waveform_mode,
|
|
240
|
+
depth_offset=depth_offset)
|
|
241
|
+
else:
|
|
242
|
+
print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
|
|
243
|
+
sys.exit(1)
|
|
244
|
+
except KeyboardInterrupt:
|
|
245
|
+
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
246
|
+
except Exception as e:
|
|
247
|
+
logging.exception("Error while processing %s", config_data['raw_path'])
|
|
248
|
+
traceback.print_exc()
|
|
249
|
+
finally:
|
|
250
|
+
client.close()
|
|
251
|
+
cluster.close()
|
|
252
|
+
status.stop()
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
@app.command()
|
|
256
|
+
def export():
|
|
257
|
+
typer.echo("Export data...")
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def main():
|
|
261
|
+
print(BANNER)
|
|
262
|
+
warnings.filterwarnings("ignore", category=UserWarning)
|
|
263
|
+
app(prog_name="oceanstream")
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
if __name__ == "__main__":
|
|
267
|
+
main()
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import pathlib
|
|
2
|
+
import xarray as xr
|
|
3
|
+
|
|
4
|
+
from typing import Dict, Union
|
|
5
|
+
from oceanstream.echodata import interpolate_sv
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from .applying_masks_handler import (
|
|
9
|
+
apply_selected_noise_masks_and_or_noise_removal as apply_selected_masks,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def apply_noise_masks(sv_with_masks: Union[xr.Dataset, str, pathlib.Path], config) -> xr.Dataset:
|
|
14
|
+
noise_masks = {}
|
|
15
|
+
|
|
16
|
+
if config["impulse"]["enabled"]:
|
|
17
|
+
noise_masks["mask_impulse"] = {"var_name": "Sv"}
|
|
18
|
+
if config["attenuation"]["enabled"]:
|
|
19
|
+
noise_masks["mask_attenuation"] = {"var_name": "Sv"}
|
|
20
|
+
if config["transient"]["enabled"]:
|
|
21
|
+
noise_masks["mask_transient"] = {"var_name": "Sv"}
|
|
22
|
+
|
|
23
|
+
ds_processed = apply_selected_masks(sv_with_masks, noise_masks)
|
|
24
|
+
|
|
25
|
+
return ds_processed
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def apply_background_noise_removal(
|
|
29
|
+
ds_processed: xr.Dataset, config
|
|
30
|
+
) -> xr.Dataset:
|
|
31
|
+
ds_interpolated = interpolate_sv(ds_processed)
|
|
32
|
+
params = config["remove_background_noise"]["parameters"]
|
|
33
|
+
|
|
34
|
+
ds_clean = apply_selected_masks(
|
|
35
|
+
ds_interpolated, processes_to_apply={"remove_background_noise": params}
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
return ds_clean
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def apply_seabed_mask(ds: xr.Dataset, config) -> xr.Dataset:
|
|
42
|
+
if config["seabed"]["enabled"] and config["false_seabed"]["enabled"]:
|
|
43
|
+
params = {"mask_seabed": {"var_name": "Sv"}, "mask_false_seabed": {"var_name": "Sv"}}
|
|
44
|
+
|
|
45
|
+
ds_processed = apply_selected_masks(ds, params)
|
|
46
|
+
|
|
47
|
+
return ds_processed
|
|
48
|
+
|
|
49
|
+
return ds
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""
|
|
2
|
+
applying_masks_handler.py
|
|
3
|
+
-------------------------
|
|
4
|
+
Description: This module provides functionalities for applying a sequence of
|
|
5
|
+
selected masks and background noise removal to echosounder datasets based on
|
|
6
|
+
specific methodologies.
|
|
7
|
+
|
|
8
|
+
Masks/Processes supported in this module include:
|
|
9
|
+
**Noise-related masks/processes, namely:**
|
|
10
|
+
- 'mask_false_seabed'
|
|
11
|
+
- 'mask_seabed'
|
|
12
|
+
- 'mask_impulse'
|
|
13
|
+
- 'mask_attenuation'
|
|
14
|
+
- 'mask_transient'
|
|
15
|
+
- 'remove_background_noise'
|
|
16
|
+
|
|
17
|
+
**Organism-related masks, namely:**
|
|
18
|
+
- 'mask_krill'
|
|
19
|
+
- 'mask_gas_bearing_organisms'
|
|
20
|
+
- 'mask_fluid_like_organisms'
|
|
21
|
+
- 'mask_shoal'
|
|
22
|
+
|
|
23
|
+
Note: If other masks that aren't supported by this module's functions are
|
|
24
|
+
attached as variables to the dataset, they can be applied using
|
|
25
|
+
`echopype.mask.apply_mask`. For details, consult the echopype documentation.
|
|
26
|
+
|
|
27
|
+
Key Functions:
|
|
28
|
+
- `apply_selected_noise_masks_and_or_noise_removal`: For applying selected
|
|
29
|
+
noise masks and/or noise removal.
|
|
30
|
+
|
|
31
|
+
- `apply_mask_organisms_in_order`: For applying organism-related masks in
|
|
32
|
+
a user-defined order.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
import pathlib
|
|
37
|
+
from typing import Union
|
|
38
|
+
|
|
39
|
+
import echopype as ep
|
|
40
|
+
import xarray as xr
|
|
41
|
+
|
|
42
|
+
from .background_noise_remover import apply_remove_background_noise
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def apply_selected_noise_masks_and_or_noise_removal(
|
|
46
|
+
ds: Union[xr.Dataset, str, pathlib.Path],
|
|
47
|
+
processes_to_apply: dict,
|
|
48
|
+
) -> xr.Dataset:
|
|
49
|
+
"""
|
|
50
|
+
Apply a sequence of selected masks and background noise removal (if provided in the processes_to_apply)
|
|
51
|
+
to the given dataset based on the specifications in the article:
|
|
52
|
+
> Haris, K., et al. "Sounding out life in the deep using acoustic data from ships of opportunity."
|
|
53
|
+
> Scientific Data 8.1 (2021): 23.
|
|
54
|
+
|
|
55
|
+
Order of cleaning process:
|
|
56
|
+
1. Impulse noise mask (if provided)
|
|
57
|
+
2. Attenuated signal mask (if provided)
|
|
58
|
+
3. Transient noise mask (if provided)
|
|
59
|
+
4. Background noise mask (if provided)
|
|
60
|
+
5. False seabed mask (if provided)
|
|
61
|
+
6. Seabed mask (if provided)
|
|
62
|
+
|
|
63
|
+
Valid processes/masks:
|
|
64
|
+
- 'mask_impulse'
|
|
65
|
+
- 'mask_attenuation'
|
|
66
|
+
- 'mask_transient'
|
|
67
|
+
- 'remove_background_noise'
|
|
68
|
+
- 'mask_false_seabed'
|
|
69
|
+
- 'mask_seabed'
|
|
70
|
+
|
|
71
|
+
Parameters:
|
|
72
|
+
- `ds`: (`xr.Dataset`, `str`, or `pathlib.Path`)
|
|
73
|
+
Points to a Dataset that contains the variable the mask should be applied to
|
|
74
|
+
and must have the masks as attached variables.
|
|
75
|
+
- `processes_to_apply` (`dict`):
|
|
76
|
+
A dictionary where keys are mask names or 'remove_background_noise' and values are specific parameters
|
|
77
|
+
(sub-dictionaries).
|
|
78
|
+
|
|
79
|
+
For more details on parameters associated with applying a mask and the `background noise removal` process,
|
|
80
|
+
those parameters found within the specific sub-dictionaries, please refer to the documentation for:
|
|
81
|
+
- `echopype.mask.apply_mask`
|
|
82
|
+
- `background_noise_remover.apply_remove_background_noise`
|
|
83
|
+
|
|
84
|
+
Returns:
|
|
85
|
+
- `xr.Dataset`: Processed dataset.
|
|
86
|
+
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
valid_processes = [
|
|
90
|
+
"mask_impulse",
|
|
91
|
+
"mask_attenuation",
|
|
92
|
+
"mask_transient",
|
|
93
|
+
"remove_background_noise",
|
|
94
|
+
"mask_false_seabed",
|
|
95
|
+
"mask_seabed",
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
# Check for unexpected masks/processes
|
|
99
|
+
for process in processes_to_apply:
|
|
100
|
+
if process not in valid_processes:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
"Unexpected mask/process. Please refer to the function documentation for valid masks/processes."
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
for process in valid_processes:
|
|
106
|
+
if process in processes_to_apply:
|
|
107
|
+
params = processes_to_apply[process]
|
|
108
|
+
if process in [
|
|
109
|
+
"mask_impulse",
|
|
110
|
+
"mask_attenuation",
|
|
111
|
+
"mask_transient",
|
|
112
|
+
"mask_false_seabed",
|
|
113
|
+
"mask_seabed",
|
|
114
|
+
]:
|
|
115
|
+
mask_data = ds[process]
|
|
116
|
+
ds = ep.mask.apply_mask(ds, mask_data, **params)
|
|
117
|
+
elif process == "remove_background_noise":
|
|
118
|
+
ds = apply_remove_background_noise(ds, **params)
|
|
119
|
+
|
|
120
|
+
return ds
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def apply_mask_organisms_in_order(ds, processes_to_apply):
|
|
124
|
+
"""
|
|
125
|
+
Apply a sequence of selected masks to the given dataset in the order that
|
|
126
|
+
the masks were provided in the input dictionary.
|
|
127
|
+
|
|
128
|
+
Valid masks:
|
|
129
|
+
'mask_krill',
|
|
130
|
+
'mask_gas_bearing_organisms',
|
|
131
|
+
'mask_fluid_like_organisms',
|
|
132
|
+
'mask_shoal'
|
|
133
|
+
|
|
134
|
+
Parameters:
|
|
135
|
+
- ds (xr.Dataset, str, or pathlib.Path):
|
|
136
|
+
Points to a Dataset that contains the variable the mask should be applied to
|
|
137
|
+
and must have the masks as attached variables.
|
|
138
|
+
- processes_to_apply (dict):
|
|
139
|
+
A dictionary where keys are mask names and values are specific parameters (sub-dictionaries).
|
|
140
|
+
|
|
141
|
+
For more details on parameters associated with applying a mask,
|
|
142
|
+
those parameters found within the specific sub-dictionaries, please refer to the documentation for:
|
|
143
|
+
- `echopype.mask.apply_mask`
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
- xr.Dataset: Processed dataset.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
recognized_masks = [
|
|
150
|
+
"mask_krill",
|
|
151
|
+
"mask_gas_bearing_organisms",
|
|
152
|
+
"mask_fluid_like_organisms",
|
|
153
|
+
"mask_shoal",
|
|
154
|
+
]
|
|
155
|
+
|
|
156
|
+
for mask_name in processes_to_apply.keys():
|
|
157
|
+
if mask_name not in recognized_masks:
|
|
158
|
+
raise ValueError(
|
|
159
|
+
"Unrecognized mask name. Please refer to the function documentation for valid masks"
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
mask_params = processes_to_apply[mask_name]
|
|
163
|
+
|
|
164
|
+
if mask_name == "mask_krill":
|
|
165
|
+
ds = ep.mask.apply_mask(ds, ds[mask_name], **mask_params)
|
|
166
|
+
elif mask_name == "mask_gas_bearing_organisms":
|
|
167
|
+
ds = ep.mask.apply_mask(ds, ds[mask_name], **mask_params)
|
|
168
|
+
elif mask_name == "mask_fluid_like_organisms":
|
|
169
|
+
ds = ep.mask.apply_mask(ds, ds[mask_name], **mask_params)
|
|
170
|
+
elif mask_name == "mask_shoal":
|
|
171
|
+
ds = ep.mask.apply_mask(ds, ds[mask_name], **mask_params)
|
|
172
|
+
|
|
173
|
+
return ds
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""
|
|
2
|
+
background_noise_remover.py
|
|
3
|
+
----------------------------
|
|
4
|
+
|
|
5
|
+
This module provides functionalities to enhance the quality of echosounder data by removing background noise.
|
|
6
|
+
|
|
7
|
+
Functions:
|
|
8
|
+
|
|
9
|
+
- `apply_remove_background_noise`:
|
|
10
|
+
Main function that applies the `remove_noise` function from echopype to a Sv dataset.
|
|
11
|
+
Computes default parameters based on dataset characteristics when not provided.
|
|
12
|
+
The dataset initially passed to the function, processed in the following manner:
|
|
13
|
+
`Sv_corrected` and `Sv_noise` variables are generated by the echopype `remove_noise` function.
|
|
14
|
+
`Sv` contains the values originally in `Sv_corrected`.
|
|
15
|
+
`Sv_with_background_noise` stores the original values of `Sv`.
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
|
|
19
|
+
To remove background noise for a given Sv dataset, `ds_Sv`, simply call:
|
|
20
|
+
`apply_remove_background_noise(ds_Sv)`
|
|
21
|
+
|
|
22
|
+
Note:
|
|
23
|
+
- Ensure that the echopype library is properly installed and imported when using this module.
|
|
24
|
+
- Sv = the volume backscattering strength
|
|
25
|
+
|
|
26
|
+
"""
|
|
27
|
+
import echopype as ep
|
|
28
|
+
import numpy as np
|
|
29
|
+
import xarray as xr
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def apply_remove_background_noise(
|
|
33
|
+
ds_Sv: xr.Dataset,
|
|
34
|
+
ping_num: int = 40,
|
|
35
|
+
range_sample_num: int = None,
|
|
36
|
+
noise_max: float = -125,
|
|
37
|
+
SNR_threshold: float = 3,
|
|
38
|
+
) -> xr.Dataset:
|
|
39
|
+
"""
|
|
40
|
+
Applies the `remove_noise` function from the echopype library to the `ds_Sv` dataset.
|
|
41
|
+
If `range_sample_num` is not provided, the function computes it based on a 10-meter vertical bin
|
|
42
|
+
for each channel, and then uses the minimum value across all channels as the default.
|
|
43
|
+
|
|
44
|
+
For detailed information on `remove_noise`, consult the documentation
|
|
45
|
+
within the `clean` module of the echopype library.
|
|
46
|
+
|
|
47
|
+
For default values and their context, refer to:
|
|
48
|
+
De Robertis & Higginbottom (2007). "A post-processing technique to estimate
|
|
49
|
+
the signal-to-noise ratio and remove echosounder background noise."
|
|
50
|
+
ICES Journal of Marine Sciences, 64(6), 1282–1291.
|
|
51
|
+
|
|
52
|
+
Parameters:
|
|
53
|
+
- ds_Sv: xr.Dataset
|
|
54
|
+
Dataset containing Sv and echo_range [m].
|
|
55
|
+
- ping_num: int, default=40
|
|
56
|
+
Number of pings to obtain noise estimates.
|
|
57
|
+
- range_sample_num: int, optional
|
|
58
|
+
Number of samples along the range_sample dimension to obtain noise estimates. If not provided,
|
|
59
|
+
it's computed based on a 10-meter vertical bin for each channel and the minimum value is used.
|
|
60
|
+
- noise_max: float, default=-125
|
|
61
|
+
The upper limit for background noise expected under the operating conditions (default=-125 dB).
|
|
62
|
+
- SNR_threshold: float, default=3 [dB]
|
|
63
|
+
Acceptable signal-to-noise ratio.
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
- ds_Sv: xr.Dataset
|
|
67
|
+
|
|
68
|
+
Notes:
|
|
69
|
+
Returns - initial xr.Dataset:
|
|
70
|
+
The dataset initially passed to the function, processed in the following manner:
|
|
71
|
+
- `Sv_corrected` and `Sv_noise` variables are generated by the echopype `remove_noise` function.
|
|
72
|
+
- `Sv` contains the values originally in `Sv_corrected`.
|
|
73
|
+
- `Sv_with_background_noise` stores the original values of `Sv`.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
# Compute range_sample_num if not provided
|
|
77
|
+
if range_sample_num is None:
|
|
78
|
+
range_sample_nums = []
|
|
79
|
+
for ch in range(ds_Sv.sizes["channel"]):
|
|
80
|
+
mean_diff = np.nanmean(np.diff(ds_Sv["echo_range"].isel(channel=ch).values[0, :]))
|
|
81
|
+
if np.isnan(mean_diff):
|
|
82
|
+
raise ValueError(
|
|
83
|
+
"The default computed value for the range_sample_num is nan, please add the range_sample_num as input parameter"
|
|
84
|
+
)
|
|
85
|
+
range_sample_nums.append(int(10 / mean_diff))
|
|
86
|
+
|
|
87
|
+
range_sample_num = min(range_sample_nums)
|
|
88
|
+
# Remove noise
|
|
89
|
+
ds_Sv_processed = ep.clean.remove_background_noise(
|
|
90
|
+
ds_Sv,
|
|
91
|
+
ping_num=ping_num,
|
|
92
|
+
range_sample_num=range_sample_num,
|
|
93
|
+
noise_max=noise_max,
|
|
94
|
+
SNR_threshold=SNR_threshold,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
return ds_Sv_processed
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import concurrent.futures
|
|
2
|
+
import time
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
from typing import Dict
|
|
5
|
+
|
|
6
|
+
import xarray
|
|
7
|
+
|
|
8
|
+
# from echopype.clean.api import (
|
|
9
|
+
# get_attenuation_mask_multichannel,
|
|
10
|
+
# get_impulse_noise_mask_multichannel,
|
|
11
|
+
# get_transient_noise_mask_multichannel,
|
|
12
|
+
# )
|
|
13
|
+
# from echopype.mask.api import get_seabed_mask_multichannel
|
|
14
|
+
|
|
15
|
+
from oceanstream.utils import add_metadata_to_mask, dict_to_formatted_list
|
|
16
|
+
|
|
17
|
+
from .types import DenoiseConfig
|
|
18
|
+
|
|
19
|
+
MASK_CREATION_FUNCTIONS = {
|
|
20
|
+
# "transient": get_transient_noise_mask_multichannel,
|
|
21
|
+
# "attenuation": get_attenuation_mask_multichannel,
|
|
22
|
+
# "impulse": get_impulse_noise_mask_multichannel,
|
|
23
|
+
# "false_seabed": get_seabed_mask_multichannel,
|
|
24
|
+
# "seabed": get_seabed_mask_multichannel,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def create_masks(source_Sv: xarray.Dataset, profiling_info: Dict, config: DenoiseConfig):
|
|
29
|
+
masks = []
|
|
30
|
+
|
|
31
|
+
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
32
|
+
future_to_mask_type = {}
|
|
33
|
+
future_to_start_time = {}
|
|
34
|
+
|
|
35
|
+
for mask_type, mask_config in config.items():
|
|
36
|
+
if is_scalar(mask_config):
|
|
37
|
+
continue
|
|
38
|
+
|
|
39
|
+
if mask_config["enabled"] and (mask_type in MASK_CREATION_FUNCTIONS):
|
|
40
|
+
profiling_info[mask_type + " mask"] = {}
|
|
41
|
+
# Map the mask type to the appropriate creation function
|
|
42
|
+
create_mask_func = MASK_CREATION_FUNCTIONS.get(mask_type)
|
|
43
|
+
|
|
44
|
+
if config["profile"]:
|
|
45
|
+
future_to_start_time[mask_type] = time.time()
|
|
46
|
+
|
|
47
|
+
future = executor.submit(
|
|
48
|
+
create_and_add_metadata, create_mask_func, source_Sv, mask_type, mask_config
|
|
49
|
+
)
|
|
50
|
+
future_to_mask_type[future] = mask_type
|
|
51
|
+
|
|
52
|
+
for future in concurrent.futures.as_completed(future_to_mask_type):
|
|
53
|
+
mask_type = future_to_mask_type[future]
|
|
54
|
+
|
|
55
|
+
try:
|
|
56
|
+
mask = future.result()
|
|
57
|
+
masks.append((mask_type, mask))
|
|
58
|
+
if config["profile"]:
|
|
59
|
+
end_time = time.time()
|
|
60
|
+
start_time = future_to_start_time[mask_type]
|
|
61
|
+
profiling_info[mask_type + " mask"]["execution_time"] = end_time - start_time
|
|
62
|
+
except Exception as exc:
|
|
63
|
+
print(f"{future_to_mask_type[future]} mask generated an exception: {exc}")
|
|
64
|
+
|
|
65
|
+
return tuple(masks), profiling_info
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def create_and_add_metadata(create_mask_func, source_Sv, mask_type, config_item):
|
|
69
|
+
mask = create_mask_func(
|
|
70
|
+
source_Sv, parameters=config_item["parameters"], method=config_item["method"]
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
return add_metadata_to_mask(
|
|
74
|
+
mask=mask,
|
|
75
|
+
metadata={
|
|
76
|
+
"mask_type": mask_type,
|
|
77
|
+
"method": config_item["method"],
|
|
78
|
+
"parameters": dict_to_formatted_list(config_item["parameters"]),
|
|
79
|
+
},
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def is_scalar(value):
|
|
84
|
+
if isinstance(value, Iterable) and not isinstance(value, str):
|
|
85
|
+
return False
|
|
86
|
+
|
|
87
|
+
return True
|