oceanstream 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. oceanstream/__init__.py +0 -0
  2. oceanstream/cli/__init__.py +3 -0
  3. oceanstream/cli/main.py +267 -0
  4. oceanstream/denoise/__init__.py +2 -0
  5. oceanstream/denoise/apply_masks.py +49 -0
  6. oceanstream/denoise/applying_masks_handler.py +173 -0
  7. oceanstream/denoise/background_noise_remover.py +97 -0
  8. oceanstream/denoise/create_masks.py +87 -0
  9. oceanstream/denoise/noise_masks.py +398 -0
  10. oceanstream/denoise/types.py +73 -0
  11. oceanstream/echodata/__init__.py +5 -0
  12. oceanstream/echodata/ensure_time_continuity.py +75 -0
  13. oceanstream/echodata/mvbs_computation.py +77 -0
  14. oceanstream/echodata/raw_handler.py +577 -0
  15. oceanstream/echodata/sv_computation.py +118 -0
  16. oceanstream/echodata/sv_dataset_extension.py +126 -0
  17. oceanstream/echodata/sv_interpolation.py +260 -0
  18. oceanstream/echodata/target_strength_computation.py +109 -0
  19. oceanstream/exports/__init__.py +2 -0
  20. oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
  21. oceanstream/exports/csv/csv_export_from_raw.py +92 -0
  22. oceanstream/exports/csv/csv_export_nasc.py +135 -0
  23. oceanstream/exports/frequency_differencing_handler.py +210 -0
  24. oceanstream/exports/nasc.py +23 -0
  25. oceanstream/exports/nasc_computation.py +62 -0
  26. oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
  27. oceanstream/exports/shoals/shoal_process.py +224 -0
  28. oceanstream/exports/shoals/shoals_handler.py +35 -0
  29. oceanstream/plot/__init__.py +2 -0
  30. oceanstream/plot/echogram.py +542 -0
  31. oceanstream/process/__init__.py +21 -0
  32. oceanstream/process/combine_zarr.py +90 -0
  33. oceanstream/process/file_processor.py +151 -0
  34. oceanstream/process/folder_processor.py +246 -0
  35. oceanstream/process/process.py +77 -0
  36. oceanstream/process/processed_data_io.py +140 -0
  37. oceanstream/settings/__init__.py +1 -0
  38. oceanstream/settings/config.py +17 -0
  39. oceanstream/utils.py +101 -0
  40. oceanstream-0.0.0.dev0.dist-info/LICENSE +21 -0
  41. oceanstream-0.0.0.dev0.dist-info/METADATA +214 -0
  42. oceanstream-0.0.0.dev0.dist-info/RECORD +46 -0
  43. oceanstream-0.0.0.dev0.dist-info/WHEEL +5 -0
  44. oceanstream-0.0.0.dev0.dist-info/dependency_links.txt +1 -0
  45. oceanstream-0.0.0.dev0.dist-info/entry_points.txt +2 -0
  46. oceanstream-0.0.0.dev0.dist-info/top_level.txt +1 -0
File without changes
@@ -0,0 +1,3 @@
1
+ """
2
+ CLI submodule for the OceanStream package.
3
+ """
@@ -0,0 +1,267 @@
1
+ import traceback
2
+
3
+ import typer
4
+ import asyncio
5
+ import os
6
+ import logging
7
+ import sys
8
+ import warnings
9
+
10
+ from pathlib import Path
11
+ from rich import print
12
+ from rich.traceback import install, Traceback
13
+ from oceanstream.settings import load_config
14
+ from dask.distributed import LocalCluster, Client
15
+ from rich.console import Console
16
+
17
+
18
+ install(show_locals=False, width=120)
19
+
20
+ DEFAULT_OUTPUT_FOLDER = "output"
21
+ DEFAULT_SONAR_MODEL = "EK60"
22
+
23
+ BANNER = """
24
+ _
25
+ | |
26
+ ___ ___ ___ __ _ _ __ ___| |_ _ __ ___ __ _ _ __ ___
27
+ / _ \ / __/ _ \/ _` | '_ \ / __| __| '__/ _ \/ _` | '_ ` _ \
28
+ | (_) | (_| __/ (_| | | | | \__ \ |_| | | __/ (_| | | | | | |
29
+ \___/ \___\___|\__,_|_| |_| |___/\__|_| \___|\__,_|_| |_| |_|
30
+
31
+ """
32
+
33
+ logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
34
+
35
+
36
+ def initialize(settings, file_path, log_level=None):
37
+ config_data = load_config(settings["config"])
38
+ config_data["raw_path"] = file_path
39
+
40
+ if log_level is not None:
41
+ logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s', force=True)
42
+ config_data["log_level"] = log_level
43
+
44
+ if 'sonar_model' in settings and settings["sonar_model"] is not None:
45
+ config_data["sonar_model"] = settings["sonar_model"]
46
+
47
+ if settings["output_folder"] is not None:
48
+ config_data["output_folder"] = settings["output_folder"]
49
+
50
+ return config_data
51
+
52
+
53
+ app = typer.Typer(help="OceanStream CLI")
54
+
55
+
56
+ @app.command()
57
+ def process(
58
+ source: str = typer.Option(..., help="Path to a raw data file"),
59
+ output: str = typer.Option(None,
60
+ help="Destination path for saving processed data. Defaults to a predefined "
61
+ "directory if not specified."),
62
+ sonar_model: str = typer.Option(None, help="Sonar model used to collect the data",
63
+ show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
64
+ plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
65
+ depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
66
+ waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
67
+ show_choices=["CW", "BB"]),
68
+ config: str = typer.Option(None, help="Path to a configuration file"),
69
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
70
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
71
+ ):
72
+ """
73
+ Process the input raw file and save the result to the specified output folder.
74
+ """
75
+ settings = {
76
+ "config": config,
77
+ "sonar_model": sonar_model,
78
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
79
+ }
80
+
81
+ filePath = Path(source)
82
+ configData = initialize(settings, filePath, log_level=log_level)
83
+
84
+ if filePath.is_file():
85
+ from oceanstream.process import process_raw_file_with_progress
86
+ asyncio.run(
87
+ process_raw_file_with_progress(configData, plot_echogram, waveform_mode=waveform_mode,
88
+ depth_offset=depth_offset))
89
+ else:
90
+ print(f"[red]❌ The provided path '{source}' is not a valid raw file.[/red]")
91
+ sys.exit(1)
92
+
93
+
94
+ @app.command()
95
+ def convert(
96
+ source: str = typer.Option(..., help="Path to a raw data file/folder"),
97
+ output: str = typer.Option(None,
98
+ help="Destination path for saving Zarr converted data. Defaults to a predefined "
99
+ "directory if not specified."),
100
+ workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
101
+ config: str = typer.Option(None, help="Path to a configuration file"),
102
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
103
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
104
+ ):
105
+ """
106
+ Convert the input raw data file(s) to Zarr and save the result to the specified output folder.
107
+ """
108
+ settings = {
109
+ "config": config,
110
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
111
+ }
112
+
113
+ filePath = Path(source)
114
+ configData = initialize(settings, filePath, log_level=log_level)
115
+
116
+ if filePath.is_file():
117
+ from oceanstream.process import convert_raw_file
118
+ convert_raw_file(filePath, configData)
119
+ print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
120
+ elif filePath.is_dir():
121
+ from oceanstream.process import convert_raw_files
122
+ convert_raw_files(configData, workers_count=workers_count)
123
+ else:
124
+ print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
125
+ sys.exit(1)
126
+
127
+
128
+ @app.command()
129
+ def combine(
130
+ source: str = typer.Option(..., help="Path to a source data folder where zarr files are located"),
131
+ output: str = typer.Option(None,
132
+ help="Destination path for saving combined zarr file. Defaults to a predefined "
133
+ "directory if not specified."),
134
+ workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
135
+ config: str = typer.Option(None, help="Path to a configuration file"),
136
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
137
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
138
+ ):
139
+ """
140
+ Process the input zarr data files and save the combined zarr file to the specified output folder.
141
+ """
142
+ settings = {
143
+ "config": config,
144
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
145
+ }
146
+
147
+ dir_path = Path(source)
148
+ configData = initialize(settings, dir_path, log_level=log_level)
149
+
150
+ if dir_path.is_dir():
151
+ console = Console()
152
+ with console.status("Processing...", spinner="dots") as status:
153
+ status.start()
154
+
155
+ cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
156
+ client = Client(cluster)
157
+
158
+ try:
159
+ from oceanstream.process import combine_zarr_files
160
+
161
+ file_name = f"{Path(dir_path).stem}-combined.zarr"
162
+ zarr_output_file = os.path.join(configData['output_folder'], file_name)
163
+ status.update(
164
+ f"Combining zarr files to {zarr_output_file}; navigate to http://localhost:8787/status for progress")
165
+
166
+ combine_zarr_files(dir_path,
167
+ zarr_output_file=zarr_output_file,
168
+ chunks=configData.get('base_chunk_sizes'))
169
+ print("\nZarr files have been combined successfully.")
170
+ except KeyboardInterrupt:
171
+ logging.info("KeyboardInterrupt received, terminating processes...")
172
+ except Exception as e:
173
+ logging.exception("Error processing folder %s", configData['raw_path'])
174
+ traceback.print_exc()
175
+ finally:
176
+ status.stop()
177
+ client.close()
178
+ cluster.close()
179
+ else:
180
+ print(f"[red]❌ The provided path '{source}' is not a valid folder.[/red]")
181
+ sys.exit(1)
182
+
183
+
184
+ @app.command()
185
+ def compute_sv(
186
+ source: str = typer.Option(..., help="Path to a Zarr root file or a directory containing Zarr files"),
187
+ output: str = typer.Option(None,
188
+ help="Destination path for saving Sv data. Defaults to a predefined directory if not specified."),
189
+ workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for Sv computation"),
190
+ sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
191
+ show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
192
+ plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
193
+ depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
194
+ waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
195
+ show_choices=["CW", "BB"]),
196
+ config: str = typer.Option(None, help="Path to a configuration file"),
197
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
198
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
199
+ ):
200
+ """
201
+ Compute the volume backscattering strength (Sv) from the Zarr file.
202
+ """
203
+ settings_dict = {
204
+ "config": config,
205
+ "sonar_model": sonar_model,
206
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
207
+ }
208
+ file_path = Path(source)
209
+ config_data = initialize(settings_dict, file_path, log_level=log_level)
210
+
211
+ console = Console()
212
+ with console.status("Processing...", spinner="dots") as status:
213
+ status.start()
214
+ cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
215
+ client = Client(cluster)
216
+
217
+ try:
218
+ if file_path.is_dir() and source.endswith(".zarr"):
219
+ status.update(
220
+ f"[blue] Computing Sv for {file_path}...[/blue] – navigate to http://localhost:8787/status for progress")
221
+ from oceanstream.process import compute_single_file
222
+
223
+ compute_single_file(config_data,
224
+ chunks=config_data.get('base_chunk_sizes'),
225
+ plot_echogram=plot_echogram,
226
+ waveform_mode=waveform_mode,
227
+ depth_offset=depth_offset)
228
+ elif file_path.is_dir():
229
+ status.update(
230
+ f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
231
+ f"http://localhost:8787/status for progress")
232
+ from oceanstream.process import process_zarr_files
233
+
234
+ process_zarr_files(config_data,
235
+ workers_count=workers_count,
236
+ status=status,
237
+ chunks=config_data.get('base_chunk_sizes'),
238
+ plot_echogram=plot_echogram,
239
+ waveform_mode=waveform_mode,
240
+ depth_offset=depth_offset)
241
+ else:
242
+ print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
243
+ sys.exit(1)
244
+ except KeyboardInterrupt:
245
+ logging.info("KeyboardInterrupt received, terminating processes...")
246
+ except Exception as e:
247
+ logging.exception("Error while processing %s", config_data['raw_path'])
248
+ traceback.print_exc()
249
+ finally:
250
+ client.close()
251
+ cluster.close()
252
+ status.stop()
253
+
254
+
255
+ @app.command()
256
+ def export():
257
+ typer.echo("Export data...")
258
+
259
+
260
+ def main():
261
+ print(BANNER)
262
+ warnings.filterwarnings("ignore", category=UserWarning)
263
+ app(prog_name="oceanstream")
264
+
265
+
266
+ if __name__ == "__main__":
267
+ main()
@@ -0,0 +1,2 @@
1
+ from .apply_masks import apply_background_noise_removal, apply_noise_masks, apply_seabed_mask
2
+ from .create_masks import create_masks
@@ -0,0 +1,49 @@
1
+ import pathlib
2
+ import xarray as xr
3
+
4
+ from typing import Dict, Union
5
+ from oceanstream.echodata import interpolate_sv
6
+
7
+
8
+ from .applying_masks_handler import (
9
+ apply_selected_noise_masks_and_or_noise_removal as apply_selected_masks,
10
+ )
11
+
12
+
13
+ def apply_noise_masks(sv_with_masks: Union[xr.Dataset, str, pathlib.Path], config) -> xr.Dataset:
14
+ noise_masks = {}
15
+
16
+ if config["impulse"]["enabled"]:
17
+ noise_masks["mask_impulse"] = {"var_name": "Sv"}
18
+ if config["attenuation"]["enabled"]:
19
+ noise_masks["mask_attenuation"] = {"var_name": "Sv"}
20
+ if config["transient"]["enabled"]:
21
+ noise_masks["mask_transient"] = {"var_name": "Sv"}
22
+
23
+ ds_processed = apply_selected_masks(sv_with_masks, noise_masks)
24
+
25
+ return ds_processed
26
+
27
+
28
+ def apply_background_noise_removal(
29
+ ds_processed: xr.Dataset, config
30
+ ) -> xr.Dataset:
31
+ ds_interpolated = interpolate_sv(ds_processed)
32
+ params = config["remove_background_noise"]["parameters"]
33
+
34
+ ds_clean = apply_selected_masks(
35
+ ds_interpolated, processes_to_apply={"remove_background_noise": params}
36
+ )
37
+
38
+ return ds_clean
39
+
40
+
41
+ def apply_seabed_mask(ds: xr.Dataset, config) -> xr.Dataset:
42
+ if config["seabed"]["enabled"] and config["false_seabed"]["enabled"]:
43
+ params = {"mask_seabed": {"var_name": "Sv"}, "mask_false_seabed": {"var_name": "Sv"}}
44
+
45
+ ds_processed = apply_selected_masks(ds, params)
46
+
47
+ return ds_processed
48
+
49
+ return ds
@@ -0,0 +1,173 @@
1
+ """
2
+ applying_masks_handler.py
3
+ -------------------------
4
+ Description: This module provides functionalities for applying a sequence of
5
+ selected masks and background noise removal to echosounder datasets based on
6
+ specific methodologies.
7
+
8
+ Masks/Processes supported in this module include:
9
+ **Noise-related masks/processes, namely:**
10
+ - 'mask_false_seabed'
11
+ - 'mask_seabed'
12
+ - 'mask_impulse'
13
+ - 'mask_attenuation'
14
+ - 'mask_transient'
15
+ - 'remove_background_noise'
16
+
17
+ **Organism-related masks, namely:**
18
+ - 'mask_krill'
19
+ - 'mask_gas_bearing_organisms'
20
+ - 'mask_fluid_like_organisms'
21
+ - 'mask_shoal'
22
+
23
+ Note: If other masks that aren't supported by this module's functions are
24
+ attached as variables to the dataset, they can be applied using
25
+ `echopype.mask.apply_mask`. For details, consult the echopype documentation.
26
+
27
+ Key Functions:
28
+ - `apply_selected_noise_masks_and_or_noise_removal`: For applying selected
29
+ noise masks and/or noise removal.
30
+
31
+ - `apply_mask_organisms_in_order`: For applying organism-related masks in
32
+ a user-defined order.
33
+ """
34
+
35
+
36
+ import pathlib
37
+ from typing import Union
38
+
39
+ import echopype as ep
40
+ import xarray as xr
41
+
42
+ from .background_noise_remover import apply_remove_background_noise
43
+
44
+
45
+ def apply_selected_noise_masks_and_or_noise_removal(
46
+ ds: Union[xr.Dataset, str, pathlib.Path],
47
+ processes_to_apply: dict,
48
+ ) -> xr.Dataset:
49
+ """
50
+ Apply a sequence of selected masks and background noise removal (if provided in the processes_to_apply)
51
+ to the given dataset based on the specifications in the article:
52
+ > Haris, K., et al. "Sounding out life in the deep using acoustic data from ships of opportunity."
53
+ > Scientific Data 8.1 (2021): 23.
54
+
55
+ Order of cleaning process:
56
+ 1. Impulse noise mask (if provided)
57
+ 2. Attenuated signal mask (if provided)
58
+ 3. Transient noise mask (if provided)
59
+ 4. Background noise mask (if provided)
60
+ 5. False seabed mask (if provided)
61
+ 6. Seabed mask (if provided)
62
+
63
+ Valid processes/masks:
64
+ - 'mask_impulse'
65
+ - 'mask_attenuation'
66
+ - 'mask_transient'
67
+ - 'remove_background_noise'
68
+ - 'mask_false_seabed'
69
+ - 'mask_seabed'
70
+
71
+ Parameters:
72
+ - `ds`: (`xr.Dataset`, `str`, or `pathlib.Path`)
73
+ Points to a Dataset that contains the variable the mask should be applied to
74
+ and must have the masks as attached variables.
75
+ - `processes_to_apply` (`dict`):
76
+ A dictionary where keys are mask names or 'remove_background_noise' and values are specific parameters
77
+ (sub-dictionaries).
78
+
79
+ For more details on parameters associated with applying a mask and the `background noise removal` process,
80
+ those parameters found within the specific sub-dictionaries, please refer to the documentation for:
81
+ - `echopype.mask.apply_mask`
82
+ - `background_noise_remover.apply_remove_background_noise`
83
+
84
+ Returns:
85
+ - `xr.Dataset`: Processed dataset.
86
+
87
+ """
88
+
89
+ valid_processes = [
90
+ "mask_impulse",
91
+ "mask_attenuation",
92
+ "mask_transient",
93
+ "remove_background_noise",
94
+ "mask_false_seabed",
95
+ "mask_seabed",
96
+ ]
97
+
98
+ # Check for unexpected masks/processes
99
+ for process in processes_to_apply:
100
+ if process not in valid_processes:
101
+ raise ValueError(
102
+ "Unexpected mask/process. Please refer to the function documentation for valid masks/processes."
103
+ )
104
+
105
+ for process in valid_processes:
106
+ if process in processes_to_apply:
107
+ params = processes_to_apply[process]
108
+ if process in [
109
+ "mask_impulse",
110
+ "mask_attenuation",
111
+ "mask_transient",
112
+ "mask_false_seabed",
113
+ "mask_seabed",
114
+ ]:
115
+ mask_data = ds[process]
116
+ ds = ep.mask.apply_mask(ds, mask_data, **params)
117
+ elif process == "remove_background_noise":
118
+ ds = apply_remove_background_noise(ds, **params)
119
+
120
+ return ds
121
+
122
+
123
+ def apply_mask_organisms_in_order(ds, processes_to_apply):
124
+ """
125
+ Apply a sequence of selected masks to the given dataset in the order that
126
+ the masks were provided in the input dictionary.
127
+
128
+ Valid masks:
129
+ 'mask_krill',
130
+ 'mask_gas_bearing_organisms',
131
+ 'mask_fluid_like_organisms',
132
+ 'mask_shoal'
133
+
134
+ Parameters:
135
+ - ds (xr.Dataset, str, or pathlib.Path):
136
+ Points to a Dataset that contains the variable the mask should be applied to
137
+ and must have the masks as attached variables.
138
+ - processes_to_apply (dict):
139
+ A dictionary where keys are mask names and values are specific parameters (sub-dictionaries).
140
+
141
+ For more details on parameters associated with applying a mask,
142
+ those parameters found within the specific sub-dictionaries, please refer to the documentation for:
143
+ - `echopype.mask.apply_mask`
144
+
145
+ Returns:
146
+ - xr.Dataset: Processed dataset.
147
+ """
148
+
149
+ recognized_masks = [
150
+ "mask_krill",
151
+ "mask_gas_bearing_organisms",
152
+ "mask_fluid_like_organisms",
153
+ "mask_shoal",
154
+ ]
155
+
156
+ for mask_name in processes_to_apply.keys():
157
+ if mask_name not in recognized_masks:
158
+ raise ValueError(
159
+ "Unrecognized mask name. Please refer to the function documentation for valid masks"
160
+ )
161
+
162
+ mask_params = processes_to_apply[mask_name]
163
+
164
+ if mask_name == "mask_krill":
165
+ ds = ep.mask.apply_mask(ds, ds[mask_name], **mask_params)
166
+ elif mask_name == "mask_gas_bearing_organisms":
167
+ ds = ep.mask.apply_mask(ds, ds[mask_name], **mask_params)
168
+ elif mask_name == "mask_fluid_like_organisms":
169
+ ds = ep.mask.apply_mask(ds, ds[mask_name], **mask_params)
170
+ elif mask_name == "mask_shoal":
171
+ ds = ep.mask.apply_mask(ds, ds[mask_name], **mask_params)
172
+
173
+ return ds
@@ -0,0 +1,97 @@
1
+ """
2
+ background_noise_remover.py
3
+ ----------------------------
4
+
5
+ This module provides functionalities to enhance the quality of echosounder data by removing background noise.
6
+
7
+ Functions:
8
+
9
+ - `apply_remove_background_noise`:
10
+ Main function that applies the `remove_noise` function from echopype to a Sv dataset.
11
+ Computes default parameters based on dataset characteristics when not provided.
12
+ The dataset initially passed to the function, processed in the following manner:
13
+ `Sv_corrected` and `Sv_noise` variables are generated by the echopype `remove_noise` function.
14
+ `Sv` contains the values originally in `Sv_corrected`.
15
+ `Sv_with_background_noise` stores the original values of `Sv`.
16
+
17
+ Usage:
18
+
19
+ To remove background noise for a given Sv dataset, `ds_Sv`, simply call:
20
+ `apply_remove_background_noise(ds_Sv)`
21
+
22
+ Note:
23
+ - Ensure that the echopype library is properly installed and imported when using this module.
24
+ - Sv = the volume backscattering strength
25
+
26
+ """
27
+ import echopype as ep
28
+ import numpy as np
29
+ import xarray as xr
30
+
31
+
32
+ def apply_remove_background_noise(
33
+ ds_Sv: xr.Dataset,
34
+ ping_num: int = 40,
35
+ range_sample_num: int = None,
36
+ noise_max: float = -125,
37
+ SNR_threshold: float = 3,
38
+ ) -> xr.Dataset:
39
+ """
40
+ Applies the `remove_noise` function from the echopype library to the `ds_Sv` dataset.
41
+ If `range_sample_num` is not provided, the function computes it based on a 10-meter vertical bin
42
+ for each channel, and then uses the minimum value across all channels as the default.
43
+
44
+ For detailed information on `remove_noise`, consult the documentation
45
+ within the `clean` module of the echopype library.
46
+
47
+ For default values and their context, refer to:
48
+ De Robertis & Higginbottom (2007). "A post-processing technique to estimate
49
+ the signal-to-noise ratio and remove echosounder background noise."
50
+ ICES Journal of Marine Sciences, 64(6), 1282–1291.
51
+
52
+ Parameters:
53
+ - ds_Sv: xr.Dataset
54
+ Dataset containing Sv and echo_range [m].
55
+ - ping_num: int, default=40
56
+ Number of pings to obtain noise estimates.
57
+ - range_sample_num: int, optional
58
+ Number of samples along the range_sample dimension to obtain noise estimates. If not provided,
59
+ it's computed based on a 10-meter vertical bin for each channel and the minimum value is used.
60
+ - noise_max: float, default=-125
61
+ The upper limit for background noise expected under the operating conditions (default=-125 dB).
62
+ - SNR_threshold: float, default=3 [dB]
63
+ Acceptable signal-to-noise ratio.
64
+
65
+ Returns:
66
+ - ds_Sv: xr.Dataset
67
+
68
+ Notes:
69
+ Returns - initial xr.Dataset:
70
+ The dataset initially passed to the function, processed in the following manner:
71
+ - `Sv_corrected` and `Sv_noise` variables are generated by the echopype `remove_noise` function.
72
+ - `Sv` contains the values originally in `Sv_corrected`.
73
+ - `Sv_with_background_noise` stores the original values of `Sv`.
74
+ """
75
+
76
+ # Compute range_sample_num if not provided
77
+ if range_sample_num is None:
78
+ range_sample_nums = []
79
+ for ch in range(ds_Sv.sizes["channel"]):
80
+ mean_diff = np.nanmean(np.diff(ds_Sv["echo_range"].isel(channel=ch).values[0, :]))
81
+ if np.isnan(mean_diff):
82
+ raise ValueError(
83
+ "The default computed value for the range_sample_num is nan, please add the range_sample_num as input parameter"
84
+ )
85
+ range_sample_nums.append(int(10 / mean_diff))
86
+
87
+ range_sample_num = min(range_sample_nums)
88
+ # Remove noise
89
+ ds_Sv_processed = ep.clean.remove_background_noise(
90
+ ds_Sv,
91
+ ping_num=ping_num,
92
+ range_sample_num=range_sample_num,
93
+ noise_max=noise_max,
94
+ SNR_threshold=SNR_threshold,
95
+ )
96
+
97
+ return ds_Sv_processed
@@ -0,0 +1,87 @@
1
+ import concurrent.futures
2
+ import time
3
+ from collections.abc import Iterable
4
+ from typing import Dict
5
+
6
+ import xarray
7
+
8
+ # from echopype.clean.api import (
9
+ # get_attenuation_mask_multichannel,
10
+ # get_impulse_noise_mask_multichannel,
11
+ # get_transient_noise_mask_multichannel,
12
+ # )
13
+ # from echopype.mask.api import get_seabed_mask_multichannel
14
+
15
+ from oceanstream.utils import add_metadata_to_mask, dict_to_formatted_list
16
+
17
+ from .types import DenoiseConfig
18
+
19
+ MASK_CREATION_FUNCTIONS = {
20
+ # "transient": get_transient_noise_mask_multichannel,
21
+ # "attenuation": get_attenuation_mask_multichannel,
22
+ # "impulse": get_impulse_noise_mask_multichannel,
23
+ # "false_seabed": get_seabed_mask_multichannel,
24
+ # "seabed": get_seabed_mask_multichannel,
25
+ }
26
+
27
+
28
+ def create_masks(source_Sv: xarray.Dataset, profiling_info: Dict, config: DenoiseConfig):
29
+ masks = []
30
+
31
+ with concurrent.futures.ThreadPoolExecutor() as executor:
32
+ future_to_mask_type = {}
33
+ future_to_start_time = {}
34
+
35
+ for mask_type, mask_config in config.items():
36
+ if is_scalar(mask_config):
37
+ continue
38
+
39
+ if mask_config["enabled"] and (mask_type in MASK_CREATION_FUNCTIONS):
40
+ profiling_info[mask_type + " mask"] = {}
41
+ # Map the mask type to the appropriate creation function
42
+ create_mask_func = MASK_CREATION_FUNCTIONS.get(mask_type)
43
+
44
+ if config["profile"]:
45
+ future_to_start_time[mask_type] = time.time()
46
+
47
+ future = executor.submit(
48
+ create_and_add_metadata, create_mask_func, source_Sv, mask_type, mask_config
49
+ )
50
+ future_to_mask_type[future] = mask_type
51
+
52
+ for future in concurrent.futures.as_completed(future_to_mask_type):
53
+ mask_type = future_to_mask_type[future]
54
+
55
+ try:
56
+ mask = future.result()
57
+ masks.append((mask_type, mask))
58
+ if config["profile"]:
59
+ end_time = time.time()
60
+ start_time = future_to_start_time[mask_type]
61
+ profiling_info[mask_type + " mask"]["execution_time"] = end_time - start_time
62
+ except Exception as exc:
63
+ print(f"{future_to_mask_type[future]} mask generated an exception: {exc}")
64
+
65
+ return tuple(masks), profiling_info
66
+
67
+
68
+ def create_and_add_metadata(create_mask_func, source_Sv, mask_type, config_item):
69
+ mask = create_mask_func(
70
+ source_Sv, parameters=config_item["parameters"], method=config_item["method"]
71
+ )
72
+
73
+ return add_metadata_to_mask(
74
+ mask=mask,
75
+ metadata={
76
+ "mask_type": mask_type,
77
+ "method": config_item["method"],
78
+ "parameters": dict_to_formatted_list(config_item["parameters"]),
79
+ },
80
+ )
81
+
82
+
83
+ def is_scalar(value):
84
+ if isinstance(value, Iterable) and not isinstance(value, str):
85
+ return False
86
+
87
+ return True