oceanstream 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanstream/__init__.py +0 -0
- oceanstream/cli/__init__.py +3 -0
- oceanstream/cli/main.py +267 -0
- oceanstream/denoise/__init__.py +2 -0
- oceanstream/denoise/apply_masks.py +49 -0
- oceanstream/denoise/applying_masks_handler.py +173 -0
- oceanstream/denoise/background_noise_remover.py +97 -0
- oceanstream/denoise/create_masks.py +87 -0
- oceanstream/denoise/noise_masks.py +398 -0
- oceanstream/denoise/types.py +73 -0
- oceanstream/echodata/__init__.py +5 -0
- oceanstream/echodata/ensure_time_continuity.py +75 -0
- oceanstream/echodata/mvbs_computation.py +77 -0
- oceanstream/echodata/raw_handler.py +577 -0
- oceanstream/echodata/sv_computation.py +118 -0
- oceanstream/echodata/sv_dataset_extension.py +126 -0
- oceanstream/echodata/sv_interpolation.py +260 -0
- oceanstream/echodata/target_strength_computation.py +109 -0
- oceanstream/exports/__init__.py +2 -0
- oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
- oceanstream/exports/csv/csv_export_from_raw.py +92 -0
- oceanstream/exports/csv/csv_export_nasc.py +135 -0
- oceanstream/exports/frequency_differencing_handler.py +210 -0
- oceanstream/exports/nasc.py +23 -0
- oceanstream/exports/nasc_computation.py +62 -0
- oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
- oceanstream/exports/shoals/shoal_process.py +224 -0
- oceanstream/exports/shoals/shoals_handler.py +35 -0
- oceanstream/plot/__init__.py +2 -0
- oceanstream/plot/echogram.py +542 -0
- oceanstream/process/__init__.py +21 -0
- oceanstream/process/combine_zarr.py +90 -0
- oceanstream/process/file_processor.py +151 -0
- oceanstream/process/folder_processor.py +246 -0
- oceanstream/process/process.py +77 -0
- oceanstream/process/processed_data_io.py +140 -0
- oceanstream/settings/__init__.py +1 -0
- oceanstream/settings/config.py +17 -0
- oceanstream/utils.py +101 -0
- oceanstream-0.0.0.dev0.dist-info/LICENSE +21 -0
- oceanstream-0.0.0.dev0.dist-info/METADATA +214 -0
- oceanstream-0.0.0.dev0.dist-info/RECORD +46 -0
- oceanstream-0.0.0.dev0.dist-info/WHEEL +5 -0
- oceanstream-0.0.0.dev0.dist-info/dependency_links.txt +1 -0
- oceanstream-0.0.0.dev0.dist-info/entry_points.txt +2 -0
- oceanstream-0.0.0.dev0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,577 @@
|
|
|
1
|
+
"""
|
|
2
|
+
raw_handler.py
|
|
3
|
+
--------------
|
|
4
|
+
Module for reading, verifying, and converting echo sounder raw data files.
|
|
5
|
+
|
|
6
|
+
This module provides functionalities to:
|
|
7
|
+
|
|
8
|
+
- Search for raw echo sounder files within specified directories or paths.
|
|
9
|
+
- Verify the integrity of these files, ensuring they are readable by echopype.
|
|
10
|
+
- Extract essential metadata from the files, such as campaign ID,\
|
|
11
|
+
date of measurement, and sonar model.
|
|
12
|
+
- Convert raw files to specified formats (netCDF or zarr) and\
|
|
13
|
+
save them to a desired location.
|
|
14
|
+
- Group similar files based on specific criteria, such as campaign ID,\
|
|
15
|
+
sonar model, and time difference.
|
|
16
|
+
|
|
17
|
+
The module supports various sonar models including
|
|
18
|
+
EK60, ES70, EK80, EA640, AZFP, and AD2CP.
|
|
19
|
+
It also defines a time threshold for determining
|
|
20
|
+
the similarity between two consecutive files.
|
|
21
|
+
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
import re
|
|
26
|
+
import echopype as ep
|
|
27
|
+
import xml.etree.ElementTree as ET
|
|
28
|
+
from datetime import datetime, timedelta
|
|
29
|
+
from typing import Dict, List, Union, Tuple, Any
|
|
30
|
+
from echopype.convert.utils.ek_raw_io import RawSimradFile
|
|
31
|
+
from .ensure_time_continuity import check_reversed_time, fix_time_reversions
|
|
32
|
+
|
|
33
|
+
SUPPORTED_SONAR_MODELS = ["EK60", "ES70", "EK80", "EA640", "AZFP", "AD2CP"]
|
|
34
|
+
TIME_BETWEEN_FILES = 30 # time in minutes between two consecutive files
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def read_file(config_data, use_swap='auto', skip_integrity_check=False, skip_time_reversion=False):
|
|
38
|
+
filename = config_data["raw_path"]
|
|
39
|
+
|
|
40
|
+
if not skip_integrity_check:
|
|
41
|
+
check = file_integrity_checking(filename)
|
|
42
|
+
if not check.get("file_integrity", False):
|
|
43
|
+
return {"Processing Error": f"File {filename} could not usable!"}
|
|
44
|
+
|
|
45
|
+
sonar_model = check.get("sonar_model", config_data["sonar_model"])
|
|
46
|
+
else:
|
|
47
|
+
sonar_model = config_data["sonar_model"]
|
|
48
|
+
|
|
49
|
+
echodata = _read_file(filename, use_swap=use_swap, sonar_model=sonar_model)
|
|
50
|
+
|
|
51
|
+
if sonar_model == "EK80":
|
|
52
|
+
encode_mode = get_encode_mode(echodata)
|
|
53
|
+
else:
|
|
54
|
+
encode_mode = "power"
|
|
55
|
+
|
|
56
|
+
# Fix time reversions if necessary
|
|
57
|
+
if not skip_time_reversion and check_reversed_time(echodata, "Sonar/Beam_group1", "ping_time"):
|
|
58
|
+
echodata = fix_time_reversions(echodata, {"Sonar/Beam_group1": "ping_time"})
|
|
59
|
+
|
|
60
|
+
return echodata, encode_mode
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def file_finder(paths: Union[str, List[str]], file_type: str = "raw") -> List[str]: # noqa: E501
|
|
64
|
+
"""
|
|
65
|
+
Finds and returns all files of a specified type from given paths.
|
|
66
|
+
|
|
67
|
+
This function searches for files of a specified type (e.g., "raw")
|
|
68
|
+
within the provided paths.
|
|
69
|
+
It can search within a single directory or
|
|
70
|
+
across multiple specified file paths.
|
|
71
|
+
|
|
72
|
+
Parameters:
|
|
73
|
+
|
|
74
|
+
- paths (str or list[str]): If a string is provided,\
|
|
75
|
+
it should be the absolute path to a directory.\
|
|
76
|
+
If a list is provided,\
|
|
77
|
+
it should contain absolute paths to individual files.
|
|
78
|
+
- file_type (str, optional): The type of files to search for.\
|
|
79
|
+
Defaults to "raw".
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
|
|
83
|
+
- list[str]: A sorted list of all found files of the specified type.
|
|
84
|
+
|
|
85
|
+
Raises:
|
|
86
|
+
|
|
87
|
+
- ValueError: If the provided paths input is neither a directory\
|
|
88
|
+
nor a list of file paths.
|
|
89
|
+
|
|
90
|
+
Example:
|
|
91
|
+
|
|
92
|
+
file_finder("/path/to/directory")
|
|
93
|
+
['/path/to/directory/file1.raw', '/path/to/directory/file2.raw']
|
|
94
|
+
|
|
95
|
+
file_finder(["/path/to/file1.raw", "/path/to/file2.raw"])
|
|
96
|
+
['/path/to/file1.raw', '/path/to/file2.raw']
|
|
97
|
+
"""
|
|
98
|
+
if file_type == "zarr" or file_type == ".zarr":
|
|
99
|
+
zarr_files = _find_zarr_root_directories(paths)
|
|
100
|
+
if isinstance(paths, str):
|
|
101
|
+
# Filter out subdirectory paths
|
|
102
|
+
zarr_files = [f for f in zarr_files if os.path.dirname(f) == paths]
|
|
103
|
+
return sorted(zarr_files)
|
|
104
|
+
|
|
105
|
+
if isinstance(paths, str) and os.path.isdir(paths):
|
|
106
|
+
ret_files = [
|
|
107
|
+
os.path.join(paths, f_p)
|
|
108
|
+
for f_p in os.listdir(paths)
|
|
109
|
+
if os.path.isfile(os.path.join(paths, f_p))
|
|
110
|
+
]
|
|
111
|
+
ret_files = file_finder(ret_files, file_type)
|
|
112
|
+
elif isinstance(paths, list):
|
|
113
|
+
ret_files = []
|
|
114
|
+
for elem in paths:
|
|
115
|
+
if "." + file_type in elem and os.path.isfile(elem):
|
|
116
|
+
ret_files.append(elem)
|
|
117
|
+
else:
|
|
118
|
+
raise ValueError(
|
|
119
|
+
"Invalid input. Provide either a directory\
|
|
120
|
+
path or a list of file paths."
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
return sorted(ret_files)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def file_integrity_checking(
|
|
127
|
+
file_path: str
|
|
128
|
+
) -> Dict[str, Union[str, datetime, bool]]: # noqa: E501
|
|
129
|
+
"""
|
|
130
|
+
Checks the integrity of a given echo sounder file.
|
|
131
|
+
|
|
132
|
+
This function verifies if the provided echo sounder file is
|
|
133
|
+
readable by echopype and extracts essential metadata such as the campaign ID, date of measurement, sonar model
|
|
134
|
+
and `use_swap` option (relevant only for raw files)
|
|
135
|
+
|
|
136
|
+
The function supports raw, netCDF, and zarr file formats.
|
|
137
|
+
|
|
138
|
+
Parameters:
|
|
139
|
+
- file_path (str): Absolute path to the echo sounder file.
|
|
140
|
+
- use_swap (bool, optional): Parameter specific to the echopype library `open_raw` function. Defaults to False\
|
|
141
|
+
If True, variables with a large memory footprint will be written to a temporary zarr store at \
|
|
142
|
+
``~/.echopype/temp_output/parsed2zarr_temp_files``\
|
|
143
|
+
Relevant only for raw files.
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
|
|
147
|
+
- dict: A dictionary containing the following keys:
|
|
148
|
+
'file_path': Absolute path to the file.
|
|
149
|
+
'campaign_id': Identifier for the measuring campaign,
|
|
150
|
+
extracted from the file name.
|
|
151
|
+
'date': Date and time when the measurement started,
|
|
152
|
+
extracted from the file name. Returns None if the date
|
|
153
|
+
and time cannot be determined.
|
|
154
|
+
'file_integrity': Boolean indicating if the file is of a supported type.
|
|
155
|
+
|
|
156
|
+
Raises:
|
|
157
|
+
- Exception: If the file type is not supported.
|
|
158
|
+
|
|
159
|
+
Example:
|
|
160
|
+
file_integrity_checking("/path/to/JR161-D20230509-T100645.raw")
|
|
161
|
+
{
|
|
162
|
+
'file_path': '/path/to/JR161-D20230509-T100645.raw',
|
|
163
|
+
'campaign_id': 'JR161',
|
|
164
|
+
'date': datetime.datetime(2023, 5, 9, 10, 6, 45),
|
|
165
|
+
'file_integrity': True
|
|
166
|
+
}
|
|
167
|
+
"""
|
|
168
|
+
return_dict = {}
|
|
169
|
+
|
|
170
|
+
# get file name from path
|
|
171
|
+
file_path = os.path.abspath(file_path)
|
|
172
|
+
_, file_name = os.path.split(file_path)
|
|
173
|
+
_, file_extension = os.path.splitext(file_path)
|
|
174
|
+
file_extension = file_extension.lower()
|
|
175
|
+
|
|
176
|
+
if file_extension not in [".raw", ".nc", ".zarr"]:
|
|
177
|
+
raise Exception("File type not supported for " + str(file_path))
|
|
178
|
+
|
|
179
|
+
campaign_id, date, sonar_model, _, file_integrity = get_campaign_metadata(file_path)
|
|
180
|
+
|
|
181
|
+
return_dict["file_path"] = file_path
|
|
182
|
+
return_dict["campaign_id"] = campaign_id
|
|
183
|
+
return_dict["date"] = date
|
|
184
|
+
return_dict["file_integrity"] = file_integrity
|
|
185
|
+
return_dict["sonar_model"] = sonar_model
|
|
186
|
+
|
|
187
|
+
return return_dict
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def get_campaign_metadata(file_path: str) -> tuple[
|
|
191
|
+
str | None | Any, datetime | None | Any, str | None, bytes | None, bool
|
|
192
|
+
]:
|
|
193
|
+
file_path = os.path.abspath(file_path)
|
|
194
|
+
_, file_name = os.path.split(file_path)
|
|
195
|
+
_, file_extension = os.path.splitext(file_path)
|
|
196
|
+
file_extension = file_extension.lower()
|
|
197
|
+
|
|
198
|
+
metadata = None
|
|
199
|
+
file_integrity = True
|
|
200
|
+
sonar_model = None
|
|
201
|
+
date = None
|
|
202
|
+
campaign_id = None
|
|
203
|
+
|
|
204
|
+
if ".raw" == file_extension:
|
|
205
|
+
metadata = _parse_metadata(file_path)
|
|
206
|
+
if metadata is not None:
|
|
207
|
+
campaign_id = metadata.get("survey_name", None)
|
|
208
|
+
date = metadata.get("timestamp", None)
|
|
209
|
+
sonar_model = detect_sonar_model(file_path, metadata=metadata)
|
|
210
|
+
|
|
211
|
+
if not metadata:
|
|
212
|
+
campaign_id = file_name.split("-")[0]
|
|
213
|
+
|
|
214
|
+
try:
|
|
215
|
+
pattern_date = r"D(\d{4})(\d{2})(\d{2})"
|
|
216
|
+
pattern_time = r"T(\d{2})(\d{2})(\d{2})"
|
|
217
|
+
|
|
218
|
+
matches_date = re.findall(pattern_date, file_name)[0]
|
|
219
|
+
matches_time = re.findall(pattern_time, file_name)[0]
|
|
220
|
+
|
|
221
|
+
year, month, day = matches_date
|
|
222
|
+
hour, minute, second = matches_time
|
|
223
|
+
|
|
224
|
+
datetime_string = f"D{year}{month}{day}-T{hour}{minute}{second}"
|
|
225
|
+
date = datetime.strptime(datetime_string, "D%Y%m%d-T%H%M%S")
|
|
226
|
+
except Exception as e:
|
|
227
|
+
e += "!"
|
|
228
|
+
file_integrity = False
|
|
229
|
+
date = None
|
|
230
|
+
|
|
231
|
+
return campaign_id, date, sonar_model, metadata, file_integrity
|
|
232
|
+
|
|
233
|
+
def read_raw_files(
|
|
234
|
+
file_dicts: List[Dict[str, Union[str, datetime, bool]]]
|
|
235
|
+
) -> List[ep.echodata.EchoData]:
|
|
236
|
+
"""
|
|
237
|
+
Reads multiple raw echo sounder files and returns a list of Datasets.
|
|
238
|
+
|
|
239
|
+
This function processes a list of file information dictionaries,
|
|
240
|
+
opens each raw file
|
|
241
|
+
using the specified sonar model,
|
|
242
|
+
and returns the corresponding datasets.
|
|
243
|
+
|
|
244
|
+
Parameters:
|
|
245
|
+
|
|
246
|
+
- file_dicts (list of dict): List of dictionaries, \
|
|
247
|
+
each containing file information \
|
|
248
|
+
as provided by the file_integrity_checking function.
|
|
249
|
+
|
|
250
|
+
Returns:
|
|
251
|
+
|
|
252
|
+
- list: List of EchoData datasets corresponding to each raw file.
|
|
253
|
+
|
|
254
|
+
"""
|
|
255
|
+
ret_list = []
|
|
256
|
+
for f_i in file_dicts:
|
|
257
|
+
opened_file = _read_file(file_path=f_i["file_path"])
|
|
258
|
+
ret_list.append(opened_file)
|
|
259
|
+
return ret_list
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def convert_raw_files(
|
|
263
|
+
file_dicts: List[Dict[str, Union[str, datetime, bool]]],
|
|
264
|
+
save_path: str = "",
|
|
265
|
+
save_file_type: str = "nc",
|
|
266
|
+
) -> List[str]:
|
|
267
|
+
"""
|
|
268
|
+
Converts multiple raw echo sounder files to the
|
|
269
|
+
specified file type and saves them.
|
|
270
|
+
|
|
271
|
+
This function processes a list of file information dictionaries,
|
|
272
|
+
converts each raw file
|
|
273
|
+
to the specified file type (netCDF or zarr),
|
|
274
|
+
and saves the converted files to the given path.
|
|
275
|
+
|
|
276
|
+
Parameters:
|
|
277
|
+
|
|
278
|
+
- file_dicts (list of dict): List of dictionaries,\
|
|
279
|
+
each containing file information.
|
|
280
|
+
- save_path (str): Directory path where\
|
|
281
|
+
the converted files will be saved.
|
|
282
|
+
- save_file_type (str): Desired file type\
|
|
283
|
+
for saving the converted files.\
|
|
284
|
+
Options are 'nc' or 'zarr'.
|
|
285
|
+
|
|
286
|
+
Returns:
|
|
287
|
+
|
|
288
|
+
- list: List of paths to the saved converted files.
|
|
289
|
+
|
|
290
|
+
"""
|
|
291
|
+
ret_list = []
|
|
292
|
+
for f_i in file_dicts:
|
|
293
|
+
opened_file = _read_file(file_path=f_i["file_path"])
|
|
294
|
+
_write_file(opened_file, save_path, save_file_type)
|
|
295
|
+
file_name = os.path.split(f_i["file_path"])[-1]
|
|
296
|
+
file_type = save_file_type
|
|
297
|
+
new_file_name = file_name.replace("raw", file_type)
|
|
298
|
+
ret_list.append(os.path.join(save_path, new_file_name))
|
|
299
|
+
return ret_list
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def split_files(
|
|
303
|
+
file_dicts: List[Dict[str, Union[str, datetime, bool]]]
|
|
304
|
+
) -> List[List[Dict[str, Union[str, datetime, bool]]]]:
|
|
305
|
+
"""
|
|
306
|
+
Splits a list of file information dictionaries into sublists based on their similarity.
|
|
307
|
+
|
|
308
|
+
This function processes a list of file information dictionaries and groups them into
|
|
309
|
+
sublists where each sublist contains files that are similar to each other based on
|
|
310
|
+
specific criteria.
|
|
311
|
+
|
|
312
|
+
Parameters:
|
|
313
|
+
- file_dicts (list of dict): List of file information dictionaries.
|
|
314
|
+
|
|
315
|
+
Returns:
|
|
316
|
+
- list of lists: List containing sublists of file dictionaries\
|
|
317
|
+
grouped by their similarity.
|
|
318
|
+
|
|
319
|
+
"""
|
|
320
|
+
list_of_lists = []
|
|
321
|
+
|
|
322
|
+
temp_list = []
|
|
323
|
+
prev_elem = file_dicts[0]
|
|
324
|
+
for elem in file_dicts:
|
|
325
|
+
if _is_similar(elem, prev_elem):
|
|
326
|
+
temp_list.append(elem)
|
|
327
|
+
else:
|
|
328
|
+
list_of_lists.append(temp_list)
|
|
329
|
+
temp_list = [elem]
|
|
330
|
+
prev_elem = elem
|
|
331
|
+
list_of_lists.append(temp_list)
|
|
332
|
+
return list_of_lists
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def concatenate_files(
|
|
336
|
+
file_dicts: List[Dict[str, Union[str, datetime, bool]]]
|
|
337
|
+
) -> ep.echodata.EchoData:
|
|
338
|
+
list_of_datasets = []
|
|
339
|
+
for file_info in file_dicts:
|
|
340
|
+
list_of_datasets.append(_read_file(file_info["file_path"]))
|
|
341
|
+
combined_dataset = ep.combine_echodata(list_of_datasets)
|
|
342
|
+
return combined_dataset
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def get_encode_mode(echo_data) -> str:
|
|
346
|
+
"""
|
|
347
|
+
For EK80:
|
|
348
|
+
If only complex data (can be BB or CW signals) exist,
|
|
349
|
+
there exists only Beam_group1 and this group may
|
|
350
|
+
contain CW or BB complex data, or a mixture of both. See example below.
|
|
351
|
+
|
|
352
|
+
If only power/angle data (only valid for CW signals) exist,
|
|
353
|
+
there exists only Beam_group1 and this group contains CW power and angle data.
|
|
354
|
+
The structure is almost identical with EK60 data above.
|
|
355
|
+
|
|
356
|
+
If both complex and power/angle data exist,
|
|
357
|
+
there exist Beam_group1 (containing complex data)
|
|
358
|
+
and Beam_group2 (containing power/angle data).
|
|
359
|
+
"""
|
|
360
|
+
beam_group1, beam_group2 = _check_beam_groups(echo_data)
|
|
361
|
+
encode_mode = "power"
|
|
362
|
+
if beam_group1:
|
|
363
|
+
imaginary_part = "backscatter_i" in echo_data.data_vars
|
|
364
|
+
if not imaginary_part:
|
|
365
|
+
return "power"
|
|
366
|
+
else:
|
|
367
|
+
return "complex"
|
|
368
|
+
if beam_group2:
|
|
369
|
+
return "complex"
|
|
370
|
+
return encode_mode
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _parse_metadata(file_path):
|
|
374
|
+
try:
|
|
375
|
+
with RawSimradFile(file_path, "r", storage_options={}) as fid:
|
|
376
|
+
config_datagram = fid.read(1)
|
|
377
|
+
return config_datagram
|
|
378
|
+
except Exception as e:
|
|
379
|
+
print(f"Error parsing metadata from {file_path}. Error: {e}")
|
|
380
|
+
return None
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _find_zarr_root_directories(paths: Union[str, List[str]]) -> List[str]:
|
|
384
|
+
"""
|
|
385
|
+
Finds and returns paths to the root directories of zarr datasets within the given paths.
|
|
386
|
+
|
|
387
|
+
Parameters:
|
|
388
|
+
- paths (str or List[str]): A single directory path or a list of directory paths.
|
|
389
|
+
|
|
390
|
+
Returns:
|
|
391
|
+
- List[str]: A list of paths to the root directories of found zarr datasets.
|
|
392
|
+
|
|
393
|
+
Raises:
|
|
394
|
+
- ValueError: If the provided paths input is not valid.
|
|
395
|
+
"""
|
|
396
|
+
zarr_roots = []
|
|
397
|
+
|
|
398
|
+
def is_zarr_root(directory: str) -> bool:
|
|
399
|
+
"""
|
|
400
|
+
Checks if a directory is the root of a zarr dataset.
|
|
401
|
+
"""
|
|
402
|
+
return any(
|
|
403
|
+
fname.endswith(".zarray") or fname.endswith(".zgroup")
|
|
404
|
+
for fname in os.listdir(directory)
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
if isinstance(paths, str):
|
|
408
|
+
if not os.path.isdir(paths):
|
|
409
|
+
raise ValueError(f"Path {paths} is not a valid directory.")
|
|
410
|
+
search_paths = [paths]
|
|
411
|
+
elif isinstance(paths, list):
|
|
412
|
+
search_paths = paths
|
|
413
|
+
else:
|
|
414
|
+
raise ValueError(
|
|
415
|
+
"Invalid input. Provide either a directory path or a list of directory paths."
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
for path in search_paths:
|
|
419
|
+
for root, dirs, _ in os.walk(path):
|
|
420
|
+
if is_zarr_root(root):
|
|
421
|
+
zarr_roots.append(root)
|
|
422
|
+
dirs[:] = [] # Skip subdirectories to avoid nested zarr datasets
|
|
423
|
+
|
|
424
|
+
return sorted(zarr_roots)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _read_file(file_path: str, sonar_model: str = None, use_swap: Union[bool, str] = 'auto',
|
|
428
|
+
max_chunk_size: str = None) -> ep.echodata.EchoData:
|
|
429
|
+
"""
|
|
430
|
+
Reads an echo sounder file and returns the corresponding Dataset.
|
|
431
|
+
|
|
432
|
+
This function determines the type of the file (raw, netCDF, or zarr) based on its extension and opens it using
|
|
433
|
+
echopype. The sonar_model and use_swap parameters are relevant only for the raw files.
|
|
434
|
+
|
|
435
|
+
Parameters:
|
|
436
|
+
|
|
437
|
+
- file_path (str): Absolute path to the echo sounder file.
|
|
438
|
+
- sonar_model (str, optional): Type of sonar model. Defaults to "EK60".\
|
|
439
|
+
Relevant only for raw files.
|
|
440
|
+
- use_swap (bool, optional): Parameter specific to the echopype library `open_raw` function. Defaults to False\
|
|
441
|
+
If True, variables with a large memory footprint will be written to a temporary zarr store at \
|
|
442
|
+
``~/.echopype/temp_output/parsed2zarr_temp_files``\
|
|
443
|
+
Relevant only for raw files.
|
|
444
|
+
|
|
445
|
+
Returns:
|
|
446
|
+
- EchoData: Dataset corresponding to the provided file.
|
|
447
|
+
|
|
448
|
+
Raises:
|
|
449
|
+
- Exception: If the file type is not supported by echopype.
|
|
450
|
+
|
|
451
|
+
"""
|
|
452
|
+
file_name = os.path.split(file_path)[-1]
|
|
453
|
+
if ".raw" in file_name:
|
|
454
|
+
if sonar_model is None:
|
|
455
|
+
sonar_model = detect_sonar_model(file_path)
|
|
456
|
+
|
|
457
|
+
ed = ep.open_raw(file_path, sonar_model=sonar_model, use_swap=use_swap, max_chunk_size=max_chunk_size) # type: ignore
|
|
458
|
+
elif ".nc" in file_name or ".zarr" in file_name:
|
|
459
|
+
ed = ep.open_converted(file_path) # create an EchoData object
|
|
460
|
+
else:
|
|
461
|
+
raise Exception("File not supported by echopype.")
|
|
462
|
+
return ed
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _write_file(
|
|
466
|
+
ed: ep.echodata.EchoData,
|
|
467
|
+
save_path: str,
|
|
468
|
+
save_file_type: str = "nc",
|
|
469
|
+
overwrite: bool = False, # noqa: E501
|
|
470
|
+
) -> str:
|
|
471
|
+
"""
|
|
472
|
+
Writes an echo sounder dataset to a specified file type and saves it.
|
|
473
|
+
|
|
474
|
+
This function takes an EchoData dataset, converts it to the specified file type
|
|
475
|
+
(netCDF or zarr), and saves the file to the provided path.
|
|
476
|
+
|
|
477
|
+
Parameters:
|
|
478
|
+
|
|
479
|
+
- ed (EchoData): echo sounder dataset to be saved.
|
|
480
|
+
- save_path (str): Directory path where the dataset will be saved.
|
|
481
|
+
- save_file_type (str, optional): Desired file type\
|
|
482
|
+
for saving the dataset. Defaults to 'nc'.\
|
|
483
|
+
Options are 'nc' or 'zarr'.
|
|
484
|
+
- overwrite (bool, optional): If True, overwrites\
|
|
485
|
+
the file if it already exists. Defaults to False.
|
|
486
|
+
|
|
487
|
+
Returns:
|
|
488
|
+
|
|
489
|
+
- str: Path to the saved file.
|
|
490
|
+
|
|
491
|
+
Raises:
|
|
492
|
+
|
|
493
|
+
- Exception: If the specified file type is not supported by echopype.
|
|
494
|
+
|
|
495
|
+
"""
|
|
496
|
+
if save_file_type == "nc":
|
|
497
|
+
ed.to_netcdf(save_path=save_path, overwrite=overwrite)
|
|
498
|
+
elif save_file_type == "zarr":
|
|
499
|
+
ed.to_zarr(save_path=save_path, overwrite=overwrite)
|
|
500
|
+
else:
|
|
501
|
+
raise Exception("File type not supported echopype.")
|
|
502
|
+
return save_path
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _is_similar(
|
|
506
|
+
file_dict1: Dict[str, Union[str, datetime, bool]],
|
|
507
|
+
file_dict2: Dict[str, Union[str, datetime, bool]],
|
|
508
|
+
) -> bool:
|
|
509
|
+
"""
|
|
510
|
+
Determines if two file information dictionaries are similar based on specific criteria.
|
|
511
|
+
|
|
512
|
+
This function checks if two file dictionaries have the same campaign ID, sonar model,
|
|
513
|
+
file integrity, and if their date difference is within a specified time range.
|
|
514
|
+
|
|
515
|
+
Parameters:
|
|
516
|
+
|
|
517
|
+
- file_dict1 (dict): First file information dictionary.
|
|
518
|
+
- file_dict2 (dict): Second file information dictionary.
|
|
519
|
+
|
|
520
|
+
Returns:
|
|
521
|
+
|
|
522
|
+
- bool: True if the file dictionaries are similar\
|
|
523
|
+
based on the criteria, False otherwise.
|
|
524
|
+
|
|
525
|
+
"""
|
|
526
|
+
if file_dict1["campaign_id"] != file_dict2["campaign_id"]:
|
|
527
|
+
return False
|
|
528
|
+
if file_dict1["sonar_model"] != file_dict2["sonar_model"]:
|
|
529
|
+
return False
|
|
530
|
+
if file_dict1["file_integrity"] != file_dict2["file_integrity"]:
|
|
531
|
+
return False
|
|
532
|
+
date_diff = file_dict1["date"] - file_dict2["date"]
|
|
533
|
+
if date_diff > timedelta(minutes=TIME_BETWEEN_FILES):
|
|
534
|
+
return False
|
|
535
|
+
return True
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def _check_beam_groups(echo_data):
|
|
539
|
+
# Check if the 'Sonar' group exists
|
|
540
|
+
if "sonar" in echo_data.group_map:
|
|
541
|
+
sonar_group = echo_data.group_map["sonar"]
|
|
542
|
+
# Checking for Beam_group1 and Beam_group2
|
|
543
|
+
has_beam_group1 = "Beam_group1" in sonar_group
|
|
544
|
+
has_beam_group2 = "Beam_group2" in sonar_group
|
|
545
|
+
|
|
546
|
+
return has_beam_group1, has_beam_group2
|
|
547
|
+
else:
|
|
548
|
+
# The 'Sonar' group does not exist in group_map
|
|
549
|
+
return False, False
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def detect_sonar_model(file_path: str, metadata=None) -> str:
|
|
553
|
+
if metadata is None:
|
|
554
|
+
metadata = _parse_metadata(file_path)
|
|
555
|
+
|
|
556
|
+
if metadata is None:
|
|
557
|
+
return None
|
|
558
|
+
|
|
559
|
+
if "sounder_name" not in metadata:
|
|
560
|
+
try:
|
|
561
|
+
xml_string = metadata.get("xml", None)
|
|
562
|
+
root = ET.fromstring(xml_string)
|
|
563
|
+
header = root.find("Header")
|
|
564
|
+
application_name = header.attrib.get("ApplicationName")
|
|
565
|
+
|
|
566
|
+
if application_name == "EK80":
|
|
567
|
+
return "EK80"
|
|
568
|
+
|
|
569
|
+
except ET.ParseError:
|
|
570
|
+
return None
|
|
571
|
+
|
|
572
|
+
return None
|
|
573
|
+
|
|
574
|
+
if metadata["sounder_name"] == "EK60" or metadata["sounder_name"] == "ER60":
|
|
575
|
+
return "EK60"
|
|
576
|
+
|
|
577
|
+
return metadata["sounder_name"]
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""
|
|
2
|
+
sv_computation.py
|
|
3
|
+
-----------------
|
|
4
|
+
|
|
5
|
+
Module for computing the volume backscattering strength (Sv) from raw data.
|
|
6
|
+
Supported Sonar Models:
|
|
7
|
+
- EK60
|
|
8
|
+
- AZFP
|
|
9
|
+
- EK80
|
|
10
|
+
Functions and Classes:
|
|
11
|
+
- `SupportedSonarModelsForSv`: An Enum containing the sonar models supported
|
|
12
|
+
for Sv computation.
|
|
13
|
+
- `WaveformMode`: Enum specifying the waveform mode ("CW" or "BB").
|
|
14
|
+
- `EncodeMode`: Enum indicating the encoding mode ("complex" or "power").
|
|
15
|
+
- `ComputeSVParams`: Class to validate and structure the parameters passed
|
|
16
|
+
to the Sv computation function.
|
|
17
|
+
- `compute_sv`: Main function to calculate Sv given an EchoData object
|
|
18
|
+
and other optional parameters. This function is based on the `echopype.calibrate.compute_Sv()` function.
|
|
19
|
+
- `compute_sv_with_encode_mode`
|
|
20
|
+
|
|
21
|
+
Usage:
|
|
22
|
+
|
|
23
|
+
To compute Sv for a given EchoData object, `ed`, simply call:
|
|
24
|
+
`compute_sv(ed)`
|
|
25
|
+
"""
|
|
26
|
+
import echopype as ep
|
|
27
|
+
import xarray as xr
|
|
28
|
+
|
|
29
|
+
from enum import Enum
|
|
30
|
+
from typing import Any, Optional
|
|
31
|
+
from echopype.echodata.echodata import EchoData
|
|
32
|
+
from pydantic import BaseModel, ValidationError, field_validator
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class SupportedSonarModelsForSv(str, Enum):
|
|
36
|
+
EK60 = "EK60"
|
|
37
|
+
AZFP = "AZFP"
|
|
38
|
+
EK80 = "EK80"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class WaveformMode(str, Enum):
|
|
42
|
+
CW = "CW"
|
|
43
|
+
BB = "BB"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class EncodeMode(str, Enum):
|
|
47
|
+
COMPLEX = "complex"
|
|
48
|
+
POWER = "power"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class ComputeSVParams(BaseModel):
|
|
52
|
+
echodata: Any
|
|
53
|
+
env_params: Optional[dict] = None
|
|
54
|
+
cal_params: Optional[dict] = None
|
|
55
|
+
waveform_mode: Optional[WaveformMode] = None
|
|
56
|
+
encode_mode: Optional[EncodeMode] = None
|
|
57
|
+
|
|
58
|
+
@field_validator("echodata")
|
|
59
|
+
def check_echodata_type(cls, value):
|
|
60
|
+
if not isinstance(value, EchoData):
|
|
61
|
+
raise ValueError("Invalid type for echodata. Expected an instance of EchoData.")
|
|
62
|
+
return value
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def compute_sv(echodata: EchoData, **kwargs) -> xr.Dataset:
|
|
66
|
+
"""
|
|
67
|
+
Computes the volume backscattering strength (Sv) from the given echodata.
|
|
68
|
+
|
|
69
|
+
Parameters:
|
|
70
|
+
- echodata (EchoData): The EchoData object containing sonar data for computation.
|
|
71
|
+
- **kwargs: Additional keyword arguments passed to the Sv computation.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
- xr.Dataset: A Dataset containing the computed Sv values.
|
|
75
|
+
|
|
76
|
+
Example:
|
|
77
|
+
>>> sv_results = compute_sv(echodata)
|
|
78
|
+
>>> print(sv_results)
|
|
79
|
+
|
|
80
|
+
Notes:
|
|
81
|
+
This function:
|
|
82
|
+
- Validates the `echodata`'s sonar model against supported models.
|
|
83
|
+
- Uses the `ComputeSVParams` pydantic model to validate parameters.
|
|
84
|
+
- Checks if the computed Sv is empty.
|
|
85
|
+
- Returns Sv only if it is not empty.
|
|
86
|
+
- Is based on the `echopype.calibrate.compute_Sv()` function.
|
|
87
|
+
|
|
88
|
+
"""
|
|
89
|
+
# Validate parameters using the pydantic model
|
|
90
|
+
try:
|
|
91
|
+
ComputeSVParams(echodata=echodata, **kwargs)
|
|
92
|
+
except ValidationError as e:
|
|
93
|
+
raise ValueError(str(e))
|
|
94
|
+
# Check if the sonar model is supported
|
|
95
|
+
sonar_model = echodata.sonar_model
|
|
96
|
+
try:
|
|
97
|
+
SupportedSonarModelsForSv(sonar_model)
|
|
98
|
+
except ValueError:
|
|
99
|
+
raise ValueError(
|
|
100
|
+
f"Sonar model '{sonar_model}'\
|
|
101
|
+
is not supported for Sv computation.\
|
|
102
|
+
Supported models are \
|
|
103
|
+
{list(SupportedSonarModelsForSv)}."
|
|
104
|
+
)
|
|
105
|
+
# Compute Sv
|
|
106
|
+
Sv = ep.calibrate.compute_Sv(echodata, **kwargs)
|
|
107
|
+
# Check if the computed Sv is empty
|
|
108
|
+
if Sv["Sv"].values.size == 0:
|
|
109
|
+
raise ValueError("Computed Sv is empty!")
|
|
110
|
+
return Sv
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def compute_sv_with_encode_mode(
|
|
114
|
+
echodata: EchoData, waveform_mode: str = "CW", encode_mode: str = "power"
|
|
115
|
+
) -> xr.Dataset:
|
|
116
|
+
sv_dataset = compute_sv(echodata, waveform_mode=waveform_mode, encode_mode=encode_mode)
|
|
117
|
+
|
|
118
|
+
return sv_dataset
|