oceanstream 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanstream/__init__.py +0 -0
- oceanstream/cli/__init__.py +3 -0
- oceanstream/cli/main.py +267 -0
- oceanstream/denoise/__init__.py +2 -0
- oceanstream/denoise/apply_masks.py +49 -0
- oceanstream/denoise/applying_masks_handler.py +173 -0
- oceanstream/denoise/background_noise_remover.py +97 -0
- oceanstream/denoise/create_masks.py +87 -0
- oceanstream/denoise/noise_masks.py +398 -0
- oceanstream/denoise/types.py +73 -0
- oceanstream/echodata/__init__.py +5 -0
- oceanstream/echodata/ensure_time_continuity.py +75 -0
- oceanstream/echodata/mvbs_computation.py +77 -0
- oceanstream/echodata/raw_handler.py +577 -0
- oceanstream/echodata/sv_computation.py +118 -0
- oceanstream/echodata/sv_dataset_extension.py +126 -0
- oceanstream/echodata/sv_interpolation.py +260 -0
- oceanstream/echodata/target_strength_computation.py +109 -0
- oceanstream/exports/__init__.py +2 -0
- oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
- oceanstream/exports/csv/csv_export_from_raw.py +92 -0
- oceanstream/exports/csv/csv_export_nasc.py +135 -0
- oceanstream/exports/frequency_differencing_handler.py +210 -0
- oceanstream/exports/nasc.py +23 -0
- oceanstream/exports/nasc_computation.py +62 -0
- oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
- oceanstream/exports/shoals/shoal_process.py +224 -0
- oceanstream/exports/shoals/shoals_handler.py +35 -0
- oceanstream/plot/__init__.py +2 -0
- oceanstream/plot/echogram.py +542 -0
- oceanstream/process/__init__.py +21 -0
- oceanstream/process/combine_zarr.py +90 -0
- oceanstream/process/file_processor.py +151 -0
- oceanstream/process/folder_processor.py +246 -0
- oceanstream/process/process.py +77 -0
- oceanstream/process/processed_data_io.py +140 -0
- oceanstream/settings/__init__.py +1 -0
- oceanstream/settings/config.py +17 -0
- oceanstream/utils.py +101 -0
- oceanstream-0.0.0.dev0.dist-info/LICENSE +21 -0
- oceanstream-0.0.0.dev0.dist-info/METADATA +214 -0
- oceanstream-0.0.0.dev0.dist-info/RECORD +46 -0
- oceanstream-0.0.0.dev0.dist-info/WHEEL +5 -0
- oceanstream-0.0.0.dev0.dist-info/dependency_links.txt +1 -0
- oceanstream-0.0.0.dev0.dist-info/entry_points.txt +2 -0
- oceanstream-0.0.0.dev0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
import time
|
|
5
|
+
import traceback
|
|
6
|
+
import echopype as ep
|
|
7
|
+
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from rich import print
|
|
10
|
+
from rich.traceback import install, Traceback
|
|
11
|
+
|
|
12
|
+
from rich.progress import Progress, TimeElapsedColumn, BarColumn, TextColumn
|
|
13
|
+
|
|
14
|
+
from oceanstream.plot import plot_sv_data_with_progress, plot_sv_data
|
|
15
|
+
from oceanstream.echodata import get_campaign_metadata
|
|
16
|
+
from .process import compute_sv, process_file_with_progress, read_file_with_progress
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
install(show_locals=True, width=120)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_chunk_sizes(var_dims, chunk_sizes):
|
|
23
|
+
return {dim: chunk_sizes[dim] for dim in var_dims if dim in chunk_sizes}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False, **kwargs):
|
|
27
|
+
"""
|
|
28
|
+
Compute Sv from echodata and save to zarr file.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
echodata:
|
|
32
|
+
config_data:
|
|
33
|
+
chunks:
|
|
34
|
+
plot_echogram:
|
|
35
|
+
**kwargs:
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
str: Path to the zarr file.
|
|
39
|
+
"""
|
|
40
|
+
file_path = config_data["raw_path"]
|
|
41
|
+
waveform_mode = kwargs.get("waveform_mode", "CW")
|
|
42
|
+
encode_mode = waveform_mode == "CW" and "power" or "complex"
|
|
43
|
+
Sv = compute_sv(echodata, encode_mode=encode_mode, **kwargs)
|
|
44
|
+
|
|
45
|
+
parent_folder = os.path.join(Path(config_data["output_folder"]), file_path.stem)
|
|
46
|
+
if not os.path.exists(parent_folder):
|
|
47
|
+
os.makedirs(parent_folder)
|
|
48
|
+
|
|
49
|
+
output_path = os.path.join(parent_folder, f"{file_path.stem}_Sv.zarr")
|
|
50
|
+
|
|
51
|
+
if chunks is not None:
|
|
52
|
+
for var in Sv.data_vars:
|
|
53
|
+
var_chunk_sizes = get_chunk_sizes(Sv[var].dims, chunks)
|
|
54
|
+
Sv[var] = Sv[var].chunk(var_chunk_sizes)
|
|
55
|
+
# Remove chunk encoding to avoid conflicts
|
|
56
|
+
if 'chunks' in Sv[var].encoding:
|
|
57
|
+
del Sv[var].encoding['chunks']
|
|
58
|
+
|
|
59
|
+
print("Removing background noise...")
|
|
60
|
+
ds_processed = Sv
|
|
61
|
+
# ds_processed = apply_background_noise_removal(Sv, config=config_data)
|
|
62
|
+
ds_processed.to_zarr(output_path)
|
|
63
|
+
|
|
64
|
+
if plot_echogram:
|
|
65
|
+
try:
|
|
66
|
+
plot_sv_data(ds_processed, file_base_name=file_path.stem, output_path=parent_folder)
|
|
67
|
+
except Exception as e:
|
|
68
|
+
logging.exception(f"Error plotting echogram for {file_path}:")
|
|
69
|
+
raise e
|
|
70
|
+
|
|
71
|
+
return output_path
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
async def process_raw_file_with_progress(config_data, plot_echogram, waveform_mode="CW", depth_offset=0):
|
|
75
|
+
try:
|
|
76
|
+
with Progress(
|
|
77
|
+
TextColumn("[progress.description]{task.description}"),
|
|
78
|
+
BarColumn(),
|
|
79
|
+
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
|
80
|
+
TimeElapsedColumn()
|
|
81
|
+
) as progress:
|
|
82
|
+
print(f"[green] Processing file: {config_data['raw_path']}[/green]")
|
|
83
|
+
read_task = progress.add_task("[cyan]Reading raw file data...", total=100)
|
|
84
|
+
|
|
85
|
+
campaign_id, date, sonar_model, metadata, _ = get_campaign_metadata(config_data['raw_path'])
|
|
86
|
+
if config_data['sonar_model'] is None:
|
|
87
|
+
config_data['sonar_model'] = sonar_model
|
|
88
|
+
|
|
89
|
+
echodata, encode_mode = await read_file_with_progress(config_data, progress, read_task)
|
|
90
|
+
echodata.to_zarr(save_path=config_data["output_folder"], overwrite=True, parallel=False)
|
|
91
|
+
progress.update(read_task, advance=100 - progress.tasks[read_task].completed)
|
|
92
|
+
|
|
93
|
+
if plot_echogram:
|
|
94
|
+
zarr_file_name = config_data['raw_path'].stem
|
|
95
|
+
|
|
96
|
+
if waveform_mode == "BB":
|
|
97
|
+
encode_mode = "complex"
|
|
98
|
+
|
|
99
|
+
compute_task = progress.add_task(
|
|
100
|
+
f"[cyan]Computing Sv with waveform_mode={waveform_mode} and encode_mode={encode_mode}...",
|
|
101
|
+
total=100)
|
|
102
|
+
|
|
103
|
+
sv_dataset = await process_file_with_progress(progress, compute_task, echodata,
|
|
104
|
+
encode_mode=encode_mode,
|
|
105
|
+
waveform_mode=waveform_mode,
|
|
106
|
+
depth_offset=depth_offset)
|
|
107
|
+
progress.update(compute_task, advance=100 - progress.tasks[compute_task].completed)
|
|
108
|
+
print(f"[blue]📝 Computed Sv and wrote zarr file to: {config_data['output_folder']}[/blue]")
|
|
109
|
+
|
|
110
|
+
print(f"[green]✅ Plotting echogram for: {config_data['raw_path']}[/green]")
|
|
111
|
+
plot_task = progress.add_task("[cyan]Plotting echogram...", total=100)
|
|
112
|
+
await plot_sv_data_with_progress(sv_dataset, output_path=config_data["output_folder"],
|
|
113
|
+
progress=progress, file_base_name=zarr_file_name, plot_task=plot_task)
|
|
114
|
+
progress.update(plot_task, advance=100 - progress.tasks[plot_task].completed)
|
|
115
|
+
print(f"[blue]📊 Plotted echogram for the data in: {config_data['output_folder']}[/blue]")
|
|
116
|
+
except Exception as e:
|
|
117
|
+
logging.exception(f"Error processing file {config_data['raw_path']}: {e}")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def convert_raw_file(file_path, config_data, progress_queue=None):
|
|
121
|
+
logging.debug("Starting processing of file: %s", file_path)
|
|
122
|
+
from oceanstream.echodata import read_file
|
|
123
|
+
|
|
124
|
+
try:
|
|
125
|
+
file_config_data = {**config_data, 'raw_path': Path(file_path)}
|
|
126
|
+
echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
|
|
127
|
+
echodata.to_zarr(save_path=config_data["output_folder"], overwrite=True, parallel=False)
|
|
128
|
+
|
|
129
|
+
if progress_queue:
|
|
130
|
+
progress_queue.put(file_path)
|
|
131
|
+
except Exception as e:
|
|
132
|
+
logging.error("Error processing file %s", file_path)
|
|
133
|
+
print(Traceback())
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def compute_single_file(config_data, **kwargs):
|
|
137
|
+
file_path = config_data["raw_path"]
|
|
138
|
+
start_time = time.time()
|
|
139
|
+
chunks = kwargs.get("chunks")
|
|
140
|
+
echodata = ep.open_converted(file_path, chunks=chunks)
|
|
141
|
+
|
|
142
|
+
try:
|
|
143
|
+
output_path = compute_Sv_to_zarr(echodata, config_data, **kwargs)
|
|
144
|
+
print(f"[blue]✅ Computed Sv and saved to: {output_path}[/blue]")
|
|
145
|
+
except Exception as e:
|
|
146
|
+
logging.error(f"Error computing Sv for {file_path}")
|
|
147
|
+
print(Traceback())
|
|
148
|
+
finally:
|
|
149
|
+
end_time = time.time()
|
|
150
|
+
total_time = end_time - start_time
|
|
151
|
+
print(f"Total time taken: {total_time:.2f} seconds")
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import logging
|
|
3
|
+
import time
|
|
4
|
+
import re
|
|
5
|
+
import signal
|
|
6
|
+
import sys
|
|
7
|
+
import traceback
|
|
8
|
+
import warnings
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from rich import print
|
|
13
|
+
from functools import partial
|
|
14
|
+
from multiprocessing import Pool, Manager, Process
|
|
15
|
+
from rich.progress import Progress, TimeElapsedColumn, BarColumn, TextColumn
|
|
16
|
+
from oceanstream.plot import plot_sv_data
|
|
17
|
+
from oceanstream.echodata import get_campaign_metadata, read_file
|
|
18
|
+
|
|
19
|
+
from .combine_zarr import read_zarr_files
|
|
20
|
+
from .process import compute_sv
|
|
21
|
+
from .processed_data_io import write_processed
|
|
22
|
+
from .file_processor import convert_raw_file, compute_single_file
|
|
23
|
+
|
|
24
|
+
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
25
|
+
warnings.filterwarnings("ignore", module="echopype")
|
|
26
|
+
pool = None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def configure_logging(log_level=logging.ERROR):
|
|
30
|
+
logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def create_progress_bar():
|
|
34
|
+
return Progress(
|
|
35
|
+
TextColumn("[progress.description]{task.description}"),
|
|
36
|
+
BarColumn(),
|
|
37
|
+
TextColumn("[progress.completed]{task.completed} of {task.total} files"),
|
|
38
|
+
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
|
39
|
+
TimeElapsedColumn()
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def populate_metadata(ed, raw_fname):
|
|
44
|
+
"""
|
|
45
|
+
Manually populate into the "ed" EchoData object
|
|
46
|
+
additional metadata about the dataset and the platform
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
# -- SONAR-netCDF4 Top-level Group attributes
|
|
50
|
+
survey_name = (
|
|
51
|
+
"2017 Joint U.S.-Canada Integrated Ecosystem and "
|
|
52
|
+
"Pacific Hake Acoustic Trawl Survey ('Pacific Hake Survey')"
|
|
53
|
+
)
|
|
54
|
+
ed['Top-level'].attrs['title'] = f"{survey_name}, file {raw_fname}"
|
|
55
|
+
ed['Top-level'].attrs['summary'] = (
|
|
56
|
+
f"EK60 raw file {raw_fname} from the {survey_name}, converted to a SONAR-netCDF4 file using echopype."
|
|
57
|
+
"Information about the survey program is available at "
|
|
58
|
+
"https://www.fisheries.noaa.gov/west-coast/science-data/"
|
|
59
|
+
"joint-us-canada-integrated-ecosystem-and-pacific-hake-acoustic-trawl-survey"
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
# -- SONAR-netCDF4 Platform Group attributes
|
|
63
|
+
# Per SONAR-netCDF4, for platform_type see https://vocab.ices.dk/?ref=311
|
|
64
|
+
ed['Platform'].attrs['platform_type'] = "Research vessel"
|
|
65
|
+
ed['Platform'].attrs['platform_name'] = "Bell M. Shimada" # A NOAA ship
|
|
66
|
+
ed['Platform'].attrs['platform_code_ICES'] = "315"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def update_progress(progress_queue, total_files, log_level):
|
|
70
|
+
configure_logging(log_level)
|
|
71
|
+
logging.debug("Initializing progress updater with total files: %d", total_files)
|
|
72
|
+
with create_progress_bar() as progress:
|
|
73
|
+
task = progress.add_task("[cyan]Processing files...", total=total_files)
|
|
74
|
+
completed_files = 0
|
|
75
|
+
|
|
76
|
+
while completed_files < total_files:
|
|
77
|
+
try:
|
|
78
|
+
message = progress_queue.get(timeout=1)
|
|
79
|
+
if message is None:
|
|
80
|
+
break
|
|
81
|
+
completed_files += 1
|
|
82
|
+
progress.update(task, advance=1)
|
|
83
|
+
logging.debug("Files completed: %d / %d", completed_files, total_files)
|
|
84
|
+
except Exception:
|
|
85
|
+
pass
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def process_single_file(file_path, config_data, progress_queue,
|
|
89
|
+
compute_sv_data=False,
|
|
90
|
+
use_distributed_dask=False,
|
|
91
|
+
plot_echogram=False, **kwargs):
|
|
92
|
+
log_level = config_data.get('log_level', logging.ERROR)
|
|
93
|
+
configure_logging(log_level)
|
|
94
|
+
logging.debug("Starting processing of file: %s", file_path)
|
|
95
|
+
try:
|
|
96
|
+
logging.debug("Reading file: %s", file_path)
|
|
97
|
+
|
|
98
|
+
file_config_data = {**config_data, 'raw_path': Path(file_path)}
|
|
99
|
+
echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
|
|
100
|
+
echodata.to_zarr(save_path=config_data["output_folder"], overwrite=True, parallel=False)
|
|
101
|
+
|
|
102
|
+
if compute_sv_data and use_distributed_dask:
|
|
103
|
+
sv_dataset = compute_sv(echodata, encode_mode, **kwargs)
|
|
104
|
+
zarr_file_name = file_config_data['raw_path'].stem
|
|
105
|
+
write_processed(sv_dataset, config_data["output_folder"], zarr_file_name, "zarr")
|
|
106
|
+
|
|
107
|
+
if plot_echogram:
|
|
108
|
+
sv_dataset = compute_sv(echodata, encode_mode, **kwargs)
|
|
109
|
+
zarr_file_name = file_config_data['raw_path'].stem
|
|
110
|
+
plot_sv_data(sv_dataset, output_path=config_data["output_folder"], file_base_name=zarr_file_name)
|
|
111
|
+
|
|
112
|
+
progress_queue.put(file_path)
|
|
113
|
+
except Exception as e:
|
|
114
|
+
logging.exception("Error processing file %s: %s", file_path, e)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def signal_handler(sig, frame):
|
|
118
|
+
global pool
|
|
119
|
+
print('Terminating processes...')
|
|
120
|
+
if pool:
|
|
121
|
+
pool.terminate()
|
|
122
|
+
pool.join()
|
|
123
|
+
sys.exit(0)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def convert_raw_files(config_data, workers_count=os.cpu_count()):
|
|
127
|
+
global pool
|
|
128
|
+
|
|
129
|
+
try:
|
|
130
|
+
dir_path = config_data['raw_path']
|
|
131
|
+
log_level = config_data.get('log_level', logging.ERROR)
|
|
132
|
+
configure_logging(log_level)
|
|
133
|
+
|
|
134
|
+
print(f"Starting to convert folder: {dir_path} to Zarr using {workers_count} parallel processes...")
|
|
135
|
+
raw_files = [os.path.join(dir_path, f) for f in os.listdir(dir_path) if
|
|
136
|
+
os.path.isfile(os.path.join(dir_path, f)) and f.endswith('.raw')]
|
|
137
|
+
file_info = []
|
|
138
|
+
|
|
139
|
+
if not raw_files:
|
|
140
|
+
logging.error("No raw files found in directory: %s", dir_path)
|
|
141
|
+
return
|
|
142
|
+
|
|
143
|
+
logging.debug("Found %d raw files in directory: %s", len(raw_files), dir_path)
|
|
144
|
+
|
|
145
|
+
for file in raw_files:
|
|
146
|
+
creation_time = from_filename(file)
|
|
147
|
+
file_info.append((file, creation_time))
|
|
148
|
+
logging.debug("File: %s, creation time: %s", file, creation_time)
|
|
149
|
+
|
|
150
|
+
if not file_info:
|
|
151
|
+
logging.error("No valid raw files with creation time found in directory: %s", dir_path)
|
|
152
|
+
return
|
|
153
|
+
|
|
154
|
+
filtered_file_info = [item for item in file_info if item[1] is not None]
|
|
155
|
+
filtered_file_info.sort(key=lambda x: x[1])
|
|
156
|
+
|
|
157
|
+
sorted_files = [file for file, _ in filtered_file_info]
|
|
158
|
+
logging.debug("Sorted files by creation time")
|
|
159
|
+
|
|
160
|
+
campaign_id, date, sonar_model, metadata, _ = get_campaign_metadata(sorted_files[0])
|
|
161
|
+
if config_data['sonar_model'] is None:
|
|
162
|
+
config_data['sonar_model'] = sonar_model
|
|
163
|
+
|
|
164
|
+
with Manager() as manager:
|
|
165
|
+
progress_queue = manager.Queue()
|
|
166
|
+
pool = Pool(processes=workers_count)
|
|
167
|
+
|
|
168
|
+
# Partial function with config_data, progress_queue and other arguments
|
|
169
|
+
process_func = partial(convert_raw_file, config_data=config_data, progress_queue=progress_queue)
|
|
170
|
+
|
|
171
|
+
# Run the progress updater in a separate process
|
|
172
|
+
progress_updater = Process(target=update_progress, args=(progress_queue, len(sorted_files), log_level))
|
|
173
|
+
progress_updater.start()
|
|
174
|
+
|
|
175
|
+
for file in sorted_files:
|
|
176
|
+
pool.apply_async(process_func, args=(file,))
|
|
177
|
+
logging.debug("Started async processing for file: %s", file)
|
|
178
|
+
|
|
179
|
+
pool.close()
|
|
180
|
+
pool.join()
|
|
181
|
+
print(f"[green]✅ All files have been converted.[/green]")
|
|
182
|
+
|
|
183
|
+
# Wait for the progress updater to finish
|
|
184
|
+
progress_queue.put(None) # Signal the progress updater to finish
|
|
185
|
+
progress_updater.join()
|
|
186
|
+
except KeyboardInterrupt:
|
|
187
|
+
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
188
|
+
if pool:
|
|
189
|
+
pool.terminate()
|
|
190
|
+
pool.join()
|
|
191
|
+
except Exception as e:
|
|
192
|
+
logging.exception("Error processing folder %s: %s", config_data['raw_path'], e)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def process_single_zarr_file(file_path, config_data, chunks=None, plot_echogram=False, waveform_mode="CW",
|
|
196
|
+
depth_offset=0):
|
|
197
|
+
file_config_data = {**config_data, 'raw_path': Path(file_path)}
|
|
198
|
+
compute_single_file(file_config_data, chunks=chunks, plot_echogram=plot_echogram,
|
|
199
|
+
waveform_mode=waveform_mode, depth_offset=depth_offset)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def process_zarr_files(config_data, workers_count=os.cpu_count(), status=None, chunks=None, plot_echogram=False,
|
|
203
|
+
waveform_mode="CW", depth_offset=0):
|
|
204
|
+
dir_path = config_data['raw_path']
|
|
205
|
+
zarr_files = read_zarr_files(dir_path)
|
|
206
|
+
|
|
207
|
+
if not zarr_files:
|
|
208
|
+
logging.error("No valid .zarr files with creation time found in directory: %s", dir_path)
|
|
209
|
+
return
|
|
210
|
+
|
|
211
|
+
logging.info(f"Found {len(zarr_files)} Zarr files in directory: {dir_path}")
|
|
212
|
+
|
|
213
|
+
# def wrapped_process_single_zarr_file(file_path):
|
|
214
|
+
# print(f"Computing Sv for {file_path}...\n")
|
|
215
|
+
# return process_single_zarr_file(file_path, config_data,
|
|
216
|
+
# chunks=chunks, plot_echogram=plot_echogram, waveform_mode=waveform_mode,
|
|
217
|
+
# depth_offset=depth_offset)
|
|
218
|
+
#
|
|
219
|
+
# # Use Dask bag to parallelize the processing
|
|
220
|
+
# bag = db.from_sequence(zarr_files, npartitions=workers_count)
|
|
221
|
+
# bag = bag.map(wrapped_process_single_zarr_file)
|
|
222
|
+
#
|
|
223
|
+
# # Compute the results
|
|
224
|
+
# results = bag.compute()
|
|
225
|
+
for file_path in zarr_files:
|
|
226
|
+
status.update(f"Computing Sv for {file_path}...\n")
|
|
227
|
+
process_single_zarr_file(file_path, config_data, chunks=chunks, plot_echogram=plot_echogram,
|
|
228
|
+
waveform_mode=waveform_mode, depth_offset=depth_offset)
|
|
229
|
+
|
|
230
|
+
logging.info("✅ All files have been processed")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def from_filename(file_name):
|
|
234
|
+
"""Extract creation time from the file name if it follows a specific pattern."""
|
|
235
|
+
pattern = r'(\d{4}[A-Z])?-D(\d{8})-T(\d{6})\.raw'
|
|
236
|
+
match = re.search(pattern, file_name)
|
|
237
|
+
if match:
|
|
238
|
+
date_str = match.group(2)
|
|
239
|
+
time_str = match.group(3)
|
|
240
|
+
creation_time = datetime.strptime(date_str + time_str, '%Y%m%d%H%M%S')
|
|
241
|
+
return creation_time
|
|
242
|
+
|
|
243
|
+
return None
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
signal.signal(signal.SIGINT, signal_handler)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
3
|
+
from oceanstream.echodata import (
|
|
4
|
+
read_file,
|
|
5
|
+
compute_sv_with_encode_mode,
|
|
6
|
+
enrich_sv_dataset,
|
|
7
|
+
regrid_dataset
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
async def process_file_with_progress(progress, compute_task_id, echodata, encode_mode="power", waveform_mode="CW",
|
|
12
|
+
depth_offset=0):
|
|
13
|
+
loop = asyncio.get_running_loop()
|
|
14
|
+
total_steps = 25
|
|
15
|
+
progress_step = 95 / total_steps
|
|
16
|
+
|
|
17
|
+
process_task_future = loop.run_in_executor(None, compute_sv_with_encode_mode, echodata, waveform_mode, encode_mode)
|
|
18
|
+
|
|
19
|
+
for step in range(total_steps):
|
|
20
|
+
if process_task_future.done():
|
|
21
|
+
break
|
|
22
|
+
await asyncio.sleep(0.1)
|
|
23
|
+
progress.update(compute_task_id, advance=progress_step)
|
|
24
|
+
|
|
25
|
+
sv_dataset = await process_task_future
|
|
26
|
+
sv_enriched = enrich_sv_dataset(
|
|
27
|
+
sv_dataset,
|
|
28
|
+
echodata,
|
|
29
|
+
waveform_mode=waveform_mode,
|
|
30
|
+
encode_mode=encode_mode,
|
|
31
|
+
depth_offset=depth_offset
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
# del echodata
|
|
35
|
+
# sv_enriched_downsampled = regrid_dataset(sv_enriched)
|
|
36
|
+
# sv_enriched = sv_enriched_downsampled
|
|
37
|
+
|
|
38
|
+
return sv_enriched
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def compute_sv(echodata, encode_mode="power", waveform_mode="CW", depth_offset=0):
|
|
42
|
+
sv_dataset = compute_sv_with_encode_mode(echodata, waveform_mode, encode_mode)
|
|
43
|
+
|
|
44
|
+
sv_enriched = enrich_sv_dataset(
|
|
45
|
+
sv_dataset,
|
|
46
|
+
echodata,
|
|
47
|
+
waveform_mode=waveform_mode,
|
|
48
|
+
encode_mode=encode_mode,
|
|
49
|
+
depth_offset=depth_offset
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
# del echodata
|
|
53
|
+
# sv_enriched_downsampled = regrid_dataset(sv_enriched)
|
|
54
|
+
# sv_enriched = sv_enriched_downsampled
|
|
55
|
+
|
|
56
|
+
return sv_enriched
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
async def read_file_with_progress(config_data, progress, read_task, waveform_mode="CW"):
|
|
60
|
+
loop = asyncio.get_running_loop()
|
|
61
|
+
with ThreadPoolExecutor() as pool:
|
|
62
|
+
total_steps = 20
|
|
63
|
+
progress_step = 95 / total_steps
|
|
64
|
+
|
|
65
|
+
read_task_future = loop.run_in_executor(pool, read_file, config_data, True)
|
|
66
|
+
|
|
67
|
+
for step in range(total_steps):
|
|
68
|
+
if read_task_future.done():
|
|
69
|
+
break
|
|
70
|
+
await asyncio.sleep(0.1)
|
|
71
|
+
progress.update(read_task, advance=progress_step)
|
|
72
|
+
|
|
73
|
+
echodata, encode_mode = await read_task_future
|
|
74
|
+
|
|
75
|
+
return echodata, encode_mode
|
|
76
|
+
|
|
77
|
+
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""
|
|
2
|
+
processed_data_io.py
|
|
3
|
+
--------------------
|
|
4
|
+
Module for handling input and output operations of processed data.
|
|
5
|
+
|
|
6
|
+
This module provides utility functions to read from and write to different
|
|
7
|
+
data formats, specifically NetCDF (".nc") and Zarr (".zarr"). It ensures that
|
|
8
|
+
the data is correctly loaded into xarray Datasets and provides error handling
|
|
9
|
+
for unsupported file formats and other potential issues.
|
|
10
|
+
|
|
11
|
+
Functions:
|
|
12
|
+
- read_processed(file_path: Union[str, Path]) -> xr.Dataset:
|
|
13
|
+
Reads and returns a xarray Dataset from a specified file path.
|
|
14
|
+
|
|
15
|
+
- write_processed(sv: xr.Dataset, file_path: Union[str, Path], file_name: str = "", file_type: str = "nc", overwrite: bool = True):
|
|
16
|
+
Saves a xarray Dataset to a specified path with a given file name and type.
|
|
17
|
+
|
|
18
|
+
Example:
|
|
19
|
+
>> ds = read_processed("/path/to/datafile.nc")
|
|
20
|
+
>> write_processed(ds, "/path/to/save", "datafile", "nc")
|
|
21
|
+
|
|
22
|
+
Note:
|
|
23
|
+
This module can be extended in the future to support more file formats or additional
|
|
24
|
+
IO operations as needed.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
import xarray as xr
|
|
28
|
+
from typing import Union
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def read_processed(file_path: Union[str, Path]) -> xr.Dataset:
|
|
33
|
+
"""
|
|
34
|
+
Read and return a xarray Dataset from a specified file path.
|
|
35
|
+
|
|
36
|
+
Parameters:
|
|
37
|
+
- file_path (Union[str, Path]): The path to the .nc or .zarr file to be read.
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
- xr.Dataset: The xarray Dataset read from the file.
|
|
41
|
+
|
|
42
|
+
Raises:
|
|
43
|
+
- ValueError: If the file does not exist or has an unsupported format.
|
|
44
|
+
- FileNotFoundError: If the file does not exist.
|
|
45
|
+
|
|
46
|
+
Supported File Types:
|
|
47
|
+
- NetCDF (".nc")
|
|
48
|
+
- Zarr (".zarr")
|
|
49
|
+
|
|
50
|
+
Example:
|
|
51
|
+
>> ds = read_processed("/path/to/datafile.nc")
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
# Convert the input to a Path object (if it isn't already)
|
|
55
|
+
file_path = Path(file_path)
|
|
56
|
+
|
|
57
|
+
# Check if the file exists
|
|
58
|
+
if not file_path.exists():
|
|
59
|
+
raise FileNotFoundError(f"File does not exist: {file_path}")
|
|
60
|
+
|
|
61
|
+
# Check if the file has a supported format
|
|
62
|
+
if file_path.suffix not in [".nc", ".zarr"]:
|
|
63
|
+
raise ValueError(f"Unsupported file format: {file_path.suffix}")
|
|
64
|
+
|
|
65
|
+
# Read the file based on its format
|
|
66
|
+
# Read the file based on its format and load data into memory
|
|
67
|
+
if file_path.suffix == ".nc":
|
|
68
|
+
with xr.open_dataset(file_path) as ds:
|
|
69
|
+
dataset = ds.load()
|
|
70
|
+
|
|
71
|
+
elif file_path.suffix == ".zarr":
|
|
72
|
+
with xr.open_zarr(file_path) as ds:
|
|
73
|
+
dataset = ds.load()
|
|
74
|
+
else:
|
|
75
|
+
raise Exception("Could not open file")
|
|
76
|
+
|
|
77
|
+
return dataset
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def write_processed(
|
|
81
|
+
sv: xr.Dataset,
|
|
82
|
+
dir_path: Union[str, Path],
|
|
83
|
+
file_name: str = "",
|
|
84
|
+
file_type: str = "nc",
|
|
85
|
+
overwrite: bool = True,
|
|
86
|
+
):
|
|
87
|
+
"""
|
|
88
|
+
Save a xarray Dataset to a specified path with a given file name and type.
|
|
89
|
+
|
|
90
|
+
Parameters:
|
|
91
|
+
- sv (xr.Dataset): The xarray Dataset to save.
|
|
92
|
+
- dir_path (Union[str, Path]): The directory path where the file should be saved.
|
|
93
|
+
- file_name (str, optional): The name of the file. Defaults to the name of the Dataset.
|
|
94
|
+
- file_type (str, optional): The type of the file, either 'nc' or 'zarr'. Defaults to 'nc'.
|
|
95
|
+
- overwrite (bool, optional): Whether to overwrite the file if it already exists. Defaults to True.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
- None
|
|
99
|
+
|
|
100
|
+
Raises:
|
|
101
|
+
- TypeError: If the provided Dataset is not an instance of xr.Dataset.
|
|
102
|
+
- ValueError: For invalid paths, unsupported file formats, or other invalid inputs.
|
|
103
|
+
|
|
104
|
+
Example:
|
|
105
|
+
>> write_processed(dataset, "/path/to/save", "datafile", "nc")
|
|
106
|
+
"""
|
|
107
|
+
if not isinstance(sv, xr.Dataset):
|
|
108
|
+
raise TypeError("Expected a xarray Dataset")
|
|
109
|
+
|
|
110
|
+
path = Path(dir_path)
|
|
111
|
+
if not path.exists():
|
|
112
|
+
try:
|
|
113
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
114
|
+
except Exception as e:
|
|
115
|
+
raise Exception(f"An error occurred while creating the directory: {e}")
|
|
116
|
+
else:
|
|
117
|
+
if not path.is_dir():
|
|
118
|
+
raise NotADirectoryError(f"Path exists but is not a directory: {path}")
|
|
119
|
+
|
|
120
|
+
if not file_name:
|
|
121
|
+
file_name = Path(sv.source_filenames[0].values.item()).stem
|
|
122
|
+
|
|
123
|
+
if "." not in file_name:
|
|
124
|
+
if file_type not in ["nc", "zarr"]:
|
|
125
|
+
raise ValueError("File type has to be one of ['nc','zarr']")
|
|
126
|
+
file_name = f"{file_name}.{file_type}"
|
|
127
|
+
elif file_name.split(".")[1] not in ["nc", "zarr"]:
|
|
128
|
+
raise ValueError(f"File name provided has unsupported format: {file_name}")
|
|
129
|
+
|
|
130
|
+
full_path = path / file_name
|
|
131
|
+
|
|
132
|
+
if full_path.is_file() and not overwrite:
|
|
133
|
+
return
|
|
134
|
+
|
|
135
|
+
if full_path.suffix == ".nc":
|
|
136
|
+
sv.to_netcdf(full_path)
|
|
137
|
+
elif full_path.suffix == ".zarr":
|
|
138
|
+
sv.to_zarr(full_path, mode="w")
|
|
139
|
+
else:
|
|
140
|
+
raise ValueError(f"Could not save file to provided path: {full_path}")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .config import load_config
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
DEFAULT_CONFIG_PATH = Path(__file__).parent / "defaults.config.json"
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def load_config(user_config_path=None):
|
|
8
|
+
with open(DEFAULT_CONFIG_PATH, "r") as file:
|
|
9
|
+
config = json.load(file)
|
|
10
|
+
|
|
11
|
+
if user_config_path and Path(user_config_path).exists():
|
|
12
|
+
with open(user_config_path, "r") as file:
|
|
13
|
+
user_config = json.load(file)
|
|
14
|
+
for key in user_config:
|
|
15
|
+
config[key] = user_config[key]
|
|
16
|
+
|
|
17
|
+
return config
|