oceanstream 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. oceanstream/__init__.py +0 -0
  2. oceanstream/cli/__init__.py +3 -0
  3. oceanstream/cli/main.py +267 -0
  4. oceanstream/denoise/__init__.py +2 -0
  5. oceanstream/denoise/apply_masks.py +49 -0
  6. oceanstream/denoise/applying_masks_handler.py +173 -0
  7. oceanstream/denoise/background_noise_remover.py +97 -0
  8. oceanstream/denoise/create_masks.py +87 -0
  9. oceanstream/denoise/noise_masks.py +398 -0
  10. oceanstream/denoise/types.py +73 -0
  11. oceanstream/echodata/__init__.py +5 -0
  12. oceanstream/echodata/ensure_time_continuity.py +75 -0
  13. oceanstream/echodata/mvbs_computation.py +77 -0
  14. oceanstream/echodata/raw_handler.py +577 -0
  15. oceanstream/echodata/sv_computation.py +118 -0
  16. oceanstream/echodata/sv_dataset_extension.py +126 -0
  17. oceanstream/echodata/sv_interpolation.py +260 -0
  18. oceanstream/echodata/target_strength_computation.py +109 -0
  19. oceanstream/exports/__init__.py +2 -0
  20. oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
  21. oceanstream/exports/csv/csv_export_from_raw.py +92 -0
  22. oceanstream/exports/csv/csv_export_nasc.py +135 -0
  23. oceanstream/exports/frequency_differencing_handler.py +210 -0
  24. oceanstream/exports/nasc.py +23 -0
  25. oceanstream/exports/nasc_computation.py +62 -0
  26. oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
  27. oceanstream/exports/shoals/shoal_process.py +224 -0
  28. oceanstream/exports/shoals/shoals_handler.py +35 -0
  29. oceanstream/plot/__init__.py +2 -0
  30. oceanstream/plot/echogram.py +542 -0
  31. oceanstream/process/__init__.py +21 -0
  32. oceanstream/process/combine_zarr.py +90 -0
  33. oceanstream/process/file_processor.py +151 -0
  34. oceanstream/process/folder_processor.py +246 -0
  35. oceanstream/process/process.py +77 -0
  36. oceanstream/process/processed_data_io.py +140 -0
  37. oceanstream/settings/__init__.py +1 -0
  38. oceanstream/settings/config.py +17 -0
  39. oceanstream/utils.py +101 -0
  40. oceanstream-0.0.0.dev0.dist-info/LICENSE +21 -0
  41. oceanstream-0.0.0.dev0.dist-info/METADATA +214 -0
  42. oceanstream-0.0.0.dev0.dist-info/RECORD +46 -0
  43. oceanstream-0.0.0.dev0.dist-info/WHEEL +5 -0
  44. oceanstream-0.0.0.dev0.dist-info/dependency_links.txt +1 -0
  45. oceanstream-0.0.0.dev0.dist-info/entry_points.txt +2 -0
  46. oceanstream-0.0.0.dev0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,151 @@
1
+ import logging
2
+ import os
3
+ import sys
4
+ import time
5
+ import traceback
6
+ import echopype as ep
7
+
8
+ from pathlib import Path
9
+ from rich import print
10
+ from rich.traceback import install, Traceback
11
+
12
+ from rich.progress import Progress, TimeElapsedColumn, BarColumn, TextColumn
13
+
14
+ from oceanstream.plot import plot_sv_data_with_progress, plot_sv_data
15
+ from oceanstream.echodata import get_campaign_metadata
16
+ from .process import compute_sv, process_file_with_progress, read_file_with_progress
17
+
18
+
19
+ install(show_locals=True, width=120)
20
+
21
+
22
+ def get_chunk_sizes(var_dims, chunk_sizes):
23
+ return {dim: chunk_sizes[dim] for dim in var_dims if dim in chunk_sizes}
24
+
25
+
26
+ def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False, **kwargs):
27
+ """
28
+ Compute Sv from echodata and save to zarr file.
29
+
30
+ Args:
31
+ echodata:
32
+ config_data:
33
+ chunks:
34
+ plot_echogram:
35
+ **kwargs:
36
+
37
+ Returns:
38
+ str: Path to the zarr file.
39
+ """
40
+ file_path = config_data["raw_path"]
41
+ waveform_mode = kwargs.get("waveform_mode", "CW")
42
+ encode_mode = waveform_mode == "CW" and "power" or "complex"
43
+ Sv = compute_sv(echodata, encode_mode=encode_mode, **kwargs)
44
+
45
+ parent_folder = os.path.join(Path(config_data["output_folder"]), file_path.stem)
46
+ if not os.path.exists(parent_folder):
47
+ os.makedirs(parent_folder)
48
+
49
+ output_path = os.path.join(parent_folder, f"{file_path.stem}_Sv.zarr")
50
+
51
+ if chunks is not None:
52
+ for var in Sv.data_vars:
53
+ var_chunk_sizes = get_chunk_sizes(Sv[var].dims, chunks)
54
+ Sv[var] = Sv[var].chunk(var_chunk_sizes)
55
+ # Remove chunk encoding to avoid conflicts
56
+ if 'chunks' in Sv[var].encoding:
57
+ del Sv[var].encoding['chunks']
58
+
59
+ print("Removing background noise...")
60
+ ds_processed = Sv
61
+ # ds_processed = apply_background_noise_removal(Sv, config=config_data)
62
+ ds_processed.to_zarr(output_path)
63
+
64
+ if plot_echogram:
65
+ try:
66
+ plot_sv_data(ds_processed, file_base_name=file_path.stem, output_path=parent_folder)
67
+ except Exception as e:
68
+ logging.exception(f"Error plotting echogram for {file_path}:")
69
+ raise e
70
+
71
+ return output_path
72
+
73
+
74
+ async def process_raw_file_with_progress(config_data, plot_echogram, waveform_mode="CW", depth_offset=0):
75
+ try:
76
+ with Progress(
77
+ TextColumn("[progress.description]{task.description}"),
78
+ BarColumn(),
79
+ TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
80
+ TimeElapsedColumn()
81
+ ) as progress:
82
+ print(f"[green] Processing file: {config_data['raw_path']}[/green]")
83
+ read_task = progress.add_task("[cyan]Reading raw file data...", total=100)
84
+
85
+ campaign_id, date, sonar_model, metadata, _ = get_campaign_metadata(config_data['raw_path'])
86
+ if config_data['sonar_model'] is None:
87
+ config_data['sonar_model'] = sonar_model
88
+
89
+ echodata, encode_mode = await read_file_with_progress(config_data, progress, read_task)
90
+ echodata.to_zarr(save_path=config_data["output_folder"], overwrite=True, parallel=False)
91
+ progress.update(read_task, advance=100 - progress.tasks[read_task].completed)
92
+
93
+ if plot_echogram:
94
+ zarr_file_name = config_data['raw_path'].stem
95
+
96
+ if waveform_mode == "BB":
97
+ encode_mode = "complex"
98
+
99
+ compute_task = progress.add_task(
100
+ f"[cyan]Computing Sv with waveform_mode={waveform_mode} and encode_mode={encode_mode}...",
101
+ total=100)
102
+
103
+ sv_dataset = await process_file_with_progress(progress, compute_task, echodata,
104
+ encode_mode=encode_mode,
105
+ waveform_mode=waveform_mode,
106
+ depth_offset=depth_offset)
107
+ progress.update(compute_task, advance=100 - progress.tasks[compute_task].completed)
108
+ print(f"[blue]📝 Computed Sv and wrote zarr file to: {config_data['output_folder']}[/blue]")
109
+
110
+ print(f"[green]✅ Plotting echogram for: {config_data['raw_path']}[/green]")
111
+ plot_task = progress.add_task("[cyan]Plotting echogram...", total=100)
112
+ await plot_sv_data_with_progress(sv_dataset, output_path=config_data["output_folder"],
113
+ progress=progress, file_base_name=zarr_file_name, plot_task=plot_task)
114
+ progress.update(plot_task, advance=100 - progress.tasks[plot_task].completed)
115
+ print(f"[blue]📊 Plotted echogram for the data in: {config_data['output_folder']}[/blue]")
116
+ except Exception as e:
117
+ logging.exception(f"Error processing file {config_data['raw_path']}: {e}")
118
+
119
+
120
+ def convert_raw_file(file_path, config_data, progress_queue=None):
121
+ logging.debug("Starting processing of file: %s", file_path)
122
+ from oceanstream.echodata import read_file
123
+
124
+ try:
125
+ file_config_data = {**config_data, 'raw_path': Path(file_path)}
126
+ echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
127
+ echodata.to_zarr(save_path=config_data["output_folder"], overwrite=True, parallel=False)
128
+
129
+ if progress_queue:
130
+ progress_queue.put(file_path)
131
+ except Exception as e:
132
+ logging.error("Error processing file %s", file_path)
133
+ print(Traceback())
134
+
135
+
136
+ def compute_single_file(config_data, **kwargs):
137
+ file_path = config_data["raw_path"]
138
+ start_time = time.time()
139
+ chunks = kwargs.get("chunks")
140
+ echodata = ep.open_converted(file_path, chunks=chunks)
141
+
142
+ try:
143
+ output_path = compute_Sv_to_zarr(echodata, config_data, **kwargs)
144
+ print(f"[blue]✅ Computed Sv and saved to: {output_path}[/blue]")
145
+ except Exception as e:
146
+ logging.error(f"Error computing Sv for {file_path}")
147
+ print(Traceback())
148
+ finally:
149
+ end_time = time.time()
150
+ total_time = end_time - start_time
151
+ print(f"Total time taken: {total_time:.2f} seconds")
@@ -0,0 +1,246 @@
1
+ import os
2
+ import logging
3
+ import time
4
+ import re
5
+ import signal
6
+ import sys
7
+ import traceback
8
+ import warnings
9
+
10
+ from pathlib import Path
11
+ from datetime import datetime
12
+ from rich import print
13
+ from functools import partial
14
+ from multiprocessing import Pool, Manager, Process
15
+ from rich.progress import Progress, TimeElapsedColumn, BarColumn, TextColumn
16
+ from oceanstream.plot import plot_sv_data
17
+ from oceanstream.echodata import get_campaign_metadata, read_file
18
+
19
+ from .combine_zarr import read_zarr_files
20
+ from .process import compute_sv
21
+ from .processed_data_io import write_processed
22
+ from .file_processor import convert_raw_file, compute_single_file
23
+
24
+ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
25
+ warnings.filterwarnings("ignore", module="echopype")
26
+ pool = None
27
+
28
+
29
+ def configure_logging(log_level=logging.ERROR):
30
+ logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s')
31
+
32
+
33
+ def create_progress_bar():
34
+ return Progress(
35
+ TextColumn("[progress.description]{task.description}"),
36
+ BarColumn(),
37
+ TextColumn("[progress.completed]{task.completed} of {task.total} files"),
38
+ TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
39
+ TimeElapsedColumn()
40
+ )
41
+
42
+
43
+ def populate_metadata(ed, raw_fname):
44
+ """
45
+ Manually populate into the "ed" EchoData object
46
+ additional metadata about the dataset and the platform
47
+ """
48
+
49
+ # -- SONAR-netCDF4 Top-level Group attributes
50
+ survey_name = (
51
+ "2017 Joint U.S.-Canada Integrated Ecosystem and "
52
+ "Pacific Hake Acoustic Trawl Survey ('Pacific Hake Survey')"
53
+ )
54
+ ed['Top-level'].attrs['title'] = f"{survey_name}, file {raw_fname}"
55
+ ed['Top-level'].attrs['summary'] = (
56
+ f"EK60 raw file {raw_fname} from the {survey_name}, converted to a SONAR-netCDF4 file using echopype."
57
+ "Information about the survey program is available at "
58
+ "https://www.fisheries.noaa.gov/west-coast/science-data/"
59
+ "joint-us-canada-integrated-ecosystem-and-pacific-hake-acoustic-trawl-survey"
60
+ )
61
+
62
+ # -- SONAR-netCDF4 Platform Group attributes
63
+ # Per SONAR-netCDF4, for platform_type see https://vocab.ices.dk/?ref=311
64
+ ed['Platform'].attrs['platform_type'] = "Research vessel"
65
+ ed['Platform'].attrs['platform_name'] = "Bell M. Shimada" # A NOAA ship
66
+ ed['Platform'].attrs['platform_code_ICES'] = "315"
67
+
68
+
69
+ def update_progress(progress_queue, total_files, log_level):
70
+ configure_logging(log_level)
71
+ logging.debug("Initializing progress updater with total files: %d", total_files)
72
+ with create_progress_bar() as progress:
73
+ task = progress.add_task("[cyan]Processing files...", total=total_files)
74
+ completed_files = 0
75
+
76
+ while completed_files < total_files:
77
+ try:
78
+ message = progress_queue.get(timeout=1)
79
+ if message is None:
80
+ break
81
+ completed_files += 1
82
+ progress.update(task, advance=1)
83
+ logging.debug("Files completed: %d / %d", completed_files, total_files)
84
+ except Exception:
85
+ pass
86
+
87
+
88
+ def process_single_file(file_path, config_data, progress_queue,
89
+ compute_sv_data=False,
90
+ use_distributed_dask=False,
91
+ plot_echogram=False, **kwargs):
92
+ log_level = config_data.get('log_level', logging.ERROR)
93
+ configure_logging(log_level)
94
+ logging.debug("Starting processing of file: %s", file_path)
95
+ try:
96
+ logging.debug("Reading file: %s", file_path)
97
+
98
+ file_config_data = {**config_data, 'raw_path': Path(file_path)}
99
+ echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
100
+ echodata.to_zarr(save_path=config_data["output_folder"], overwrite=True, parallel=False)
101
+
102
+ if compute_sv_data and use_distributed_dask:
103
+ sv_dataset = compute_sv(echodata, encode_mode, **kwargs)
104
+ zarr_file_name = file_config_data['raw_path'].stem
105
+ write_processed(sv_dataset, config_data["output_folder"], zarr_file_name, "zarr")
106
+
107
+ if plot_echogram:
108
+ sv_dataset = compute_sv(echodata, encode_mode, **kwargs)
109
+ zarr_file_name = file_config_data['raw_path'].stem
110
+ plot_sv_data(sv_dataset, output_path=config_data["output_folder"], file_base_name=zarr_file_name)
111
+
112
+ progress_queue.put(file_path)
113
+ except Exception as e:
114
+ logging.exception("Error processing file %s: %s", file_path, e)
115
+
116
+
117
+ def signal_handler(sig, frame):
118
+ global pool
119
+ print('Terminating processes...')
120
+ if pool:
121
+ pool.terminate()
122
+ pool.join()
123
+ sys.exit(0)
124
+
125
+
126
+ def convert_raw_files(config_data, workers_count=os.cpu_count()):
127
+ global pool
128
+
129
+ try:
130
+ dir_path = config_data['raw_path']
131
+ log_level = config_data.get('log_level', logging.ERROR)
132
+ configure_logging(log_level)
133
+
134
+ print(f"Starting to convert folder: {dir_path} to Zarr using {workers_count} parallel processes...")
135
+ raw_files = [os.path.join(dir_path, f) for f in os.listdir(dir_path) if
136
+ os.path.isfile(os.path.join(dir_path, f)) and f.endswith('.raw')]
137
+ file_info = []
138
+
139
+ if not raw_files:
140
+ logging.error("No raw files found in directory: %s", dir_path)
141
+ return
142
+
143
+ logging.debug("Found %d raw files in directory: %s", len(raw_files), dir_path)
144
+
145
+ for file in raw_files:
146
+ creation_time = from_filename(file)
147
+ file_info.append((file, creation_time))
148
+ logging.debug("File: %s, creation time: %s", file, creation_time)
149
+
150
+ if not file_info:
151
+ logging.error("No valid raw files with creation time found in directory: %s", dir_path)
152
+ return
153
+
154
+ filtered_file_info = [item for item in file_info if item[1] is not None]
155
+ filtered_file_info.sort(key=lambda x: x[1])
156
+
157
+ sorted_files = [file for file, _ in filtered_file_info]
158
+ logging.debug("Sorted files by creation time")
159
+
160
+ campaign_id, date, sonar_model, metadata, _ = get_campaign_metadata(sorted_files[0])
161
+ if config_data['sonar_model'] is None:
162
+ config_data['sonar_model'] = sonar_model
163
+
164
+ with Manager() as manager:
165
+ progress_queue = manager.Queue()
166
+ pool = Pool(processes=workers_count)
167
+
168
+ # Partial function with config_data, progress_queue and other arguments
169
+ process_func = partial(convert_raw_file, config_data=config_data, progress_queue=progress_queue)
170
+
171
+ # Run the progress updater in a separate process
172
+ progress_updater = Process(target=update_progress, args=(progress_queue, len(sorted_files), log_level))
173
+ progress_updater.start()
174
+
175
+ for file in sorted_files:
176
+ pool.apply_async(process_func, args=(file,))
177
+ logging.debug("Started async processing for file: %s", file)
178
+
179
+ pool.close()
180
+ pool.join()
181
+ print(f"[green]✅ All files have been converted.[/green]")
182
+
183
+ # Wait for the progress updater to finish
184
+ progress_queue.put(None) # Signal the progress updater to finish
185
+ progress_updater.join()
186
+ except KeyboardInterrupt:
187
+ logging.info("KeyboardInterrupt received, terminating processes...")
188
+ if pool:
189
+ pool.terminate()
190
+ pool.join()
191
+ except Exception as e:
192
+ logging.exception("Error processing folder %s: %s", config_data['raw_path'], e)
193
+
194
+
195
+ def process_single_zarr_file(file_path, config_data, chunks=None, plot_echogram=False, waveform_mode="CW",
196
+ depth_offset=0):
197
+ file_config_data = {**config_data, 'raw_path': Path(file_path)}
198
+ compute_single_file(file_config_data, chunks=chunks, plot_echogram=plot_echogram,
199
+ waveform_mode=waveform_mode, depth_offset=depth_offset)
200
+
201
+
202
+ def process_zarr_files(config_data, workers_count=os.cpu_count(), status=None, chunks=None, plot_echogram=False,
203
+ waveform_mode="CW", depth_offset=0):
204
+ dir_path = config_data['raw_path']
205
+ zarr_files = read_zarr_files(dir_path)
206
+
207
+ if not zarr_files:
208
+ logging.error("No valid .zarr files with creation time found in directory: %s", dir_path)
209
+ return
210
+
211
+ logging.info(f"Found {len(zarr_files)} Zarr files in directory: {dir_path}")
212
+
213
+ # def wrapped_process_single_zarr_file(file_path):
214
+ # print(f"Computing Sv for {file_path}...\n")
215
+ # return process_single_zarr_file(file_path, config_data,
216
+ # chunks=chunks, plot_echogram=plot_echogram, waveform_mode=waveform_mode,
217
+ # depth_offset=depth_offset)
218
+ #
219
+ # # Use Dask bag to parallelize the processing
220
+ # bag = db.from_sequence(zarr_files, npartitions=workers_count)
221
+ # bag = bag.map(wrapped_process_single_zarr_file)
222
+ #
223
+ # # Compute the results
224
+ # results = bag.compute()
225
+ for file_path in zarr_files:
226
+ status.update(f"Computing Sv for {file_path}...\n")
227
+ process_single_zarr_file(file_path, config_data, chunks=chunks, plot_echogram=plot_echogram,
228
+ waveform_mode=waveform_mode, depth_offset=depth_offset)
229
+
230
+ logging.info("✅ All files have been processed")
231
+
232
+
233
+ def from_filename(file_name):
234
+ """Extract creation time from the file name if it follows a specific pattern."""
235
+ pattern = r'(\d{4}[A-Z])?-D(\d{8})-T(\d{6})\.raw'
236
+ match = re.search(pattern, file_name)
237
+ if match:
238
+ date_str = match.group(2)
239
+ time_str = match.group(3)
240
+ creation_time = datetime.strptime(date_str + time_str, '%Y%m%d%H%M%S')
241
+ return creation_time
242
+
243
+ return None
244
+
245
+
246
+ signal.signal(signal.SIGINT, signal_handler)
@@ -0,0 +1,77 @@
1
+ import asyncio
2
+ from concurrent.futures import ThreadPoolExecutor
3
+ from oceanstream.echodata import (
4
+ read_file,
5
+ compute_sv_with_encode_mode,
6
+ enrich_sv_dataset,
7
+ regrid_dataset
8
+ )
9
+
10
+
11
+ async def process_file_with_progress(progress, compute_task_id, echodata, encode_mode="power", waveform_mode="CW",
12
+ depth_offset=0):
13
+ loop = asyncio.get_running_loop()
14
+ total_steps = 25
15
+ progress_step = 95 / total_steps
16
+
17
+ process_task_future = loop.run_in_executor(None, compute_sv_with_encode_mode, echodata, waveform_mode, encode_mode)
18
+
19
+ for step in range(total_steps):
20
+ if process_task_future.done():
21
+ break
22
+ await asyncio.sleep(0.1)
23
+ progress.update(compute_task_id, advance=progress_step)
24
+
25
+ sv_dataset = await process_task_future
26
+ sv_enriched = enrich_sv_dataset(
27
+ sv_dataset,
28
+ echodata,
29
+ waveform_mode=waveform_mode,
30
+ encode_mode=encode_mode,
31
+ depth_offset=depth_offset
32
+ )
33
+
34
+ # del echodata
35
+ # sv_enriched_downsampled = regrid_dataset(sv_enriched)
36
+ # sv_enriched = sv_enriched_downsampled
37
+
38
+ return sv_enriched
39
+
40
+
41
+ def compute_sv(echodata, encode_mode="power", waveform_mode="CW", depth_offset=0):
42
+ sv_dataset = compute_sv_with_encode_mode(echodata, waveform_mode, encode_mode)
43
+
44
+ sv_enriched = enrich_sv_dataset(
45
+ sv_dataset,
46
+ echodata,
47
+ waveform_mode=waveform_mode,
48
+ encode_mode=encode_mode,
49
+ depth_offset=depth_offset
50
+ )
51
+
52
+ # del echodata
53
+ # sv_enriched_downsampled = regrid_dataset(sv_enriched)
54
+ # sv_enriched = sv_enriched_downsampled
55
+
56
+ return sv_enriched
57
+
58
+
59
+ async def read_file_with_progress(config_data, progress, read_task, waveform_mode="CW"):
60
+ loop = asyncio.get_running_loop()
61
+ with ThreadPoolExecutor() as pool:
62
+ total_steps = 20
63
+ progress_step = 95 / total_steps
64
+
65
+ read_task_future = loop.run_in_executor(pool, read_file, config_data, True)
66
+
67
+ for step in range(total_steps):
68
+ if read_task_future.done():
69
+ break
70
+ await asyncio.sleep(0.1)
71
+ progress.update(read_task, advance=progress_step)
72
+
73
+ echodata, encode_mode = await read_task_future
74
+
75
+ return echodata, encode_mode
76
+
77
+
@@ -0,0 +1,140 @@
1
+ """
2
+ processed_data_io.py
3
+ --------------------
4
+ Module for handling input and output operations of processed data.
5
+
6
+ This module provides utility functions to read from and write to different
7
+ data formats, specifically NetCDF (".nc") and Zarr (".zarr"). It ensures that
8
+ the data is correctly loaded into xarray Datasets and provides error handling
9
+ for unsupported file formats and other potential issues.
10
+
11
+ Functions:
12
+ - read_processed(file_path: Union[str, Path]) -> xr.Dataset:
13
+ Reads and returns a xarray Dataset from a specified file path.
14
+
15
+ - write_processed(sv: xr.Dataset, file_path: Union[str, Path], file_name: str = "", file_type: str = "nc", overwrite: bool = True):
16
+ Saves a xarray Dataset to a specified path with a given file name and type.
17
+
18
+ Example:
19
+ >> ds = read_processed("/path/to/datafile.nc")
20
+ >> write_processed(ds, "/path/to/save", "datafile", "nc")
21
+
22
+ Note:
23
+ This module can be extended in the future to support more file formats or additional
24
+ IO operations as needed.
25
+ """
26
+
27
+ import xarray as xr
28
+ from typing import Union
29
+ from pathlib import Path
30
+
31
+
32
+ def read_processed(file_path: Union[str, Path]) -> xr.Dataset:
33
+ """
34
+ Read and return a xarray Dataset from a specified file path.
35
+
36
+ Parameters:
37
+ - file_path (Union[str, Path]): The path to the .nc or .zarr file to be read.
38
+
39
+ Returns:
40
+ - xr.Dataset: The xarray Dataset read from the file.
41
+
42
+ Raises:
43
+ - ValueError: If the file does not exist or has an unsupported format.
44
+ - FileNotFoundError: If the file does not exist.
45
+
46
+ Supported File Types:
47
+ - NetCDF (".nc")
48
+ - Zarr (".zarr")
49
+
50
+ Example:
51
+ >> ds = read_processed("/path/to/datafile.nc")
52
+ """
53
+
54
+ # Convert the input to a Path object (if it isn't already)
55
+ file_path = Path(file_path)
56
+
57
+ # Check if the file exists
58
+ if not file_path.exists():
59
+ raise FileNotFoundError(f"File does not exist: {file_path}")
60
+
61
+ # Check if the file has a supported format
62
+ if file_path.suffix not in [".nc", ".zarr"]:
63
+ raise ValueError(f"Unsupported file format: {file_path.suffix}")
64
+
65
+ # Read the file based on its format
66
+ # Read the file based on its format and load data into memory
67
+ if file_path.suffix == ".nc":
68
+ with xr.open_dataset(file_path) as ds:
69
+ dataset = ds.load()
70
+
71
+ elif file_path.suffix == ".zarr":
72
+ with xr.open_zarr(file_path) as ds:
73
+ dataset = ds.load()
74
+ else:
75
+ raise Exception("Could not open file")
76
+
77
+ return dataset
78
+
79
+
80
+ def write_processed(
81
+ sv: xr.Dataset,
82
+ dir_path: Union[str, Path],
83
+ file_name: str = "",
84
+ file_type: str = "nc",
85
+ overwrite: bool = True,
86
+ ):
87
+ """
88
+ Save a xarray Dataset to a specified path with a given file name and type.
89
+
90
+ Parameters:
91
+ - sv (xr.Dataset): The xarray Dataset to save.
92
+ - dir_path (Union[str, Path]): The directory path where the file should be saved.
93
+ - file_name (str, optional): The name of the file. Defaults to the name of the Dataset.
94
+ - file_type (str, optional): The type of the file, either 'nc' or 'zarr'. Defaults to 'nc'.
95
+ - overwrite (bool, optional): Whether to overwrite the file if it already exists. Defaults to True.
96
+
97
+ Returns:
98
+ - None
99
+
100
+ Raises:
101
+ - TypeError: If the provided Dataset is not an instance of xr.Dataset.
102
+ - ValueError: For invalid paths, unsupported file formats, or other invalid inputs.
103
+
104
+ Example:
105
+ >> write_processed(dataset, "/path/to/save", "datafile", "nc")
106
+ """
107
+ if not isinstance(sv, xr.Dataset):
108
+ raise TypeError("Expected a xarray Dataset")
109
+
110
+ path = Path(dir_path)
111
+ if not path.exists():
112
+ try:
113
+ path.mkdir(parents=True, exist_ok=True)
114
+ except Exception as e:
115
+ raise Exception(f"An error occurred while creating the directory: {e}")
116
+ else:
117
+ if not path.is_dir():
118
+ raise NotADirectoryError(f"Path exists but is not a directory: {path}")
119
+
120
+ if not file_name:
121
+ file_name = Path(sv.source_filenames[0].values.item()).stem
122
+
123
+ if "." not in file_name:
124
+ if file_type not in ["nc", "zarr"]:
125
+ raise ValueError("File type has to be one of ['nc','zarr']")
126
+ file_name = f"{file_name}.{file_type}"
127
+ elif file_name.split(".")[1] not in ["nc", "zarr"]:
128
+ raise ValueError(f"File name provided has unsupported format: {file_name}")
129
+
130
+ full_path = path / file_name
131
+
132
+ if full_path.is_file() and not overwrite:
133
+ return
134
+
135
+ if full_path.suffix == ".nc":
136
+ sv.to_netcdf(full_path)
137
+ elif full_path.suffix == ".zarr":
138
+ sv.to_zarr(full_path, mode="w")
139
+ else:
140
+ raise ValueError(f"Could not save file to provided path: {full_path}")
@@ -0,0 +1 @@
1
+ from .config import load_config
@@ -0,0 +1,17 @@
1
+ import json
2
+ from pathlib import Path
3
+
4
+ DEFAULT_CONFIG_PATH = Path(__file__).parent / "defaults.config.json"
5
+
6
+
7
+ def load_config(user_config_path=None):
8
+ with open(DEFAULT_CONFIG_PATH, "r") as file:
9
+ config = json.load(file)
10
+
11
+ if user_config_path and Path(user_config_path).exists():
12
+ with open(user_config_path, "r") as file:
13
+ user_config = json.load(file)
14
+ for key in user_config:
15
+ config[key] = user_config[key]
16
+
17
+ return config