oceanstream 0.0.0.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. oceanstream-0.0.0.dev0/LICENSE +21 -0
  2. oceanstream-0.0.0.dev0/MANIFEST.in +9 -0
  3. oceanstream-0.0.0.dev0/PKG-INFO +197 -0
  4. oceanstream-0.0.0.dev0/README.md +152 -0
  5. oceanstream-0.0.0.dev0/oceanstream/__init__.py +0 -0
  6. oceanstream-0.0.0.dev0/oceanstream/cli/__init__.py +3 -0
  7. oceanstream-0.0.0.dev0/oceanstream/cli/main.py +267 -0
  8. oceanstream-0.0.0.dev0/oceanstream/denoise/__init__.py +2 -0
  9. oceanstream-0.0.0.dev0/oceanstream/denoise/apply_masks.py +49 -0
  10. oceanstream-0.0.0.dev0/oceanstream/denoise/applying_masks_handler.py +173 -0
  11. oceanstream-0.0.0.dev0/oceanstream/denoise/background_noise_remover.py +97 -0
  12. oceanstream-0.0.0.dev0/oceanstream/denoise/create_masks.py +87 -0
  13. oceanstream-0.0.0.dev0/oceanstream/denoise/noise_masks.py +398 -0
  14. oceanstream-0.0.0.dev0/oceanstream/denoise/types.py +73 -0
  15. oceanstream-0.0.0.dev0/oceanstream/echodata/__init__.py +5 -0
  16. oceanstream-0.0.0.dev0/oceanstream/echodata/ensure_time_continuity.py +75 -0
  17. oceanstream-0.0.0.dev0/oceanstream/echodata/mvbs_computation.py +77 -0
  18. oceanstream-0.0.0.dev0/oceanstream/echodata/raw_handler.py +577 -0
  19. oceanstream-0.0.0.dev0/oceanstream/echodata/sv_computation.py +118 -0
  20. oceanstream-0.0.0.dev0/oceanstream/echodata/sv_dataset_extension.py +126 -0
  21. oceanstream-0.0.0.dev0/oceanstream/echodata/sv_interpolation.py +260 -0
  22. oceanstream-0.0.0.dev0/oceanstream/echodata/target_strength_computation.py +109 -0
  23. oceanstream-0.0.0.dev0/oceanstream/exports/__init__.py +2 -0
  24. oceanstream-0.0.0.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
  25. oceanstream-0.0.0.dev0/oceanstream/exports/csv/csv_export_from_raw.py +92 -0
  26. oceanstream-0.0.0.dev0/oceanstream/exports/csv/csv_export_nasc.py +135 -0
  27. oceanstream-0.0.0.dev0/oceanstream/exports/frequency_differencing_handler.py +210 -0
  28. oceanstream-0.0.0.dev0/oceanstream/exports/nasc.py +23 -0
  29. oceanstream-0.0.0.dev0/oceanstream/exports/nasc_computation.py +62 -0
  30. oceanstream-0.0.0.dev0/oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
  31. oceanstream-0.0.0.dev0/oceanstream/exports/shoals/shoal_process.py +224 -0
  32. oceanstream-0.0.0.dev0/oceanstream/exports/shoals/shoals_handler.py +35 -0
  33. oceanstream-0.0.0.dev0/oceanstream/plot/__init__.py +2 -0
  34. oceanstream-0.0.0.dev0/oceanstream/plot/echogram.py +542 -0
  35. oceanstream-0.0.0.dev0/oceanstream/process/__init__.py +21 -0
  36. oceanstream-0.0.0.dev0/oceanstream/process/combine_zarr.py +90 -0
  37. oceanstream-0.0.0.dev0/oceanstream/process/file_processor.py +151 -0
  38. oceanstream-0.0.0.dev0/oceanstream/process/folder_processor.py +246 -0
  39. oceanstream-0.0.0.dev0/oceanstream/process/process.py +77 -0
  40. oceanstream-0.0.0.dev0/oceanstream/process/processed_data_io.py +140 -0
  41. oceanstream-0.0.0.dev0/oceanstream/settings/__init__.py +1 -0
  42. oceanstream-0.0.0.dev0/oceanstream/settings/config.py +17 -0
  43. oceanstream-0.0.0.dev0/oceanstream/utils.py +101 -0
  44. oceanstream-0.0.0.dev0/oceanstream.egg-info/PKG-INFO +197 -0
  45. oceanstream-0.0.0.dev0/oceanstream.egg-info/SOURCES.txt +54 -0
  46. oceanstream-0.0.0.dev0/oceanstream.egg-info/dependency_links.txt +1 -0
  47. oceanstream-0.0.0.dev0/oceanstream.egg-info/entry_points.txt +2 -0
  48. oceanstream-0.0.0.dev0/oceanstream.egg-info/requires.txt +31 -0
  49. oceanstream-0.0.0.dev0/oceanstream.egg-info/top_level.txt +1 -0
  50. oceanstream-0.0.0.dev0/pyproject.toml +46 -0
  51. oceanstream-0.0.0.dev0/requirements-dev.txt +52 -0
  52. oceanstream-0.0.0.dev0/requirements-local.txt +4 -0
  53. oceanstream-0.0.0.dev0/requirements.txt +4 -0
  54. oceanstream-0.0.0.dev0/setup.cfg +53 -0
  55. oceanstream-0.0.0.dev0/setup.py +71 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Pine View Software AS
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,9 @@
1
+ prune .github
2
+ prune build
3
+ prune docs
4
+ exclude *.ini
5
+ exclude *.yaml
6
+ exclude *.yml
7
+ exclude *.json
8
+ include *.txt
9
+ prune tests
@@ -0,0 +1,197 @@
1
+ Metadata-Version: 2.1
2
+ Name: oceanstream
3
+ Version: 0.0.0.dev0
4
+ Summary: OceanStream: process raw sonar data at scale
5
+ Home-page: https://github.com/OceanStreamIO/oceanstream
6
+ Author: Pineview Labs
7
+ Author-email: Pineview Labs <hello@pineview.io>
8
+ License: MIT License
9
+
10
+ Copyright (c) 2023 Pine View Software AS
11
+
12
+ Permission is hereby granted, free of charge, to any person obtaining a copy
13
+ of this software and associated documentation files (the "Software"), to deal
14
+ in the Software without restriction, including without limitation the rights
15
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
16
+ copies of the Software, and to permit persons to whom the Software is
17
+ furnished to do so, subject to the following conditions:
18
+
19
+ The above copyright notice and this permission notice shall be included in all
20
+ copies or substantial portions of the Software.
21
+
22
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
25
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
27
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
28
+ SOFTWARE.
29
+
30
+ Classifier: Programming Language :: Python :: 3
31
+ Classifier: Programming Language :: Python :: 3.10
32
+ Classifier: Programming Language :: Python :: 3.11
33
+ Classifier: License :: OSI Approved :: MIT License
34
+ Classifier: Operating System :: OS Independent
35
+ Requires-Python: >=3.11
36
+ Description-Content-Type: text/markdown
37
+ Provides-Extra: cli
38
+ Provides-Extra: echodata
39
+ Provides-Extra: plot
40
+ Provides-Extra: denoise
41
+ Provides-Extra: process
42
+ Provides-Extra: exports
43
+ Provides-Extra: complete
44
+ License-File: LICENSE
45
+
46
+ # OceanStream
47
+
48
+ [OceanStream](https://oceanstream.io/) is an integrated cloud data platform used for hydroacoustic data collection and analysis in real-time. It is built on [Azure IoT Edge](https://azure.microsoft.com/en-us/products/iot-edge) infrastructure.
49
+
50
+ <a href="https://oceanstream.io/" target="_blank"><img width="1537" alt="OceanStream.io" src="https://github.com/OceanStreamIO/oceanstream-cli/assets/419506/a1e5ecc6-77d6-45ea-ac4c-51ca1378acd2"></a>
51
+
52
+ ## How to Install
53
+
54
+ ### Prerequisites
55
+
56
+ - **Python 3.11**: Ensure Python 3.11 is installed on your system.
57
+
58
+ ### Setup
59
+
60
+ To contribute, clone the OceanStream repository's dev branch, which contains the latest development changes:
61
+
62
+ git clone -b dev https://github.com/OceanStreamIO/oceanstream.git
63
+
64
+ Navigate to the oceanstream directory:
65
+
66
+ cd oceanstream
67
+
68
+ Now, you can install dependencies, run tests, or start development! Direct all pull requests to the dev branch.
69
+
70
+ ### Using Conda/Mamba
71
+
72
+ - **Create a virtual environment to use the package:**
73
+ ```bash
74
+ conda env create -f environment.yml
75
+ ```
76
+ This command creates an environment named `oceanstream`. To activate it:
77
+ ```
78
+ conda activate oceanstream
79
+ ```
80
+ - **Create a virtual environment to contribute to the package:**
81
+ ```bash
82
+ conda env create -f environment-dev.yml
83
+ ```
84
+ Again, activate the environment with:
85
+ ```
86
+ conda activate oceanstream
87
+ ```
88
+ ### Using Pip
89
+
90
+ 1. **Create a Virtual Environment:**
91
+ ```bash
92
+ python -m venv .venv
93
+ ```
94
+ This command creates a virtual environment in the `.venv` directory using Python 3.9.
95
+
96
+ - **Example using `pyenv`:**
97
+
98
+ - Install Python 3.9 with `pyenv`:
99
+ ```bash
100
+ pyenv install 3.9
101
+ ```
102
+ - Navigate to your project directory:
103
+ ```bash
104
+ cd /path/to/your/project_directory
105
+ ```
106
+ - Create the virtual environment using the Python 3.9 executable managed by `pyenv`:
107
+ ```bash
108
+ ~/.pyenv/versions/3.11/bin/python -m venv .venv
109
+ ```
110
+ 2. **Activate the Virtual Environment:**
111
+
112
+ - On Linux and MacOS:
113
+ ```bash
114
+ source .venv/bin/activate
115
+ ```
116
+ - On Windows:
117
+ ```bash
118
+ .venv\Scripts\activate.bat # In cmd.exe
119
+ .venv\Scripts\Activate.ps1 # In PowerShell
120
+ ```
121
+ 3. **Install the Dependencies:**
122
+
123
+ - To use the package:
124
+ ```bash
125
+ pip install -r requirements.txt
126
+ ```
127
+ - To contribute to the package:
128
+ ```bash
129
+ pip install -r requirements-dev.txt
130
+ ```
131
+ ## Running Pre-Commit Locally
132
+
133
+ ### Installation
134
+
135
+ Install the pre-commit tool using pip:
136
+
137
+ pip install pre-commit
138
+
139
+ ### Installing the Git Hook Scripts
140
+
141
+ Navigate to your repository where the `.pre-commit-config.yaml` file is located. Install the Git hook scripts with:
142
+
143
+ pre-commit install
144
+
145
+ ### Running Pre-Commit
146
+
147
+ Pre-commit will now run automatically every time you attempt to make a commit. If any hooks fail, the commit will be blocked, and you'll be prompted to fix the issues before committing again.
148
+
149
+ To manually run all hooks against all the files, use:
150
+
151
+ pre-commit run --all-files
152
+
153
+ ### Updating Hooks
154
+
155
+ To update your hooks to the latest versions, use:
156
+
157
+ pre-commit autoupdate
158
+
159
+ ### Skipping Hooks
160
+
161
+ To bypass the hooks for a particular commit, use the `-n` or `--no-verify` option:
162
+
163
+ git commit -m "Your commit message" -n
164
+
165
+ ### Uninstalling Pre-Commit
166
+
167
+ To uninstall the Git hook scripts, use:
168
+
169
+ pre-commit uninstall
170
+
171
+ ## Building Documentation Locally
172
+
173
+ ### Prerequisites
174
+
175
+ Ensure Python and pip are installed on your system.
176
+
177
+ ### Steps
178
+
179
+ 1. **Install Sphinx:**
180
+
181
+ pip install sphinx
182
+
183
+ 2. **Navigate to Your Project's docs Directory:**
184
+
185
+ cd /path/to/your/docs_directory
186
+
187
+ 3. **Build the Documentation:**
188
+
189
+ - For Linux/Mac:
190
+
191
+ make html
192
+
193
+ - For Windows:
194
+
195
+ make.bat html
196
+
197
+ This command will generate the HTML documentation in the `build/html` directory within your docs folder.
@@ -0,0 +1,152 @@
1
+ # OceanStream
2
+
3
+ [OceanStream](https://oceanstream.io/) is an integrated cloud data platform used for hydroacoustic data collection and analysis in real-time. It is built on [Azure IoT Edge](https://azure.microsoft.com/en-us/products/iot-edge) infrastructure.
4
+
5
+ <a href="https://oceanstream.io/" target="_blank"><img width="1537" alt="OceanStream.io" src="https://github.com/OceanStreamIO/oceanstream-cli/assets/419506/a1e5ecc6-77d6-45ea-ac4c-51ca1378acd2"></a>
6
+
7
+ ## How to Install
8
+
9
+ ### Prerequisites
10
+
11
+ - **Python 3.11**: Ensure Python 3.11 is installed on your system.
12
+
13
+ ### Setup
14
+
15
+ To contribute, clone the OceanStream repository's dev branch, which contains the latest development changes:
16
+
17
+ git clone -b dev https://github.com/OceanStreamIO/oceanstream.git
18
+
19
+ Navigate to the oceanstream directory:
20
+
21
+ cd oceanstream
22
+
23
+ Now, you can install dependencies, run tests, or start development! Direct all pull requests to the dev branch.
24
+
25
+ ### Using Conda/Mamba
26
+
27
+ - **Create a virtual environment to use the package:**
28
+ ```bash
29
+ conda env create -f environment.yml
30
+ ```
31
+ This command creates an environment named `oceanstream`. To activate it:
32
+ ```
33
+ conda activate oceanstream
34
+ ```
35
+ - **Create a virtual environment to contribute to the package:**
36
+ ```bash
37
+ conda env create -f environment-dev.yml
38
+ ```
39
+ Again, activate the environment with:
40
+ ```
41
+ conda activate oceanstream
42
+ ```
43
+ ### Using Pip
44
+
45
+ 1. **Create a Virtual Environment:**
46
+ ```bash
47
+ python -m venv .venv
48
+ ```
49
+ This command creates a virtual environment in the `.venv` directory using Python 3.9.
50
+
51
+ - **Example using `pyenv`:**
52
+
53
+ - Install Python 3.9 with `pyenv`:
54
+ ```bash
55
+ pyenv install 3.9
56
+ ```
57
+ - Navigate to your project directory:
58
+ ```bash
59
+ cd /path/to/your/project_directory
60
+ ```
61
+ - Create the virtual environment using the Python 3.9 executable managed by `pyenv`:
62
+ ```bash
63
+ ~/.pyenv/versions/3.11/bin/python -m venv .venv
64
+ ```
65
+ 2. **Activate the Virtual Environment:**
66
+
67
+ - On Linux and MacOS:
68
+ ```bash
69
+ source .venv/bin/activate
70
+ ```
71
+ - On Windows:
72
+ ```bash
73
+ .venv\Scripts\activate.bat # In cmd.exe
74
+ .venv\Scripts\Activate.ps1 # In PowerShell
75
+ ```
76
+ 3. **Install the Dependencies:**
77
+
78
+ - To use the package:
79
+ ```bash
80
+ pip install -r requirements.txt
81
+ ```
82
+ - To contribute to the package:
83
+ ```bash
84
+ pip install -r requirements-dev.txt
85
+ ```
86
+ ## Running Pre-Commit Locally
87
+
88
+ ### Installation
89
+
90
+ Install the pre-commit tool using pip:
91
+
92
+ pip install pre-commit
93
+
94
+ ### Installing the Git Hook Scripts
95
+
96
+ Navigate to your repository where the `.pre-commit-config.yaml` file is located. Install the Git hook scripts with:
97
+
98
+ pre-commit install
99
+
100
+ ### Running Pre-Commit
101
+
102
+ Pre-commit will now run automatically every time you attempt to make a commit. If any hooks fail, the commit will be blocked, and you'll be prompted to fix the issues before committing again.
103
+
104
+ To manually run all hooks against all the files, use:
105
+
106
+ pre-commit run --all-files
107
+
108
+ ### Updating Hooks
109
+
110
+ To update your hooks to the latest versions, use:
111
+
112
+ pre-commit autoupdate
113
+
114
+ ### Skipping Hooks
115
+
116
+ To bypass the hooks for a particular commit, use the `-n` or `--no-verify` option:
117
+
118
+ git commit -m "Your commit message" -n
119
+
120
+ ### Uninstalling Pre-Commit
121
+
122
+ To uninstall the Git hook scripts, use:
123
+
124
+ pre-commit uninstall
125
+
126
+ ## Building Documentation Locally
127
+
128
+ ### Prerequisites
129
+
130
+ Ensure Python and pip are installed on your system.
131
+
132
+ ### Steps
133
+
134
+ 1. **Install Sphinx:**
135
+
136
+ pip install sphinx
137
+
138
+ 2. **Navigate to Your Project's docs Directory:**
139
+
140
+ cd /path/to/your/docs_directory
141
+
142
+ 3. **Build the Documentation:**
143
+
144
+ - For Linux/Mac:
145
+
146
+ make html
147
+
148
+ - For Windows:
149
+
150
+ make.bat html
151
+
152
+ This command will generate the HTML documentation in the `build/html` directory within your docs folder.
File without changes
@@ -0,0 +1,3 @@
1
+ """
2
+ CLI submodule for the OceanStream package.
3
+ """
@@ -0,0 +1,267 @@
1
+ import traceback
2
+
3
+ import typer
4
+ import asyncio
5
+ import os
6
+ import logging
7
+ import sys
8
+ import warnings
9
+
10
+ from pathlib import Path
11
+ from rich import print
12
+ from rich.traceback import install, Traceback
13
+ from oceanstream.settings import load_config
14
+ from dask.distributed import LocalCluster, Client
15
+ from rich.console import Console
16
+
17
+
18
+ install(show_locals=False, width=120)
19
+
20
+ DEFAULT_OUTPUT_FOLDER = "output"
21
+ DEFAULT_SONAR_MODEL = "EK60"
22
+
23
+ BANNER = """
24
+ _
25
+ | |
26
+ ___ ___ ___ __ _ _ __ ___| |_ _ __ ___ __ _ _ __ ___
27
+ / _ \ / __/ _ \/ _` | '_ \ / __| __| '__/ _ \/ _` | '_ ` _ \
28
+ | (_) | (_| __/ (_| | | | | \__ \ |_| | | __/ (_| | | | | | |
29
+ \___/ \___\___|\__,_|_| |_| |___/\__|_| \___|\__,_|_| |_| |_|
30
+
31
+ """
32
+
33
+ logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
34
+
35
+
36
+ def initialize(settings, file_path, log_level=None):
37
+ config_data = load_config(settings["config"])
38
+ config_data["raw_path"] = file_path
39
+
40
+ if log_level is not None:
41
+ logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s', force=True)
42
+ config_data["log_level"] = log_level
43
+
44
+ if 'sonar_model' in settings and settings["sonar_model"] is not None:
45
+ config_data["sonar_model"] = settings["sonar_model"]
46
+
47
+ if settings["output_folder"] is not None:
48
+ config_data["output_folder"] = settings["output_folder"]
49
+
50
+ return config_data
51
+
52
+
53
+ app = typer.Typer(help="OceanStream CLI")
54
+
55
+
56
+ @app.command()
57
+ def process(
58
+ source: str = typer.Option(..., help="Path to a raw data file"),
59
+ output: str = typer.Option(None,
60
+ help="Destination path for saving processed data. Defaults to a predefined "
61
+ "directory if not specified."),
62
+ sonar_model: str = typer.Option(None, help="Sonar model used to collect the data",
63
+ show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
64
+ plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
65
+ depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
66
+ waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
67
+ show_choices=["CW", "BB"]),
68
+ config: str = typer.Option(None, help="Path to a configuration file"),
69
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
70
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
71
+ ):
72
+ """
73
+ Process the input raw file and save the result to the specified output folder.
74
+ """
75
+ settings = {
76
+ "config": config,
77
+ "sonar_model": sonar_model,
78
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
79
+ }
80
+
81
+ filePath = Path(source)
82
+ configData = initialize(settings, filePath, log_level=log_level)
83
+
84
+ if filePath.is_file():
85
+ from oceanstream.process import process_raw_file_with_progress
86
+ asyncio.run(
87
+ process_raw_file_with_progress(configData, plot_echogram, waveform_mode=waveform_mode,
88
+ depth_offset=depth_offset))
89
+ else:
90
+ print(f"[red]❌ The provided path '{source}' is not a valid raw file.[/red]")
91
+ sys.exit(1)
92
+
93
+
94
+ @app.command()
95
+ def convert(
96
+ source: str = typer.Option(..., help="Path to a raw data file/folder"),
97
+ output: str = typer.Option(None,
98
+ help="Destination path for saving Zarr converted data. Defaults to a predefined "
99
+ "directory if not specified."),
100
+ workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
101
+ config: str = typer.Option(None, help="Path to a configuration file"),
102
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
103
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
104
+ ):
105
+ """
106
+ Convert the input raw data file(s) to Zarr and save the result to the specified output folder.
107
+ """
108
+ settings = {
109
+ "config": config,
110
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
111
+ }
112
+
113
+ filePath = Path(source)
114
+ configData = initialize(settings, filePath, log_level=log_level)
115
+
116
+ if filePath.is_file():
117
+ from oceanstream.process import convert_raw_file
118
+ convert_raw_file(filePath, configData)
119
+ print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
120
+ elif filePath.is_dir():
121
+ from oceanstream.process import convert_raw_files
122
+ convert_raw_files(configData, workers_count=workers_count)
123
+ else:
124
+ print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
125
+ sys.exit(1)
126
+
127
+
128
+ @app.command()
129
+ def combine(
130
+ source: str = typer.Option(..., help="Path to a source data folder where zarr files are located"),
131
+ output: str = typer.Option(None,
132
+ help="Destination path for saving combined zarr file. Defaults to a predefined "
133
+ "directory if not specified."),
134
+ workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
135
+ config: str = typer.Option(None, help="Path to a configuration file"),
136
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
137
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
138
+ ):
139
+ """
140
+ Process the input zarr data files and save the combined zarr file to the specified output folder.
141
+ """
142
+ settings = {
143
+ "config": config,
144
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
145
+ }
146
+
147
+ dir_path = Path(source)
148
+ configData = initialize(settings, dir_path, log_level=log_level)
149
+
150
+ if dir_path.is_dir():
151
+ console = Console()
152
+ with console.status("Processing...", spinner="dots") as status:
153
+ status.start()
154
+
155
+ cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
156
+ client = Client(cluster)
157
+
158
+ try:
159
+ from oceanstream.process import combine_zarr_files
160
+
161
+ file_name = f"{Path(dir_path).stem}-combined.zarr"
162
+ zarr_output_file = os.path.join(configData['output_folder'], file_name)
163
+ status.update(
164
+ f"Combining zarr files to {zarr_output_file}; navigate to http://localhost:8787/status for progress")
165
+
166
+ combine_zarr_files(dir_path,
167
+ zarr_output_file=zarr_output_file,
168
+ chunks=configData.get('base_chunk_sizes'))
169
+ print("\nZarr files have been combined successfully.")
170
+ except KeyboardInterrupt:
171
+ logging.info("KeyboardInterrupt received, terminating processes...")
172
+ except Exception as e:
173
+ logging.exception("Error processing folder %s", configData['raw_path'])
174
+ traceback.print_exc()
175
+ finally:
176
+ status.stop()
177
+ client.close()
178
+ cluster.close()
179
+ else:
180
+ print(f"[red]❌ The provided path '{source}' is not a valid folder.[/red]")
181
+ sys.exit(1)
182
+
183
+
184
+ @app.command()
185
+ def compute_sv(
186
+ source: str = typer.Option(..., help="Path to a Zarr root file or a directory containing Zarr files"),
187
+ output: str = typer.Option(None,
188
+ help="Destination path for saving Sv data. Defaults to a predefined directory if not specified."),
189
+ workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for Sv computation"),
190
+ sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
191
+ show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
192
+ plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
193
+ depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
194
+ waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
195
+ show_choices=["CW", "BB"]),
196
+ config: str = typer.Option(None, help="Path to a configuration file"),
197
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
198
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
199
+ ):
200
+ """
201
+ Compute the volume backscattering strength (Sv) from the Zarr file.
202
+ """
203
+ settings_dict = {
204
+ "config": config,
205
+ "sonar_model": sonar_model,
206
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
207
+ }
208
+ file_path = Path(source)
209
+ config_data = initialize(settings_dict, file_path, log_level=log_level)
210
+
211
+ console = Console()
212
+ with console.status("Processing...", spinner="dots") as status:
213
+ status.start()
214
+ cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
215
+ client = Client(cluster)
216
+
217
+ try:
218
+ if file_path.is_dir() and source.endswith(".zarr"):
219
+ status.update(
220
+ f"[blue] Computing Sv for {file_path}...[/blue] – navigate to http://localhost:8787/status for progress")
221
+ from oceanstream.process import compute_single_file
222
+
223
+ compute_single_file(config_data,
224
+ chunks=config_data.get('base_chunk_sizes'),
225
+ plot_echogram=plot_echogram,
226
+ waveform_mode=waveform_mode,
227
+ depth_offset=depth_offset)
228
+ elif file_path.is_dir():
229
+ status.update(
230
+ f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
231
+ f"http://localhost:8787/status for progress")
232
+ from oceanstream.process import process_zarr_files
233
+
234
+ process_zarr_files(config_data,
235
+ workers_count=workers_count,
236
+ status=status,
237
+ chunks=config_data.get('base_chunk_sizes'),
238
+ plot_echogram=plot_echogram,
239
+ waveform_mode=waveform_mode,
240
+ depth_offset=depth_offset)
241
+ else:
242
+ print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
243
+ sys.exit(1)
244
+ except KeyboardInterrupt:
245
+ logging.info("KeyboardInterrupt received, terminating processes...")
246
+ except Exception as e:
247
+ logging.exception("Error while processing %s", config_data['raw_path'])
248
+ traceback.print_exc()
249
+ finally:
250
+ client.close()
251
+ cluster.close()
252
+ status.stop()
253
+
254
+
255
+ @app.command()
256
+ def export():
257
+ typer.echo("Export data...")
258
+
259
+
260
+ def main():
261
+ print(BANNER)
262
+ warnings.filterwarnings("ignore", category=UserWarning)
263
+ app(prog_name="oceanstream")
264
+
265
+
266
+ if __name__ == "__main__":
267
+ main()
@@ -0,0 +1,2 @@
1
+ from .apply_masks import apply_background_noise_removal, apply_noise_masks, apply_seabed_mask
2
+ from .create_masks import create_masks