oceanstream 0.0.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanstream-0.0.0.dev0/LICENSE +21 -0
- oceanstream-0.0.0.dev0/MANIFEST.in +9 -0
- oceanstream-0.0.0.dev0/PKG-INFO +197 -0
- oceanstream-0.0.0.dev0/README.md +152 -0
- oceanstream-0.0.0.dev0/oceanstream/__init__.py +0 -0
- oceanstream-0.0.0.dev0/oceanstream/cli/__init__.py +3 -0
- oceanstream-0.0.0.dev0/oceanstream/cli/main.py +267 -0
- oceanstream-0.0.0.dev0/oceanstream/denoise/__init__.py +2 -0
- oceanstream-0.0.0.dev0/oceanstream/denoise/apply_masks.py +49 -0
- oceanstream-0.0.0.dev0/oceanstream/denoise/applying_masks_handler.py +173 -0
- oceanstream-0.0.0.dev0/oceanstream/denoise/background_noise_remover.py +97 -0
- oceanstream-0.0.0.dev0/oceanstream/denoise/create_masks.py +87 -0
- oceanstream-0.0.0.dev0/oceanstream/denoise/noise_masks.py +398 -0
- oceanstream-0.0.0.dev0/oceanstream/denoise/types.py +73 -0
- oceanstream-0.0.0.dev0/oceanstream/echodata/__init__.py +5 -0
- oceanstream-0.0.0.dev0/oceanstream/echodata/ensure_time_continuity.py +75 -0
- oceanstream-0.0.0.dev0/oceanstream/echodata/mvbs_computation.py +77 -0
- oceanstream-0.0.0.dev0/oceanstream/echodata/raw_handler.py +577 -0
- oceanstream-0.0.0.dev0/oceanstream/echodata/sv_computation.py +118 -0
- oceanstream-0.0.0.dev0/oceanstream/echodata/sv_dataset_extension.py +126 -0
- oceanstream-0.0.0.dev0/oceanstream/echodata/sv_interpolation.py +260 -0
- oceanstream-0.0.0.dev0/oceanstream/echodata/target_strength_computation.py +109 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/__init__.py +2 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +89 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/csv/csv_export_from_raw.py +92 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/csv/csv_export_nasc.py +135 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/frequency_differencing_handler.py +210 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/nasc.py +23 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/nasc_computation.py +62 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/shoals/shoal_detection_handler.py +109 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/shoals/shoal_process.py +224 -0
- oceanstream-0.0.0.dev0/oceanstream/exports/shoals/shoals_handler.py +35 -0
- oceanstream-0.0.0.dev0/oceanstream/plot/__init__.py +2 -0
- oceanstream-0.0.0.dev0/oceanstream/plot/echogram.py +542 -0
- oceanstream-0.0.0.dev0/oceanstream/process/__init__.py +21 -0
- oceanstream-0.0.0.dev0/oceanstream/process/combine_zarr.py +90 -0
- oceanstream-0.0.0.dev0/oceanstream/process/file_processor.py +151 -0
- oceanstream-0.0.0.dev0/oceanstream/process/folder_processor.py +246 -0
- oceanstream-0.0.0.dev0/oceanstream/process/process.py +77 -0
- oceanstream-0.0.0.dev0/oceanstream/process/processed_data_io.py +140 -0
- oceanstream-0.0.0.dev0/oceanstream/settings/__init__.py +1 -0
- oceanstream-0.0.0.dev0/oceanstream/settings/config.py +17 -0
- oceanstream-0.0.0.dev0/oceanstream/utils.py +101 -0
- oceanstream-0.0.0.dev0/oceanstream.egg-info/PKG-INFO +197 -0
- oceanstream-0.0.0.dev0/oceanstream.egg-info/SOURCES.txt +54 -0
- oceanstream-0.0.0.dev0/oceanstream.egg-info/dependency_links.txt +1 -0
- oceanstream-0.0.0.dev0/oceanstream.egg-info/entry_points.txt +2 -0
- oceanstream-0.0.0.dev0/oceanstream.egg-info/requires.txt +31 -0
- oceanstream-0.0.0.dev0/oceanstream.egg-info/top_level.txt +1 -0
- oceanstream-0.0.0.dev0/pyproject.toml +46 -0
- oceanstream-0.0.0.dev0/requirements-dev.txt +52 -0
- oceanstream-0.0.0.dev0/requirements-local.txt +4 -0
- oceanstream-0.0.0.dev0/requirements.txt +4 -0
- oceanstream-0.0.0.dev0/setup.cfg +53 -0
- oceanstream-0.0.0.dev0/setup.py +71 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Pine View Software AS
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: oceanstream
|
|
3
|
+
Version: 0.0.0.dev0
|
|
4
|
+
Summary: OceanStream: process raw sonar data at scale
|
|
5
|
+
Home-page: https://github.com/OceanStreamIO/oceanstream
|
|
6
|
+
Author: Pineview Labs
|
|
7
|
+
Author-email: Pineview Labs <hello@pineview.io>
|
|
8
|
+
License: MIT License
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2023 Pine View Software AS
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
13
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
14
|
+
in the Software without restriction, including without limitation the rights
|
|
15
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
16
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
17
|
+
furnished to do so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
|
|
30
|
+
Classifier: Programming Language :: Python :: 3
|
|
31
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
32
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Operating System :: OS Independent
|
|
35
|
+
Requires-Python: >=3.11
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
Provides-Extra: cli
|
|
38
|
+
Provides-Extra: echodata
|
|
39
|
+
Provides-Extra: plot
|
|
40
|
+
Provides-Extra: denoise
|
|
41
|
+
Provides-Extra: process
|
|
42
|
+
Provides-Extra: exports
|
|
43
|
+
Provides-Extra: complete
|
|
44
|
+
License-File: LICENSE
|
|
45
|
+
|
|
46
|
+
# OceanStream
|
|
47
|
+
|
|
48
|
+
[OceanStream](https://oceanstream.io/) is an integrated cloud data platform used for hydroacoustic data collection and analysis in real-time. It is built on [Azure IoT Edge](https://azure.microsoft.com/en-us/products/iot-edge) infrastructure.
|
|
49
|
+
|
|
50
|
+
<a href="https://oceanstream.io/" target="_blank"><img width="1537" alt="OceanStream.io" src="https://github.com/OceanStreamIO/oceanstream-cli/assets/419506/a1e5ecc6-77d6-45ea-ac4c-51ca1378acd2"></a>
|
|
51
|
+
|
|
52
|
+
## How to Install
|
|
53
|
+
|
|
54
|
+
### Prerequisites
|
|
55
|
+
|
|
56
|
+
- **Python 3.11**: Ensure Python 3.11 is installed on your system.
|
|
57
|
+
|
|
58
|
+
### Setup
|
|
59
|
+
|
|
60
|
+
To contribute, clone the OceanStream repository's dev branch, which contains the latest development changes:
|
|
61
|
+
|
|
62
|
+
git clone -b dev https://github.com/OceanStreamIO/oceanstream.git
|
|
63
|
+
|
|
64
|
+
Navigate to the oceanstream directory:
|
|
65
|
+
|
|
66
|
+
cd oceanstream
|
|
67
|
+
|
|
68
|
+
Now, you can install dependencies, run tests, or start development! Direct all pull requests to the dev branch.
|
|
69
|
+
|
|
70
|
+
### Using Conda/Mamba
|
|
71
|
+
|
|
72
|
+
- **Create a virtual environment to use the package:**
|
|
73
|
+
```bash
|
|
74
|
+
conda env create -f environment.yml
|
|
75
|
+
```
|
|
76
|
+
This command creates an environment named `oceanstream`. To activate it:
|
|
77
|
+
```
|
|
78
|
+
conda activate oceanstream
|
|
79
|
+
```
|
|
80
|
+
- **Create a virtual environment to contribute to the package:**
|
|
81
|
+
```bash
|
|
82
|
+
conda env create -f environment-dev.yml
|
|
83
|
+
```
|
|
84
|
+
Again, activate the environment with:
|
|
85
|
+
```
|
|
86
|
+
conda activate oceanstream
|
|
87
|
+
```
|
|
88
|
+
### Using Pip
|
|
89
|
+
|
|
90
|
+
1. **Create a Virtual Environment:**
|
|
91
|
+
```bash
|
|
92
|
+
python -m venv .venv
|
|
93
|
+
```
|
|
94
|
+
This command creates a virtual environment in the `.venv` directory using Python 3.9.
|
|
95
|
+
|
|
96
|
+
- **Example using `pyenv`:**
|
|
97
|
+
|
|
98
|
+
- Install Python 3.9 with `pyenv`:
|
|
99
|
+
```bash
|
|
100
|
+
pyenv install 3.9
|
|
101
|
+
```
|
|
102
|
+
- Navigate to your project directory:
|
|
103
|
+
```bash
|
|
104
|
+
cd /path/to/your/project_directory
|
|
105
|
+
```
|
|
106
|
+
- Create the virtual environment using the Python 3.9 executable managed by `pyenv`:
|
|
107
|
+
```bash
|
|
108
|
+
~/.pyenv/versions/3.11/bin/python -m venv .venv
|
|
109
|
+
```
|
|
110
|
+
2. **Activate the Virtual Environment:**
|
|
111
|
+
|
|
112
|
+
- On Linux and MacOS:
|
|
113
|
+
```bash
|
|
114
|
+
source .venv/bin/activate
|
|
115
|
+
```
|
|
116
|
+
- On Windows:
|
|
117
|
+
```bash
|
|
118
|
+
.venv\Scripts\activate.bat # In cmd.exe
|
|
119
|
+
.venv\Scripts\Activate.ps1 # In PowerShell
|
|
120
|
+
```
|
|
121
|
+
3. **Install the Dependencies:**
|
|
122
|
+
|
|
123
|
+
- To use the package:
|
|
124
|
+
```bash
|
|
125
|
+
pip install -r requirements.txt
|
|
126
|
+
```
|
|
127
|
+
- To contribute to the package:
|
|
128
|
+
```bash
|
|
129
|
+
pip install -r requirements-dev.txt
|
|
130
|
+
```
|
|
131
|
+
## Running Pre-Commit Locally
|
|
132
|
+
|
|
133
|
+
### Installation
|
|
134
|
+
|
|
135
|
+
Install the pre-commit tool using pip:
|
|
136
|
+
|
|
137
|
+
pip install pre-commit
|
|
138
|
+
|
|
139
|
+
### Installing the Git Hook Scripts
|
|
140
|
+
|
|
141
|
+
Navigate to your repository where the `.pre-commit-config.yaml` file is located. Install the Git hook scripts with:
|
|
142
|
+
|
|
143
|
+
pre-commit install
|
|
144
|
+
|
|
145
|
+
### Running Pre-Commit
|
|
146
|
+
|
|
147
|
+
Pre-commit will now run automatically every time you attempt to make a commit. If any hooks fail, the commit will be blocked, and you'll be prompted to fix the issues before committing again.
|
|
148
|
+
|
|
149
|
+
To manually run all hooks against all the files, use:
|
|
150
|
+
|
|
151
|
+
pre-commit run --all-files
|
|
152
|
+
|
|
153
|
+
### Updating Hooks
|
|
154
|
+
|
|
155
|
+
To update your hooks to the latest versions, use:
|
|
156
|
+
|
|
157
|
+
pre-commit autoupdate
|
|
158
|
+
|
|
159
|
+
### Skipping Hooks
|
|
160
|
+
|
|
161
|
+
To bypass the hooks for a particular commit, use the `-n` or `--no-verify` option:
|
|
162
|
+
|
|
163
|
+
git commit -m "Your commit message" -n
|
|
164
|
+
|
|
165
|
+
### Uninstalling Pre-Commit
|
|
166
|
+
|
|
167
|
+
To uninstall the Git hook scripts, use:
|
|
168
|
+
|
|
169
|
+
pre-commit uninstall
|
|
170
|
+
|
|
171
|
+
## Building Documentation Locally
|
|
172
|
+
|
|
173
|
+
### Prerequisites
|
|
174
|
+
|
|
175
|
+
Ensure Python and pip are installed on your system.
|
|
176
|
+
|
|
177
|
+
### Steps
|
|
178
|
+
|
|
179
|
+
1. **Install Sphinx:**
|
|
180
|
+
|
|
181
|
+
pip install sphinx
|
|
182
|
+
|
|
183
|
+
2. **Navigate to Your Project's docs Directory:**
|
|
184
|
+
|
|
185
|
+
cd /path/to/your/docs_directory
|
|
186
|
+
|
|
187
|
+
3. **Build the Documentation:**
|
|
188
|
+
|
|
189
|
+
- For Linux/Mac:
|
|
190
|
+
|
|
191
|
+
make html
|
|
192
|
+
|
|
193
|
+
- For Windows:
|
|
194
|
+
|
|
195
|
+
make.bat html
|
|
196
|
+
|
|
197
|
+
This command will generate the HTML documentation in the `build/html` directory within your docs folder.
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# OceanStream
|
|
2
|
+
|
|
3
|
+
[OceanStream](https://oceanstream.io/) is an integrated cloud data platform used for hydroacoustic data collection and analysis in real-time. It is built on [Azure IoT Edge](https://azure.microsoft.com/en-us/products/iot-edge) infrastructure.
|
|
4
|
+
|
|
5
|
+
<a href="https://oceanstream.io/" target="_blank"><img width="1537" alt="OceanStream.io" src="https://github.com/OceanStreamIO/oceanstream-cli/assets/419506/a1e5ecc6-77d6-45ea-ac4c-51ca1378acd2"></a>
|
|
6
|
+
|
|
7
|
+
## How to Install
|
|
8
|
+
|
|
9
|
+
### Prerequisites
|
|
10
|
+
|
|
11
|
+
- **Python 3.11**: Ensure Python 3.11 is installed on your system.
|
|
12
|
+
|
|
13
|
+
### Setup
|
|
14
|
+
|
|
15
|
+
To contribute, clone the OceanStream repository's dev branch, which contains the latest development changes:
|
|
16
|
+
|
|
17
|
+
git clone -b dev https://github.com/OceanStreamIO/oceanstream.git
|
|
18
|
+
|
|
19
|
+
Navigate to the oceanstream directory:
|
|
20
|
+
|
|
21
|
+
cd oceanstream
|
|
22
|
+
|
|
23
|
+
Now, you can install dependencies, run tests, or start development! Direct all pull requests to the dev branch.
|
|
24
|
+
|
|
25
|
+
### Using Conda/Mamba
|
|
26
|
+
|
|
27
|
+
- **Create a virtual environment to use the package:**
|
|
28
|
+
```bash
|
|
29
|
+
conda env create -f environment.yml
|
|
30
|
+
```
|
|
31
|
+
This command creates an environment named `oceanstream`. To activate it:
|
|
32
|
+
```
|
|
33
|
+
conda activate oceanstream
|
|
34
|
+
```
|
|
35
|
+
- **Create a virtual environment to contribute to the package:**
|
|
36
|
+
```bash
|
|
37
|
+
conda env create -f environment-dev.yml
|
|
38
|
+
```
|
|
39
|
+
Again, activate the environment with:
|
|
40
|
+
```
|
|
41
|
+
conda activate oceanstream
|
|
42
|
+
```
|
|
43
|
+
### Using Pip
|
|
44
|
+
|
|
45
|
+
1. **Create a Virtual Environment:**
|
|
46
|
+
```bash
|
|
47
|
+
python -m venv .venv
|
|
48
|
+
```
|
|
49
|
+
This command creates a virtual environment in the `.venv` directory using Python 3.9.
|
|
50
|
+
|
|
51
|
+
- **Example using `pyenv`:**
|
|
52
|
+
|
|
53
|
+
- Install Python 3.9 with `pyenv`:
|
|
54
|
+
```bash
|
|
55
|
+
pyenv install 3.9
|
|
56
|
+
```
|
|
57
|
+
- Navigate to your project directory:
|
|
58
|
+
```bash
|
|
59
|
+
cd /path/to/your/project_directory
|
|
60
|
+
```
|
|
61
|
+
- Create the virtual environment using the Python 3.9 executable managed by `pyenv`:
|
|
62
|
+
```bash
|
|
63
|
+
~/.pyenv/versions/3.11/bin/python -m venv .venv
|
|
64
|
+
```
|
|
65
|
+
2. **Activate the Virtual Environment:**
|
|
66
|
+
|
|
67
|
+
- On Linux and MacOS:
|
|
68
|
+
```bash
|
|
69
|
+
source .venv/bin/activate
|
|
70
|
+
```
|
|
71
|
+
- On Windows:
|
|
72
|
+
```bash
|
|
73
|
+
.venv\Scripts\activate.bat # In cmd.exe
|
|
74
|
+
.venv\Scripts\Activate.ps1 # In PowerShell
|
|
75
|
+
```
|
|
76
|
+
3. **Install the Dependencies:**
|
|
77
|
+
|
|
78
|
+
- To use the package:
|
|
79
|
+
```bash
|
|
80
|
+
pip install -r requirements.txt
|
|
81
|
+
```
|
|
82
|
+
- To contribute to the package:
|
|
83
|
+
```bash
|
|
84
|
+
pip install -r requirements-dev.txt
|
|
85
|
+
```
|
|
86
|
+
## Running Pre-Commit Locally
|
|
87
|
+
|
|
88
|
+
### Installation
|
|
89
|
+
|
|
90
|
+
Install the pre-commit tool using pip:
|
|
91
|
+
|
|
92
|
+
pip install pre-commit
|
|
93
|
+
|
|
94
|
+
### Installing the Git Hook Scripts
|
|
95
|
+
|
|
96
|
+
Navigate to your repository where the `.pre-commit-config.yaml` file is located. Install the Git hook scripts with:
|
|
97
|
+
|
|
98
|
+
pre-commit install
|
|
99
|
+
|
|
100
|
+
### Running Pre-Commit
|
|
101
|
+
|
|
102
|
+
Pre-commit will now run automatically every time you attempt to make a commit. If any hooks fail, the commit will be blocked, and you'll be prompted to fix the issues before committing again.
|
|
103
|
+
|
|
104
|
+
To manually run all hooks against all the files, use:
|
|
105
|
+
|
|
106
|
+
pre-commit run --all-files
|
|
107
|
+
|
|
108
|
+
### Updating Hooks
|
|
109
|
+
|
|
110
|
+
To update your hooks to the latest versions, use:
|
|
111
|
+
|
|
112
|
+
pre-commit autoupdate
|
|
113
|
+
|
|
114
|
+
### Skipping Hooks
|
|
115
|
+
|
|
116
|
+
To bypass the hooks for a particular commit, use the `-n` or `--no-verify` option:
|
|
117
|
+
|
|
118
|
+
git commit -m "Your commit message" -n
|
|
119
|
+
|
|
120
|
+
### Uninstalling Pre-Commit
|
|
121
|
+
|
|
122
|
+
To uninstall the Git hook scripts, use:
|
|
123
|
+
|
|
124
|
+
pre-commit uninstall
|
|
125
|
+
|
|
126
|
+
## Building Documentation Locally
|
|
127
|
+
|
|
128
|
+
### Prerequisites
|
|
129
|
+
|
|
130
|
+
Ensure Python and pip are installed on your system.
|
|
131
|
+
|
|
132
|
+
### Steps
|
|
133
|
+
|
|
134
|
+
1. **Install Sphinx:**
|
|
135
|
+
|
|
136
|
+
pip install sphinx
|
|
137
|
+
|
|
138
|
+
2. **Navigate to Your Project's docs Directory:**
|
|
139
|
+
|
|
140
|
+
cd /path/to/your/docs_directory
|
|
141
|
+
|
|
142
|
+
3. **Build the Documentation:**
|
|
143
|
+
|
|
144
|
+
- For Linux/Mac:
|
|
145
|
+
|
|
146
|
+
make html
|
|
147
|
+
|
|
148
|
+
- For Windows:
|
|
149
|
+
|
|
150
|
+
make.bat html
|
|
151
|
+
|
|
152
|
+
This command will generate the HTML documentation in the `build/html` directory within your docs folder.
|
|
File without changes
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import traceback
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
import asyncio
|
|
5
|
+
import os
|
|
6
|
+
import logging
|
|
7
|
+
import sys
|
|
8
|
+
import warnings
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from rich import print
|
|
12
|
+
from rich.traceback import install, Traceback
|
|
13
|
+
from oceanstream.settings import load_config
|
|
14
|
+
from dask.distributed import LocalCluster, Client
|
|
15
|
+
from rich.console import Console
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
install(show_locals=False, width=120)
|
|
19
|
+
|
|
20
|
+
DEFAULT_OUTPUT_FOLDER = "output"
|
|
21
|
+
DEFAULT_SONAR_MODEL = "EK60"
|
|
22
|
+
|
|
23
|
+
BANNER = """
|
|
24
|
+
_
|
|
25
|
+
| |
|
|
26
|
+
___ ___ ___ __ _ _ __ ___| |_ _ __ ___ __ _ _ __ ___
|
|
27
|
+
/ _ \ / __/ _ \/ _` | '_ \ / __| __| '__/ _ \/ _` | '_ ` _ \
|
|
28
|
+
| (_) | (_| __/ (_| | | | | \__ \ |_| | | __/ (_| | | | | | |
|
|
29
|
+
\___/ \___\___|\__,_|_| |_| |___/\__|_| \___|\__,_|_| |_| |_|
|
|
30
|
+
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def initialize(settings, file_path, log_level=None):
|
|
37
|
+
config_data = load_config(settings["config"])
|
|
38
|
+
config_data["raw_path"] = file_path
|
|
39
|
+
|
|
40
|
+
if log_level is not None:
|
|
41
|
+
logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s', force=True)
|
|
42
|
+
config_data["log_level"] = log_level
|
|
43
|
+
|
|
44
|
+
if 'sonar_model' in settings and settings["sonar_model"] is not None:
|
|
45
|
+
config_data["sonar_model"] = settings["sonar_model"]
|
|
46
|
+
|
|
47
|
+
if settings["output_folder"] is not None:
|
|
48
|
+
config_data["output_folder"] = settings["output_folder"]
|
|
49
|
+
|
|
50
|
+
return config_data
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
app = typer.Typer(help="OceanStream CLI")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@app.command()
|
|
57
|
+
def process(
|
|
58
|
+
source: str = typer.Option(..., help="Path to a raw data file"),
|
|
59
|
+
output: str = typer.Option(None,
|
|
60
|
+
help="Destination path for saving processed data. Defaults to a predefined "
|
|
61
|
+
"directory if not specified."),
|
|
62
|
+
sonar_model: str = typer.Option(None, help="Sonar model used to collect the data",
|
|
63
|
+
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
64
|
+
plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
|
|
65
|
+
depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
|
|
66
|
+
waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
|
|
67
|
+
show_choices=["CW", "BB"]),
|
|
68
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
69
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
70
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
71
|
+
):
|
|
72
|
+
"""
|
|
73
|
+
Process the input raw file and save the result to the specified output folder.
|
|
74
|
+
"""
|
|
75
|
+
settings = {
|
|
76
|
+
"config": config,
|
|
77
|
+
"sonar_model": sonar_model,
|
|
78
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
filePath = Path(source)
|
|
82
|
+
configData = initialize(settings, filePath, log_level=log_level)
|
|
83
|
+
|
|
84
|
+
if filePath.is_file():
|
|
85
|
+
from oceanstream.process import process_raw_file_with_progress
|
|
86
|
+
asyncio.run(
|
|
87
|
+
process_raw_file_with_progress(configData, plot_echogram, waveform_mode=waveform_mode,
|
|
88
|
+
depth_offset=depth_offset))
|
|
89
|
+
else:
|
|
90
|
+
print(f"[red]❌ The provided path '{source}' is not a valid raw file.[/red]")
|
|
91
|
+
sys.exit(1)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@app.command()
|
|
95
|
+
def convert(
|
|
96
|
+
source: str = typer.Option(..., help="Path to a raw data file/folder"),
|
|
97
|
+
output: str = typer.Option(None,
|
|
98
|
+
help="Destination path for saving Zarr converted data. Defaults to a predefined "
|
|
99
|
+
"directory if not specified."),
|
|
100
|
+
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
|
|
101
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
102
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
103
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
104
|
+
):
|
|
105
|
+
"""
|
|
106
|
+
Convert the input raw data file(s) to Zarr and save the result to the specified output folder.
|
|
107
|
+
"""
|
|
108
|
+
settings = {
|
|
109
|
+
"config": config,
|
|
110
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
filePath = Path(source)
|
|
114
|
+
configData = initialize(settings, filePath, log_level=log_level)
|
|
115
|
+
|
|
116
|
+
if filePath.is_file():
|
|
117
|
+
from oceanstream.process import convert_raw_file
|
|
118
|
+
convert_raw_file(filePath, configData)
|
|
119
|
+
print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
|
|
120
|
+
elif filePath.is_dir():
|
|
121
|
+
from oceanstream.process import convert_raw_files
|
|
122
|
+
convert_raw_files(configData, workers_count=workers_count)
|
|
123
|
+
else:
|
|
124
|
+
print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
|
|
125
|
+
sys.exit(1)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@app.command()
|
|
129
|
+
def combine(
|
|
130
|
+
source: str = typer.Option(..., help="Path to a source data folder where zarr files are located"),
|
|
131
|
+
output: str = typer.Option(None,
|
|
132
|
+
help="Destination path for saving combined zarr file. Defaults to a predefined "
|
|
133
|
+
"directory if not specified."),
|
|
134
|
+
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
|
|
135
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
136
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
137
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
138
|
+
):
|
|
139
|
+
"""
|
|
140
|
+
Process the input zarr data files and save the combined zarr file to the specified output folder.
|
|
141
|
+
"""
|
|
142
|
+
settings = {
|
|
143
|
+
"config": config,
|
|
144
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
dir_path = Path(source)
|
|
148
|
+
configData = initialize(settings, dir_path, log_level=log_level)
|
|
149
|
+
|
|
150
|
+
if dir_path.is_dir():
|
|
151
|
+
console = Console()
|
|
152
|
+
with console.status("Processing...", spinner="dots") as status:
|
|
153
|
+
status.start()
|
|
154
|
+
|
|
155
|
+
cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
|
|
156
|
+
client = Client(cluster)
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
from oceanstream.process import combine_zarr_files
|
|
160
|
+
|
|
161
|
+
file_name = f"{Path(dir_path).stem}-combined.zarr"
|
|
162
|
+
zarr_output_file = os.path.join(configData['output_folder'], file_name)
|
|
163
|
+
status.update(
|
|
164
|
+
f"Combining zarr files to {zarr_output_file}; navigate to http://localhost:8787/status for progress")
|
|
165
|
+
|
|
166
|
+
combine_zarr_files(dir_path,
|
|
167
|
+
zarr_output_file=zarr_output_file,
|
|
168
|
+
chunks=configData.get('base_chunk_sizes'))
|
|
169
|
+
print("\nZarr files have been combined successfully.")
|
|
170
|
+
except KeyboardInterrupt:
|
|
171
|
+
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
172
|
+
except Exception as e:
|
|
173
|
+
logging.exception("Error processing folder %s", configData['raw_path'])
|
|
174
|
+
traceback.print_exc()
|
|
175
|
+
finally:
|
|
176
|
+
status.stop()
|
|
177
|
+
client.close()
|
|
178
|
+
cluster.close()
|
|
179
|
+
else:
|
|
180
|
+
print(f"[red]❌ The provided path '{source}' is not a valid folder.[/red]")
|
|
181
|
+
sys.exit(1)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@app.command()
|
|
185
|
+
def compute_sv(
|
|
186
|
+
source: str = typer.Option(..., help="Path to a Zarr root file or a directory containing Zarr files"),
|
|
187
|
+
output: str = typer.Option(None,
|
|
188
|
+
help="Destination path for saving Sv data. Defaults to a predefined directory if not specified."),
|
|
189
|
+
workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for Sv computation"),
|
|
190
|
+
sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
|
|
191
|
+
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
192
|
+
plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
|
|
193
|
+
depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
|
|
194
|
+
waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
|
|
195
|
+
show_choices=["CW", "BB"]),
|
|
196
|
+
config: str = typer.Option(None, help="Path to a configuration file"),
|
|
197
|
+
log_level: str = typer.Option("WARNING", help="Set the logging level",
|
|
198
|
+
show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
|
|
199
|
+
):
|
|
200
|
+
"""
|
|
201
|
+
Compute the volume backscattering strength (Sv) from the Zarr file.
|
|
202
|
+
"""
|
|
203
|
+
settings_dict = {
|
|
204
|
+
"config": config,
|
|
205
|
+
"sonar_model": sonar_model,
|
|
206
|
+
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
207
|
+
}
|
|
208
|
+
file_path = Path(source)
|
|
209
|
+
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
210
|
+
|
|
211
|
+
console = Console()
|
|
212
|
+
with console.status("Processing...", spinner="dots") as status:
|
|
213
|
+
status.start()
|
|
214
|
+
cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
|
|
215
|
+
client = Client(cluster)
|
|
216
|
+
|
|
217
|
+
try:
|
|
218
|
+
if file_path.is_dir() and source.endswith(".zarr"):
|
|
219
|
+
status.update(
|
|
220
|
+
f"[blue] Computing Sv for {file_path}...[/blue] – navigate to http://localhost:8787/status for progress")
|
|
221
|
+
from oceanstream.process import compute_single_file
|
|
222
|
+
|
|
223
|
+
compute_single_file(config_data,
|
|
224
|
+
chunks=config_data.get('base_chunk_sizes'),
|
|
225
|
+
plot_echogram=plot_echogram,
|
|
226
|
+
waveform_mode=waveform_mode,
|
|
227
|
+
depth_offset=depth_offset)
|
|
228
|
+
elif file_path.is_dir():
|
|
229
|
+
status.update(
|
|
230
|
+
f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
|
|
231
|
+
f"http://localhost:8787/status for progress")
|
|
232
|
+
from oceanstream.process import process_zarr_files
|
|
233
|
+
|
|
234
|
+
process_zarr_files(config_data,
|
|
235
|
+
workers_count=workers_count,
|
|
236
|
+
status=status,
|
|
237
|
+
chunks=config_data.get('base_chunk_sizes'),
|
|
238
|
+
plot_echogram=plot_echogram,
|
|
239
|
+
waveform_mode=waveform_mode,
|
|
240
|
+
depth_offset=depth_offset)
|
|
241
|
+
else:
|
|
242
|
+
print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
|
|
243
|
+
sys.exit(1)
|
|
244
|
+
except KeyboardInterrupt:
|
|
245
|
+
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
246
|
+
except Exception as e:
|
|
247
|
+
logging.exception("Error while processing %s", config_data['raw_path'])
|
|
248
|
+
traceback.print_exc()
|
|
249
|
+
finally:
|
|
250
|
+
client.close()
|
|
251
|
+
cluster.close()
|
|
252
|
+
status.stop()
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
@app.command()
|
|
256
|
+
def export():
|
|
257
|
+
typer.echo("Export data...")
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def main():
|
|
261
|
+
print(BANNER)
|
|
262
|
+
warnings.filterwarnings("ignore", category=UserWarning)
|
|
263
|
+
app(prog_name="oceanstream")
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
if __name__ == "__main__":
|
|
267
|
+
main()
|