water-column-sonar-processing 25.11.1__py3-none-any.whl → 26.1.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of water-column-sonar-processing might be problematic. Click here for more details.
- water_column_sonar_processing/aws/s3_manager.py +2 -4
- water_column_sonar_processing/aws/s3fs_manager.py +1 -9
- water_column_sonar_processing/cruise/create_empty_zarr_store.py +19 -81
- water_column_sonar_processing/cruise/resample_regrid.py +88 -104
- water_column_sonar_processing/geometry/__init__.py +2 -0
- water_column_sonar_processing/geometry/elevation_manager.py +2 -2
- water_column_sonar_processing/geometry/geometry_manager.py +11 -13
- water_column_sonar_processing/geometry/line_simplification.py +10 -10
- water_column_sonar_processing/geometry/pmtile_generation.py +8 -3
- water_column_sonar_processing/geometry/spatiotemporal.py +106 -0
- water_column_sonar_processing/index/index_manager.py +43 -46
- water_column_sonar_processing/model/zarr_manager.py +533 -514
- water_column_sonar_processing/processing/raw_to_zarr.py +45 -139
- water_column_sonar_processing/utility/cleaner.py +2 -1
- water_column_sonar_processing/utility/constants.py +29 -29
- water_column_sonar_processing-26.1.14.dist-info/METADATA +240 -0
- {water_column_sonar_processing-25.11.1.dist-info → water_column_sonar_processing-26.1.14.dist-info}/RECORD +20 -20
- water_column_sonar_processing/process.py +0 -149
- water_column_sonar_processing-25.11.1.dist-info/METADATA +0 -182
- {water_column_sonar_processing-25.11.1.dist-info → water_column_sonar_processing-26.1.14.dist-info}/WHEEL +0 -0
- {water_column_sonar_processing-25.11.1.dist-info → water_column_sonar_processing-26.1.14.dist-info}/licenses/LICENSE +0 -0
- {water_column_sonar_processing-25.11.1.dist-info → water_column_sonar_processing-26.1.14.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: water-column-sonar-processing
|
|
3
|
+
Version: 26.1.14
|
|
4
|
+
Summary: Processing tool for water column sonar data.
|
|
5
|
+
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
|
+
Maintainer-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://echo.fish
|
|
9
|
+
Project-URL: Repository, https://github.com/CI-CMG/water-column-sonar-processing
|
|
10
|
+
Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
|
|
11
|
+
Keywords: ocean,sonar,water column,zarr
|
|
12
|
+
Requires-Python: >=3.12
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: boto3==1.41.5
|
|
16
|
+
Requires-Dist: boto3-stubs
|
|
17
|
+
Requires-Dist: botocore==1.41.5
|
|
18
|
+
Requires-Dist: build
|
|
19
|
+
Requires-Dist: echopype==0.11.0
|
|
20
|
+
Requires-Dist: fiona==1.10.1
|
|
21
|
+
Requires-Dist: geopandas==1.1.2
|
|
22
|
+
Requires-Dist: importlib==1.0.4
|
|
23
|
+
Requires-Dist: mock
|
|
24
|
+
Requires-Dist: moto[all]==5.1.19
|
|
25
|
+
Requires-Dist: moto[server]==5.1.19
|
|
26
|
+
Requires-Dist: multidict==6.7.0
|
|
27
|
+
Requires-Dist: netcdf4==1.7.3
|
|
28
|
+
Requires-Dist: numpy==2.4.1
|
|
29
|
+
Requires-Dist: pandas==2.3.3
|
|
30
|
+
Requires-Dist: pandas-stubs
|
|
31
|
+
Requires-Dist: pooch==1.8.2
|
|
32
|
+
Requires-Dist: pyarrow==22.0.0
|
|
33
|
+
Requires-Dist: pykalman==0.11.1
|
|
34
|
+
Requires-Dist: python-dotenv
|
|
35
|
+
Requires-Dist: requests==2.32.5
|
|
36
|
+
Requires-Dist: ruff>=0.14.13
|
|
37
|
+
Requires-Dist: s3fs==2026.1.0
|
|
38
|
+
Requires-Dist: scipy==1.17.0
|
|
39
|
+
Requires-Dist: setuptools==80.9.0
|
|
40
|
+
Requires-Dist: shapely==2.1.2
|
|
41
|
+
Requires-Dist: xarray==2025.12.0
|
|
42
|
+
Requires-Dist: zarr==3.1.5
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# Water Column Sonar Processing
|
|
46
|
+
|
|
47
|
+
Processing tool for converting Level_0 water column sonar data to Level_1 and Level_2 derived data sets as well as
|
|
48
|
+
generating geospatial information.
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
  
|
|
52
|
+
|
|
53
|
+
# Setting up the Python Environment
|
|
54
|
+
|
|
55
|
+
> uv python install --reinstall
|
|
56
|
+
> uv venv
|
|
57
|
+
> Python 3.10.12 # 3.13.1
|
|
58
|
+
|
|
59
|
+
# Installing Dependencies
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
source .venv/bin/activate
|
|
63
|
+
# or ".venv\Scripts\activate" in windows
|
|
64
|
+
|
|
65
|
+
uv sync --all-groups
|
|
66
|
+
|
|
67
|
+
uv pip install --upgrade pip
|
|
68
|
+
|
|
69
|
+
uv run pre-commit install
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
# Pytest
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
uv run pytest --cache-clear tests # -W ignore::DeprecationWarning
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
or
|
|
79
|
+
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
80
|
+
|
|
81
|
+
# Test Coverage
|
|
82
|
+
|
|
83
|
+
```commandline
|
|
84
|
+
uv run pytest --cov=water_column_sonar_processing
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
With line numbers
|
|
88
|
+
|
|
89
|
+
```commandline
|
|
90
|
+
uv run pytest tests/geometry --cov=water_column_sonar_processing --cov-report term-missing
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Current status:
|
|
94
|
+
|
|
95
|
+
```commandline
|
|
96
|
+
======================================================================================= tests coverage =======================================================================================
|
|
97
|
+
_____________________________________________________________________ coverage: platform darwin, python 3.12.12-final-0 ______________________________________________________________________
|
|
98
|
+
|
|
99
|
+
Name Stmts Miss Cover
|
|
100
|
+
-------------------------------------------------------------------------------------
|
|
101
|
+
water_column_sonar_processing/__init__.py 3 0 100%
|
|
102
|
+
water_column_sonar_processing/aws/__init__.py 6 0 100%
|
|
103
|
+
water_column_sonar_processing/aws/dynamodb_manager.py 44 4 91%
|
|
104
|
+
water_column_sonar_processing/aws/s3_manager.py 161 27 83%
|
|
105
|
+
water_column_sonar_processing/aws/s3fs_manager.py 18 0 100%
|
|
106
|
+
water_column_sonar_processing/aws/sns_manager.py 18 1 94%
|
|
107
|
+
water_column_sonar_processing/aws/sqs_manager.py 17 2 88%
|
|
108
|
+
water_column_sonar_processing/cruise/__init__.py 3 0 100%
|
|
109
|
+
water_column_sonar_processing/cruise/create_empty_zarr_store.py 38 2 95%
|
|
110
|
+
water_column_sonar_processing/cruise/datatree_manager.py 0 0 100%
|
|
111
|
+
water_column_sonar_processing/cruise/resample_regrid.py 87 6 93%
|
|
112
|
+
water_column_sonar_processing/geometry/__init__.py 6 0 100%
|
|
113
|
+
water_column_sonar_processing/geometry/elevation_manager.py 29 1 97%
|
|
114
|
+
water_column_sonar_processing/geometry/geometry_manager.py 72 33 54%
|
|
115
|
+
water_column_sonar_processing/geometry/line_simplification.py 38 4 89%
|
|
116
|
+
water_column_sonar_processing/geometry/pmtile_generation.py 80 58 28%
|
|
117
|
+
water_column_sonar_processing/geometry/spatiotemporal.py 42 2 95%
|
|
118
|
+
water_column_sonar_processing/index/__init__.py 2 0 100%
|
|
119
|
+
water_column_sonar_processing/index/index_manager.py 118 91 23%
|
|
120
|
+
water_column_sonar_processing/model/__init__.py 2 0 100%
|
|
121
|
+
water_column_sonar_processing/model/zarr_manager.py 103 8 92%
|
|
122
|
+
water_column_sonar_processing/processing/__init__.py 3 0 100%
|
|
123
|
+
water_column_sonar_processing/processing/raw_to_netcdf.py 85 24 72%
|
|
124
|
+
water_column_sonar_processing/processing/raw_to_zarr.py 88 5 94%
|
|
125
|
+
water_column_sonar_processing/utility/__init__.py 5 0 100%
|
|
126
|
+
water_column_sonar_processing/utility/cleaner.py 14 0 100%
|
|
127
|
+
water_column_sonar_processing/utility/constants.py 62 0 100%
|
|
128
|
+
water_column_sonar_processing/utility/pipeline_status.py 42 0 100%
|
|
129
|
+
water_column_sonar_processing/utility/timestamp.py 5 0 100%
|
|
130
|
+
-------------------------------------------------------------------------------------
|
|
131
|
+
TOTAL 1191 268 77%
|
|
132
|
+
=================================================================== 47 passed, 4 skipped, 21 warnings in 337.77s (0:05:37)
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
# Instructions
|
|
136
|
+
|
|
137
|
+
Following this tutorial:
|
|
138
|
+
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
139
|
+
|
|
140
|
+
# Pre Commit Hook
|
|
141
|
+
|
|
142
|
+
see here for installation: https://pre-commit.com/
|
|
143
|
+
https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
|
|
144
|
+
|
|
145
|
+
```
|
|
146
|
+
uv run pre-commit install --allow-missing-config
|
|
147
|
+
# or
|
|
148
|
+
uv run pre-commit install
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
# Black
|
|
152
|
+
|
|
153
|
+
https://ljvmiranda921.github.io/notebook/2018/06/21/precommits-using-black-and-flake8/
|
|
154
|
+
|
|
155
|
+
```
|
|
156
|
+
Settings > Black
|
|
157
|
+
Execution mode: Package
|
|
158
|
+
Python Interpreter: .../.venv/bin/python
|
|
159
|
+
Use Black Formatter: X On Code reformat, X On Save
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
# Linting
|
|
163
|
+
|
|
164
|
+
Ruff
|
|
165
|
+
https://plugins.jetbrains.com/plugin/20574-ruff
|
|
166
|
+
|
|
167
|
+
# Colab Test
|
|
168
|
+
|
|
169
|
+
https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
|
|
170
|
+
|
|
171
|
+
# Tag a Release
|
|
172
|
+
|
|
173
|
+
Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
|
|
174
|
+
|
|
175
|
+
```commandline
|
|
176
|
+
git tag -a v26.1.13 -m "Releasing v26.1.13"
|
|
177
|
+
git push origin --tags
|
|
178
|
+
#gh release create v26.1.12
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
# To Publish To PROD
|
|
182
|
+
|
|
183
|
+
```
|
|
184
|
+
uv build --no-sources
|
|
185
|
+
uv publish
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
# TODO:
|
|
189
|
+
|
|
190
|
+
add https://pypi.org/project/setuptools-scm/
|
|
191
|
+
for extracting the version
|
|
192
|
+
|
|
193
|
+
# Security scanning
|
|
194
|
+
|
|
195
|
+
> bandit -r water_column_sonar_processing/
|
|
196
|
+
|
|
197
|
+
# Data Debugging
|
|
198
|
+
|
|
199
|
+
Experimental Plotting in Xarray (hvPlot):
|
|
200
|
+
https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
|
|
201
|
+
|
|
202
|
+
HB0707 Zoomable Cruise:
|
|
203
|
+
https://hb0707.s3.us-east-1.amazonaws.com/index.html
|
|
204
|
+
|
|
205
|
+
# UV Debugging
|
|
206
|
+
|
|
207
|
+
```
|
|
208
|
+
uv pip install --upgrade pip
|
|
209
|
+
#uv sync --all-groups
|
|
210
|
+
uv run pre-commit install
|
|
211
|
+
uv lock --check
|
|
212
|
+
uv lock
|
|
213
|
+
uv sync --all-groups
|
|
214
|
+
uv run pytest --cache-clear tests
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
# Fixing S3FS Problems
|
|
218
|
+
|
|
219
|
+
```commandline
|
|
220
|
+
To enable/disa asyncio for the debugger, follow the steps:
|
|
221
|
+
Open PyCharm
|
|
222
|
+
Use Shift + Shift (Search Everywhere)
|
|
223
|
+
In the popup type: Registry and press Enter
|
|
224
|
+
Find "Registry" in the list of results and click on it.
|
|
225
|
+
In the new popup find python.debug.asyncio.repl line and check the respective checkbox
|
|
226
|
+
Press Close.
|
|
227
|
+
Restart the IDE.
|
|
228
|
+
The asyncio support will be enabled in the debugger.
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Another useful trick is to turn off "gevent" to speed up debugging:
|
|
232
|
+
|
|
233
|
+
```commandline
|
|
234
|
+
Python > Debugger > "Gevent Compatible"
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
# Fixing windows/wsl/ubuntu/mac git compatability
|
|
238
|
+
|
|
239
|
+
> git config --global core.filemode false
|
|
240
|
+
> git config --global core.autocrlf true
|
|
@@ -1,34 +1,34 @@
|
|
|
1
1
|
water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
|
|
2
|
-
water_column_sonar_processing/process.py,sha256=eatdHHc7_mPapTjxsOyD24OlEWeqj56MmVTMPjxF2QM,5290
|
|
3
2
|
water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
|
|
4
3
|
water_column_sonar_processing/aws/dynamodb_manager.py,sha256=EwSc4UTVQTR6tXsrx2eNdbhvexxfrz3GkkvO97F_9fM,14017
|
|
5
|
-
water_column_sonar_processing/aws/s3_manager.py,sha256=
|
|
6
|
-
water_column_sonar_processing/aws/s3fs_manager.py,sha256=
|
|
4
|
+
water_column_sonar_processing/aws/s3_manager.py,sha256=SI1X1ju7ueYJbtEn6DVjUuXNN6so1EKy9xQwCW3rqp4,16183
|
|
5
|
+
water_column_sonar_processing/aws/s3fs_manager.py,sha256=NJoky3uvMgt7m04LBZhrRkR-w5y18MIPP22EOWQos9E,2275
|
|
7
6
|
water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
|
|
8
7
|
water_column_sonar_processing/aws/sqs_manager.py,sha256=j_DGOKble3i-DlHT_uGxCFEmHVbYhFrpbhXdJKLtBSo,1600
|
|
9
8
|
water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
|
|
10
|
-
water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=
|
|
9
|
+
water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=4sZ_6ThlFs7pfnbHsbdpnlkpnGr9lBDM4-S4hRtYi0Q,5125
|
|
11
10
|
water_column_sonar_processing/cruise/datatree_manager.py,sha256=ed_lY04MyaXKADl7EaqHyEZ8JQMiyIAXVhvtE7YnWcg,662
|
|
12
|
-
water_column_sonar_processing/cruise/resample_regrid.py,sha256=
|
|
13
|
-
water_column_sonar_processing/geometry/__init__.py,sha256=
|
|
14
|
-
water_column_sonar_processing/geometry/elevation_manager.py,sha256=
|
|
15
|
-
water_column_sonar_processing/geometry/geometry_manager.py,sha256=
|
|
16
|
-
water_column_sonar_processing/geometry/line_simplification.py,sha256=
|
|
17
|
-
water_column_sonar_processing/geometry/pmtile_generation.py,sha256=
|
|
11
|
+
water_column_sonar_processing/cruise/resample_regrid.py,sha256=R1l5XjksaH5xqD3x3jdD-sJpe7xuXu4YKh3uCzfCumM,14541
|
|
12
|
+
water_column_sonar_processing/geometry/__init__.py,sha256=VfUXTH_OnjQy6EWCw1Hn2ua7OYGI-6EITeH37b4Wjxc,371
|
|
13
|
+
water_column_sonar_processing/geometry/elevation_manager.py,sha256=2TBiDB3qNkM5ZRUEWhQCjrqRYagur0z8hT102Sts0p8,4251
|
|
14
|
+
water_column_sonar_processing/geometry/geometry_manager.py,sha256=cQhLGMwHZskvuKU83gdLunQbM62s57k8qEKkuDWRlPE,10929
|
|
15
|
+
water_column_sonar_processing/geometry/line_simplification.py,sha256=tsUPu3XeMqBHZ8zVbtXI-l_leosj2ENd7JWJSzz_2ws,9288
|
|
16
|
+
water_column_sonar_processing/geometry/pmtile_generation.py,sha256=6dKLbpyZEEj8KfYmqjzUe8rrtNKAFhUK94BhnepFbSg,9577
|
|
17
|
+
water_column_sonar_processing/geometry/spatiotemporal.py,sha256=KnNqQiuon15EDafUMxO80Grnu_yTxq1aYEKE1u1pXdU,4095
|
|
18
18
|
water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
|
|
19
|
-
water_column_sonar_processing/index/index_manager.py,sha256=
|
|
19
|
+
water_column_sonar_processing/index/index_manager.py,sha256=Dino5EO58Aq2tDhoz8sya3YFXUpqOQMsSoKEnJZMgdI,16170
|
|
20
20
|
water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
|
|
21
|
-
water_column_sonar_processing/model/zarr_manager.py,sha256=
|
|
21
|
+
water_column_sonar_processing/model/zarr_manager.py,sha256=Ghxsd4lI01khbOP0Pl1T5IJNZUxgWdxXHQiYzTZck50,32420
|
|
22
22
|
water_column_sonar_processing/processing/__init__.py,sha256=TDnA4_Nyb8MnU8HaixaCzEdTbFkXsUXA0ekvxtYm1-U,150
|
|
23
23
|
water_column_sonar_processing/processing/raw_to_netcdf.py,sha256=3UdglrWXFLxa_jHkhvC9P9NUkp-iKmp6MvnXFnXi4ns,13098
|
|
24
|
-
water_column_sonar_processing/processing/raw_to_zarr.py,sha256=
|
|
24
|
+
water_column_sonar_processing/processing/raw_to_zarr.py,sha256=XhNCQe3YXUhlhQAzGTcktpJVX7t6Gf_yi29Vpsp4L4k,13558
|
|
25
25
|
water_column_sonar_processing/utility/__init__.py,sha256=5gV8sJcnQGrV2AecwkBFZljOC-fQwLuDX70DxX59x2o,289
|
|
26
|
-
water_column_sonar_processing/utility/cleaner.py,sha256=
|
|
27
|
-
water_column_sonar_processing/utility/constants.py,sha256=
|
|
26
|
+
water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
|
|
27
|
+
water_column_sonar_processing/utility/constants.py,sha256=atCf0MAJJFt2QdS-8mN0NLgmVOMTmyaryWKXPmDTimY,3530
|
|
28
28
|
water_column_sonar_processing/utility/pipeline_status.py,sha256=xbl6-4ePq1Krfo18Mgr0jzWz9C_g2Kbey_QZNJuwdkI,4406
|
|
29
29
|
water_column_sonar_processing/utility/timestamp.py,sha256=rn8SDbGYjRvanDUOrvKpNjp3-AK6-KlU2NaCk6Ok8rc,337
|
|
30
|
-
water_column_sonar_processing-
|
|
31
|
-
water_column_sonar_processing-
|
|
32
|
-
water_column_sonar_processing-
|
|
33
|
-
water_column_sonar_processing-
|
|
34
|
-
water_column_sonar_processing-
|
|
30
|
+
water_column_sonar_processing-26.1.14.dist-info/licenses/LICENSE,sha256=TosqaZpJgYvhgXIyYBti-ggJaO8rxRg3FtThY08s9Aw,1110
|
|
31
|
+
water_column_sonar_processing-26.1.14.dist-info/METADATA,sha256=3Huf0Mv53tOFWZitSynNCvBG03zmVIsv3hyLcMx7RI4,8432
|
|
32
|
+
water_column_sonar_processing-26.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
33
|
+
water_column_sonar_processing-26.1.14.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
|
|
34
|
+
water_column_sonar_processing-26.1.14.dist-info/RECORD,,
|
|
@@ -1,149 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import os
|
|
3
|
-
|
|
4
|
-
import numpy as np
|
|
5
|
-
|
|
6
|
-
from water_column_sonar_processing.aws import (
|
|
7
|
-
DynamoDBManager,
|
|
8
|
-
S3FSManager,
|
|
9
|
-
S3Manager,
|
|
10
|
-
SNSManager,
|
|
11
|
-
)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
###########################################################
|
|
15
|
-
class Process:
|
|
16
|
-
#######################################################
|
|
17
|
-
def __init__(
|
|
18
|
-
self,
|
|
19
|
-
):
|
|
20
|
-
self.input_bucket_name = os.environ["INPUT_BUCKET_NAME"]
|
|
21
|
-
self.output_bucket_name = os.environ["OUTPUT_BUCKET_NAME"]
|
|
22
|
-
self.table_name = os.environ["TABLE_NAME"]
|
|
23
|
-
self.topic_arn = os.environ["TOPIC_ARN"]
|
|
24
|
-
# self.output_bucket_access_key = ?
|
|
25
|
-
# self.output_bucket_secret_access_key = ?
|
|
26
|
-
|
|
27
|
-
def execute(self):
|
|
28
|
-
# input_s3_manager = (
|
|
29
|
-
# S3Manager()
|
|
30
|
-
# ) # TODO: Need to allow passing in of credentials when writing to protected bucket
|
|
31
|
-
s3fs_manager = S3FSManager() # TODO: delete this
|
|
32
|
-
print(s3fs_manager) # TODO: delete this
|
|
33
|
-
output_s3_manager = S3Manager()
|
|
34
|
-
# TODO: s3fs?
|
|
35
|
-
sns_manager = SNSManager()
|
|
36
|
-
ddb_manager = DynamoDBManager()
|
|
37
|
-
|
|
38
|
-
# [1 of 5] Update Pipeline Status in DynamoDB
|
|
39
|
-
# self.dynamodb.update_ status ()
|
|
40
|
-
|
|
41
|
-
# [2 of 5] Download Object From Input Bucket
|
|
42
|
-
# return_value = input_s3_manager.download_file(
|
|
43
|
-
# bucket_name=self.input_bucket_name,
|
|
44
|
-
# key="the_input_key",
|
|
45
|
-
# file_name="the_input_key",
|
|
46
|
-
# )
|
|
47
|
-
# print(return_value)
|
|
48
|
-
|
|
49
|
-
# [3 of 5] Update Entry in DynamoDB
|
|
50
|
-
ship_name = "David_Starr_Jordan" # TODO: get this from input sns message
|
|
51
|
-
cruise_name = "DS0604"
|
|
52
|
-
sensor_name = "EK60"
|
|
53
|
-
file_name = "DSJ0604-D20060406-T113407.raw"
|
|
54
|
-
|
|
55
|
-
test_channels = [
|
|
56
|
-
"GPT 38 kHz 009072055a7f 2 ES38B",
|
|
57
|
-
"GPT 70 kHz 00907203400a 3 ES70-7C",
|
|
58
|
-
"GPT 120 kHz 009072034d52 1 ES120-7",
|
|
59
|
-
"GPT 200 kHz 0090720564e4 4 ES200-7C",
|
|
60
|
-
]
|
|
61
|
-
test_frequencies = [38_000, 70_000, 120_000, 200_000]
|
|
62
|
-
ddb_manager.update_item(
|
|
63
|
-
table_name=self.table_name,
|
|
64
|
-
key={
|
|
65
|
-
"FILE_NAME": {"S": file_name}, # Partition Key
|
|
66
|
-
"CRUISE_NAME": {"S": cruise_name}, # Sort Key
|
|
67
|
-
},
|
|
68
|
-
expression_attribute_names={
|
|
69
|
-
"#CH": "CHANNELS",
|
|
70
|
-
"#ET": "END_TIME",
|
|
71
|
-
"#ED": "ERROR_DETAIL",
|
|
72
|
-
"#FR": "FREQUENCIES",
|
|
73
|
-
"#MA": "MAX_ECHO_RANGE",
|
|
74
|
-
"#MI": "MIN_ECHO_RANGE",
|
|
75
|
-
"#ND": "NUM_PING_TIME_DROPNA",
|
|
76
|
-
"#PS": "PIPELINE_STATUS", # testing this updated
|
|
77
|
-
"#PT": "PIPELINE_TIME", # testing this updated
|
|
78
|
-
"#SE": "SENSOR_NAME",
|
|
79
|
-
"#SH": "SHIP_NAME",
|
|
80
|
-
"#ST": "START_TIME",
|
|
81
|
-
# "#ZB": "ZARR_BUCKET",
|
|
82
|
-
# "#ZP": "ZARR_PATH",
|
|
83
|
-
},
|
|
84
|
-
expression_attribute_values={
|
|
85
|
-
":ch": {"L": [{"S": i} for i in test_channels]},
|
|
86
|
-
":et": {"S": "2006-04-06T13:35:28.688Z"},
|
|
87
|
-
":ed": {"S": ""},
|
|
88
|
-
":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
|
|
89
|
-
":ma": {"N": str(np.round(499.7653, 4))},
|
|
90
|
-
":mi": {"N": str(np.round(0.25, 4))},
|
|
91
|
-
":nd": {"N": str(2458)},
|
|
92
|
-
":ps": {"S": "SUCCESS_AGGREGATOR"},
|
|
93
|
-
":pt": {"S": "2023-10-02T08:54:43Z"},
|
|
94
|
-
":se": {"S": sensor_name},
|
|
95
|
-
":sh": {"S": ship_name},
|
|
96
|
-
":st": {"S": "2006-04-06T11:34:07.288Z"},
|
|
97
|
-
# ":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
|
|
98
|
-
# ":zp": {
|
|
99
|
-
# "S": "level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.model"
|
|
100
|
-
# },
|
|
101
|
-
},
|
|
102
|
-
update_expression=(
|
|
103
|
-
"SET "
|
|
104
|
-
"#CH = :ch, "
|
|
105
|
-
"#ET = :et, "
|
|
106
|
-
"#ED = :ed, "
|
|
107
|
-
"#FR = :fr, "
|
|
108
|
-
"#MA = :ma, "
|
|
109
|
-
"#MI = :mi, "
|
|
110
|
-
"#ND = :nd, "
|
|
111
|
-
"#PS = :ps, "
|
|
112
|
-
"#PT = :pt, "
|
|
113
|
-
"#SE = :se, "
|
|
114
|
-
"#SH = :sh, "
|
|
115
|
-
"#ST = :st, "
|
|
116
|
-
"#ZB = :zb, "
|
|
117
|
-
"#ZP = :zp"
|
|
118
|
-
),
|
|
119
|
-
)
|
|
120
|
-
|
|
121
|
-
# [4 of 5] Write Object to Output Bucket
|
|
122
|
-
output_s3_manager.put(
|
|
123
|
-
bucket_name=self.output_bucket_name, key="123", body="456"
|
|
124
|
-
)
|
|
125
|
-
|
|
126
|
-
# [_ of _] Read file-level Zarr store from bucket, Create GeoJSON, Write to bucket
|
|
127
|
-
# [_ of _] Create empty cruise-level Zarr store
|
|
128
|
-
# [_ of _] Resample and write to cruise-level Zarr Store
|
|
129
|
-
|
|
130
|
-
# [5 of 5] Publish Done Message
|
|
131
|
-
success_message = {
|
|
132
|
-
"default": {
|
|
133
|
-
"shipName": ship_name,
|
|
134
|
-
"cruiseName": cruise_name,
|
|
135
|
-
"sensorName": sensor_name,
|
|
136
|
-
"fileName": file_name,
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
sns_manager.publish(
|
|
140
|
-
topic_arn=self.topic_arn,
|
|
141
|
-
message=json.dumps(success_message),
|
|
142
|
-
)
|
|
143
|
-
print("done...")
|
|
144
|
-
|
|
145
|
-
#######################################################
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
###########################################################
|
|
149
|
-
###########################################################
|
|
@@ -1,182 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: water-column-sonar-processing
|
|
3
|
-
Version: 25.11.1
|
|
4
|
-
Summary: Processing tool for water column sonar data.
|
|
5
|
-
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
|
-
Maintainer-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
7
|
-
License-Expression: MIT
|
|
8
|
-
Project-URL: Homepage, https://echo.fish
|
|
9
|
-
Project-URL: Repository, https://github.com/CI-CMG/water-column-sonar-processing
|
|
10
|
-
Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
|
|
11
|
-
Keywords: ocean,sonar,water column,zarr
|
|
12
|
-
Requires-Python: >=3.12
|
|
13
|
-
Description-Content-Type: text/markdown
|
|
14
|
-
License-File: LICENSE
|
|
15
|
-
Requires-Dist: bandit==1.8.6
|
|
16
|
-
Requires-Dist: black>=25.11.0
|
|
17
|
-
Requires-Dist: boto3==1.40.49
|
|
18
|
-
Requires-Dist: boto3-stubs==1.40.49
|
|
19
|
-
Requires-Dist: botocore==1.40.49
|
|
20
|
-
Requires-Dist: build>=1.3.0
|
|
21
|
-
Requires-Dist: echopype==0.11.0
|
|
22
|
-
Requires-Dist: fiona==1.10.1
|
|
23
|
-
Requires-Dist: geopandas==1.1.1
|
|
24
|
-
Requires-Dist: importlib
|
|
25
|
-
Requires-Dist: mock==5.2.0
|
|
26
|
-
Requires-Dist: moto[all]==5.1.16
|
|
27
|
-
Requires-Dist: moto[server]==5.1.16
|
|
28
|
-
Requires-Dist: multidict==6.7.0
|
|
29
|
-
Requires-Dist: netcdf4==1.7.3
|
|
30
|
-
Requires-Dist: numpy==2.3.4
|
|
31
|
-
Requires-Dist: pandas==2.3.3
|
|
32
|
-
Requires-Dist: pandas-stubs==2.3.2.250926
|
|
33
|
-
Requires-Dist: pooch==1.8.2
|
|
34
|
-
Requires-Dist: pyarrow==22.0.0
|
|
35
|
-
Requires-Dist: pykalman==0.10.2
|
|
36
|
-
Requires-Dist: python-dotenv==1.2.1
|
|
37
|
-
Requires-Dist: requests==2.32.5
|
|
38
|
-
Requires-Dist: s3fs==2025.10.0
|
|
39
|
-
Requires-Dist: scipy==1.16.3
|
|
40
|
-
Requires-Dist: setuptools==80.9.0
|
|
41
|
-
Requires-Dist: shapely==2.1.2
|
|
42
|
-
Requires-Dist: xarray==2025.10.1
|
|
43
|
-
Requires-Dist: zarr==3.1.3
|
|
44
|
-
Dynamic: license-file
|
|
45
|
-
|
|
46
|
-
# Water Column Sonar Processing
|
|
47
|
-
|
|
48
|
-
Processing tool for converting Level_0 water column sonar data to Level_1 and Level_2 derived data sets as well as
|
|
49
|
-
generating geospatial information.
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
  
|
|
53
|
-
|
|
54
|
-
# Setting up the Python Environment
|
|
55
|
-
|
|
56
|
-
> Python 3.12.12
|
|
57
|
-
|
|
58
|
-
# Installing Dependencies
|
|
59
|
-
|
|
60
|
-
```
|
|
61
|
-
source .venv/bin/activate
|
|
62
|
-
# or ".venv\Scripts\activate" in windows
|
|
63
|
-
|
|
64
|
-
uv pip install --upgrade pip
|
|
65
|
-
|
|
66
|
-
uv sync --all-groups
|
|
67
|
-
|
|
68
|
-
uv run pre-commit install
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
# Pytest
|
|
72
|
-
|
|
73
|
-
```
|
|
74
|
-
uv run pytest --cache-clear tests # -W ignore::DeprecationWarning
|
|
75
|
-
```
|
|
76
|
-
|
|
77
|
-
or
|
|
78
|
-
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
79
|
-
|
|
80
|
-
# Instructions
|
|
81
|
-
|
|
82
|
-
Following this tutorial:
|
|
83
|
-
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
84
|
-
|
|
85
|
-
# Pre Commit Hook
|
|
86
|
-
|
|
87
|
-
see here for installation: https://pre-commit.com/
|
|
88
|
-
https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
|
|
89
|
-
|
|
90
|
-
```
|
|
91
|
-
uv run pre-commit install --allow-missing-config
|
|
92
|
-
# or
|
|
93
|
-
uv run pre-commit install
|
|
94
|
-
```
|
|
95
|
-
|
|
96
|
-
# Black
|
|
97
|
-
|
|
98
|
-
https://ljvmiranda921.github.io/notebook/2018/06/21/precommits-using-black-and-flake8/
|
|
99
|
-
|
|
100
|
-
```
|
|
101
|
-
Settings > Black
|
|
102
|
-
Execution mode: Package
|
|
103
|
-
Python Interpreter: .../.venv/bin/python
|
|
104
|
-
Use Black Formatter: X On Code reformat, X On Save
|
|
105
|
-
```
|
|
106
|
-
|
|
107
|
-
# Linting
|
|
108
|
-
|
|
109
|
-
Ruff
|
|
110
|
-
https://plugins.jetbrains.com/plugin/20574-ruff
|
|
111
|
-
|
|
112
|
-
# Colab Test
|
|
113
|
-
|
|
114
|
-
https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
|
|
115
|
-
|
|
116
|
-
# Test Coverage
|
|
117
|
-
|
|
118
|
-
TODO
|
|
119
|
-
|
|
120
|
-
# Tag a Release
|
|
121
|
-
|
|
122
|
-
Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
|
|
123
|
-
|
|
124
|
-
```commandline
|
|
125
|
-
git tag -a v25.11.1 -m "Releasing v25.11.1"
|
|
126
|
-
git push origin --tags
|
|
127
|
-
```
|
|
128
|
-
|
|
129
|
-
# To Publish To PROD
|
|
130
|
-
|
|
131
|
-
```
|
|
132
|
-
uv build --no-sources
|
|
133
|
-
uv publish
|
|
134
|
-
```
|
|
135
|
-
|
|
136
|
-
# TODO:
|
|
137
|
-
|
|
138
|
-
add https://pypi.org/project/setuptools-scm/
|
|
139
|
-
for extracting the version
|
|
140
|
-
|
|
141
|
-
# Security scanning
|
|
142
|
-
|
|
143
|
-
> bandit -r water_column_sonar_processing/
|
|
144
|
-
|
|
145
|
-
# Data Debugging
|
|
146
|
-
|
|
147
|
-
Experimental Plotting in Xarray (hvPlot):
|
|
148
|
-
https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
|
|
149
|
-
|
|
150
|
-
HB0707 Zoomable Cruise:
|
|
151
|
-
https://hb0707.s3.us-east-1.amazonaws.com/index.html
|
|
152
|
-
|
|
153
|
-
# UV Debugging
|
|
154
|
-
|
|
155
|
-
```
|
|
156
|
-
uv pip install --upgrade pip
|
|
157
|
-
uv sync --all-groups
|
|
158
|
-
uv run pre-commit install
|
|
159
|
-
uv lock --check
|
|
160
|
-
uv lock
|
|
161
|
-
uv sync --all-groups
|
|
162
|
-
uv run pytest --cache-clear tests
|
|
163
|
-
```
|
|
164
|
-
|
|
165
|
-
# Fixing S3FS Problems
|
|
166
|
-
|
|
167
|
-
```commandline
|
|
168
|
-
To enable/disa asyncio for the debugger, follow the steps:
|
|
169
|
-
Open PyCharm
|
|
170
|
-
Use Shift + Shift (Search Everywhere)
|
|
171
|
-
In the popup type: Registry and press Enter
|
|
172
|
-
Find "Registry" in the list of results and click on it.
|
|
173
|
-
In the new popup find python.debug.asyncio.repl line and check the respective checkbox
|
|
174
|
-
Press Close.
|
|
175
|
-
Restart the IDE.
|
|
176
|
-
The asyncio support will be enabled in the debugger.
|
|
177
|
-
```
|
|
178
|
-
|
|
179
|
-
# Fixing windows/wsl/ubuntu/mac git compatability
|
|
180
|
-
|
|
181
|
-
> git config --global core.filemode false
|
|
182
|
-
> git config --global core.autocrlf true
|
|
File without changes
|
|
File without changes
|
|
File without changes
|