water-column-sonar-processing 0.0.9__py3-none-any.whl → 26.1.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- water_column_sonar_processing/aws/dynamodb_manager.py +138 -59
- water_column_sonar_processing/aws/s3_manager.py +179 -141
- water_column_sonar_processing/aws/s3fs_manager.py +29 -33
- water_column_sonar_processing/aws/sqs_manager.py +1 -1
- water_column_sonar_processing/cruise/create_empty_zarr_store.py +35 -96
- water_column_sonar_processing/cruise/datatree_manager.py +21 -0
- water_column_sonar_processing/cruise/resample_regrid.py +142 -127
- water_column_sonar_processing/geometry/__init__.py +10 -2
- water_column_sonar_processing/geometry/elevation_manager.py +111 -0
- water_column_sonar_processing/geometry/geometry_manager.py +50 -49
- water_column_sonar_processing/geometry/line_simplification.py +176 -0
- water_column_sonar_processing/geometry/pmtile_generation.py +227 -223
- water_column_sonar_processing/geometry/spatiotemporal.py +106 -0
- water_column_sonar_processing/index/index_manager.py +151 -33
- water_column_sonar_processing/model/zarr_manager.py +665 -262
- water_column_sonar_processing/processing/__init__.py +3 -3
- water_column_sonar_processing/processing/raw_to_netcdf.py +320 -0
- water_column_sonar_processing/processing/raw_to_zarr.py +206 -214
- water_column_sonar_processing/utility/__init__.py +9 -2
- water_column_sonar_processing/utility/constants.py +69 -18
- water_column_sonar_processing/utility/pipeline_status.py +11 -15
- water_column_sonar_processing/utility/timestamp.py +3 -4
- water_column_sonar_processing-26.1.9.dist-info/METADATA +239 -0
- water_column_sonar_processing-26.1.9.dist-info/RECORD +34 -0
- {water_column_sonar_processing-0.0.9.dist-info → water_column_sonar_processing-26.1.9.dist-info}/WHEEL +1 -1
- {water_column_sonar_processing-0.0.9.dist-info → water_column_sonar_processing-26.1.9.dist-info/licenses}/LICENSE +1 -1
- water_column_sonar_processing/geometry/geometry_simplification.py +0 -82
- water_column_sonar_processing/process.py +0 -147
- water_column_sonar_processing/processing/cruise_sampler.py +0 -342
- water_column_sonar_processing-0.0.9.dist-info/METADATA +0 -134
- water_column_sonar_processing-0.0.9.dist-info/RECORD +0 -32
- {water_column_sonar_processing-0.0.9.dist-info → water_column_sonar_processing-26.1.9.dist-info}/top_level.txt +0 -0
|
@@ -1,22 +1,47 @@
|
|
|
1
|
-
from enum import Enum,
|
|
1
|
+
from enum import Enum, unique
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
2
4
|
|
|
3
5
|
|
|
4
6
|
@unique
|
|
5
|
-
class
|
|
6
|
-
|
|
7
|
+
class Instruments(Enum):
|
|
8
|
+
# Values are determined using scan of the fist byte of data
|
|
9
|
+
EK60 = "EK60"
|
|
10
|
+
EK80 = "EK80"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# @unique
|
|
14
|
+
class Constants(Enum):
|
|
15
|
+
"""
|
|
16
|
+
See here for data type support: https://github.com/zarr-developers/zarr-extensions/tree/main/data-types
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
TILE_SIZE = 512
|
|
20
|
+
|
|
7
21
|
# Average https://noaa-wcsd-zarr-pds.s3.us-east-1.amazonaws.com/level_2/Henry_B._Bigelow/HB0902/EK60/HB0902.zarr/time/927
|
|
8
|
-
# chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon
|
|
9
|
-
# NOTE: larger value here will speed up the TurfJS download of
|
|
10
|
-
|
|
22
|
+
# chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon dataset
|
|
23
|
+
# NOTE: larger value here will speed up the TurfJS download of dataset in the UI
|
|
24
|
+
# Problem interpolating the dataset: cannot reshape array of size 65536 into shape...
|
|
25
|
+
# TODO: needs to be enum
|
|
26
|
+
SPATIOTEMPORAL_CHUNK_SIZE = int(1e6) # int(2 ** 16) - 1024
|
|
27
|
+
# TODO: create test for SPATIOTEMPORAL_CHUNK_SIZE with requirement!
|
|
28
|
+
|
|
29
|
+
LEVEL_0 = "raw"
|
|
30
|
+
LEVEL_1 = "level_1" # from bucket path
|
|
31
|
+
LEVEL_2 = "level_2a" # updating zarr store path for zarr v3
|
|
32
|
+
LEVEL_3 = "level_3"
|
|
33
|
+
|
|
34
|
+
EK60 = "EK60" # TODO: use for "instrument"
|
|
35
|
+
EK80 = "EK80"
|
|
36
|
+
# INSTRUMENT = EK60 | EK80
|
|
11
37
|
|
|
12
38
|
|
|
13
39
|
class Coordinates(Enum):
|
|
14
40
|
"""
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
standard_name — name in lowercase and snake_case
|
|
41
|
+
dtype: data type
|
|
42
|
+
units: netcdf defined units
|
|
43
|
+
long_name: most readable description of variable
|
|
44
|
+
standard_name: name in lowercase and snake_case
|
|
20
45
|
"""
|
|
21
46
|
|
|
22
47
|
PROJECT_NAME = "echofish"
|
|
@@ -27,41 +52,67 @@ class Coordinates(Enum):
|
|
|
27
52
|
DEPTH_LONG_NAME = "Depth below surface"
|
|
28
53
|
DEPTH_STANDARD_NAME = "depth"
|
|
29
54
|
|
|
55
|
+
# https://cfconventions.org/Data/cf-conventions/cf-conventions-1.8/cf-conventions.html#table-supported-units
|
|
30
56
|
TIME = "time"
|
|
31
|
-
TIME_DTYPE = "
|
|
57
|
+
TIME_DTYPE = "datetime64[ns]"
|
|
32
58
|
# Note: units and calendar are used downstream by Xarray
|
|
33
|
-
TIME_UNITS = "
|
|
59
|
+
TIME_UNITS = "nanoseconds since 1970-01-01"
|
|
34
60
|
TIME_LONG_NAME = "Timestamp of each ping"
|
|
35
61
|
TIME_STANDARD_NAME = "time"
|
|
36
62
|
TIME_CALENDAR = "proleptic_gregorian"
|
|
37
63
|
# TODO: create test for reading out timestamps in Xarray
|
|
38
64
|
|
|
39
65
|
FREQUENCY = "frequency"
|
|
40
|
-
FREQUENCY_DTYPE =
|
|
66
|
+
FREQUENCY_DTYPE = np.uint64
|
|
41
67
|
FREQUENCY_UNITS = "Hz"
|
|
42
68
|
FREQUENCY_LONG_NAME = "Transducer frequency"
|
|
43
69
|
FREQUENCY_STANDARD_NAME = "sound_frequency"
|
|
44
70
|
|
|
45
71
|
LATITUDE = "latitude"
|
|
46
|
-
LATITUDE_DTYPE =
|
|
72
|
+
LATITUDE_DTYPE = np.float32
|
|
47
73
|
LATITUDE_UNITS = "degrees_north"
|
|
48
74
|
LATITUDE_LONG_NAME = "Latitude"
|
|
49
75
|
LATITUDE_STANDARD_NAME = "latitude"
|
|
50
76
|
|
|
51
77
|
LONGITUDE = "longitude"
|
|
52
|
-
LONGITUDE_DTYPE =
|
|
78
|
+
LONGITUDE_DTYPE = np.float32
|
|
53
79
|
LONGITUDE_UNITS = "degrees_east"
|
|
54
80
|
LONGITUDE_LONG_NAME = "Longitude"
|
|
55
81
|
LONGITUDE_STANDARD_NAME = "longitude"
|
|
56
82
|
|
|
57
83
|
BOTTOM = "bottom"
|
|
58
|
-
BOTTOM_DTYPE =
|
|
84
|
+
BOTTOM_DTYPE = np.float32
|
|
59
85
|
BOTTOM_UNITS = "m"
|
|
60
86
|
BOTTOM_LONG_NAME = "Detected sea floor depth"
|
|
61
87
|
BOTTOM_STANDARD_NAME = "bottom"
|
|
62
88
|
|
|
89
|
+
SPEED = "speed"
|
|
90
|
+
SPEED_DTYPE = np.float32
|
|
91
|
+
SPEED_UNITS = "Knots"
|
|
92
|
+
SPEED_LONG_NAME = "Nautical miles per hour"
|
|
93
|
+
SPEED_STANDARD_NAME = "speed"
|
|
94
|
+
|
|
95
|
+
# This is the width of each 'pixel' of the water columns
|
|
96
|
+
DISTANCE = "distance"
|
|
97
|
+
DISTANCE_DTYPE = np.float32
|
|
98
|
+
DISTANCE_UNITS = "m"
|
|
99
|
+
DISTANCE_LONG_NAME = "GPS distance"
|
|
100
|
+
DISTANCE_STANDARD_NAME = "distance"
|
|
101
|
+
|
|
63
102
|
SV = "Sv"
|
|
64
|
-
SV_DTYPE =
|
|
103
|
+
SV_DTYPE = np.float32
|
|
65
104
|
SV_UNITS = "dB"
|
|
66
105
|
SV_LONG_NAME = "Volume backscattering strength (Sv re 1 m-1)"
|
|
67
106
|
SV_STANDARD_NAME = "volume_backscattering_strength"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# TODO: delete this
|
|
110
|
+
# class BatchShape(Enum):
|
|
111
|
+
# """
|
|
112
|
+
# The tensor shape of a machine learning sample.
|
|
113
|
+
# """
|
|
114
|
+
#
|
|
115
|
+
# DEPTH = 2
|
|
116
|
+
# TIME = 3
|
|
117
|
+
# FREQUENCY = 4
|
|
118
|
+
# BATCH_SIZE = 5
|
|
@@ -107,19 +107,15 @@ class PipelineStatus(Flag):
|
|
|
107
107
|
# Status.LEVEL_1_PROCESSING.value < Status.LEVEL_2_PROCESSING.value
|
|
108
108
|
|
|
109
109
|
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cloudformation-stack.html
|
|
110
|
-
"""
|
|
111
|
-
CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
|
|
112
|
-
ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
|
|
113
|
-
DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
|
|
114
|
-
UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
|
|
115
|
-
UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
|
|
116
|
-
UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
|
|
117
|
-
REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
|
|
118
|
-
IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
|
|
119
110
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
111
|
+
# CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
|
|
112
|
+
# ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
|
|
113
|
+
# DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
|
|
114
|
+
# UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
|
|
115
|
+
# UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
|
|
116
|
+
# UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
|
|
117
|
+
# REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
|
|
118
|
+
# IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
|
|
119
|
+
# failure - noun -
|
|
120
|
+
# failed - verb - "verbs should be avoided"
|
|
121
|
+
# success - noun
|
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
|
|
1
|
+
import datetime
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
###########################################################
|
|
5
5
|
class Timestamp:
|
|
6
6
|
@staticmethod
|
|
7
7
|
def get_timestamp():
|
|
8
|
-
# return timestamp in form:
|
|
9
|
-
|
|
10
|
-
return f"{datetime.utcnow().isoformat()[:23]}Z"
|
|
8
|
+
# return timestamp in form: '2024-03-29T19:36:52.433Z'
|
|
9
|
+
return f"{datetime.datetime.now(datetime.UTC).isoformat()[:23]}Z"
|
|
11
10
|
|
|
12
11
|
|
|
13
12
|
###########################################################
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: water-column-sonar-processing
|
|
3
|
+
Version: 26.1.9
|
|
4
|
+
Summary: Processing tool for water column sonar data.
|
|
5
|
+
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
|
+
Maintainer-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://echo.fish
|
|
9
|
+
Project-URL: Repository, https://github.com/CI-CMG/water-column-sonar-processing
|
|
10
|
+
Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
|
|
11
|
+
Keywords: ocean,sonar,water column,zarr
|
|
12
|
+
Requires-Python: >=3.12
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: boto3==1.41.5
|
|
16
|
+
Requires-Dist: boto3-stubs
|
|
17
|
+
Requires-Dist: botocore==1.41.5
|
|
18
|
+
Requires-Dist: build
|
|
19
|
+
Requires-Dist: echopype==0.11.0
|
|
20
|
+
Requires-Dist: fiona==1.10.1
|
|
21
|
+
Requires-Dist: geopandas==1.1.2
|
|
22
|
+
Requires-Dist: importlib==1.0.4
|
|
23
|
+
Requires-Dist: mock
|
|
24
|
+
Requires-Dist: moto[all]==5.1.19
|
|
25
|
+
Requires-Dist: moto[server]==5.1.19
|
|
26
|
+
Requires-Dist: multidict==6.7.0
|
|
27
|
+
Requires-Dist: netcdf4==1.7.3
|
|
28
|
+
Requires-Dist: numpy==2.4.1
|
|
29
|
+
Requires-Dist: pandas==2.3.3
|
|
30
|
+
Requires-Dist: pandas-stubs
|
|
31
|
+
Requires-Dist: pooch==1.8.2
|
|
32
|
+
Requires-Dist: pyarrow==22.0.0
|
|
33
|
+
Requires-Dist: pykalman==0.11.1
|
|
34
|
+
Requires-Dist: python-dotenv
|
|
35
|
+
Requires-Dist: requests==2.32.5
|
|
36
|
+
Requires-Dist: s3fs==2026.1.0
|
|
37
|
+
Requires-Dist: scipy==1.17.0
|
|
38
|
+
Requires-Dist: setuptools==80.9.0
|
|
39
|
+
Requires-Dist: shapely==2.1.2
|
|
40
|
+
Requires-Dist: xarray==2025.12.0
|
|
41
|
+
Requires-Dist: zarr==3.1.5
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
# Water Column Sonar Processing
|
|
45
|
+
|
|
46
|
+
Processing tool for converting Level_0 water column sonar data to Level_1 and Level_2 derived data sets as well as
|
|
47
|
+
generating geospatial information.
|
|
48
|
+
|
|
49
|
+

|
|
50
|
+
  
|
|
51
|
+
|
|
52
|
+
# Setting up the Python Environment
|
|
53
|
+
|
|
54
|
+
> uv python install --reinstall
|
|
55
|
+
> uv venv
|
|
56
|
+
> Python 3.10.12 # 3.13.1
|
|
57
|
+
|
|
58
|
+
# Installing Dependencies
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
source .venv/bin/activate
|
|
62
|
+
# or ".venv\Scripts\activate" in windows
|
|
63
|
+
|
|
64
|
+
uv sync --all-groups
|
|
65
|
+
|
|
66
|
+
uv pip install --upgrade pip
|
|
67
|
+
|
|
68
|
+
uv run pre-commit install
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
# Pytest
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
uv run pytest --cache-clear tests # -W ignore::DeprecationWarning
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
or
|
|
78
|
+
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
79
|
+
|
|
80
|
+
# Test Coverage
|
|
81
|
+
|
|
82
|
+
```commandline
|
|
83
|
+
uv run pytest --cov=water_column_sonar_processing
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
With line numbers
|
|
87
|
+
|
|
88
|
+
```commandline
|
|
89
|
+
uv run pytest tests/geometry --cov=water_column_sonar_processing --cov-report term-missing
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Current status:
|
|
93
|
+
|
|
94
|
+
```commandline
|
|
95
|
+
======================================================================================= tests coverage =======================================================================================
|
|
96
|
+
_____________________________________________________________________ coverage: platform darwin, python 3.12.12-final-0 ______________________________________________________________________
|
|
97
|
+
|
|
98
|
+
Name Stmts Miss Cover
|
|
99
|
+
-------------------------------------------------------------------------------------
|
|
100
|
+
water_column_sonar_processing/__init__.py 3 0 100%
|
|
101
|
+
water_column_sonar_processing/aws/__init__.py 6 0 100%
|
|
102
|
+
water_column_sonar_processing/aws/dynamodb_manager.py 44 4 91%
|
|
103
|
+
water_column_sonar_processing/aws/s3_manager.py 161 27 83%
|
|
104
|
+
water_column_sonar_processing/aws/s3fs_manager.py 18 0 100%
|
|
105
|
+
water_column_sonar_processing/aws/sns_manager.py 18 1 94%
|
|
106
|
+
water_column_sonar_processing/aws/sqs_manager.py 17 2 88%
|
|
107
|
+
water_column_sonar_processing/cruise/__init__.py 3 0 100%
|
|
108
|
+
water_column_sonar_processing/cruise/create_empty_zarr_store.py 38 2 95%
|
|
109
|
+
water_column_sonar_processing/cruise/datatree_manager.py 0 0 100%
|
|
110
|
+
water_column_sonar_processing/cruise/resample_regrid.py 87 6 93%
|
|
111
|
+
water_column_sonar_processing/geometry/__init__.py 6 0 100%
|
|
112
|
+
water_column_sonar_processing/geometry/elevation_manager.py 29 1 97%
|
|
113
|
+
water_column_sonar_processing/geometry/geometry_manager.py 72 33 54%
|
|
114
|
+
water_column_sonar_processing/geometry/line_simplification.py 38 4 89%
|
|
115
|
+
water_column_sonar_processing/geometry/pmtile_generation.py 80 58 28%
|
|
116
|
+
water_column_sonar_processing/geometry/spatiotemporal.py 42 2 95%
|
|
117
|
+
water_column_sonar_processing/index/__init__.py 2 0 100%
|
|
118
|
+
water_column_sonar_processing/index/index_manager.py 118 91 23%
|
|
119
|
+
water_column_sonar_processing/model/__init__.py 2 0 100%
|
|
120
|
+
water_column_sonar_processing/model/zarr_manager.py 103 8 92%
|
|
121
|
+
water_column_sonar_processing/processing/__init__.py 3 0 100%
|
|
122
|
+
water_column_sonar_processing/processing/raw_to_netcdf.py 85 24 72%
|
|
123
|
+
water_column_sonar_processing/processing/raw_to_zarr.py 88 5 94%
|
|
124
|
+
water_column_sonar_processing/utility/__init__.py 5 0 100%
|
|
125
|
+
water_column_sonar_processing/utility/cleaner.py 14 0 100%
|
|
126
|
+
water_column_sonar_processing/utility/constants.py 62 0 100%
|
|
127
|
+
water_column_sonar_processing/utility/pipeline_status.py 42 0 100%
|
|
128
|
+
water_column_sonar_processing/utility/timestamp.py 5 0 100%
|
|
129
|
+
-------------------------------------------------------------------------------------
|
|
130
|
+
TOTAL 1191 268 77%
|
|
131
|
+
=================================================================== 47 passed, 4 skipped, 21 warnings in 337.77s (0:05:37)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
# Instructions
|
|
135
|
+
|
|
136
|
+
Following this tutorial:
|
|
137
|
+
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
138
|
+
|
|
139
|
+
# Pre Commit Hook
|
|
140
|
+
|
|
141
|
+
see here for installation: https://pre-commit.com/
|
|
142
|
+
https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
|
|
143
|
+
|
|
144
|
+
```
|
|
145
|
+
uv run pre-commit install --allow-missing-config
|
|
146
|
+
# or
|
|
147
|
+
uv run pre-commit install
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
# Black
|
|
151
|
+
|
|
152
|
+
https://ljvmiranda921.github.io/notebook/2018/06/21/precommits-using-black-and-flake8/
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
Settings > Black
|
|
156
|
+
Execution mode: Package
|
|
157
|
+
Python Interpreter: .../.venv/bin/python
|
|
158
|
+
Use Black Formatter: X On Code reformat, X On Save
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
# Linting
|
|
162
|
+
|
|
163
|
+
Ruff
|
|
164
|
+
https://plugins.jetbrains.com/plugin/20574-ruff
|
|
165
|
+
|
|
166
|
+
# Colab Test
|
|
167
|
+
|
|
168
|
+
https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
|
|
169
|
+
|
|
170
|
+
# Tag a Release
|
|
171
|
+
|
|
172
|
+
Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
|
|
173
|
+
|
|
174
|
+
```commandline
|
|
175
|
+
git tag -a v26.1.8 -m "Releasing v26.1.8"
|
|
176
|
+
git push origin --tags
|
|
177
|
+
#gh release create v26.1.7
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
# To Publish To PROD
|
|
181
|
+
|
|
182
|
+
```
|
|
183
|
+
uv build --no-sources
|
|
184
|
+
uv publish
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
# TODO:
|
|
188
|
+
|
|
189
|
+
add https://pypi.org/project/setuptools-scm/
|
|
190
|
+
for extracting the version
|
|
191
|
+
|
|
192
|
+
# Security scanning
|
|
193
|
+
|
|
194
|
+
> bandit -r water_column_sonar_processing/
|
|
195
|
+
|
|
196
|
+
# Data Debugging
|
|
197
|
+
|
|
198
|
+
Experimental Plotting in Xarray (hvPlot):
|
|
199
|
+
https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
|
|
200
|
+
|
|
201
|
+
HB0707 Zoomable Cruise:
|
|
202
|
+
https://hb0707.s3.us-east-1.amazonaws.com/index.html
|
|
203
|
+
|
|
204
|
+
# UV Debugging
|
|
205
|
+
|
|
206
|
+
```
|
|
207
|
+
uv pip install --upgrade pip
|
|
208
|
+
#uv sync --all-groups
|
|
209
|
+
uv run pre-commit install
|
|
210
|
+
uv lock --check
|
|
211
|
+
uv lock
|
|
212
|
+
uv sync --all-groups
|
|
213
|
+
uv run pytest --cache-clear tests
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
# Fixing S3FS Problems
|
|
217
|
+
|
|
218
|
+
```commandline
|
|
219
|
+
To enable/disa asyncio for the debugger, follow the steps:
|
|
220
|
+
Open PyCharm
|
|
221
|
+
Use Shift + Shift (Search Everywhere)
|
|
222
|
+
In the popup type: Registry and press Enter
|
|
223
|
+
Find "Registry" in the list of results and click on it.
|
|
224
|
+
In the new popup find python.debug.asyncio.repl line and check the respective checkbox
|
|
225
|
+
Press Close.
|
|
226
|
+
Restart the IDE.
|
|
227
|
+
The asyncio support will be enabled in the debugger.
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
Another useful trick is to turn off "gevent" to speed up debugging:
|
|
231
|
+
|
|
232
|
+
```commandline
|
|
233
|
+
Python > Debugger > "Gevent Compatible"
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
# Fixing windows/wsl/ubuntu/mac git compatability
|
|
237
|
+
|
|
238
|
+
> git config --global core.filemode false
|
|
239
|
+
> git config --global core.autocrlf true
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
|
|
2
|
+
water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
|
|
3
|
+
water_column_sonar_processing/aws/dynamodb_manager.py,sha256=EwSc4UTVQTR6tXsrx2eNdbhvexxfrz3GkkvO97F_9fM,14017
|
|
4
|
+
water_column_sonar_processing/aws/s3_manager.py,sha256=SI1X1ju7ueYJbtEn6DVjUuXNN6so1EKy9xQwCW3rqp4,16183
|
|
5
|
+
water_column_sonar_processing/aws/s3fs_manager.py,sha256=NJoky3uvMgt7m04LBZhrRkR-w5y18MIPP22EOWQos9E,2275
|
|
6
|
+
water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
|
|
7
|
+
water_column_sonar_processing/aws/sqs_manager.py,sha256=j_DGOKble3i-DlHT_uGxCFEmHVbYhFrpbhXdJKLtBSo,1600
|
|
8
|
+
water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
|
|
9
|
+
water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=x_xw6I7RVpnlt9BpLE6f9RjArfkK8WKaSEsBoknKvY4,4939
|
|
10
|
+
water_column_sonar_processing/cruise/datatree_manager.py,sha256=ed_lY04MyaXKADl7EaqHyEZ8JQMiyIAXVhvtE7YnWcg,662
|
|
11
|
+
water_column_sonar_processing/cruise/resample_regrid.py,sha256=HLyBAjX3EYUa09B3zSWZTqW6peZ9p2z0bkQDNI41s68,13662
|
|
12
|
+
water_column_sonar_processing/geometry/__init__.py,sha256=VfUXTH_OnjQy6EWCw1Hn2ua7OYGI-6EITeH37b4Wjxc,371
|
|
13
|
+
water_column_sonar_processing/geometry/elevation_manager.py,sha256=2TBiDB3qNkM5ZRUEWhQCjrqRYagur0z8hT102Sts0p8,4251
|
|
14
|
+
water_column_sonar_processing/geometry/geometry_manager.py,sha256=cQhLGMwHZskvuKU83gdLunQbM62s57k8qEKkuDWRlPE,10929
|
|
15
|
+
water_column_sonar_processing/geometry/line_simplification.py,sha256=tsUPu3XeMqBHZ8zVbtXI-l_leosj2ENd7JWJSzz_2ws,9288
|
|
16
|
+
water_column_sonar_processing/geometry/pmtile_generation.py,sha256=6dKLbpyZEEj8KfYmqjzUe8rrtNKAFhUK94BhnepFbSg,9577
|
|
17
|
+
water_column_sonar_processing/geometry/spatiotemporal.py,sha256=KnNqQiuon15EDafUMxO80Grnu_yTxq1aYEKE1u1pXdU,4095
|
|
18
|
+
water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
|
|
19
|
+
water_column_sonar_processing/index/index_manager.py,sha256=Dino5EO58Aq2tDhoz8sya3YFXUpqOQMsSoKEnJZMgdI,16170
|
|
20
|
+
water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
|
|
21
|
+
water_column_sonar_processing/model/zarr_manager.py,sha256=DfTt0Azou8N3eaiR2hO7hcZrM5yRoeQEWQoAmzCgBlA,32303
|
|
22
|
+
water_column_sonar_processing/processing/__init__.py,sha256=TDnA4_Nyb8MnU8HaixaCzEdTbFkXsUXA0ekvxtYm1-U,150
|
|
23
|
+
water_column_sonar_processing/processing/raw_to_netcdf.py,sha256=3UdglrWXFLxa_jHkhvC9P9NUkp-iKmp6MvnXFnXi4ns,13098
|
|
24
|
+
water_column_sonar_processing/processing/raw_to_zarr.py,sha256=KYW2SW9HJanonQ6dRC4BSX6y4wmroU9LXOkKUrp4cwQ,14302
|
|
25
|
+
water_column_sonar_processing/utility/__init__.py,sha256=5gV8sJcnQGrV2AecwkBFZljOC-fQwLuDX70DxX59x2o,289
|
|
26
|
+
water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
|
|
27
|
+
water_column_sonar_processing/utility/constants.py,sha256=atCf0MAJJFt2QdS-8mN0NLgmVOMTmyaryWKXPmDTimY,3530
|
|
28
|
+
water_column_sonar_processing/utility/pipeline_status.py,sha256=xbl6-4ePq1Krfo18Mgr0jzWz9C_g2Kbey_QZNJuwdkI,4406
|
|
29
|
+
water_column_sonar_processing/utility/timestamp.py,sha256=rn8SDbGYjRvanDUOrvKpNjp3-AK6-KlU2NaCk6Ok8rc,337
|
|
30
|
+
water_column_sonar_processing-26.1.9.dist-info/licenses/LICENSE,sha256=TosqaZpJgYvhgXIyYBti-ggJaO8rxRg3FtThY08s9Aw,1110
|
|
31
|
+
water_column_sonar_processing-26.1.9.dist-info/METADATA,sha256=ZXbr-XGWO-_mYuRBVg_BF_ZBWpvorNKZCRAwrRNx6PM,8399
|
|
32
|
+
water_column_sonar_processing-26.1.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
33
|
+
water_column_sonar_processing-26.1.9.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
|
|
34
|
+
water_column_sonar_processing-26.1.9.dist-info/RECORD,,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
MIT License
|
|
2
2
|
|
|
3
|
-
Copyright (c)
|
|
3
|
+
Copyright (c) 2025 Cooperative Institutes, Coastal and Marine Geophysics
|
|
4
4
|
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
# import json
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
# lambda for timestamp in form "yyyy-MM-ddTHH:mm:ssZ"
|
|
5
|
-
# dt = lambda: datetime.now().isoformat(timespec="seconds") + "Z"
|
|
6
|
-
|
|
7
|
-
# https://shapely.readthedocs.io/en/stable/reference/shapely.MultiLineString.html#shapely.MultiLineString
|
|
8
|
-
"""
|
|
9
|
-
// [Decimal / Places / Degrees / Object that can be recognized at scale / N/S or E/W at equator, E/W at 23N/S, E/W at 45N/S, E/W at 67N/S]
|
|
10
|
-
// 0 1.0 1° 00′ 0″ country or large region 111.32 km 102.47 km 78.71 km 43.496 km
|
|
11
|
-
// 1 0.1 0° 06′ 0″ large city or district 11.132 km 10.247 km 7.871 km 4.3496 km
|
|
12
|
-
// 2 0.01 0° 00′ 36″ town or village 1.1132 km 1.0247 km 787.1 m 434.96 m
|
|
13
|
-
// 3 0.001 0° 00′ 3.6″ neighborhood, street 111.32 m 102.47 m 78.71 m 43.496 m
|
|
14
|
-
// 4 0.0001 0° 00′ 0.36″ individual street, land parcel 11.132 m 10.247 m 7.871 m 4.3496 m
|
|
15
|
-
// 5 0.00001 0° 00′ 0.036″ individual trees, door entrance 1.1132 m 1.0247 m 787.1 mm 434.96 mm
|
|
16
|
-
// 6 0.000001 0° 00′ 0.0036″ individual humans 111.32 mm 102.47 mm 78.71 mm 43.496 mm
|
|
17
|
-
// 7 0.0000001 0° 00′ 0.00036″ practical limit of commercial surveying 11.132 mm 10.247 mm 7.871 mm 4.3496 mm
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
|
-
"""
|
|
21
|
-
private static final int SRID = 8307;
|
|
22
|
-
private static final double simplificationTolerance = 0.0001;
|
|
23
|
-
private static final long splitGeometryMs = 900000L;
|
|
24
|
-
private static final int batchSize = 10000;
|
|
25
|
-
private static final int geoJsonPrecision = 5;
|
|
26
|
-
final int geoJsonPrecision = 5;
|
|
27
|
-
final double simplificationTolerance = 0.0001;
|
|
28
|
-
final int simplifierBatchSize = 3000;
|
|
29
|
-
final long maxCount = 0;
|
|
30
|
-
private static final double maxAllowedSpeedKnts = 60D;
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
"""
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class GeometrySimplification:
|
|
37
|
-
# TODO: in the future move to standalone library
|
|
38
|
-
#######################################################
|
|
39
|
-
def __init__(
|
|
40
|
-
self,
|
|
41
|
-
):
|
|
42
|
-
pass
|
|
43
|
-
|
|
44
|
-
#######################################################
|
|
45
|
-
def speed_check(
|
|
46
|
-
self,
|
|
47
|
-
speed_knots=50,
|
|
48
|
-
) -> None:
|
|
49
|
-
print(speed_knots)
|
|
50
|
-
pass
|
|
51
|
-
|
|
52
|
-
def remove_null_island_values(
|
|
53
|
-
self,
|
|
54
|
-
epsilon=1e-5,
|
|
55
|
-
) -> None:
|
|
56
|
-
print(epsilon)
|
|
57
|
-
pass
|
|
58
|
-
|
|
59
|
-
def stream_geometry(
|
|
60
|
-
self,
|
|
61
|
-
) -> None:
|
|
62
|
-
pass
|
|
63
|
-
|
|
64
|
-
def break_linestring_into_multi_linestring(
|
|
65
|
-
self,
|
|
66
|
-
) -> None:
|
|
67
|
-
# For any line-strings across the antimeridian, break into multilinestring
|
|
68
|
-
pass
|
|
69
|
-
|
|
70
|
-
def simplify(
|
|
71
|
-
self,
|
|
72
|
-
) -> None:
|
|
73
|
-
pass
|
|
74
|
-
|
|
75
|
-
def kalman_filter(self):
|
|
76
|
-
# for cruises with bad signal, filter so that
|
|
77
|
-
pass
|
|
78
|
-
|
|
79
|
-
#######################################################
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
###########################################################
|