water-column-sonar-processing 0.0.1__py3-none-any.whl → 26.1.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of water-column-sonar-processing might be problematic. Click here for more details.
- water_column_sonar_processing/__init__.py +13 -0
- water_column_sonar_processing/aws/__init__.py +7 -0
- water_column_sonar_processing/aws/dynamodb_manager.py +355 -0
- water_column_sonar_processing/aws/s3_manager.py +418 -0
- water_column_sonar_processing/aws/s3fs_manager.py +64 -0
- {model → water_column_sonar_processing}/aws/sns_manager.py +10 -21
- {model → water_column_sonar_processing}/aws/sqs_manager.py +11 -19
- water_column_sonar_processing/cruise/__init__.py +4 -0
- water_column_sonar_processing/cruise/create_empty_zarr_store.py +129 -0
- water_column_sonar_processing/cruise/datatree_manager.py +21 -0
- water_column_sonar_processing/cruise/resample_regrid.py +323 -0
- water_column_sonar_processing/geometry/__init__.py +13 -0
- water_column_sonar_processing/geometry/elevation_manager.py +111 -0
- water_column_sonar_processing/geometry/geometry_manager.py +241 -0
- water_column_sonar_processing/geometry/line_simplification.py +176 -0
- water_column_sonar_processing/geometry/pmtile_generation.py +266 -0
- water_column_sonar_processing/geometry/spatiotemporal.py +106 -0
- water_column_sonar_processing/index/__init__.py +3 -0
- water_column_sonar_processing/index/index_manager.py +381 -0
- water_column_sonar_processing/model/__init__.py +3 -0
- water_column_sonar_processing/model/zarr_manager.py +741 -0
- water_column_sonar_processing/processing/__init__.py +4 -0
- water_column_sonar_processing/processing/raw_to_netcdf.py +320 -0
- water_column_sonar_processing/processing/raw_to_zarr.py +331 -0
- water_column_sonar_processing/utility/__init__.py +13 -0
- {model → water_column_sonar_processing}/utility/cleaner.py +7 -7
- water_column_sonar_processing/utility/constants.py +118 -0
- {model → water_column_sonar_processing}/utility/pipeline_status.py +47 -24
- water_column_sonar_processing/utility/timestamp.py +12 -0
- water_column_sonar_processing-26.1.14.dist-info/METADATA +240 -0
- water_column_sonar_processing-26.1.14.dist-info/RECORD +34 -0
- {water_column_sonar_processing-0.0.1.dist-info → water_column_sonar_processing-26.1.14.dist-info}/WHEEL +1 -1
- {water_column_sonar_processing-0.0.1.dist-info → water_column_sonar_processing-26.1.14.dist-info/licenses}/LICENSE +1 -1
- water_column_sonar_processing-26.1.14.dist-info/top_level.txt +1 -0
- __init__.py +0 -0
- model/__init__.py +0 -0
- model/aws/__init__.py +0 -0
- model/aws/dynamodb_manager.py +0 -149
- model/aws/s3_manager.py +0 -356
- model/aws/s3fs_manager.py +0 -74
- model/cruise/__init__.py +0 -0
- model/cruise/create_empty_zarr_store.py +0 -166
- model/cruise/resample_regrid.py +0 -248
- model/geospatial/__init__.py +0 -0
- model/geospatial/geometry_manager.py +0 -194
- model/geospatial/geometry_simplification.py +0 -81
- model/geospatial/pmtile_generation.py +0 -74
- model/index/__init__.py +0 -0
- model/index/index.py +0 -228
- model/model.py +0 -138
- model/utility/__init__.py +0 -0
- model/utility/constants.py +0 -56
- model/utility/timestamp.py +0 -12
- model/zarr/__init__.py +0 -0
- model/zarr/bar.py +0 -28
- model/zarr/foo.py +0 -11
- model/zarr/zarr_manager.py +0 -298
- water_column_sonar_processing-0.0.1.dist-info/METADATA +0 -89
- water_column_sonar_processing-0.0.1.dist-info/RECORD +0 -32
- water_column_sonar_processing-0.0.1.dist-info/top_level.txt +0 -2
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import os
|
|
2
1
|
import glob
|
|
2
|
+
import os
|
|
3
3
|
import shutil
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
###########################################################
|
|
7
7
|
class Cleaner:
|
|
8
8
|
@staticmethod
|
|
9
|
-
def delete_local_files(
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
print('Deleting all local raw and zarr files')
|
|
9
|
+
def delete_local_files(file_types=["*.raw*", "*.model"]): # '*.json'
|
|
10
|
+
# TODO: add .zarr to this
|
|
11
|
+
print("Deleting all local raw and model files")
|
|
13
12
|
for i in file_types:
|
|
14
13
|
for j in glob.glob(i):
|
|
15
14
|
if os.path.isdir(j):
|
|
16
15
|
shutil.rmtree(j, ignore_errors=True)
|
|
17
16
|
elif os.path.isfile(j):
|
|
18
17
|
os.remove(j)
|
|
19
|
-
print(
|
|
18
|
+
print("done deleting")
|
|
20
19
|
|
|
21
|
-
|
|
20
|
+
|
|
21
|
+
###########################################################
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
from enum import Enum, unique
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@unique
|
|
7
|
+
class Instruments(Enum):
|
|
8
|
+
# Values are determined using scan of the fist byte of data
|
|
9
|
+
EK60 = "EK60"
|
|
10
|
+
EK80 = "EK80"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# @unique
|
|
14
|
+
class Constants(Enum):
|
|
15
|
+
"""
|
|
16
|
+
See here for data type support: https://github.com/zarr-developers/zarr-extensions/tree/main/data-types
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
TILE_SIZE = 512
|
|
20
|
+
|
|
21
|
+
# Average https://noaa-wcsd-zarr-pds.s3.us-east-1.amazonaws.com/level_2/Henry_B._Bigelow/HB0902/EK60/HB0902.zarr/time/927
|
|
22
|
+
# chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon dataset
|
|
23
|
+
# NOTE: larger value here will speed up the TurfJS download of dataset in the UI
|
|
24
|
+
# Problem interpolating the dataset: cannot reshape array of size 65536 into shape...
|
|
25
|
+
# TODO: needs to be enum
|
|
26
|
+
SPATIOTEMPORAL_CHUNK_SIZE = int(1e6) # int(2 ** 16) - 1024
|
|
27
|
+
# TODO: create test for SPATIOTEMPORAL_CHUNK_SIZE with requirement!
|
|
28
|
+
|
|
29
|
+
LEVEL_0 = "raw"
|
|
30
|
+
LEVEL_1 = "level_1" # from bucket path
|
|
31
|
+
LEVEL_2 = "level_2a" # updating zarr store path for zarr v3
|
|
32
|
+
LEVEL_3 = "level_3"
|
|
33
|
+
|
|
34
|
+
EK60 = "EK60" # TODO: use for "instrument"
|
|
35
|
+
EK80 = "EK80"
|
|
36
|
+
# INSTRUMENT = EK60 | EK80
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Coordinates(Enum):
|
|
40
|
+
"""
|
|
41
|
+
dtype: data type
|
|
42
|
+
units: netcdf defined units
|
|
43
|
+
long_name: most readable description of variable
|
|
44
|
+
standard_name: name in lowercase and snake_case
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
PROJECT_NAME = "echofish"
|
|
48
|
+
|
|
49
|
+
DEPTH = "depth"
|
|
50
|
+
DEPTH_DTYPE = "float32"
|
|
51
|
+
DEPTH_UNITS = "m" # TODO: Pint? <https://pint.readthedocs.io/en/stable/>
|
|
52
|
+
DEPTH_LONG_NAME = "Depth below surface"
|
|
53
|
+
DEPTH_STANDARD_NAME = "depth"
|
|
54
|
+
|
|
55
|
+
# https://cfconventions.org/Data/cf-conventions/cf-conventions-1.8/cf-conventions.html#table-supported-units
|
|
56
|
+
TIME = "time"
|
|
57
|
+
TIME_DTYPE = "datetime64[ns]"
|
|
58
|
+
# Note: units and calendar are used downstream by Xarray
|
|
59
|
+
TIME_UNITS = "nanoseconds since 1970-01-01"
|
|
60
|
+
TIME_LONG_NAME = "Timestamp of each ping"
|
|
61
|
+
TIME_STANDARD_NAME = "time"
|
|
62
|
+
TIME_CALENDAR = "proleptic_gregorian"
|
|
63
|
+
# TODO: create test for reading out timestamps in Xarray
|
|
64
|
+
|
|
65
|
+
FREQUENCY = "frequency"
|
|
66
|
+
FREQUENCY_DTYPE = np.uint64
|
|
67
|
+
FREQUENCY_UNITS = "Hz"
|
|
68
|
+
FREQUENCY_LONG_NAME = "Transducer frequency"
|
|
69
|
+
FREQUENCY_STANDARD_NAME = "sound_frequency"
|
|
70
|
+
|
|
71
|
+
LATITUDE = "latitude"
|
|
72
|
+
LATITUDE_DTYPE = np.float32
|
|
73
|
+
LATITUDE_UNITS = "degrees_north"
|
|
74
|
+
LATITUDE_LONG_NAME = "Latitude"
|
|
75
|
+
LATITUDE_STANDARD_NAME = "latitude"
|
|
76
|
+
|
|
77
|
+
LONGITUDE = "longitude"
|
|
78
|
+
LONGITUDE_DTYPE = np.float32
|
|
79
|
+
LONGITUDE_UNITS = "degrees_east"
|
|
80
|
+
LONGITUDE_LONG_NAME = "Longitude"
|
|
81
|
+
LONGITUDE_STANDARD_NAME = "longitude"
|
|
82
|
+
|
|
83
|
+
BOTTOM = "bottom"
|
|
84
|
+
BOTTOM_DTYPE = np.float32
|
|
85
|
+
BOTTOM_UNITS = "m"
|
|
86
|
+
BOTTOM_LONG_NAME = "Detected sea floor depth"
|
|
87
|
+
BOTTOM_STANDARD_NAME = "bottom"
|
|
88
|
+
|
|
89
|
+
SPEED = "speed"
|
|
90
|
+
SPEED_DTYPE = np.float32
|
|
91
|
+
SPEED_UNITS = "Knots"
|
|
92
|
+
SPEED_LONG_NAME = "Nautical miles per hour"
|
|
93
|
+
SPEED_STANDARD_NAME = "speed"
|
|
94
|
+
|
|
95
|
+
# This is the width of each 'pixel' of the water columns
|
|
96
|
+
DISTANCE = "distance"
|
|
97
|
+
DISTANCE_DTYPE = np.float32
|
|
98
|
+
DISTANCE_UNITS = "m"
|
|
99
|
+
DISTANCE_LONG_NAME = "GPS distance"
|
|
100
|
+
DISTANCE_STANDARD_NAME = "distance"
|
|
101
|
+
|
|
102
|
+
SV = "Sv"
|
|
103
|
+
SV_DTYPE = np.float32
|
|
104
|
+
SV_UNITS = "dB"
|
|
105
|
+
SV_LONG_NAME = "Volume backscattering strength (Sv re 1 m-1)"
|
|
106
|
+
SV_STANDARD_NAME = "volume_backscattering_strength"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# TODO: delete this
|
|
110
|
+
# class BatchShape(Enum):
|
|
111
|
+
# """
|
|
112
|
+
# The tensor shape of a machine learning sample.
|
|
113
|
+
# """
|
|
114
|
+
#
|
|
115
|
+
# DEPTH = 2
|
|
116
|
+
# TIME = 3
|
|
117
|
+
# FREQUENCY = 4
|
|
118
|
+
# BATCH_SIZE = 5
|
|
@@ -16,13 +16,17 @@ class PipelineStatus(Flag):
|
|
|
16
16
|
PROCESSING_RAW_PROCESSOR = auto()
|
|
17
17
|
FAILURE_RAW_PROCESSOR = auto()
|
|
18
18
|
SUCCESS_RAW_PROCESSOR = auto()
|
|
19
|
-
RAW_PROCESSOR =
|
|
19
|
+
RAW_PROCESSOR = (
|
|
20
|
+
PROCESSING_RAW_PROCESSOR | FAILURE_RAW_PROCESSOR | SUCCESS_RAW_PROCESSOR
|
|
21
|
+
)
|
|
20
22
|
#
|
|
21
23
|
# RAW_AGGREGATOR --> AGGREGATOR (Scatter-Gather EIP)
|
|
22
24
|
PROCESSING_RAW_AGGREGATOR = auto()
|
|
23
25
|
FAILURE_RAW_AGGREGATOR = auto()
|
|
24
26
|
SUCCESS_RAW_AGGREGATOR = auto()
|
|
25
|
-
RAW_AGGREGATOR =
|
|
27
|
+
RAW_AGGREGATOR = (
|
|
28
|
+
PROCESSING_RAW_AGGREGATOR | FAILURE_RAW_AGGREGATOR | SUCCESS_RAW_AGGREGATOR
|
|
29
|
+
)
|
|
26
30
|
#
|
|
27
31
|
LEVEL_1_PROCESSING = RAW_SPLITTER | RAW_PROCESSOR | RAW_AGGREGATOR
|
|
28
32
|
#
|
|
@@ -35,27 +39,43 @@ class PipelineStatus(Flag):
|
|
|
35
39
|
PROCESSING_CRUISE_INITIALIZER = auto()
|
|
36
40
|
FAILURE_CRUISE_INITIALIZER = auto()
|
|
37
41
|
SUCCESS_CRUISE_INITIALIZER = auto()
|
|
38
|
-
CRUISE_INITIALIZER =
|
|
42
|
+
CRUISE_INITIALIZER = (
|
|
43
|
+
PROCESSING_CRUISE_INITIALIZER
|
|
44
|
+
| FAILURE_CRUISE_INITIALIZER
|
|
45
|
+
| SUCCESS_CRUISE_INITIALIZER
|
|
46
|
+
)
|
|
39
47
|
#
|
|
40
48
|
# CRUISE_SPLITTER --> SPLITTER
|
|
41
49
|
PROCESSING_CRUISE_SPLITTER = auto()
|
|
42
50
|
FAILURE_CRUISE_SPLITTER = auto()
|
|
43
51
|
SUCCESS_CRUISE_SPLITTER = auto()
|
|
44
|
-
CRUISE_SPLITTER =
|
|
52
|
+
CRUISE_SPLITTER = (
|
|
53
|
+
PROCESSING_CRUISE_SPLITTER | FAILURE_CRUISE_SPLITTER | SUCCESS_CRUISE_SPLITTER
|
|
54
|
+
)
|
|
45
55
|
#
|
|
46
56
|
# CRUISE_PROCESSOR --> PROCESSOR <-- Note: these need to run sequentially now
|
|
47
57
|
PROCESSING_CRUISE_PROCESSOR = auto()
|
|
48
58
|
FAILURE_CRUISE_PROCESSOR = auto()
|
|
49
59
|
SUCCESS_CRUISE_PROCESSOR = auto()
|
|
50
|
-
CRUISE_PROCESSOR =
|
|
60
|
+
CRUISE_PROCESSOR = (
|
|
61
|
+
PROCESSING_CRUISE_PROCESSOR
|
|
62
|
+
| FAILURE_CRUISE_PROCESSOR
|
|
63
|
+
| SUCCESS_CRUISE_PROCESSOR
|
|
64
|
+
)
|
|
51
65
|
#
|
|
52
66
|
# CRUISE_AGGREGATOR --> AGGREGATOR
|
|
53
67
|
PROCESSING_CRUISE_AGGREGATOR = auto()
|
|
54
68
|
FAILURE_CRUISE_AGGREGATOR = auto()
|
|
55
69
|
SUCCESS_CRUISE_AGGREGATOR = auto()
|
|
56
|
-
CRUISE_AGGREGATOR =
|
|
70
|
+
CRUISE_AGGREGATOR = (
|
|
71
|
+
PROCESSING_CRUISE_AGGREGATOR
|
|
72
|
+
| FAILURE_CRUISE_AGGREGATOR
|
|
73
|
+
| SUCCESS_CRUISE_AGGREGATOR
|
|
74
|
+
)
|
|
57
75
|
#
|
|
58
|
-
LEVEL_2_PROCESSING =
|
|
76
|
+
LEVEL_2_PROCESSING = (
|
|
77
|
+
CRUISE_INITIALIZER | CRUISE_SPLITTER | CRUISE_PROCESSOR | CRUISE_AGGREGATOR
|
|
78
|
+
)
|
|
59
79
|
#
|
|
60
80
|
# --------------------------------------------------- #
|
|
61
81
|
# --- Level 3 Data ---------------------------------- #
|
|
@@ -64,35 +84,38 @@ class PipelineStatus(Flag):
|
|
|
64
84
|
PROCESSING_TILE_PROCESSOR = auto()
|
|
65
85
|
FAILURE_TILE_PROCESSOR = auto()
|
|
66
86
|
SUCCESS_TILE_PROCESSOR = auto()
|
|
67
|
-
TILE_PROCESSOR =
|
|
87
|
+
TILE_PROCESSOR = (
|
|
88
|
+
PROCESSING_TILE_PROCESSOR | FAILURE_TILE_PROCESSOR | SUCCESS_TILE_PROCESSOR
|
|
89
|
+
)
|
|
68
90
|
#
|
|
69
91
|
# GEOHASH_PROCESSOR
|
|
70
92
|
PROCESSING_GEOHASH_PROCESSOR = auto()
|
|
71
93
|
FAILURE_GEOHASH_PROCESSOR = auto()
|
|
72
94
|
SUCCESS_GEOHASH_PROCESSOR = auto()
|
|
73
|
-
GEOHASH_PROCESSOR =
|
|
95
|
+
GEOHASH_PROCESSOR = (
|
|
96
|
+
PROCESSING_GEOHASH_PROCESSOR
|
|
97
|
+
| FAILURE_GEOHASH_PROCESSOR
|
|
98
|
+
| SUCCESS_GEOHASH_PROCESSOR
|
|
99
|
+
)
|
|
74
100
|
#
|
|
75
101
|
LEVEL_3_PROCESSING = TILE_PROCESSOR | GEOHASH_PROCESSOR
|
|
76
102
|
# --------------------------------------------------- #
|
|
77
103
|
# --------------------------------------------------- #
|
|
78
104
|
|
|
105
|
+
|
|
79
106
|
# Status.PROCESSING_RAW_AGGREGATOR in Status.LEVEL_1_PROCESSING
|
|
80
107
|
# Status.LEVEL_1_PROCESSING.value < Status.LEVEL_2_PROCESSING.value
|
|
81
108
|
|
|
82
109
|
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cloudformation-stack.html
|
|
83
|
-
"""
|
|
84
|
-
CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
|
|
85
|
-
ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
|
|
86
|
-
DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
|
|
87
|
-
UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
|
|
88
|
-
UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
|
|
89
|
-
UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
|
|
90
|
-
REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
|
|
91
|
-
IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
|
|
92
|
-
|
|
93
|
-
failure - noun -
|
|
94
|
-
failed - verb - "verbs should be avoided"
|
|
95
|
-
|
|
96
|
-
success - noun
|
|
97
110
|
|
|
98
|
-
|
|
111
|
+
# CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
|
|
112
|
+
# ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
|
|
113
|
+
# DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
|
|
114
|
+
# UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
|
|
115
|
+
# UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
|
|
116
|
+
# UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
|
|
117
|
+
# REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
|
|
118
|
+
# IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
|
|
119
|
+
# failure - noun -
|
|
120
|
+
# failed - verb - "verbs should be avoided"
|
|
121
|
+
# success - noun
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
###########################################################
|
|
5
|
+
class Timestamp:
|
|
6
|
+
@staticmethod
|
|
7
|
+
def get_timestamp():
|
|
8
|
+
# return timestamp in form: '2024-03-29T19:36:52.433Z'
|
|
9
|
+
return f"{datetime.datetime.now(datetime.UTC).isoformat()[:23]}Z"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
###########################################################
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: water-column-sonar-processing
|
|
3
|
+
Version: 26.1.14
|
|
4
|
+
Summary: Processing tool for water column sonar data.
|
|
5
|
+
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
|
+
Maintainer-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://echo.fish
|
|
9
|
+
Project-URL: Repository, https://github.com/CI-CMG/water-column-sonar-processing
|
|
10
|
+
Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
|
|
11
|
+
Keywords: ocean,sonar,water column,zarr
|
|
12
|
+
Requires-Python: >=3.12
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: boto3==1.41.5
|
|
16
|
+
Requires-Dist: boto3-stubs
|
|
17
|
+
Requires-Dist: botocore==1.41.5
|
|
18
|
+
Requires-Dist: build
|
|
19
|
+
Requires-Dist: echopype==0.11.0
|
|
20
|
+
Requires-Dist: fiona==1.10.1
|
|
21
|
+
Requires-Dist: geopandas==1.1.2
|
|
22
|
+
Requires-Dist: importlib==1.0.4
|
|
23
|
+
Requires-Dist: mock
|
|
24
|
+
Requires-Dist: moto[all]==5.1.19
|
|
25
|
+
Requires-Dist: moto[server]==5.1.19
|
|
26
|
+
Requires-Dist: multidict==6.7.0
|
|
27
|
+
Requires-Dist: netcdf4==1.7.3
|
|
28
|
+
Requires-Dist: numpy==2.4.1
|
|
29
|
+
Requires-Dist: pandas==2.3.3
|
|
30
|
+
Requires-Dist: pandas-stubs
|
|
31
|
+
Requires-Dist: pooch==1.8.2
|
|
32
|
+
Requires-Dist: pyarrow==22.0.0
|
|
33
|
+
Requires-Dist: pykalman==0.11.1
|
|
34
|
+
Requires-Dist: python-dotenv
|
|
35
|
+
Requires-Dist: requests==2.32.5
|
|
36
|
+
Requires-Dist: ruff>=0.14.13
|
|
37
|
+
Requires-Dist: s3fs==2026.1.0
|
|
38
|
+
Requires-Dist: scipy==1.17.0
|
|
39
|
+
Requires-Dist: setuptools==80.9.0
|
|
40
|
+
Requires-Dist: shapely==2.1.2
|
|
41
|
+
Requires-Dist: xarray==2025.12.0
|
|
42
|
+
Requires-Dist: zarr==3.1.5
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# Water Column Sonar Processing
|
|
46
|
+
|
|
47
|
+
Processing tool for converting Level_0 water column sonar data to Level_1 and Level_2 derived data sets as well as
|
|
48
|
+
generating geospatial information.
|
|
49
|
+
|
|
50
|
+

|
|
51
|
+
  
|
|
52
|
+
|
|
53
|
+
# Setting up the Python Environment
|
|
54
|
+
|
|
55
|
+
> uv python install --reinstall
|
|
56
|
+
> uv venv
|
|
57
|
+
> Python 3.10.12 # 3.13.1
|
|
58
|
+
|
|
59
|
+
# Installing Dependencies
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
source .venv/bin/activate
|
|
63
|
+
# or ".venv\Scripts\activate" in windows
|
|
64
|
+
|
|
65
|
+
uv sync --all-groups
|
|
66
|
+
|
|
67
|
+
uv pip install --upgrade pip
|
|
68
|
+
|
|
69
|
+
uv run pre-commit install
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
# Pytest
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
uv run pytest --cache-clear tests # -W ignore::DeprecationWarning
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
or
|
|
79
|
+
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
80
|
+
|
|
81
|
+
# Test Coverage
|
|
82
|
+
|
|
83
|
+
```commandline
|
|
84
|
+
uv run pytest --cov=water_column_sonar_processing
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
With line numbers
|
|
88
|
+
|
|
89
|
+
```commandline
|
|
90
|
+
uv run pytest tests/geometry --cov=water_column_sonar_processing --cov-report term-missing
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Current status:
|
|
94
|
+
|
|
95
|
+
```commandline
|
|
96
|
+
======================================================================================= tests coverage =======================================================================================
|
|
97
|
+
_____________________________________________________________________ coverage: platform darwin, python 3.12.12-final-0 ______________________________________________________________________
|
|
98
|
+
|
|
99
|
+
Name Stmts Miss Cover
|
|
100
|
+
-------------------------------------------------------------------------------------
|
|
101
|
+
water_column_sonar_processing/__init__.py 3 0 100%
|
|
102
|
+
water_column_sonar_processing/aws/__init__.py 6 0 100%
|
|
103
|
+
water_column_sonar_processing/aws/dynamodb_manager.py 44 4 91%
|
|
104
|
+
water_column_sonar_processing/aws/s3_manager.py 161 27 83%
|
|
105
|
+
water_column_sonar_processing/aws/s3fs_manager.py 18 0 100%
|
|
106
|
+
water_column_sonar_processing/aws/sns_manager.py 18 1 94%
|
|
107
|
+
water_column_sonar_processing/aws/sqs_manager.py 17 2 88%
|
|
108
|
+
water_column_sonar_processing/cruise/__init__.py 3 0 100%
|
|
109
|
+
water_column_sonar_processing/cruise/create_empty_zarr_store.py 38 2 95%
|
|
110
|
+
water_column_sonar_processing/cruise/datatree_manager.py 0 0 100%
|
|
111
|
+
water_column_sonar_processing/cruise/resample_regrid.py 87 6 93%
|
|
112
|
+
water_column_sonar_processing/geometry/__init__.py 6 0 100%
|
|
113
|
+
water_column_sonar_processing/geometry/elevation_manager.py 29 1 97%
|
|
114
|
+
water_column_sonar_processing/geometry/geometry_manager.py 72 33 54%
|
|
115
|
+
water_column_sonar_processing/geometry/line_simplification.py 38 4 89%
|
|
116
|
+
water_column_sonar_processing/geometry/pmtile_generation.py 80 58 28%
|
|
117
|
+
water_column_sonar_processing/geometry/spatiotemporal.py 42 2 95%
|
|
118
|
+
water_column_sonar_processing/index/__init__.py 2 0 100%
|
|
119
|
+
water_column_sonar_processing/index/index_manager.py 118 91 23%
|
|
120
|
+
water_column_sonar_processing/model/__init__.py 2 0 100%
|
|
121
|
+
water_column_sonar_processing/model/zarr_manager.py 103 8 92%
|
|
122
|
+
water_column_sonar_processing/processing/__init__.py 3 0 100%
|
|
123
|
+
water_column_sonar_processing/processing/raw_to_netcdf.py 85 24 72%
|
|
124
|
+
water_column_sonar_processing/processing/raw_to_zarr.py 88 5 94%
|
|
125
|
+
water_column_sonar_processing/utility/__init__.py 5 0 100%
|
|
126
|
+
water_column_sonar_processing/utility/cleaner.py 14 0 100%
|
|
127
|
+
water_column_sonar_processing/utility/constants.py 62 0 100%
|
|
128
|
+
water_column_sonar_processing/utility/pipeline_status.py 42 0 100%
|
|
129
|
+
water_column_sonar_processing/utility/timestamp.py 5 0 100%
|
|
130
|
+
-------------------------------------------------------------------------------------
|
|
131
|
+
TOTAL 1191 268 77%
|
|
132
|
+
=================================================================== 47 passed, 4 skipped, 21 warnings in 337.77s (0:05:37)
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
# Instructions
|
|
136
|
+
|
|
137
|
+
Following this tutorial:
|
|
138
|
+
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
139
|
+
|
|
140
|
+
# Pre Commit Hook
|
|
141
|
+
|
|
142
|
+
see here for installation: https://pre-commit.com/
|
|
143
|
+
https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
|
|
144
|
+
|
|
145
|
+
```
|
|
146
|
+
uv run pre-commit install --allow-missing-config
|
|
147
|
+
# or
|
|
148
|
+
uv run pre-commit install
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
# Black
|
|
152
|
+
|
|
153
|
+
https://ljvmiranda921.github.io/notebook/2018/06/21/precommits-using-black-and-flake8/
|
|
154
|
+
|
|
155
|
+
```
|
|
156
|
+
Settings > Black
|
|
157
|
+
Execution mode: Package
|
|
158
|
+
Python Interpreter: .../.venv/bin/python
|
|
159
|
+
Use Black Formatter: X On Code reformat, X On Save
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
# Linting
|
|
163
|
+
|
|
164
|
+
Ruff
|
|
165
|
+
https://plugins.jetbrains.com/plugin/20574-ruff
|
|
166
|
+
|
|
167
|
+
# Colab Test
|
|
168
|
+
|
|
169
|
+
https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
|
|
170
|
+
|
|
171
|
+
# Tag a Release
|
|
172
|
+
|
|
173
|
+
Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
|
|
174
|
+
|
|
175
|
+
```commandline
|
|
176
|
+
git tag -a v26.1.13 -m "Releasing v26.1.13"
|
|
177
|
+
git push origin --tags
|
|
178
|
+
#gh release create v26.1.12
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
# To Publish To PROD
|
|
182
|
+
|
|
183
|
+
```
|
|
184
|
+
uv build --no-sources
|
|
185
|
+
uv publish
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
# TODO:
|
|
189
|
+
|
|
190
|
+
add https://pypi.org/project/setuptools-scm/
|
|
191
|
+
for extracting the version
|
|
192
|
+
|
|
193
|
+
# Security scanning
|
|
194
|
+
|
|
195
|
+
> bandit -r water_column_sonar_processing/
|
|
196
|
+
|
|
197
|
+
# Data Debugging
|
|
198
|
+
|
|
199
|
+
Experimental Plotting in Xarray (hvPlot):
|
|
200
|
+
https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
|
|
201
|
+
|
|
202
|
+
HB0707 Zoomable Cruise:
|
|
203
|
+
https://hb0707.s3.us-east-1.amazonaws.com/index.html
|
|
204
|
+
|
|
205
|
+
# UV Debugging
|
|
206
|
+
|
|
207
|
+
```
|
|
208
|
+
uv pip install --upgrade pip
|
|
209
|
+
#uv sync --all-groups
|
|
210
|
+
uv run pre-commit install
|
|
211
|
+
uv lock --check
|
|
212
|
+
uv lock
|
|
213
|
+
uv sync --all-groups
|
|
214
|
+
uv run pytest --cache-clear tests
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
# Fixing S3FS Problems
|
|
218
|
+
|
|
219
|
+
```commandline
|
|
220
|
+
To enable/disa asyncio for the debugger, follow the steps:
|
|
221
|
+
Open PyCharm
|
|
222
|
+
Use Shift + Shift (Search Everywhere)
|
|
223
|
+
In the popup type: Registry and press Enter
|
|
224
|
+
Find "Registry" in the list of results and click on it.
|
|
225
|
+
In the new popup find python.debug.asyncio.repl line and check the respective checkbox
|
|
226
|
+
Press Close.
|
|
227
|
+
Restart the IDE.
|
|
228
|
+
The asyncio support will be enabled in the debugger.
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Another useful trick is to turn off "gevent" to speed up debugging:
|
|
232
|
+
|
|
233
|
+
```commandline
|
|
234
|
+
Python > Debugger > "Gevent Compatible"
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
# Fixing windows/wsl/ubuntu/mac git compatability
|
|
238
|
+
|
|
239
|
+
> git config --global core.filemode false
|
|
240
|
+
> git config --global core.autocrlf true
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
|
|
2
|
+
water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
|
|
3
|
+
water_column_sonar_processing/aws/dynamodb_manager.py,sha256=EwSc4UTVQTR6tXsrx2eNdbhvexxfrz3GkkvO97F_9fM,14017
|
|
4
|
+
water_column_sonar_processing/aws/s3_manager.py,sha256=SI1X1ju7ueYJbtEn6DVjUuXNN6so1EKy9xQwCW3rqp4,16183
|
|
5
|
+
water_column_sonar_processing/aws/s3fs_manager.py,sha256=NJoky3uvMgt7m04LBZhrRkR-w5y18MIPP22EOWQos9E,2275
|
|
6
|
+
water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
|
|
7
|
+
water_column_sonar_processing/aws/sqs_manager.py,sha256=j_DGOKble3i-DlHT_uGxCFEmHVbYhFrpbhXdJKLtBSo,1600
|
|
8
|
+
water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
|
|
9
|
+
water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=4sZ_6ThlFs7pfnbHsbdpnlkpnGr9lBDM4-S4hRtYi0Q,5125
|
|
10
|
+
water_column_sonar_processing/cruise/datatree_manager.py,sha256=ed_lY04MyaXKADl7EaqHyEZ8JQMiyIAXVhvtE7YnWcg,662
|
|
11
|
+
water_column_sonar_processing/cruise/resample_regrid.py,sha256=R1l5XjksaH5xqD3x3jdD-sJpe7xuXu4YKh3uCzfCumM,14541
|
|
12
|
+
water_column_sonar_processing/geometry/__init__.py,sha256=VfUXTH_OnjQy6EWCw1Hn2ua7OYGI-6EITeH37b4Wjxc,371
|
|
13
|
+
water_column_sonar_processing/geometry/elevation_manager.py,sha256=2TBiDB3qNkM5ZRUEWhQCjrqRYagur0z8hT102Sts0p8,4251
|
|
14
|
+
water_column_sonar_processing/geometry/geometry_manager.py,sha256=cQhLGMwHZskvuKU83gdLunQbM62s57k8qEKkuDWRlPE,10929
|
|
15
|
+
water_column_sonar_processing/geometry/line_simplification.py,sha256=tsUPu3XeMqBHZ8zVbtXI-l_leosj2ENd7JWJSzz_2ws,9288
|
|
16
|
+
water_column_sonar_processing/geometry/pmtile_generation.py,sha256=6dKLbpyZEEj8KfYmqjzUe8rrtNKAFhUK94BhnepFbSg,9577
|
|
17
|
+
water_column_sonar_processing/geometry/spatiotemporal.py,sha256=KnNqQiuon15EDafUMxO80Grnu_yTxq1aYEKE1u1pXdU,4095
|
|
18
|
+
water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
|
|
19
|
+
water_column_sonar_processing/index/index_manager.py,sha256=Dino5EO58Aq2tDhoz8sya3YFXUpqOQMsSoKEnJZMgdI,16170
|
|
20
|
+
water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
|
|
21
|
+
water_column_sonar_processing/model/zarr_manager.py,sha256=Ghxsd4lI01khbOP0Pl1T5IJNZUxgWdxXHQiYzTZck50,32420
|
|
22
|
+
water_column_sonar_processing/processing/__init__.py,sha256=TDnA4_Nyb8MnU8HaixaCzEdTbFkXsUXA0ekvxtYm1-U,150
|
|
23
|
+
water_column_sonar_processing/processing/raw_to_netcdf.py,sha256=3UdglrWXFLxa_jHkhvC9P9NUkp-iKmp6MvnXFnXi4ns,13098
|
|
24
|
+
water_column_sonar_processing/processing/raw_to_zarr.py,sha256=XhNCQe3YXUhlhQAzGTcktpJVX7t6Gf_yi29Vpsp4L4k,13558
|
|
25
|
+
water_column_sonar_processing/utility/__init__.py,sha256=5gV8sJcnQGrV2AecwkBFZljOC-fQwLuDX70DxX59x2o,289
|
|
26
|
+
water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
|
|
27
|
+
water_column_sonar_processing/utility/constants.py,sha256=atCf0MAJJFt2QdS-8mN0NLgmVOMTmyaryWKXPmDTimY,3530
|
|
28
|
+
water_column_sonar_processing/utility/pipeline_status.py,sha256=xbl6-4ePq1Krfo18Mgr0jzWz9C_g2Kbey_QZNJuwdkI,4406
|
|
29
|
+
water_column_sonar_processing/utility/timestamp.py,sha256=rn8SDbGYjRvanDUOrvKpNjp3-AK6-KlU2NaCk6Ok8rc,337
|
|
30
|
+
water_column_sonar_processing-26.1.14.dist-info/licenses/LICENSE,sha256=TosqaZpJgYvhgXIyYBti-ggJaO8rxRg3FtThY08s9Aw,1110
|
|
31
|
+
water_column_sonar_processing-26.1.14.dist-info/METADATA,sha256=3Huf0Mv53tOFWZitSynNCvBG03zmVIsv3hyLcMx7RI4,8432
|
|
32
|
+
water_column_sonar_processing-26.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
33
|
+
water_column_sonar_processing-26.1.14.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
|
|
34
|
+
water_column_sonar_processing-26.1.14.dist-info/RECORD,,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
MIT License
|
|
2
2
|
|
|
3
|
-
Copyright (c)
|
|
3
|
+
Copyright (c) 2025 Cooperative Institutes, Coastal and Marine Geophysics
|
|
4
4
|
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
water_column_sonar_processing
|
__init__.py
DELETED
|
File without changes
|
model/__init__.py
DELETED
|
File without changes
|
model/aws/__init__.py
DELETED
|
File without changes
|