water-column-sonar-processing 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- water_column_sonar_processing/__init__.py +16 -0
- water_column_sonar_processing/aws/__init__.py +7 -0
- {aws_manager → water_column_sonar_processing/aws}/dynamodb_manager.py +71 -50
- {aws_manager → water_column_sonar_processing/aws}/s3_manager.py +120 -130
- {aws_manager → water_column_sonar_processing/aws}/s3fs_manager.py +13 -19
- {aws_manager → water_column_sonar_processing/aws}/sns_manager.py +10 -21
- {aws_manager → water_column_sonar_processing/aws}/sqs_manager.py +10 -18
- water_column_sonar_processing/cruise/__init__.py +4 -0
- {cruise → water_column_sonar_processing/cruise}/create_empty_zarr_store.py +62 -44
- {cruise → water_column_sonar_processing/cruise}/resample_regrid.py +117 -66
- water_column_sonar_processing/geometry/__init__.py +5 -0
- {geometry_manager → water_column_sonar_processing/geometry}/geometry_manager.py +80 -49
- {geometry_manager → water_column_sonar_processing/geometry}/geometry_simplification.py +13 -12
- {geometry_manager → water_column_sonar_processing/geometry}/pmtile_generation.py +25 -24
- water_column_sonar_processing/index/__init__.py +3 -0
- {index_manager → water_column_sonar_processing/index}/index_manager.py +106 -82
- water_column_sonar_processing/model/__init__.py +3 -0
- {zarr_manager → water_column_sonar_processing/model}/zarr_manager.py +119 -83
- water_column_sonar_processing/process.py +147 -0
- water_column_sonar_processing/utility/__init__.py +6 -0
- {utility → water_column_sonar_processing/utility}/cleaner.py +6 -7
- water_column_sonar_processing/utility/constants.py +63 -0
- {utility → water_column_sonar_processing/utility}/pipeline_status.py +37 -10
- {utility → water_column_sonar_processing/utility}/timestamp.py +3 -2
- {water_column_sonar_processing-0.0.4.dist-info → water_column_sonar_processing-0.0.6.dist-info}/METADATA +31 -1
- water_column_sonar_processing-0.0.6.dist-info/RECORD +29 -0
- water_column_sonar_processing-0.0.6.dist-info/top_level.txt +1 -0
- __init__.py +0 -0
- aws_manager/__init__.py +0 -4
- cruise/__init__.py +0 -0
- geometry_manager/__init__.py +0 -0
- index_manager/__init__.py +0 -0
- model.py +0 -140
- utility/__init__.py +0 -0
- utility/constants.py +0 -56
- water_column_sonar_processing-0.0.4.dist-info/RECORD +0 -29
- water_column_sonar_processing-0.0.4.dist-info/top_level.txt +0 -8
- zarr_manager/__init__.py +0 -0
- {water_column_sonar_processing-0.0.4.dist-info → water_column_sonar_processing-0.0.6.dist-info}/LICENSE +0 -0
- {water_column_sonar_processing-0.0.4.dist-info → water_column_sonar_processing-0.0.6.dist-info}/WHEEL +0 -0
|
@@ -16,13 +16,17 @@ class PipelineStatus(Flag):
|
|
|
16
16
|
PROCESSING_RAW_PROCESSOR = auto()
|
|
17
17
|
FAILURE_RAW_PROCESSOR = auto()
|
|
18
18
|
SUCCESS_RAW_PROCESSOR = auto()
|
|
19
|
-
RAW_PROCESSOR =
|
|
19
|
+
RAW_PROCESSOR = (
|
|
20
|
+
PROCESSING_RAW_PROCESSOR | FAILURE_RAW_PROCESSOR | SUCCESS_RAW_PROCESSOR
|
|
21
|
+
)
|
|
20
22
|
#
|
|
21
23
|
# RAW_AGGREGATOR --> AGGREGATOR (Scatter-Gather EIP)
|
|
22
24
|
PROCESSING_RAW_AGGREGATOR = auto()
|
|
23
25
|
FAILURE_RAW_AGGREGATOR = auto()
|
|
24
26
|
SUCCESS_RAW_AGGREGATOR = auto()
|
|
25
|
-
RAW_AGGREGATOR =
|
|
27
|
+
RAW_AGGREGATOR = (
|
|
28
|
+
PROCESSING_RAW_AGGREGATOR | FAILURE_RAW_AGGREGATOR | SUCCESS_RAW_AGGREGATOR
|
|
29
|
+
)
|
|
26
30
|
#
|
|
27
31
|
LEVEL_1_PROCESSING = RAW_SPLITTER | RAW_PROCESSOR | RAW_AGGREGATOR
|
|
28
32
|
#
|
|
@@ -35,27 +39,43 @@ class PipelineStatus(Flag):
|
|
|
35
39
|
PROCESSING_CRUISE_INITIALIZER = auto()
|
|
36
40
|
FAILURE_CRUISE_INITIALIZER = auto()
|
|
37
41
|
SUCCESS_CRUISE_INITIALIZER = auto()
|
|
38
|
-
CRUISE_INITIALIZER =
|
|
42
|
+
CRUISE_INITIALIZER = (
|
|
43
|
+
PROCESSING_CRUISE_INITIALIZER
|
|
44
|
+
| FAILURE_CRUISE_INITIALIZER
|
|
45
|
+
| SUCCESS_CRUISE_INITIALIZER
|
|
46
|
+
)
|
|
39
47
|
#
|
|
40
48
|
# CRUISE_SPLITTER --> SPLITTER
|
|
41
49
|
PROCESSING_CRUISE_SPLITTER = auto()
|
|
42
50
|
FAILURE_CRUISE_SPLITTER = auto()
|
|
43
51
|
SUCCESS_CRUISE_SPLITTER = auto()
|
|
44
|
-
CRUISE_SPLITTER =
|
|
52
|
+
CRUISE_SPLITTER = (
|
|
53
|
+
PROCESSING_CRUISE_SPLITTER | FAILURE_CRUISE_SPLITTER | SUCCESS_CRUISE_SPLITTER
|
|
54
|
+
)
|
|
45
55
|
#
|
|
46
56
|
# CRUISE_PROCESSOR --> PROCESSOR <-- Note: these need to run sequentially now
|
|
47
57
|
PROCESSING_CRUISE_PROCESSOR = auto()
|
|
48
58
|
FAILURE_CRUISE_PROCESSOR = auto()
|
|
49
59
|
SUCCESS_CRUISE_PROCESSOR = auto()
|
|
50
|
-
CRUISE_PROCESSOR =
|
|
60
|
+
CRUISE_PROCESSOR = (
|
|
61
|
+
PROCESSING_CRUISE_PROCESSOR
|
|
62
|
+
| FAILURE_CRUISE_PROCESSOR
|
|
63
|
+
| SUCCESS_CRUISE_PROCESSOR
|
|
64
|
+
)
|
|
51
65
|
#
|
|
52
66
|
# CRUISE_AGGREGATOR --> AGGREGATOR
|
|
53
67
|
PROCESSING_CRUISE_AGGREGATOR = auto()
|
|
54
68
|
FAILURE_CRUISE_AGGREGATOR = auto()
|
|
55
69
|
SUCCESS_CRUISE_AGGREGATOR = auto()
|
|
56
|
-
CRUISE_AGGREGATOR =
|
|
70
|
+
CRUISE_AGGREGATOR = (
|
|
71
|
+
PROCESSING_CRUISE_AGGREGATOR
|
|
72
|
+
| FAILURE_CRUISE_AGGREGATOR
|
|
73
|
+
| SUCCESS_CRUISE_AGGREGATOR
|
|
74
|
+
)
|
|
57
75
|
#
|
|
58
|
-
LEVEL_2_PROCESSING =
|
|
76
|
+
LEVEL_2_PROCESSING = (
|
|
77
|
+
CRUISE_INITIALIZER | CRUISE_SPLITTER | CRUISE_PROCESSOR | CRUISE_AGGREGATOR
|
|
78
|
+
)
|
|
59
79
|
#
|
|
60
80
|
# --------------------------------------------------- #
|
|
61
81
|
# --- Level 3 Data ---------------------------------- #
|
|
@@ -64,18 +84,25 @@ class PipelineStatus(Flag):
|
|
|
64
84
|
PROCESSING_TILE_PROCESSOR = auto()
|
|
65
85
|
FAILURE_TILE_PROCESSOR = auto()
|
|
66
86
|
SUCCESS_TILE_PROCESSOR = auto()
|
|
67
|
-
TILE_PROCESSOR =
|
|
87
|
+
TILE_PROCESSOR = (
|
|
88
|
+
PROCESSING_TILE_PROCESSOR | FAILURE_TILE_PROCESSOR | SUCCESS_TILE_PROCESSOR
|
|
89
|
+
)
|
|
68
90
|
#
|
|
69
91
|
# GEOHASH_PROCESSOR
|
|
70
92
|
PROCESSING_GEOHASH_PROCESSOR = auto()
|
|
71
93
|
FAILURE_GEOHASH_PROCESSOR = auto()
|
|
72
94
|
SUCCESS_GEOHASH_PROCESSOR = auto()
|
|
73
|
-
GEOHASH_PROCESSOR =
|
|
95
|
+
GEOHASH_PROCESSOR = (
|
|
96
|
+
PROCESSING_GEOHASH_PROCESSOR
|
|
97
|
+
| FAILURE_GEOHASH_PROCESSOR
|
|
98
|
+
| SUCCESS_GEOHASH_PROCESSOR
|
|
99
|
+
)
|
|
74
100
|
#
|
|
75
101
|
LEVEL_3_PROCESSING = TILE_PROCESSOR | GEOHASH_PROCESSOR
|
|
76
102
|
# --------------------------------------------------- #
|
|
77
103
|
# --------------------------------------------------- #
|
|
78
104
|
|
|
105
|
+
|
|
79
106
|
# Status.PROCESSING_RAW_AGGREGATOR in Status.LEVEL_1_PROCESSING
|
|
80
107
|
# Status.LEVEL_1_PROCESSING.value < Status.LEVEL_2_PROCESSING.value
|
|
81
108
|
|
|
@@ -95,4 +122,4 @@ failed - verb - "verbs should be avoided"
|
|
|
95
122
|
|
|
96
123
|
success - noun
|
|
97
124
|
|
|
98
|
-
"""
|
|
125
|
+
"""
|
|
@@ -7,6 +7,7 @@ class Timestamp:
|
|
|
7
7
|
def get_timestamp():
|
|
8
8
|
# return timestamp in form:
|
|
9
9
|
# PIPELINE_TIME = '2024-03-29T19:36:52.433Z'
|
|
10
|
-
return f
|
|
10
|
+
return f"{datetime.utcnow().isoformat()[:23]}Z"
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
|
|
13
|
+
###########################################################
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: water_column_sonar_processing
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.6
|
|
4
4
|
Summary: A processing tool for water column sonar data.
|
|
5
5
|
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
|
|
@@ -11,6 +11,26 @@ Classifier: Operating System :: OS Independent
|
|
|
11
11
|
Requires-Python: >=3.10
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
|
+
Requires-Dist: aiobotocore~=2.9.0
|
|
15
|
+
Requires-Dist: aiohttp~=3.10.10
|
|
16
|
+
Requires-Dist: boto3==1.33.13
|
|
17
|
+
Requires-Dist: botocore~=1.33.13
|
|
18
|
+
Requires-Dist: echopype==0.9.0
|
|
19
|
+
Requires-Dist: geopandas==1.0.1
|
|
20
|
+
Requires-Dist: mock~=5.1.0
|
|
21
|
+
Requires-Dist: moto~=5.0.18
|
|
22
|
+
Requires-Dist: numcodecs==0.13.1
|
|
23
|
+
Requires-Dist: numpy==1.26.4
|
|
24
|
+
Requires-Dist: pandas==2.2.3
|
|
25
|
+
Requires-Dist: pytest~=8.3.3
|
|
26
|
+
Requires-Dist: python-dotenv==1.0.0
|
|
27
|
+
Requires-Dist: requests==2.32.3
|
|
28
|
+
Requires-Dist: s3fs==2024.10.0
|
|
29
|
+
Requires-Dist: scipy==1.14.1
|
|
30
|
+
Requires-Dist: shapely==2.0.3
|
|
31
|
+
Requires-Dist: typing-extensions==4.10.0
|
|
32
|
+
Requires-Dist: xarray==2022.12.0
|
|
33
|
+
Requires-Dist: zarr==2.18.3
|
|
14
34
|
|
|
15
35
|
# Water Column Sonar Processing
|
|
16
36
|
Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
|
|
@@ -72,6 +92,8 @@ https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
|
72
92
|
# To Publish To TEST
|
|
73
93
|
```commandline
|
|
74
94
|
python -m build
|
|
95
|
+
# python -m build --sdist
|
|
96
|
+
# python -m build --wheel
|
|
75
97
|
python -m twine upload --repository testpypi dist/*
|
|
76
98
|
pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
|
|
77
99
|
python
|
|
@@ -87,7 +109,15 @@ python -m build
|
|
|
87
109
|
python -m twine upload --repository pypi dist/*
|
|
88
110
|
```
|
|
89
111
|
|
|
112
|
+
# Pre Commit Hook
|
|
113
|
+
https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
|
|
114
|
+
```
|
|
115
|
+
pre-commit install --allow-missing-config
|
|
116
|
+
```
|
|
90
117
|
|
|
91
118
|
# Linting
|
|
92
119
|
Ruff
|
|
93
120
|
https://plugins.jetbrains.com/plugin/20574-ruff
|
|
121
|
+
|
|
122
|
+
# Colab Test
|
|
123
|
+
https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
water_column_sonar_processing/__init__.py,sha256=S11loSDNOmZRyiTMbTwiQ1PP59SSPq2q2P61e5gPrW4,295
|
|
2
|
+
water_column_sonar_processing/process.py,sha256=AHoCehc95XqGI1IZkvfcbnYpjr8J-OQfCBGs0-1O9_w,5439
|
|
3
|
+
water_column_sonar_processing/aws/__init__.py,sha256=u5J-TOEVgAQsMdc5LMo1igUESRclzV8gf-b0jUaJ9Gg,277
|
|
4
|
+
water_column_sonar_processing/aws/dynamodb_manager.py,sha256=sPa_5RlTNjcFavZ74LF0UINKSyrbgJNROXbe16hCz7U,5936
|
|
5
|
+
water_column_sonar_processing/aws/s3_manager.py,sha256=P-0ZFsa6D6obIy-iNnWF1BrcoHOfAvYM-Qn9cA0e5H0,13142
|
|
6
|
+
water_column_sonar_processing/aws/s3fs_manager.py,sha256=thVJPQKhbvF1g-Ue3BYgwazFOFDYOICIEJx4zkXBQ1E,2381
|
|
7
|
+
water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
|
|
8
|
+
water_column_sonar_processing/aws/sqs_manager.py,sha256=NSUrWmnSC8h8Gf7gT0U8zFaQQ-yX89h0Q0mDLKGqp2Y,1597
|
|
9
|
+
water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
|
|
10
|
+
water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=w2lxfPS2HRVlNCr6OWBRxIyGaPUnxAnWrYgmu0EgOL0,7275
|
|
11
|
+
water_column_sonar_processing/cruise/resample_regrid.py,sha256=HeRN4EOZch-HhiK-s5LjX0bz4sKXKpea5YVIMEZOAGg,12356
|
|
12
|
+
water_column_sonar_processing/geometry/__init__.py,sha256=_ol5nI8AL30pYXeAh5rtP7YmQggitPC6LA_kuTfPJ0Q,231
|
|
13
|
+
water_column_sonar_processing/geometry/geometry_manager.py,sha256=RJ5pljMLqjNdJbc8Q0s-HGphMy6B7e_7R7ZZfdxEY-k,9913
|
|
14
|
+
water_column_sonar_processing/geometry/geometry_simplification.py,sha256=im1HG9nfYIerQv3w-PUHzphw2B7aGgnsA3Zcdy2oTmA,3016
|
|
15
|
+
water_column_sonar_processing/geometry/pmtile_generation.py,sha256=KI5y3CVBXSrxYoenQPHBUhPPAvcw1bkqCg1LZ9xvlzY,2772
|
|
16
|
+
water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
|
|
17
|
+
water_column_sonar_processing/index/index_manager.py,sha256=2D4mbWK2D3p0wsZqeoxieCtz9HsDkm5Mvt9u2t12BFI,11107
|
|
18
|
+
water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
|
|
19
|
+
water_column_sonar_processing/model/zarr_manager.py,sha256=IvA2mTZ6YrfTKL3U6NVgmjeqYPUXMSI3V-HI5ESgAwg,13408
|
|
20
|
+
water_column_sonar_processing/utility/__init__.py,sha256=nyqPobcvwftr6T4MNxNtQtfbWzW9Kgpbp6JO7Gr5IZI,206
|
|
21
|
+
water_column_sonar_processing/utility/cleaner.py,sha256=jaNDHbZxqQv2I8VfFkbwlQoCtIHi1pQonqt3XtioVq0,585
|
|
22
|
+
water_column_sonar_processing/utility/constants.py,sha256=qSuxPg4dnyI4DDXWc1OYqTBTChxZ9iWdUrFfqLtH21c,1793
|
|
23
|
+
water_column_sonar_processing/utility/pipeline_status.py,sha256=O-0SySqdRGJ6bs3zQe1NV9vkOpmsRM7zj5QoHgzYioY,4395
|
|
24
|
+
water_column_sonar_processing/utility/timestamp.py,sha256=bO0oir7KxxoEHPGRkz9FCBfOligkocUyRiWRzAq8fnU,361
|
|
25
|
+
water_column_sonar_processing-0.0.6.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
|
|
26
|
+
water_column_sonar_processing-0.0.6.dist-info/METADATA,sha256=nQW75RMHtE7D1faQBnzZ2L9IL7hlAITv_oHfnur8D0Y,4109
|
|
27
|
+
water_column_sonar_processing-0.0.6.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
28
|
+
water_column_sonar_processing-0.0.6.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
|
|
29
|
+
water_column_sonar_processing-0.0.6.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
water_column_sonar_processing
|
__init__.py
DELETED
|
File without changes
|
aws_manager/__init__.py
DELETED
cruise/__init__.py
DELETED
|
File without changes
|
geometry_manager/__init__.py
DELETED
|
File without changes
|
index_manager/__init__.py
DELETED
|
File without changes
|
model.py
DELETED
|
@@ -1,140 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import json
|
|
3
|
-
import numpy as np
|
|
4
|
-
from aws_manager.s3_manager import S3Manager
|
|
5
|
-
from aws_manager.s3fs_manager import S3FSManager
|
|
6
|
-
from aws_manager.sns_manager import SNSManager
|
|
7
|
-
from aws_manager.dynamodb_manager import DynamoDBManager
|
|
8
|
-
|
|
9
|
-
###########################################################
|
|
10
|
-
class Model:
|
|
11
|
-
#######################################################
|
|
12
|
-
def __init__(
|
|
13
|
-
self,
|
|
14
|
-
):
|
|
15
|
-
self.input_bucket_name = os.environ['INPUT_BUCKET_NAME']
|
|
16
|
-
self.output_bucket_name = os.environ['OUTPUT_BUCKET_NAME']
|
|
17
|
-
self.table_name = os.environ['TABLE_NAME']
|
|
18
|
-
self.topic_arn = os.environ['TOPIC_ARN']
|
|
19
|
-
# self.output_bucket_access_key = ?
|
|
20
|
-
# self.output_bucket_secret_access_key = ?
|
|
21
|
-
|
|
22
|
-
def execute(self):
|
|
23
|
-
input_s3_manager = S3Manager() # TODO: Need to allow passing in of credentials when writing to protected bucket
|
|
24
|
-
s3fs_manager = S3FSManager() # TODO: delete this
|
|
25
|
-
print(s3fs_manager) # TODO: delete this
|
|
26
|
-
output_s3_manager = S3Manager()
|
|
27
|
-
# TODO: s3fs?
|
|
28
|
-
sns_manager = SNSManager()
|
|
29
|
-
ddb_manager = DynamoDBManager()
|
|
30
|
-
|
|
31
|
-
# [1 of 5] Update Pipeline Status in DynamoDB
|
|
32
|
-
#self.dynamodb.update_ status ()
|
|
33
|
-
|
|
34
|
-
# [2 of 5] Download Object From Input Bucket
|
|
35
|
-
return_value = input_s3_manager.get(
|
|
36
|
-
bucket_name=self.input_bucket_name,
|
|
37
|
-
key='the_input_key'
|
|
38
|
-
)
|
|
39
|
-
print(return_value)
|
|
40
|
-
|
|
41
|
-
# [3 of 5] Update Entry in DynamoDB
|
|
42
|
-
ship_name = 'David_Starr_Jordan' # TODO: get this from input sns message
|
|
43
|
-
cruise_name = 'DS0604'
|
|
44
|
-
sensor_name = 'EK60'
|
|
45
|
-
file_name = "DSJ0604-D20060406-T113407.raw"
|
|
46
|
-
|
|
47
|
-
test_channels = [
|
|
48
|
-
"GPT 38 kHz 009072055a7f 2 ES38B",
|
|
49
|
-
"GPT 70 kHz 00907203400a 3 ES70-7C",
|
|
50
|
-
"GPT 120 kHz 009072034d52 1 ES120-7",
|
|
51
|
-
"GPT 200 kHz 0090720564e4 4 ES200-7C"
|
|
52
|
-
]
|
|
53
|
-
test_frequencies = [38_000, 70_000, 120_000, 200_000]
|
|
54
|
-
ddb_manager.update_item(
|
|
55
|
-
table_name=self.table_name,
|
|
56
|
-
key={
|
|
57
|
-
'FILE_NAME': {'S': file_name}, # Partition Key
|
|
58
|
-
'CRUISE_NAME': {'S': cruise_name}, # Sort Key
|
|
59
|
-
},
|
|
60
|
-
expression_attribute_names={
|
|
61
|
-
'#CH': 'CHANNELS',
|
|
62
|
-
'#ET': 'END_TIME',
|
|
63
|
-
'#ED': 'ERROR_DETAIL',
|
|
64
|
-
'#FR': 'FREQUENCIES',
|
|
65
|
-
'#MA': 'MAX_ECHO_RANGE',
|
|
66
|
-
'#MI': 'MIN_ECHO_RANGE',
|
|
67
|
-
'#ND': 'NUM_PING_TIME_DROPNA',
|
|
68
|
-
'#PS': 'PIPELINE_STATUS', # testing this updated
|
|
69
|
-
'#PT': 'PIPELINE_TIME', # testing this updated
|
|
70
|
-
'#SE': 'SENSOR_NAME',
|
|
71
|
-
'#SH': 'SHIP_NAME',
|
|
72
|
-
'#ST': 'START_TIME',
|
|
73
|
-
'#ZB': 'ZARR_BUCKET',
|
|
74
|
-
'#ZP': 'ZARR_PATH',
|
|
75
|
-
},
|
|
76
|
-
expression_attribute_values={
|
|
77
|
-
':ch': {'L': [{'S': i} for i in test_channels]},
|
|
78
|
-
':et': {'S': '2006-04-06T13:35:28.688Z'},
|
|
79
|
-
':ed': {'S': ''},
|
|
80
|
-
':fr': {'L': [{'N': str(i)} for i in test_frequencies]},
|
|
81
|
-
':ma': {'N': str(np.round(499.7653, 4))},
|
|
82
|
-
':mi': {'N': str(np.round(0.25, 4))},
|
|
83
|
-
':nd': {'N': str(2458)},
|
|
84
|
-
':ps': {'S': 'SUCCESS_AGGREGATOR'},
|
|
85
|
-
':pt': {'S': '2023-10-02T08:54:43Z'},
|
|
86
|
-
':se': {'S': sensor_name},
|
|
87
|
-
':sh': {'S': ship_name},
|
|
88
|
-
':st': {'S': '2006-04-06T11:34:07.288Z'},
|
|
89
|
-
':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
|
|
90
|
-
':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.zarr_manager'},
|
|
91
|
-
},
|
|
92
|
-
update_expression=(
|
|
93
|
-
'SET '
|
|
94
|
-
'#CH = :ch, '
|
|
95
|
-
'#ET = :et, '
|
|
96
|
-
'#ED = :ed, '
|
|
97
|
-
'#FR = :fr, '
|
|
98
|
-
'#MA = :ma, '
|
|
99
|
-
'#MI = :mi, '
|
|
100
|
-
'#ND = :nd, '
|
|
101
|
-
'#PS = :ps, '
|
|
102
|
-
'#PT = :pt, '
|
|
103
|
-
'#SE = :se, '
|
|
104
|
-
'#SH = :sh, '
|
|
105
|
-
'#ST = :st, '
|
|
106
|
-
'#ZB = :zb, '
|
|
107
|
-
'#ZP = :zp'
|
|
108
|
-
)
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
# [4 of 5] Write Object to Output Bucket
|
|
112
|
-
output_s3_manager.put(
|
|
113
|
-
bucket_name=self.output_bucket_name,
|
|
114
|
-
key='123',
|
|
115
|
-
body='456'
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
# [_ of _] Read file-level Zarr store from bucket, Create GeoJSON, Write to bucket
|
|
119
|
-
# [_ of _] Create empty cruise-level Zarr store
|
|
120
|
-
# [_ of _] Resample and write to cruise-level Zarr Store
|
|
121
|
-
|
|
122
|
-
# [5 of 5] Publish Done Message
|
|
123
|
-
success_message = {
|
|
124
|
-
"default": {
|
|
125
|
-
"shipName": ship_name,
|
|
126
|
-
"cruiseName": cruise_name,
|
|
127
|
-
"sensorName": sensor_name,
|
|
128
|
-
"fileName": file_name,
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
sns_manager.publish(
|
|
132
|
-
topic_arn=self.topic_arn,
|
|
133
|
-
message=json.dumps(success_message),
|
|
134
|
-
)
|
|
135
|
-
print("done...")
|
|
136
|
-
|
|
137
|
-
#######################################################
|
|
138
|
-
|
|
139
|
-
###########################################################
|
|
140
|
-
###########################################################
|
utility/__init__.py
DELETED
|
File without changes
|
utility/constants.py
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
from enum import Enum, Flag, unique
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
@unique
|
|
5
|
-
class Constants(Flag):
|
|
6
|
-
TILE_SIZE = 512
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class Coordinates(Enum):
|
|
10
|
-
"""
|
|
11
|
-
Should try to specify
|
|
12
|
-
dtype
|
|
13
|
-
units
|
|
14
|
-
long_name — most readable description of variable
|
|
15
|
-
standard_name — name in lowercase and snake_case
|
|
16
|
-
"""
|
|
17
|
-
PROJECT_NAME = 'echofish'
|
|
18
|
-
|
|
19
|
-
DEPTH = 'depth'
|
|
20
|
-
DEPTH_DTYPE = 'float32'
|
|
21
|
-
DEPTH_UNITS = 'm' # TODO: Pint? <https://pint.readthedocs.io/en/stable/>
|
|
22
|
-
DEPTH_LONG_NAME = 'Depth below surface'
|
|
23
|
-
DEPTH_STANDARD_NAME = 'depth'
|
|
24
|
-
|
|
25
|
-
TIME = 'time'
|
|
26
|
-
TIME_DTYPE = 'float64'
|
|
27
|
-
# Note: units and calendar are used downstream by Xarray
|
|
28
|
-
TIME_UNITS = 'seconds since 1970-01-01 00:00:00'
|
|
29
|
-
TIME_LONG_NAME = 'Timestamp of each ping'
|
|
30
|
-
TIME_STANDARD_NAME = 'time'
|
|
31
|
-
TIME_CALENDAR = 'proleptic_gregorian'
|
|
32
|
-
# TODO: create test for reading out timestamps in Xarray
|
|
33
|
-
|
|
34
|
-
FREQUENCY = 'frequency'
|
|
35
|
-
FREQUENCY_DTYPE = 'int'
|
|
36
|
-
FREQUENCY_UNITS = 'Hz'
|
|
37
|
-
FREQUENCY_LONG_NAME = 'Transducer frequency'
|
|
38
|
-
FREQUENCY_STANDARD_NAME = 'sound_frequency'
|
|
39
|
-
|
|
40
|
-
LATITUDE = 'latitude'
|
|
41
|
-
LATITUDE_DTYPE = 'float32'
|
|
42
|
-
LATITUDE_UNITS = 'degrees_north'
|
|
43
|
-
LATITUDE_LONG_NAME = 'Latitude'
|
|
44
|
-
LATITUDE_STANDARD_NAME = 'latitude'
|
|
45
|
-
|
|
46
|
-
LONGITUDE = 'longitude'
|
|
47
|
-
LONGITUDE_DTYPE = 'float32'
|
|
48
|
-
LONGITUDE_UNITS = 'degrees_east'
|
|
49
|
-
LONGITUDE_LONG_NAME = 'Longitude'
|
|
50
|
-
LONGITUDE_STANDARD_NAME = 'longitude'
|
|
51
|
-
|
|
52
|
-
SV = 'Sv'
|
|
53
|
-
SV_DTYPE = 'float32' # TODO: experiment with dtype of int
|
|
54
|
-
SV_UNITS = 'dB'
|
|
55
|
-
SV_LONG_NAME = 'Volume backscattering strength (Sv re 1 m-1)'
|
|
56
|
-
SV_STANDARD_NAME = 'volume_backscattering_strength'
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
model.py,sha256=r5S-ZZEV1lJdpEU7yzGKD9WIWuRRjaw1d7gdGQoBaTo,5254
|
|
3
|
-
aws_manager/__init__.py,sha256=pYvL00VjrwCe9k5jfof2k-nc9IG1N-hy1k95W7VlKOs,162
|
|
4
|
-
aws_manager/dynamodb_manager.py,sha256=dy2IUqQMlyXbnSGgJWt4ggxUnu2heb_Z2jgueaUkCzQ,5346
|
|
5
|
-
aws_manager/s3_manager.py,sha256=r5CJ0XIvA_i4azFvLF4xKdcCuhKBJRNFOL2luvpIBnM,13481
|
|
6
|
-
aws_manager/s3fs_manager.py,sha256=k4nW48E0reQwj1t4pe_ybK5kX0H1SHeG8urddHelGZ4,2463
|
|
7
|
-
aws_manager/sns_manager.py,sha256=IR5Y3MvwYFjd-eSg4tEjoNyABGxdJb85wOj0I-vckX0,2204
|
|
8
|
-
aws_manager/sqs_manager.py,sha256=VQSkxYtdhH4zfZqQl5FiNE2_svczN0hU-UBgbviwvtc,1696
|
|
9
|
-
cruise/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
-
cruise/create_empty_zarr_store.py,sha256=K_Jfm_7Rt8snBXLQOJOjYilPH3pUXHveql9hay8HvCE,7012
|
|
11
|
-
cruise/resample_regrid.py,sha256=Do6j-OhU8t4n10EkHRSyPC5MfqbAwA829q9S_cd4W1U,11311
|
|
12
|
-
geometry_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
-
geometry_manager/geometry_manager.py,sha256=tCNatSleMCJhGxLp9H_WYr1MquBU9sBCFWsZ1o0rTL4,9409
|
|
14
|
-
geometry_manager/geometry_simplification.py,sha256=_TCrlxqWlhO05Od3ZB4Sfgob9p3tQr3r1vjseG0qMNQ,3071
|
|
15
|
-
geometry_manager/pmtile_generation.py,sha256=7v1XZKprlNV_fFWXHWD6mn4z0xf21rpUMqj3RLIAgy8,2798
|
|
16
|
-
index_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
-
index_manager/index_manager.py,sha256=spw3OHAgacr2WsTgCg4VpFbCg-brg6S29fd44xi1Lqc,10718
|
|
18
|
-
utility/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
-
utility/cleaner.py,sha256=QaB1-y7cBYWgZn_H9whYQSpguGcLKN5LZVtfIwMNRvw,615
|
|
20
|
-
utility/constants.py,sha256=uTuSyH9YGG8bDd5xNbT4zA5O_lu874C-8m1MRvVppPw,1631
|
|
21
|
-
utility/pipeline_status.py,sha256=iRLyzI3gsesyz0I4NukbJwqiRmXgSZBOdB3C1x2eMSg,4185
|
|
22
|
-
utility/timestamp.py,sha256=i4ub7ob24edHU2yt4aOmvle_Nmlf1F_9HVGa9TJ5HgM,359
|
|
23
|
-
zarr_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
-
zarr_manager/zarr_manager.py,sha256=QK6M7YJ-wxcxRp3lxAc8mhdUtTplKgE-9WZRyyFYwdw,12408
|
|
25
|
-
water_column_sonar_processing-0.0.4.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
|
|
26
|
-
water_column_sonar_processing-0.0.4.dist-info/METADATA,sha256=PRJRr9pCM25QSMUu5cK-A398FRZJeYchGsqQfzS-3rg,3157
|
|
27
|
-
water_column_sonar_processing-0.0.4.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
28
|
-
water_column_sonar_processing-0.0.4.dist-info/top_level.txt,sha256=l_orjhw-0yuQ3IW_ZgpmaFmAC6ngnS20SYXWqneatVI,86
|
|
29
|
-
water_column_sonar_processing-0.0.4.dist-info/RECORD,,
|
zarr_manager/__init__.py
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|