water-column-sonar-processing 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. water_column_sonar_processing/__init__.py +16 -0
  2. water_column_sonar_processing/aws/__init__.py +7 -0
  3. {aws_manager → water_column_sonar_processing/aws}/dynamodb_manager.py +71 -50
  4. {aws_manager → water_column_sonar_processing/aws}/s3_manager.py +120 -130
  5. {aws_manager → water_column_sonar_processing/aws}/s3fs_manager.py +13 -19
  6. {aws_manager → water_column_sonar_processing/aws}/sns_manager.py +10 -21
  7. {aws_manager → water_column_sonar_processing/aws}/sqs_manager.py +10 -18
  8. water_column_sonar_processing/cruise/__init__.py +4 -0
  9. {cruise → water_column_sonar_processing/cruise}/create_empty_zarr_store.py +62 -44
  10. {cruise → water_column_sonar_processing/cruise}/resample_regrid.py +117 -66
  11. water_column_sonar_processing/geometry/__init__.py +5 -0
  12. {geometry_manager → water_column_sonar_processing/geometry}/geometry_manager.py +80 -49
  13. {geometry_manager → water_column_sonar_processing/geometry}/geometry_simplification.py +13 -12
  14. {geometry_manager → water_column_sonar_processing/geometry}/pmtile_generation.py +25 -24
  15. water_column_sonar_processing/index/__init__.py +3 -0
  16. {index_manager → water_column_sonar_processing/index}/index_manager.py +106 -82
  17. water_column_sonar_processing/model/__init__.py +3 -0
  18. {zarr_manager → water_column_sonar_processing/model}/zarr_manager.py +119 -83
  19. water_column_sonar_processing/process.py +147 -0
  20. water_column_sonar_processing/utility/__init__.py +6 -0
  21. {utility → water_column_sonar_processing/utility}/cleaner.py +6 -7
  22. water_column_sonar_processing/utility/constants.py +63 -0
  23. {utility → water_column_sonar_processing/utility}/pipeline_status.py +37 -10
  24. {utility → water_column_sonar_processing/utility}/timestamp.py +3 -2
  25. {water_column_sonar_processing-0.0.4.dist-info → water_column_sonar_processing-0.0.6.dist-info}/METADATA +31 -1
  26. water_column_sonar_processing-0.0.6.dist-info/RECORD +29 -0
  27. water_column_sonar_processing-0.0.6.dist-info/top_level.txt +1 -0
  28. __init__.py +0 -0
  29. aws_manager/__init__.py +0 -4
  30. cruise/__init__.py +0 -0
  31. geometry_manager/__init__.py +0 -0
  32. index_manager/__init__.py +0 -0
  33. model.py +0 -140
  34. utility/__init__.py +0 -0
  35. utility/constants.py +0 -56
  36. water_column_sonar_processing-0.0.4.dist-info/RECORD +0 -29
  37. water_column_sonar_processing-0.0.4.dist-info/top_level.txt +0 -8
  38. zarr_manager/__init__.py +0 -0
  39. {water_column_sonar_processing-0.0.4.dist-info → water_column_sonar_processing-0.0.6.dist-info}/LICENSE +0 -0
  40. {water_column_sonar_processing-0.0.4.dist-info → water_column_sonar_processing-0.0.6.dist-info}/WHEEL +0 -0
@@ -16,13 +16,17 @@ class PipelineStatus(Flag):
16
16
  PROCESSING_RAW_PROCESSOR = auto()
17
17
  FAILURE_RAW_PROCESSOR = auto()
18
18
  SUCCESS_RAW_PROCESSOR = auto()
19
- RAW_PROCESSOR = PROCESSING_RAW_PROCESSOR | FAILURE_RAW_PROCESSOR | SUCCESS_RAW_PROCESSOR
19
+ RAW_PROCESSOR = (
20
+ PROCESSING_RAW_PROCESSOR | FAILURE_RAW_PROCESSOR | SUCCESS_RAW_PROCESSOR
21
+ )
20
22
  #
21
23
  # RAW_AGGREGATOR --> AGGREGATOR (Scatter-Gather EIP)
22
24
  PROCESSING_RAW_AGGREGATOR = auto()
23
25
  FAILURE_RAW_AGGREGATOR = auto()
24
26
  SUCCESS_RAW_AGGREGATOR = auto()
25
- RAW_AGGREGATOR = PROCESSING_RAW_AGGREGATOR | FAILURE_RAW_AGGREGATOR | SUCCESS_RAW_AGGREGATOR
27
+ RAW_AGGREGATOR = (
28
+ PROCESSING_RAW_AGGREGATOR | FAILURE_RAW_AGGREGATOR | SUCCESS_RAW_AGGREGATOR
29
+ )
26
30
  #
27
31
  LEVEL_1_PROCESSING = RAW_SPLITTER | RAW_PROCESSOR | RAW_AGGREGATOR
28
32
  #
@@ -35,27 +39,43 @@ class PipelineStatus(Flag):
35
39
  PROCESSING_CRUISE_INITIALIZER = auto()
36
40
  FAILURE_CRUISE_INITIALIZER = auto()
37
41
  SUCCESS_CRUISE_INITIALIZER = auto()
38
- CRUISE_INITIALIZER = PROCESSING_CRUISE_INITIALIZER | FAILURE_CRUISE_INITIALIZER | SUCCESS_CRUISE_INITIALIZER
42
+ CRUISE_INITIALIZER = (
43
+ PROCESSING_CRUISE_INITIALIZER
44
+ | FAILURE_CRUISE_INITIALIZER
45
+ | SUCCESS_CRUISE_INITIALIZER
46
+ )
39
47
  #
40
48
  # CRUISE_SPLITTER --> SPLITTER
41
49
  PROCESSING_CRUISE_SPLITTER = auto()
42
50
  FAILURE_CRUISE_SPLITTER = auto()
43
51
  SUCCESS_CRUISE_SPLITTER = auto()
44
- CRUISE_SPLITTER = PROCESSING_CRUISE_SPLITTER | FAILURE_CRUISE_SPLITTER | SUCCESS_CRUISE_SPLITTER
52
+ CRUISE_SPLITTER = (
53
+ PROCESSING_CRUISE_SPLITTER | FAILURE_CRUISE_SPLITTER | SUCCESS_CRUISE_SPLITTER
54
+ )
45
55
  #
46
56
  # CRUISE_PROCESSOR --> PROCESSOR <-- Note: these need to run sequentially now
47
57
  PROCESSING_CRUISE_PROCESSOR = auto()
48
58
  FAILURE_CRUISE_PROCESSOR = auto()
49
59
  SUCCESS_CRUISE_PROCESSOR = auto()
50
- CRUISE_PROCESSOR = PROCESSING_CRUISE_PROCESSOR | FAILURE_CRUISE_PROCESSOR | SUCCESS_CRUISE_PROCESSOR
60
+ CRUISE_PROCESSOR = (
61
+ PROCESSING_CRUISE_PROCESSOR
62
+ | FAILURE_CRUISE_PROCESSOR
63
+ | SUCCESS_CRUISE_PROCESSOR
64
+ )
51
65
  #
52
66
  # CRUISE_AGGREGATOR --> AGGREGATOR
53
67
  PROCESSING_CRUISE_AGGREGATOR = auto()
54
68
  FAILURE_CRUISE_AGGREGATOR = auto()
55
69
  SUCCESS_CRUISE_AGGREGATOR = auto()
56
- CRUISE_AGGREGATOR = PROCESSING_CRUISE_AGGREGATOR | FAILURE_CRUISE_AGGREGATOR | SUCCESS_CRUISE_AGGREGATOR
70
+ CRUISE_AGGREGATOR = (
71
+ PROCESSING_CRUISE_AGGREGATOR
72
+ | FAILURE_CRUISE_AGGREGATOR
73
+ | SUCCESS_CRUISE_AGGREGATOR
74
+ )
57
75
  #
58
- LEVEL_2_PROCESSING = CRUISE_INITIALIZER | CRUISE_SPLITTER | CRUISE_PROCESSOR | CRUISE_AGGREGATOR
76
+ LEVEL_2_PROCESSING = (
77
+ CRUISE_INITIALIZER | CRUISE_SPLITTER | CRUISE_PROCESSOR | CRUISE_AGGREGATOR
78
+ )
59
79
  #
60
80
  # --------------------------------------------------- #
61
81
  # --- Level 3 Data ---------------------------------- #
@@ -64,18 +84,25 @@ class PipelineStatus(Flag):
64
84
  PROCESSING_TILE_PROCESSOR = auto()
65
85
  FAILURE_TILE_PROCESSOR = auto()
66
86
  SUCCESS_TILE_PROCESSOR = auto()
67
- TILE_PROCESSOR = PROCESSING_TILE_PROCESSOR | FAILURE_TILE_PROCESSOR | SUCCESS_TILE_PROCESSOR
87
+ TILE_PROCESSOR = (
88
+ PROCESSING_TILE_PROCESSOR | FAILURE_TILE_PROCESSOR | SUCCESS_TILE_PROCESSOR
89
+ )
68
90
  #
69
91
  # GEOHASH_PROCESSOR
70
92
  PROCESSING_GEOHASH_PROCESSOR = auto()
71
93
  FAILURE_GEOHASH_PROCESSOR = auto()
72
94
  SUCCESS_GEOHASH_PROCESSOR = auto()
73
- GEOHASH_PROCESSOR = PROCESSING_GEOHASH_PROCESSOR | FAILURE_GEOHASH_PROCESSOR | SUCCESS_GEOHASH_PROCESSOR
95
+ GEOHASH_PROCESSOR = (
96
+ PROCESSING_GEOHASH_PROCESSOR
97
+ | FAILURE_GEOHASH_PROCESSOR
98
+ | SUCCESS_GEOHASH_PROCESSOR
99
+ )
74
100
  #
75
101
  LEVEL_3_PROCESSING = TILE_PROCESSOR | GEOHASH_PROCESSOR
76
102
  # --------------------------------------------------- #
77
103
  # --------------------------------------------------- #
78
104
 
105
+
79
106
  # Status.PROCESSING_RAW_AGGREGATOR in Status.LEVEL_1_PROCESSING
80
107
  # Status.LEVEL_1_PROCESSING.value < Status.LEVEL_2_PROCESSING.value
81
108
 
@@ -95,4 +122,4 @@ failed - verb - "verbs should be avoided"
95
122
 
96
123
  success - noun
97
124
 
98
- """
125
+ """
@@ -7,6 +7,7 @@ class Timestamp:
7
7
  def get_timestamp():
8
8
  # return timestamp in form:
9
9
  # PIPELINE_TIME = '2024-03-29T19:36:52.433Z'
10
- return f'{datetime.utcnow().isoformat()[:23]}Z'
10
+ return f"{datetime.utcnow().isoformat()[:23]}Z"
11
11
 
12
- ###########################################################
12
+
13
+ ###########################################################
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: water_column_sonar_processing
3
- Version: 0.0.4
3
+ Version: 0.0.6
4
4
  Summary: A processing tool for water column sonar data.
5
5
  Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
6
6
  Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
@@ -11,6 +11,26 @@ Classifier: Operating System :: OS Independent
11
11
  Requires-Python: >=3.10
12
12
  Description-Content-Type: text/markdown
13
13
  License-File: LICENSE
14
+ Requires-Dist: aiobotocore~=2.9.0
15
+ Requires-Dist: aiohttp~=3.10.10
16
+ Requires-Dist: boto3==1.33.13
17
+ Requires-Dist: botocore~=1.33.13
18
+ Requires-Dist: echopype==0.9.0
19
+ Requires-Dist: geopandas==1.0.1
20
+ Requires-Dist: mock~=5.1.0
21
+ Requires-Dist: moto~=5.0.18
22
+ Requires-Dist: numcodecs==0.13.1
23
+ Requires-Dist: numpy==1.26.4
24
+ Requires-Dist: pandas==2.2.3
25
+ Requires-Dist: pytest~=8.3.3
26
+ Requires-Dist: python-dotenv==1.0.0
27
+ Requires-Dist: requests==2.32.3
28
+ Requires-Dist: s3fs==2024.10.0
29
+ Requires-Dist: scipy==1.14.1
30
+ Requires-Dist: shapely==2.0.3
31
+ Requires-Dist: typing-extensions==4.10.0
32
+ Requires-Dist: xarray==2022.12.0
33
+ Requires-Dist: zarr==2.18.3
14
34
 
15
35
  # Water Column Sonar Processing
16
36
  Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
@@ -72,6 +92,8 @@ https://packaging.python.org/en/latest/tutorials/packaging-projects/
72
92
  # To Publish To TEST
73
93
  ```commandline
74
94
  python -m build
95
+ # python -m build --sdist
96
+ # python -m build --wheel
75
97
  python -m twine upload --repository testpypi dist/*
76
98
  pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
77
99
  python
@@ -87,7 +109,15 @@ python -m build
87
109
  python -m twine upload --repository pypi dist/*
88
110
  ```
89
111
 
112
+ # Pre Commit Hook
113
+ https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
114
+ ```
115
+ pre-commit install --allow-missing-config
116
+ ```
90
117
 
91
118
  # Linting
92
119
  Ruff
93
120
  https://plugins.jetbrains.com/plugin/20574-ruff
121
+
122
+ # Colab Test
123
+ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
@@ -0,0 +1,29 @@
1
+ water_column_sonar_processing/__init__.py,sha256=S11loSDNOmZRyiTMbTwiQ1PP59SSPq2q2P61e5gPrW4,295
2
+ water_column_sonar_processing/process.py,sha256=AHoCehc95XqGI1IZkvfcbnYpjr8J-OQfCBGs0-1O9_w,5439
3
+ water_column_sonar_processing/aws/__init__.py,sha256=u5J-TOEVgAQsMdc5LMo1igUESRclzV8gf-b0jUaJ9Gg,277
4
+ water_column_sonar_processing/aws/dynamodb_manager.py,sha256=sPa_5RlTNjcFavZ74LF0UINKSyrbgJNROXbe16hCz7U,5936
5
+ water_column_sonar_processing/aws/s3_manager.py,sha256=P-0ZFsa6D6obIy-iNnWF1BrcoHOfAvYM-Qn9cA0e5H0,13142
6
+ water_column_sonar_processing/aws/s3fs_manager.py,sha256=thVJPQKhbvF1g-Ue3BYgwazFOFDYOICIEJx4zkXBQ1E,2381
7
+ water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
8
+ water_column_sonar_processing/aws/sqs_manager.py,sha256=NSUrWmnSC8h8Gf7gT0U8zFaQQ-yX89h0Q0mDLKGqp2Y,1597
9
+ water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
10
+ water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=w2lxfPS2HRVlNCr6OWBRxIyGaPUnxAnWrYgmu0EgOL0,7275
11
+ water_column_sonar_processing/cruise/resample_regrid.py,sha256=HeRN4EOZch-HhiK-s5LjX0bz4sKXKpea5YVIMEZOAGg,12356
12
+ water_column_sonar_processing/geometry/__init__.py,sha256=_ol5nI8AL30pYXeAh5rtP7YmQggitPC6LA_kuTfPJ0Q,231
13
+ water_column_sonar_processing/geometry/geometry_manager.py,sha256=RJ5pljMLqjNdJbc8Q0s-HGphMy6B7e_7R7ZZfdxEY-k,9913
14
+ water_column_sonar_processing/geometry/geometry_simplification.py,sha256=im1HG9nfYIerQv3w-PUHzphw2B7aGgnsA3Zcdy2oTmA,3016
15
+ water_column_sonar_processing/geometry/pmtile_generation.py,sha256=KI5y3CVBXSrxYoenQPHBUhPPAvcw1bkqCg1LZ9xvlzY,2772
16
+ water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
17
+ water_column_sonar_processing/index/index_manager.py,sha256=2D4mbWK2D3p0wsZqeoxieCtz9HsDkm5Mvt9u2t12BFI,11107
18
+ water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
19
+ water_column_sonar_processing/model/zarr_manager.py,sha256=IvA2mTZ6YrfTKL3U6NVgmjeqYPUXMSI3V-HI5ESgAwg,13408
20
+ water_column_sonar_processing/utility/__init__.py,sha256=nyqPobcvwftr6T4MNxNtQtfbWzW9Kgpbp6JO7Gr5IZI,206
21
+ water_column_sonar_processing/utility/cleaner.py,sha256=jaNDHbZxqQv2I8VfFkbwlQoCtIHi1pQonqt3XtioVq0,585
22
+ water_column_sonar_processing/utility/constants.py,sha256=qSuxPg4dnyI4DDXWc1OYqTBTChxZ9iWdUrFfqLtH21c,1793
23
+ water_column_sonar_processing/utility/pipeline_status.py,sha256=O-0SySqdRGJ6bs3zQe1NV9vkOpmsRM7zj5QoHgzYioY,4395
24
+ water_column_sonar_processing/utility/timestamp.py,sha256=bO0oir7KxxoEHPGRkz9FCBfOligkocUyRiWRzAq8fnU,361
25
+ water_column_sonar_processing-0.0.6.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
26
+ water_column_sonar_processing-0.0.6.dist-info/METADATA,sha256=nQW75RMHtE7D1faQBnzZ2L9IL7hlAITv_oHfnur8D0Y,4109
27
+ water_column_sonar_processing-0.0.6.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
28
+ water_column_sonar_processing-0.0.6.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
29
+ water_column_sonar_processing-0.0.6.dist-info/RECORD,,
@@ -0,0 +1 @@
1
+ water_column_sonar_processing
__init__.py DELETED
File without changes
aws_manager/__init__.py DELETED
@@ -1,4 +0,0 @@
1
- # from .dynamodb_manager import DynamoDBManager
2
- # from .s3_manager import S3Manager
3
- # from .s3fs_manager import S3FSManager
4
- # from .sns_manager import SNSManager
cruise/__init__.py DELETED
File without changes
File without changes
index_manager/__init__.py DELETED
File without changes
model.py DELETED
@@ -1,140 +0,0 @@
1
- import os
2
- import json
3
- import numpy as np
4
- from aws_manager.s3_manager import S3Manager
5
- from aws_manager.s3fs_manager import S3FSManager
6
- from aws_manager.sns_manager import SNSManager
7
- from aws_manager.dynamodb_manager import DynamoDBManager
8
-
9
- ###########################################################
10
- class Model:
11
- #######################################################
12
- def __init__(
13
- self,
14
- ):
15
- self.input_bucket_name = os.environ['INPUT_BUCKET_NAME']
16
- self.output_bucket_name = os.environ['OUTPUT_BUCKET_NAME']
17
- self.table_name = os.environ['TABLE_NAME']
18
- self.topic_arn = os.environ['TOPIC_ARN']
19
- # self.output_bucket_access_key = ?
20
- # self.output_bucket_secret_access_key = ?
21
-
22
- def execute(self):
23
- input_s3_manager = S3Manager() # TODO: Need to allow passing in of credentials when writing to protected bucket
24
- s3fs_manager = S3FSManager() # TODO: delete this
25
- print(s3fs_manager) # TODO: delete this
26
- output_s3_manager = S3Manager()
27
- # TODO: s3fs?
28
- sns_manager = SNSManager()
29
- ddb_manager = DynamoDBManager()
30
-
31
- # [1 of 5] Update Pipeline Status in DynamoDB
32
- #self.dynamodb.update_ status ()
33
-
34
- # [2 of 5] Download Object From Input Bucket
35
- return_value = input_s3_manager.get(
36
- bucket_name=self.input_bucket_name,
37
- key='the_input_key'
38
- )
39
- print(return_value)
40
-
41
- # [3 of 5] Update Entry in DynamoDB
42
- ship_name = 'David_Starr_Jordan' # TODO: get this from input sns message
43
- cruise_name = 'DS0604'
44
- sensor_name = 'EK60'
45
- file_name = "DSJ0604-D20060406-T113407.raw"
46
-
47
- test_channels = [
48
- "GPT 38 kHz 009072055a7f 2 ES38B",
49
- "GPT 70 kHz 00907203400a 3 ES70-7C",
50
- "GPT 120 kHz 009072034d52 1 ES120-7",
51
- "GPT 200 kHz 0090720564e4 4 ES200-7C"
52
- ]
53
- test_frequencies = [38_000, 70_000, 120_000, 200_000]
54
- ddb_manager.update_item(
55
- table_name=self.table_name,
56
- key={
57
- 'FILE_NAME': {'S': file_name}, # Partition Key
58
- 'CRUISE_NAME': {'S': cruise_name}, # Sort Key
59
- },
60
- expression_attribute_names={
61
- '#CH': 'CHANNELS',
62
- '#ET': 'END_TIME',
63
- '#ED': 'ERROR_DETAIL',
64
- '#FR': 'FREQUENCIES',
65
- '#MA': 'MAX_ECHO_RANGE',
66
- '#MI': 'MIN_ECHO_RANGE',
67
- '#ND': 'NUM_PING_TIME_DROPNA',
68
- '#PS': 'PIPELINE_STATUS', # testing this updated
69
- '#PT': 'PIPELINE_TIME', # testing this updated
70
- '#SE': 'SENSOR_NAME',
71
- '#SH': 'SHIP_NAME',
72
- '#ST': 'START_TIME',
73
- '#ZB': 'ZARR_BUCKET',
74
- '#ZP': 'ZARR_PATH',
75
- },
76
- expression_attribute_values={
77
- ':ch': {'L': [{'S': i} for i in test_channels]},
78
- ':et': {'S': '2006-04-06T13:35:28.688Z'},
79
- ':ed': {'S': ''},
80
- ':fr': {'L': [{'N': str(i)} for i in test_frequencies]},
81
- ':ma': {'N': str(np.round(499.7653, 4))},
82
- ':mi': {'N': str(np.round(0.25, 4))},
83
- ':nd': {'N': str(2458)},
84
- ':ps': {'S': 'SUCCESS_AGGREGATOR'},
85
- ':pt': {'S': '2023-10-02T08:54:43Z'},
86
- ':se': {'S': sensor_name},
87
- ':sh': {'S': ship_name},
88
- ':st': {'S': '2006-04-06T11:34:07.288Z'},
89
- ':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
90
- ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.zarr_manager'},
91
- },
92
- update_expression=(
93
- 'SET '
94
- '#CH = :ch, '
95
- '#ET = :et, '
96
- '#ED = :ed, '
97
- '#FR = :fr, '
98
- '#MA = :ma, '
99
- '#MI = :mi, '
100
- '#ND = :nd, '
101
- '#PS = :ps, '
102
- '#PT = :pt, '
103
- '#SE = :se, '
104
- '#SH = :sh, '
105
- '#ST = :st, '
106
- '#ZB = :zb, '
107
- '#ZP = :zp'
108
- )
109
- )
110
-
111
- # [4 of 5] Write Object to Output Bucket
112
- output_s3_manager.put(
113
- bucket_name=self.output_bucket_name,
114
- key='123',
115
- body='456'
116
- )
117
-
118
- # [_ of _] Read file-level Zarr store from bucket, Create GeoJSON, Write to bucket
119
- # [_ of _] Create empty cruise-level Zarr store
120
- # [_ of _] Resample and write to cruise-level Zarr Store
121
-
122
- # [5 of 5] Publish Done Message
123
- success_message = {
124
- "default": {
125
- "shipName": ship_name,
126
- "cruiseName": cruise_name,
127
- "sensorName": sensor_name,
128
- "fileName": file_name,
129
- }
130
- }
131
- sns_manager.publish(
132
- topic_arn=self.topic_arn,
133
- message=json.dumps(success_message),
134
- )
135
- print("done...")
136
-
137
- #######################################################
138
-
139
- ###########################################################
140
- ###########################################################
utility/__init__.py DELETED
File without changes
utility/constants.py DELETED
@@ -1,56 +0,0 @@
1
- from enum import Enum, Flag, unique
2
-
3
-
4
- @unique
5
- class Constants(Flag):
6
- TILE_SIZE = 512
7
-
8
-
9
- class Coordinates(Enum):
10
- """
11
- Should try to specify
12
- dtype
13
- units
14
- long_name — most readable description of variable
15
- standard_name — name in lowercase and snake_case
16
- """
17
- PROJECT_NAME = 'echofish'
18
-
19
- DEPTH = 'depth'
20
- DEPTH_DTYPE = 'float32'
21
- DEPTH_UNITS = 'm' # TODO: Pint? <https://pint.readthedocs.io/en/stable/>
22
- DEPTH_LONG_NAME = 'Depth below surface'
23
- DEPTH_STANDARD_NAME = 'depth'
24
-
25
- TIME = 'time'
26
- TIME_DTYPE = 'float64'
27
- # Note: units and calendar are used downstream by Xarray
28
- TIME_UNITS = 'seconds since 1970-01-01 00:00:00'
29
- TIME_LONG_NAME = 'Timestamp of each ping'
30
- TIME_STANDARD_NAME = 'time'
31
- TIME_CALENDAR = 'proleptic_gregorian'
32
- # TODO: create test for reading out timestamps in Xarray
33
-
34
- FREQUENCY = 'frequency'
35
- FREQUENCY_DTYPE = 'int'
36
- FREQUENCY_UNITS = 'Hz'
37
- FREQUENCY_LONG_NAME = 'Transducer frequency'
38
- FREQUENCY_STANDARD_NAME = 'sound_frequency'
39
-
40
- LATITUDE = 'latitude'
41
- LATITUDE_DTYPE = 'float32'
42
- LATITUDE_UNITS = 'degrees_north'
43
- LATITUDE_LONG_NAME = 'Latitude'
44
- LATITUDE_STANDARD_NAME = 'latitude'
45
-
46
- LONGITUDE = 'longitude'
47
- LONGITUDE_DTYPE = 'float32'
48
- LONGITUDE_UNITS = 'degrees_east'
49
- LONGITUDE_LONG_NAME = 'Longitude'
50
- LONGITUDE_STANDARD_NAME = 'longitude'
51
-
52
- SV = 'Sv'
53
- SV_DTYPE = 'float32' # TODO: experiment with dtype of int
54
- SV_UNITS = 'dB'
55
- SV_LONG_NAME = 'Volume backscattering strength (Sv re 1 m-1)'
56
- SV_STANDARD_NAME = 'volume_backscattering_strength'
@@ -1,29 +0,0 @@
1
- __init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- model.py,sha256=r5S-ZZEV1lJdpEU7yzGKD9WIWuRRjaw1d7gdGQoBaTo,5254
3
- aws_manager/__init__.py,sha256=pYvL00VjrwCe9k5jfof2k-nc9IG1N-hy1k95W7VlKOs,162
4
- aws_manager/dynamodb_manager.py,sha256=dy2IUqQMlyXbnSGgJWt4ggxUnu2heb_Z2jgueaUkCzQ,5346
5
- aws_manager/s3_manager.py,sha256=r5CJ0XIvA_i4azFvLF4xKdcCuhKBJRNFOL2luvpIBnM,13481
6
- aws_manager/s3fs_manager.py,sha256=k4nW48E0reQwj1t4pe_ybK5kX0H1SHeG8urddHelGZ4,2463
7
- aws_manager/sns_manager.py,sha256=IR5Y3MvwYFjd-eSg4tEjoNyABGxdJb85wOj0I-vckX0,2204
8
- aws_manager/sqs_manager.py,sha256=VQSkxYtdhH4zfZqQl5FiNE2_svczN0hU-UBgbviwvtc,1696
9
- cruise/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
- cruise/create_empty_zarr_store.py,sha256=K_Jfm_7Rt8snBXLQOJOjYilPH3pUXHveql9hay8HvCE,7012
11
- cruise/resample_regrid.py,sha256=Do6j-OhU8t4n10EkHRSyPC5MfqbAwA829q9S_cd4W1U,11311
12
- geometry_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
13
- geometry_manager/geometry_manager.py,sha256=tCNatSleMCJhGxLp9H_WYr1MquBU9sBCFWsZ1o0rTL4,9409
14
- geometry_manager/geometry_simplification.py,sha256=_TCrlxqWlhO05Od3ZB4Sfgob9p3tQr3r1vjseG0qMNQ,3071
15
- geometry_manager/pmtile_generation.py,sha256=7v1XZKprlNV_fFWXHWD6mn4z0xf21rpUMqj3RLIAgy8,2798
16
- index_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
17
- index_manager/index_manager.py,sha256=spw3OHAgacr2WsTgCg4VpFbCg-brg6S29fd44xi1Lqc,10718
18
- utility/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
- utility/cleaner.py,sha256=QaB1-y7cBYWgZn_H9whYQSpguGcLKN5LZVtfIwMNRvw,615
20
- utility/constants.py,sha256=uTuSyH9YGG8bDd5xNbT4zA5O_lu874C-8m1MRvVppPw,1631
21
- utility/pipeline_status.py,sha256=iRLyzI3gsesyz0I4NukbJwqiRmXgSZBOdB3C1x2eMSg,4185
22
- utility/timestamp.py,sha256=i4ub7ob24edHU2yt4aOmvle_Nmlf1F_9HVGa9TJ5HgM,359
23
- zarr_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
24
- zarr_manager/zarr_manager.py,sha256=QK6M7YJ-wxcxRp3lxAc8mhdUtTplKgE-9WZRyyFYwdw,12408
25
- water_column_sonar_processing-0.0.4.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
26
- water_column_sonar_processing-0.0.4.dist-info/METADATA,sha256=PRJRr9pCM25QSMUu5cK-A398FRZJeYchGsqQfzS-3rg,3157
27
- water_column_sonar_processing-0.0.4.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
28
- water_column_sonar_processing-0.0.4.dist-info/top_level.txt,sha256=l_orjhw-0yuQ3IW_ZgpmaFmAC6ngnS20SYXWqneatVI,86
29
- water_column_sonar_processing-0.0.4.dist-info/RECORD,,
@@ -1,8 +0,0 @@
1
- __init__
2
- aws_manager
3
- cruise
4
- geometry_manager
5
- index_manager
6
- model
7
- utility
8
- zarr_manager
zarr_manager/__init__.py DELETED
File without changes