water-column-sonar-processing 0.0.7__tar.gz → 0.0.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {water_column_sonar_processing-0.0.7/src/water_column_sonar_processing.egg-info → water_column_sonar_processing-0.0.9}/PKG-INFO +20 -10
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/README.md +10 -1
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/pyproject.toml +19 -9
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/requirements.txt +12 -11
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_create_empty_zarr_store.py +10 -6
- water_column_sonar_processing-0.0.9/tests/test_dynamodb_manager.py +376 -0
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_geometry_manager.py +10 -6
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_index.py +34 -9
- water_column_sonar_processing-0.0.9/tests/test_pmtile_generation.py +266 -0
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_process.py +5 -21
- water_column_sonar_processing-0.0.9/tests/test_raw_to_zarr.py +146 -0
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_resample_regrid.py +2 -2
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_s3_manager.py +27 -7
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_s3fs_manager.py +4 -17
- water_column_sonar_processing-0.0.9/tests/test_s3fs_with_moto.py +78 -0
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_sns_sqs_manager.py +1 -2
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_zarr_manager.py +31 -23
- water_column_sonar_processing-0.0.9/water_column_sonar_processing/__init__.py +13 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/__init__.py +2 -2
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/dynamodb_manager.py +149 -43
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/s3_manager.py +71 -37
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/cruise/create_empty_zarr_store.py +6 -4
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/cruise/resample_regrid.py +3 -3
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/geometry/geometry_manager.py +21 -6
- water_column_sonar_processing-0.0.9/water_column_sonar_processing/geometry/pmtile_generation.py +262 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/index/index_manager.py +25 -13
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/model/zarr_manager.py +27 -25
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/process.py +4 -4
- water_column_sonar_processing-0.0.9/water_column_sonar_processing/processing/__init__.py +4 -0
- water_column_sonar_processing-0.0.9/water_column_sonar_processing/processing/cruise_sampler.py +342 -0
- water_column_sonar_processing-0.0.9/water_column_sonar_processing/processing/raw_to_zarr.py +349 -0
- water_column_sonar_processing-0.0.9/water_column_sonar_processing/utility/__init__.py +6 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/utility/cleaner.py +1 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/utility/constants.py +6 -2
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9/water_column_sonar_processing.egg-info}/PKG-INFO +20 -10
- water_column_sonar_processing-0.0.9/water_column_sonar_processing.egg-info/SOURCES.txt +50 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing.egg-info/requires.txt +10 -9
- water_column_sonar_processing-0.0.7/src/water_column_sonar_processing/__init__.py +0 -16
- water_column_sonar_processing-0.0.7/src/water_column_sonar_processing/geometry/pmtile_generation.py +0 -75
- water_column_sonar_processing-0.0.7/src/water_column_sonar_processing/utility/__init__.py +0 -6
- water_column_sonar_processing-0.0.7/src/water_column_sonar_processing.egg-info/SOURCES.txt +0 -45
- water_column_sonar_processing-0.0.7/tests/test_dynamodb_manager.py +0 -204
- water_column_sonar_processing-0.0.7/tests/test_s3fs_with_moto.py +0 -31
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/LICENSE +0 -0
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/setup.cfg +0 -0
- {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_geometry_simplification.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/s3fs_manager.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/sns_manager.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/sqs_manager.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/cruise/__init__.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/geometry/__init__.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/geometry/geometry_simplification.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/index/__init__.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/model/__init__.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/utility/pipeline_status.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/utility/timestamp.py +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing.egg-info/dependency_links.txt +0 -0
- {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: water_column_sonar_processing
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.9
|
|
4
4
|
Summary: A processing tool for water column sonar data.
|
|
5
5
|
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
|
|
@@ -11,26 +11,27 @@ Classifier: Operating System :: OS Independent
|
|
|
11
11
|
Requires-Python: >=3.10
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
|
-
Requires-Dist: aiobotocore
|
|
15
|
-
Requires-Dist:
|
|
16
|
-
Requires-Dist:
|
|
17
|
-
Requires-Dist: botocore~=1.33.13
|
|
14
|
+
Requires-Dist: aiobotocore==2.15.2
|
|
15
|
+
Requires-Dist: boto3==1.35.36
|
|
16
|
+
Requires-Dist: botocore==1.35.36
|
|
18
17
|
Requires-Dist: echopype==0.9.0
|
|
19
18
|
Requires-Dist: fiona==1.10.1
|
|
20
19
|
Requires-Dist: geopandas==1.0.1
|
|
21
|
-
Requires-Dist: mock
|
|
22
|
-
Requires-Dist: moto
|
|
20
|
+
Requires-Dist: mock==5.1.0
|
|
21
|
+
Requires-Dist: moto[all]==5.0.21
|
|
22
|
+
Requires-Dist: moto[server]==5.0.21
|
|
23
23
|
Requires-Dist: numcodecs==0.13.1
|
|
24
24
|
Requires-Dist: numpy==1.26.4
|
|
25
25
|
Requires-Dist: pandas==2.2.3
|
|
26
|
-
Requires-Dist:
|
|
26
|
+
Requires-Dist: pyarrow==18.1.0
|
|
27
27
|
Requires-Dist: python-dotenv==1.0.0
|
|
28
28
|
Requires-Dist: requests==2.32.3
|
|
29
|
-
Requires-Dist: s3fs==
|
|
29
|
+
Requires-Dist: s3fs==2023.12.1
|
|
30
30
|
Requires-Dist: scipy==1.14.1
|
|
31
|
+
Requires-Dist: setuptools==75.6.0
|
|
31
32
|
Requires-Dist: shapely==2.0.3
|
|
32
33
|
Requires-Dist: typing-extensions==4.10.0
|
|
33
|
-
Requires-Dist: xarray==
|
|
34
|
+
Requires-Dist: xarray==2024.10.0
|
|
34
35
|
Requires-Dist: zarr==2.18.3
|
|
35
36
|
|
|
36
37
|
# Water Column Sonar Processing
|
|
@@ -85,6 +86,8 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
85
86
|
```commandline
|
|
86
87
|
pytest --disable-warnings
|
|
87
88
|
```
|
|
89
|
+
or
|
|
90
|
+
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
88
91
|
|
|
89
92
|
# Instructions
|
|
90
93
|
Following this tutorial:
|
|
@@ -122,3 +125,10 @@ https://plugins.jetbrains.com/plugin/20574-ruff
|
|
|
122
125
|
|
|
123
126
|
# Colab Test
|
|
124
127
|
https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
|
|
128
|
+
|
|
129
|
+
# Test Coverage
|
|
130
|
+
20241124
|
|
131
|
+
8 failed, 32 passed, 3 skipped, 1 warning in 6.92s
|
|
132
|
+
20241125
|
|
133
|
+
5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
|
|
134
|
+
3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
|
|
@@ -50,6 +50,8 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
50
50
|
```commandline
|
|
51
51
|
pytest --disable-warnings
|
|
52
52
|
```
|
|
53
|
+
or
|
|
54
|
+
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
53
55
|
|
|
54
56
|
# Instructions
|
|
55
57
|
Following this tutorial:
|
|
@@ -86,4 +88,11 @@ Ruff
|
|
|
86
88
|
https://plugins.jetbrains.com/plugin/20574-ruff
|
|
87
89
|
|
|
88
90
|
# Colab Test
|
|
89
|
-
https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
|
|
91
|
+
https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
|
|
92
|
+
|
|
93
|
+
# Test Coverage
|
|
94
|
+
20241124
|
|
95
|
+
8 failed, 32 passed, 3 skipped, 1 warning in 6.92s
|
|
96
|
+
20241125
|
|
97
|
+
5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
|
|
98
|
+
3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
|
|
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "water_column_sonar_processing"
|
|
10
|
-
version = "0.0.
|
|
10
|
+
version = "0.0.9"
|
|
11
11
|
authors = [
|
|
12
12
|
{ name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
|
|
13
13
|
]
|
|
@@ -25,19 +25,29 @@ dynamic = ["dependencies"]
|
|
|
25
25
|
Homepage = "https://github.com/CI-CMG/water-column-sonar-processing"
|
|
26
26
|
Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
|
|
27
27
|
|
|
28
|
-
#[pytest]
|
|
28
|
+
#[tool.pytest.ini_options]
|
|
29
|
+
#filterwarnings = [
|
|
30
|
+
# "error",
|
|
31
|
+
# "ignore::UserWarning",
|
|
32
|
+
# # note the use of single quote below to denote "raw" strings in TOML
|
|
33
|
+
# 'ignore:function ham\(\) is deprecated:DeprecationWarning',
|
|
34
|
+
#]
|
|
29
35
|
#pythonpath = "src"
|
|
30
36
|
#testpaths = "tests"
|
|
37
|
+
#[pytest]
|
|
38
|
+
#addopts = [
|
|
39
|
+
# "-p no:warnings"
|
|
40
|
+
#]
|
|
31
41
|
|
|
32
|
-
[tool.pytest.ini_options]
|
|
33
|
-
minversion = "6.0"
|
|
34
|
-
|
|
35
|
-
testpaths = [
|
|
36
|
-
"tests",
|
|
37
|
-
]
|
|
42
|
+
#[tool.pytest.ini_options]
|
|
43
|
+
#minversion = "6.0"
|
|
44
|
+
##addopts = "-ra -q"
|
|
45
|
+
#testpaths = [
|
|
46
|
+
# "tests",
|
|
47
|
+
#]
|
|
38
48
|
|
|
39
49
|
[tool.setuptools.dynamic]
|
|
40
50
|
dependencies = {file = ["requirements.txt"]}
|
|
41
|
-
optional-dependencies = {dev = { file = ["
|
|
51
|
+
optional-dependencies = {dev = { file = ["requirements_dev.txt"] }}
|
|
42
52
|
|
|
43
53
|
# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
|
{water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/requirements.txt
RENAMED
|
@@ -2,26 +2,27 @@
|
|
|
2
2
|
# defined for Python 3.12
|
|
3
3
|
# Note: be careful with conversions for pandas >=2.0.0, timestamps will have a lot of problems
|
|
4
4
|
|
|
5
|
-
aiobotocore
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
botocore~=1.33.13
|
|
5
|
+
aiobotocore==2.15.2
|
|
6
|
+
boto3==1.35.36
|
|
7
|
+
botocore==1.35.36
|
|
9
8
|
echopype==0.9.0
|
|
10
9
|
fiona==1.10.1
|
|
10
|
+
# Alternative to geopandas: pyogrio
|
|
11
11
|
geopandas==1.0.1
|
|
12
|
-
mock
|
|
13
|
-
moto
|
|
12
|
+
mock==5.1.0
|
|
13
|
+
moto[all]==5.0.21
|
|
14
|
+
moto[server]==5.0.21
|
|
14
15
|
numcodecs==0.13.1
|
|
15
16
|
numpy==1.26.4
|
|
16
17
|
pandas==2.2.3
|
|
17
|
-
|
|
18
|
+
pyarrow==18.1.0
|
|
18
19
|
python-dotenv==1.0.0
|
|
19
20
|
requests==2.32.3
|
|
20
|
-
#s3fs==
|
|
21
|
-
s3fs==
|
|
21
|
+
#s3fs==2024.10.0 # this version creates problems
|
|
22
|
+
s3fs==2023.12.1
|
|
22
23
|
scipy==1.14.1
|
|
24
|
+
setuptools==75.6.0
|
|
23
25
|
shapely==2.0.3
|
|
24
26
|
typing-extensions==4.10.0
|
|
25
|
-
xarray==
|
|
26
|
-
#zarr==2.16.1
|
|
27
|
+
xarray==2024.10.0
|
|
27
28
|
zarr==2.18.3
|
|
@@ -4,9 +4,11 @@ from dotenv import find_dotenv, load_dotenv
|
|
|
4
4
|
from moto import mock_aws
|
|
5
5
|
|
|
6
6
|
from water_column_sonar_processing.aws import DynamoDBManager, S3Manager
|
|
7
|
-
from water_column_sonar_processing.cruise
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
from water_column_sonar_processing.cruise import CreateEmptyZarrStore
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
# from water_column_sonar_processing import DynamoDBManager, S3Manager
|
|
11
|
+
# from water_column_sonar_processing import CreateEmptyZarrStore
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
# TEMPDIR = "/tmp"
|
|
@@ -25,7 +27,7 @@ def teardown_module():
|
|
|
25
27
|
|
|
26
28
|
#######################################################
|
|
27
29
|
@mock_aws
|
|
28
|
-
def test_create_empty_zarr_store():
|
|
30
|
+
def test_create_empty_zarr_store(): # PASSING, needs modification at end
|
|
29
31
|
dynamo_db_manager = DynamoDBManager()
|
|
30
32
|
s3_manager = S3Manager()
|
|
31
33
|
|
|
@@ -39,9 +41,11 @@ def test_create_empty_zarr_store():
|
|
|
39
41
|
# [0] create bucket with test files
|
|
40
42
|
input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
|
|
41
43
|
output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
|
|
44
|
+
|
|
42
45
|
s3_manager.create_bucket(bucket_name=input_bucket_name)
|
|
43
46
|
s3_manager.create_bucket(bucket_name=output_bucket_name)
|
|
44
47
|
print(s3_manager.list_buckets())
|
|
48
|
+
|
|
45
49
|
# TODO: put at the path where it needs to be deleted
|
|
46
50
|
s3_manager.put(bucket_name=input_bucket_name, key="the_key", body="the_body")
|
|
47
51
|
|
|
@@ -80,7 +84,7 @@ def test_create_empty_zarr_store():
|
|
|
80
84
|
249.792,
|
|
81
85
|
249.792,
|
|
82
86
|
249.792,
|
|
83
|
-
999.744,
|
|
87
|
+
999.744, # note: different depth
|
|
84
88
|
249.792,
|
|
85
89
|
249.792,
|
|
86
90
|
249.792,
|
|
@@ -211,7 +215,7 @@ def test_create_empty_zarr_store():
|
|
|
211
215
|
tempdir="/tmp",
|
|
212
216
|
)
|
|
213
217
|
|
|
214
|
-
assert os.path.exists(f"/tmp/{cruise_name}.zarr")
|
|
218
|
+
assert os.path.exists(f"/tmp/{cruise_name}.zarr") # TODO: create better tmp directory for testing
|
|
215
219
|
# TODO: should actually assert in the bucket
|
|
216
220
|
# mount and verify:
|
|
217
221
|
# shape
|
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from dotenv import find_dotenv, load_dotenv
|
|
3
|
+
from moto import mock_aws
|
|
4
|
+
|
|
5
|
+
from water_column_sonar_processing.aws import DynamoDBManager
|
|
6
|
+
from water_column_sonar_processing.utility import PipelineStatus
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
#######################################################
|
|
10
|
+
def setup_module():
|
|
11
|
+
print("setup")
|
|
12
|
+
env_file = find_dotenv(".env-test")
|
|
13
|
+
load_dotenv(dotenv_path=env_file, override=True)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def teardown_module():
|
|
17
|
+
print("teardown")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
#######################################################
|
|
21
|
+
def test_serializer_deserializer():
|
|
22
|
+
# ---Initialize--- #
|
|
23
|
+
low_level_data = {
|
|
24
|
+
"ACTIVE": {"BOOL": True},
|
|
25
|
+
"CRC": {"N": "-1600155180"},
|
|
26
|
+
"ID": {"S": "bewfv43843b"},
|
|
27
|
+
"params": {"M": {"customer": {"S": "TEST"}, "index": {"N": "1"}}},
|
|
28
|
+
"THIS_STATUS": {"N": "10"},
|
|
29
|
+
"TYPE": {"N": "22"},
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
# Lazy-eval the dynamodb attribute (boto3 is dynamic!)
|
|
33
|
+
dynamo_db_manager = DynamoDBManager()
|
|
34
|
+
|
|
35
|
+
# To go from low-level format to python
|
|
36
|
+
deserializer = dynamo_db_manager.type_deserializer
|
|
37
|
+
python_data = {k: deserializer.deserialize(v) for k, v in low_level_data.items()}
|
|
38
|
+
|
|
39
|
+
assert python_data["ACTIVE"]
|
|
40
|
+
assert python_data["CRC"] == -1600155180
|
|
41
|
+
assert python_data["ID"]
|
|
42
|
+
assert python_data["params"] == {"customer": "TEST", "index": 1}
|
|
43
|
+
assert python_data["THIS_STATUS"] == 10
|
|
44
|
+
assert python_data["TYPE"] == 22
|
|
45
|
+
|
|
46
|
+
# To go from python to low-level format
|
|
47
|
+
serializer = dynamo_db_manager.type_serializer
|
|
48
|
+
low_level_copy = {k: serializer.serialize(v) for k, v in python_data.items()}
|
|
49
|
+
|
|
50
|
+
assert low_level_data == low_level_copy
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
#######################################################
|
|
54
|
+
@mock_aws
|
|
55
|
+
def test_dynamodb_manager(): # PASSING
|
|
56
|
+
# ---Initialize--- #
|
|
57
|
+
table_name = "test_table"
|
|
58
|
+
dynamo_db_manager = DynamoDBManager()
|
|
59
|
+
|
|
60
|
+
# ---Create Table--- #
|
|
61
|
+
dynamo_db_manager.create_water_column_sonar_table(table_name=table_name)
|
|
62
|
+
|
|
63
|
+
# ---Add Items to Table--- #
|
|
64
|
+
test_channels = [
|
|
65
|
+
"GPT 38 kHz 009072055a7f 2 ES38B",
|
|
66
|
+
"GPT 70 kHz 00907203400a 3 ES70-7C",
|
|
67
|
+
"GPT 120 kHz 009072034d52 1 ES120-7",
|
|
68
|
+
"GPT 200 kHz 0090720564e4 4 ES200-7C",
|
|
69
|
+
]
|
|
70
|
+
test_frequencies = [38_000, 70_000, 120_000, 200_000]
|
|
71
|
+
ship_name = "David_Starr_Jordan"
|
|
72
|
+
cruise_name = "DS0604"
|
|
73
|
+
sensor_name = "EK60"
|
|
74
|
+
file_name = "DSJ0604-D20060419-T184612.raw"
|
|
75
|
+
|
|
76
|
+
dynamo_db_manager.update_item(
|
|
77
|
+
table_name=table_name,
|
|
78
|
+
key={
|
|
79
|
+
"FILE_NAME": {"S": file_name}, # Partition Key
|
|
80
|
+
"CRUISE_NAME": {"S": cruise_name}, # Sort Key
|
|
81
|
+
},
|
|
82
|
+
expression_attribute_names={
|
|
83
|
+
"#CH": "CHANNELS",
|
|
84
|
+
"#ET": "END_TIME",
|
|
85
|
+
"#ED": "ERROR_DETAIL",
|
|
86
|
+
"#FR": "FREQUENCIES",
|
|
87
|
+
"#MA": "MAX_ECHO_RANGE",
|
|
88
|
+
"#MI": "MIN_ECHO_RANGE",
|
|
89
|
+
"#ND": "NUM_PING_TIME_DROPNA",
|
|
90
|
+
"#PS": "PIPELINE_STATUS", # testing this updated
|
|
91
|
+
"#PT": "PIPELINE_TIME", # testing this updated
|
|
92
|
+
"#SE": "SENSOR_NAME",
|
|
93
|
+
"#SH": "SHIP_NAME",
|
|
94
|
+
"#ST": "START_TIME",
|
|
95
|
+
"#ZB": "ZARR_BUCKET",
|
|
96
|
+
"#ZP": "ZARR_PATH",
|
|
97
|
+
},
|
|
98
|
+
expression_attribute_values={
|
|
99
|
+
":ch": {"L": [{"S": i} for i in test_channels]},
|
|
100
|
+
":et": {"S": "2006-04-19T20:07:33.623Z"},
|
|
101
|
+
":ed": {"S": ""},
|
|
102
|
+
":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
|
|
103
|
+
":ma": {"N": str(np.round(500.785201, 4))},
|
|
104
|
+
":mi": {"N": str(np.round(0.25001, 4))},
|
|
105
|
+
":nd": {"N": str(2428)},
|
|
106
|
+
":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
|
|
107
|
+
":pt": {"S": "2023-10-02T08:08:08Z"},
|
|
108
|
+
":se": {"S": sensor_name},
|
|
109
|
+
":sh": {"S": ship_name},
|
|
110
|
+
":st": {"S": "2006-04-19T18:46:12.564Z"},
|
|
111
|
+
":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
|
|
112
|
+
":zp": {
|
|
113
|
+
"S": f"level_1/{ship_name}/{cruise_name}/{sensor_name}/DSJ0604-D20060419-T184612.zarr"
|
|
114
|
+
},
|
|
115
|
+
},
|
|
116
|
+
update_expression=(
|
|
117
|
+
"SET "
|
|
118
|
+
"#CH = :ch, "
|
|
119
|
+
"#ET = :et, "
|
|
120
|
+
"#ED = :ed, "
|
|
121
|
+
"#FR = :fr, "
|
|
122
|
+
"#MA = :ma, "
|
|
123
|
+
"#MI = :mi, "
|
|
124
|
+
"#ND = :nd, "
|
|
125
|
+
"#PS = :ps, "
|
|
126
|
+
"#PT = :pt, "
|
|
127
|
+
"#SE = :se, "
|
|
128
|
+
"#SH = :sh, "
|
|
129
|
+
"#ST = :st, "
|
|
130
|
+
"#ZB = :zb, "
|
|
131
|
+
"#ZP = :zp"
|
|
132
|
+
),
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
# ---Read From Table--- #
|
|
136
|
+
response = dynamo_db_manager.get_table_item(
|
|
137
|
+
table_name=table_name,
|
|
138
|
+
key={"FILE_NAME": "DSJ0604-D20060419-T184612.raw", "CRUISE_NAME": "DS0604"},
|
|
139
|
+
)
|
|
140
|
+
assert (
|
|
141
|
+
response["Item"]["PIPELINE_STATUS"]
|
|
142
|
+
== "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
# ---Change Items in Table (using src)--- #
|
|
146
|
+
dynamo_db_manager.update_item(
|
|
147
|
+
table_name=table_name,
|
|
148
|
+
key={
|
|
149
|
+
"FILE_NAME": {
|
|
150
|
+
"S": "DSJ0604-D20060419-T184612.raw",
|
|
151
|
+
},
|
|
152
|
+
"CRUISE_NAME": {
|
|
153
|
+
"S": "DS0604",
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
expression_attribute_names={
|
|
157
|
+
"#PS": "PIPELINE_STATUS",
|
|
158
|
+
"#PT": "PIPELINE_TIME",
|
|
159
|
+
},
|
|
160
|
+
expression_attribute_values={
|
|
161
|
+
":ps": {"S": PipelineStatus.SUCCESS_CRUISE_PROCESSOR.name},
|
|
162
|
+
":pt": {"S": "2023-10-02T09:09:09Z"},
|
|
163
|
+
},
|
|
164
|
+
update_expression=("SET " "#PS = :ps, " "#PT = :pt"),
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
# ---Read From Table Again--- #
|
|
168
|
+
response = dynamo_db_manager.get_table_item(
|
|
169
|
+
table_name=table_name,
|
|
170
|
+
key={"FILE_NAME": "DSJ0604-D20060419-T184612.raw", "CRUISE_NAME": "DS0604"},
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
assert response["Item"]["PIPELINE_STATUS"] == "SUCCESS_CRUISE_PROCESSOR"
|
|
174
|
+
|
|
175
|
+
# TODO: get the table as a dataframe
|
|
176
|
+
df = dynamo_db_manager.get_table_as_df(
|
|
177
|
+
table_name=table_name,
|
|
178
|
+
ship_name="David_Starr_Jordan",
|
|
179
|
+
cruise_name="DS0604",
|
|
180
|
+
sensor_name="EK60",
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
assert df.shape[1] == 16
|
|
184
|
+
# TODO: check fields
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
#######################################################
|
|
188
|
+
def update_dynamodb_item(
|
|
189
|
+
dynamo_db_manager,
|
|
190
|
+
table_name,
|
|
191
|
+
ship_name,
|
|
192
|
+
sensor_name,
|
|
193
|
+
cruise_name,
|
|
194
|
+
file_name,
|
|
195
|
+
test_channels,
|
|
196
|
+
test_frequencies,
|
|
197
|
+
):
|
|
198
|
+
dynamo_db_manager.update_item(
|
|
199
|
+
table_name=table_name,
|
|
200
|
+
key={
|
|
201
|
+
"FILE_NAME": {"S": file_name}, # Partition Key
|
|
202
|
+
"CRUISE_NAME": {"S": cruise_name}, # Sort Key
|
|
203
|
+
# TODO: should be FILE_NAME & SENSOR_NAME? so they are truly unique for when two sensors are processed within one cruise
|
|
204
|
+
},
|
|
205
|
+
expression_attribute_names={
|
|
206
|
+
"#CH": "CHANNELS",
|
|
207
|
+
"#ET": "END_TIME",
|
|
208
|
+
"#ED": "ERROR_DETAIL",
|
|
209
|
+
"#FR": "FREQUENCIES",
|
|
210
|
+
"#MA": "MAX_ECHO_RANGE",
|
|
211
|
+
"#MI": "MIN_ECHO_RANGE",
|
|
212
|
+
"#ND": "NUM_PING_TIME_DROPNA",
|
|
213
|
+
"#PS": "PIPELINE_STATUS", # testing this updated
|
|
214
|
+
"#PT": "PIPELINE_TIME", # testing this updated
|
|
215
|
+
"#SE": "SENSOR_NAME",
|
|
216
|
+
"#SH": "SHIP_NAME",
|
|
217
|
+
"#ST": "START_TIME",
|
|
218
|
+
"#ZB": "ZARR_BUCKET",
|
|
219
|
+
"#ZP": "ZARR_PATH",
|
|
220
|
+
},
|
|
221
|
+
expression_attribute_values={
|
|
222
|
+
":ch": {"L": [{"S": i} for i in test_channels]},
|
|
223
|
+
":et": {"S": "2006-04-19T20:07:33.623Z"},
|
|
224
|
+
":ed": {"S": ""},
|
|
225
|
+
":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
|
|
226
|
+
":ma": {"N": str(np.round(500.785201, 4))},
|
|
227
|
+
":mi": {"N": str(np.round(0.25001, 4))},
|
|
228
|
+
":nd": {"N": str(2428)},
|
|
229
|
+
":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
|
|
230
|
+
":pt": {"S": "2023-10-02T08:08:080Z"}, # datetime.now().isoformat(timespec="seconds") + "Z"
|
|
231
|
+
":se": {"S": sensor_name},
|
|
232
|
+
":sh": {"S": ship_name},
|
|
233
|
+
":st": {"S": "2006-04-19T18:46:12.564Z"},
|
|
234
|
+
":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
|
|
235
|
+
":zp": {
|
|
236
|
+
"S": f"level_1/{ship_name}/{cruise_name}/{sensor_name}/DSJ0604-D20060419-T184612.zarr"
|
|
237
|
+
},
|
|
238
|
+
},
|
|
239
|
+
update_expression=(
|
|
240
|
+
"SET "
|
|
241
|
+
"#CH = :ch, "
|
|
242
|
+
"#ET = :et, "
|
|
243
|
+
"#ED = :ed, "
|
|
244
|
+
"#FR = :fr, "
|
|
245
|
+
"#MA = :ma, "
|
|
246
|
+
"#MI = :mi, "
|
|
247
|
+
"#ND = :nd, "
|
|
248
|
+
"#PS = :ps, "
|
|
249
|
+
"#PT = :pt, "
|
|
250
|
+
"#SE = :se, "
|
|
251
|
+
"#SH = :sh, "
|
|
252
|
+
"#ST = :st, "
|
|
253
|
+
"#ZB = :zb, "
|
|
254
|
+
"#ZP = :zp"
|
|
255
|
+
),
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
@mock_aws
|
|
259
|
+
def test_delete_item(): # PASSING
|
|
260
|
+
# ---Initialize--- #
|
|
261
|
+
table_name = "test_table"
|
|
262
|
+
dynamo_db_manager = DynamoDBManager()
|
|
263
|
+
|
|
264
|
+
# ---Create Table--- #
|
|
265
|
+
dynamo_db_manager.create_water_column_sonar_table(table_name=table_name)
|
|
266
|
+
|
|
267
|
+
# ---Add Items to Table--- #
|
|
268
|
+
test_channels = [
|
|
269
|
+
"GPT 38 kHz 009072055a7f 2 ES38B",
|
|
270
|
+
"GPT 70 kHz 00907203400a 3 ES70-7C",
|
|
271
|
+
"GPT 120 kHz 009072034d52 1 ES120-7",
|
|
272
|
+
"GPT 200 kHz 0090720564e4 4 ES200-7C",
|
|
273
|
+
]
|
|
274
|
+
test_frequencies = [38_000, 70_000, 120_000, 200_000]
|
|
275
|
+
|
|
276
|
+
sensor_name = "EK60"
|
|
277
|
+
|
|
278
|
+
# First Cruise
|
|
279
|
+
ship_name1 = "David_Starr_Jordan"
|
|
280
|
+
cruise_name1 = "DS0604"
|
|
281
|
+
file_name1a = "DSJ0604-D20060406-T035914.raw"
|
|
282
|
+
file_name1b = "DSJ0604-D20060406-T050022.raw"
|
|
283
|
+
|
|
284
|
+
# Second Cruise
|
|
285
|
+
ship_name2 = "David_Starr_Jordan"
|
|
286
|
+
cruise_name2 = "DS0704"
|
|
287
|
+
file_name2a = "DSJ270307-D20070327-T225442.raw"
|
|
288
|
+
file_name2b = "DSJ270307-D20070328-T001728.raw"
|
|
289
|
+
|
|
290
|
+
update_dynamodb_item(
|
|
291
|
+
dynamo_db_manager=dynamo_db_manager,
|
|
292
|
+
table_name=table_name,
|
|
293
|
+
ship_name=ship_name1,
|
|
294
|
+
sensor_name=sensor_name,
|
|
295
|
+
cruise_name=cruise_name1,
|
|
296
|
+
file_name=file_name1a,
|
|
297
|
+
test_channels=test_channels,
|
|
298
|
+
test_frequencies=test_frequencies,
|
|
299
|
+
)
|
|
300
|
+
update_dynamodb_item(
|
|
301
|
+
dynamo_db_manager=dynamo_db_manager,
|
|
302
|
+
table_name=table_name,
|
|
303
|
+
ship_name=ship_name1,
|
|
304
|
+
sensor_name=sensor_name,
|
|
305
|
+
cruise_name=cruise_name1,
|
|
306
|
+
file_name=file_name1b,
|
|
307
|
+
test_channels=test_channels,
|
|
308
|
+
test_frequencies=test_frequencies,
|
|
309
|
+
)
|
|
310
|
+
update_dynamodb_item(
|
|
311
|
+
dynamo_db_manager=dynamo_db_manager,
|
|
312
|
+
table_name=table_name,
|
|
313
|
+
ship_name=ship_name2,
|
|
314
|
+
sensor_name=sensor_name,
|
|
315
|
+
cruise_name=cruise_name2,
|
|
316
|
+
file_name=file_name2a,
|
|
317
|
+
test_channels=test_channels,
|
|
318
|
+
test_frequencies=test_frequencies,
|
|
319
|
+
)
|
|
320
|
+
update_dynamodb_item(
|
|
321
|
+
dynamo_db_manager=dynamo_db_manager,
|
|
322
|
+
table_name=table_name,
|
|
323
|
+
ship_name=ship_name2,
|
|
324
|
+
sensor_name=sensor_name,
|
|
325
|
+
cruise_name=cruise_name2,
|
|
326
|
+
file_name=file_name2b,
|
|
327
|
+
test_channels=test_channels,
|
|
328
|
+
test_frequencies=test_frequencies,
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
# ---Read From Table, verify four items--- #
|
|
332
|
+
response = dynamo_db_manager.describe_table(table_name=table_name)
|
|
333
|
+
assert response['Table']['ItemCount'] == 4
|
|
334
|
+
df1 = dynamo_db_manager.get_table_as_df(
|
|
335
|
+
ship_name=ship_name1,
|
|
336
|
+
cruise_name=cruise_name1,
|
|
337
|
+
sensor_name="EK60",
|
|
338
|
+
table_name=table_name,
|
|
339
|
+
)
|
|
340
|
+
df2 = dynamo_db_manager.get_table_as_df(
|
|
341
|
+
ship_name=ship_name2,
|
|
342
|
+
cruise_name=cruise_name2,
|
|
343
|
+
sensor_name="EK60",
|
|
344
|
+
table_name=table_name,
|
|
345
|
+
)
|
|
346
|
+
assert df1.shape[0] == 2 # TODO: check actual filenames
|
|
347
|
+
assert df2.shape[0] == 2
|
|
348
|
+
|
|
349
|
+
# ---Delete cruise From Table Again--- #
|
|
350
|
+
dynamo_db_manager.delete_item(
|
|
351
|
+
table_name=table_name,
|
|
352
|
+
cruise_name=cruise_name2,
|
|
353
|
+
file_name=file_name2a,
|
|
354
|
+
)
|
|
355
|
+
dynamo_db_manager.delete_item(
|
|
356
|
+
table_name=table_name,
|
|
357
|
+
cruise_name=cruise_name2,
|
|
358
|
+
file_name=file_name2b,
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
# ---Get count of items again---#
|
|
362
|
+
response = dynamo_db_manager.describe_table(table_name=table_name)
|
|
363
|
+
assert response['Table']['ItemCount'] == 2
|
|
364
|
+
|
|
365
|
+
df4 = dynamo_db_manager.get_table_as_df(
|
|
366
|
+
table_name=table_name,
|
|
367
|
+
ship_name=ship_name2,
|
|
368
|
+
cruise_name=cruise_name2,
|
|
369
|
+
sensor_name="EK60",
|
|
370
|
+
)
|
|
371
|
+
assert df4.shape == (0, 0)
|
|
372
|
+
|
|
373
|
+
# assert df.shape[1] == 16
|
|
374
|
+
# TODO: check fields
|
|
375
|
+
|
|
376
|
+
#######################################################
|
|
@@ -2,7 +2,7 @@ import echopype as ep
|
|
|
2
2
|
import numpy as np
|
|
3
3
|
from dotenv import find_dotenv, load_dotenv
|
|
4
4
|
|
|
5
|
-
from water_column_sonar_processing.geometry
|
|
5
|
+
from water_column_sonar_processing.geometry import GeometryManager
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
#######################################################
|
|
@@ -18,10 +18,13 @@ def teardown_module():
|
|
|
18
18
|
|
|
19
19
|
#######################################################
|
|
20
20
|
|
|
21
|
-
|
|
22
21
|
# @mock_s3
|
|
23
22
|
def test_geometry_manager(tmp_path):
|
|
24
|
-
|
|
23
|
+
"""
|
|
24
|
+
# TODO: need to find a small file to test with, put into test bucket, read from there into
|
|
25
|
+
"""
|
|
26
|
+
input_bucket_name = "noaa-wcsd-pds"
|
|
27
|
+
output_bucket_name = "noaa-wcsd-zarr-pds"
|
|
25
28
|
# file_name = 'D20070719-T232718.raw' # too big
|
|
26
29
|
# file_name = 'D20070720-T224031.raw' # has >4 points in dataset
|
|
27
30
|
file_name = "D20070724-T042400.raw"
|
|
@@ -30,7 +33,7 @@ def test_geometry_manager(tmp_path):
|
|
|
30
33
|
cruise_name = "HB0706"
|
|
31
34
|
sensor_name = "EK60"
|
|
32
35
|
|
|
33
|
-
s3_path = f"s3://{
|
|
36
|
+
s3_path = f"s3://{input_bucket_name}/data/raw/{ship_name}/{cruise_name}/{sensor_name}/{file_name}"
|
|
34
37
|
# s3_path = f"r2d2-testing-level-2-data/level_2/Henry_B._Bigelow/HB0707/EK60/HB0707.model"
|
|
35
38
|
|
|
36
39
|
print(s3_path)
|
|
@@ -44,13 +47,14 @@ def test_geometry_manager(tmp_path):
|
|
|
44
47
|
|
|
45
48
|
geometry_manager = GeometryManager()
|
|
46
49
|
|
|
47
|
-
time, lat, lon = geometry_manager.read_echodata_gps_data(
|
|
50
|
+
time, lat, lon = geometry_manager.read_echodata_gps_data( # gps_df.index.values, gps_df.latitude.values, gps_df.longitude.values
|
|
48
51
|
echodata=echodata,
|
|
52
|
+
output_bucket_name=output_bucket_name,
|
|
49
53
|
ship_name=ship_name,
|
|
50
54
|
cruise_name=cruise_name,
|
|
51
55
|
sensor_name=sensor_name,
|
|
52
56
|
file_name=file_name,
|
|
53
|
-
write_geojson=False
|
|
57
|
+
write_geojson=False
|
|
54
58
|
)
|
|
55
59
|
# NOTE CHECK FOR NULL ISLAND ON RETURN
|
|
56
60
|
null_island_indices = list(
|