water-column-sonar-processing 0.0.7__tar.gz → 0.0.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. {water_column_sonar_processing-0.0.7/src/water_column_sonar_processing.egg-info → water_column_sonar_processing-0.0.9}/PKG-INFO +20 -10
  2. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/README.md +10 -1
  3. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/pyproject.toml +19 -9
  4. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/requirements.txt +12 -11
  5. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_create_empty_zarr_store.py +10 -6
  6. water_column_sonar_processing-0.0.9/tests/test_dynamodb_manager.py +376 -0
  7. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_geometry_manager.py +10 -6
  8. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_index.py +34 -9
  9. water_column_sonar_processing-0.0.9/tests/test_pmtile_generation.py +266 -0
  10. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_process.py +5 -21
  11. water_column_sonar_processing-0.0.9/tests/test_raw_to_zarr.py +146 -0
  12. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_resample_regrid.py +2 -2
  13. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_s3_manager.py +27 -7
  14. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_s3fs_manager.py +4 -17
  15. water_column_sonar_processing-0.0.9/tests/test_s3fs_with_moto.py +78 -0
  16. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_sns_sqs_manager.py +1 -2
  17. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_zarr_manager.py +31 -23
  18. water_column_sonar_processing-0.0.9/water_column_sonar_processing/__init__.py +13 -0
  19. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/__init__.py +2 -2
  20. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/dynamodb_manager.py +149 -43
  21. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/s3_manager.py +71 -37
  22. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/cruise/create_empty_zarr_store.py +6 -4
  23. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/cruise/resample_regrid.py +3 -3
  24. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/geometry/geometry_manager.py +21 -6
  25. water_column_sonar_processing-0.0.9/water_column_sonar_processing/geometry/pmtile_generation.py +262 -0
  26. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/index/index_manager.py +25 -13
  27. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/model/zarr_manager.py +27 -25
  28. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/process.py +4 -4
  29. water_column_sonar_processing-0.0.9/water_column_sonar_processing/processing/__init__.py +4 -0
  30. water_column_sonar_processing-0.0.9/water_column_sonar_processing/processing/cruise_sampler.py +342 -0
  31. water_column_sonar_processing-0.0.9/water_column_sonar_processing/processing/raw_to_zarr.py +349 -0
  32. water_column_sonar_processing-0.0.9/water_column_sonar_processing/utility/__init__.py +6 -0
  33. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/utility/cleaner.py +1 -0
  34. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/utility/constants.py +6 -2
  35. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9/water_column_sonar_processing.egg-info}/PKG-INFO +20 -10
  36. water_column_sonar_processing-0.0.9/water_column_sonar_processing.egg-info/SOURCES.txt +50 -0
  37. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing.egg-info/requires.txt +10 -9
  38. water_column_sonar_processing-0.0.7/src/water_column_sonar_processing/__init__.py +0 -16
  39. water_column_sonar_processing-0.0.7/src/water_column_sonar_processing/geometry/pmtile_generation.py +0 -75
  40. water_column_sonar_processing-0.0.7/src/water_column_sonar_processing/utility/__init__.py +0 -6
  41. water_column_sonar_processing-0.0.7/src/water_column_sonar_processing.egg-info/SOURCES.txt +0 -45
  42. water_column_sonar_processing-0.0.7/tests/test_dynamodb_manager.py +0 -204
  43. water_column_sonar_processing-0.0.7/tests/test_s3fs_with_moto.py +0 -31
  44. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/LICENSE +0 -0
  45. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/setup.cfg +0 -0
  46. {water_column_sonar_processing-0.0.7 → water_column_sonar_processing-0.0.9}/tests/test_geometry_simplification.py +0 -0
  47. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/s3fs_manager.py +0 -0
  48. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/sns_manager.py +0 -0
  49. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/aws/sqs_manager.py +0 -0
  50. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/cruise/__init__.py +0 -0
  51. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/geometry/__init__.py +0 -0
  52. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/geometry/geometry_simplification.py +0 -0
  53. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/index/__init__.py +0 -0
  54. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/model/__init__.py +0 -0
  55. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/utility/pipeline_status.py +0 -0
  56. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing/utility/timestamp.py +0 -0
  57. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing.egg-info/dependency_links.txt +0 -0
  58. {water_column_sonar_processing-0.0.7/src → water_column_sonar_processing-0.0.9}/water_column_sonar_processing.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: water_column_sonar_processing
3
- Version: 0.0.7
3
+ Version: 0.0.9
4
4
  Summary: A processing tool for water column sonar data.
5
5
  Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
6
6
  Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
@@ -11,26 +11,27 @@ Classifier: Operating System :: OS Independent
11
11
  Requires-Python: >=3.10
12
12
  Description-Content-Type: text/markdown
13
13
  License-File: LICENSE
14
- Requires-Dist: aiobotocore~=2.9.0
15
- Requires-Dist: aiohttp~=3.10.10
16
- Requires-Dist: boto3==1.33.13
17
- Requires-Dist: botocore~=1.33.13
14
+ Requires-Dist: aiobotocore==2.15.2
15
+ Requires-Dist: boto3==1.35.36
16
+ Requires-Dist: botocore==1.35.36
18
17
  Requires-Dist: echopype==0.9.0
19
18
  Requires-Dist: fiona==1.10.1
20
19
  Requires-Dist: geopandas==1.0.1
21
- Requires-Dist: mock~=5.1.0
22
- Requires-Dist: moto~=5.0.18
20
+ Requires-Dist: mock==5.1.0
21
+ Requires-Dist: moto[all]==5.0.21
22
+ Requires-Dist: moto[server]==5.0.21
23
23
  Requires-Dist: numcodecs==0.13.1
24
24
  Requires-Dist: numpy==1.26.4
25
25
  Requires-Dist: pandas==2.2.3
26
- Requires-Dist: pytest~=8.3.3
26
+ Requires-Dist: pyarrow==18.1.0
27
27
  Requires-Dist: python-dotenv==1.0.0
28
28
  Requires-Dist: requests==2.32.3
29
- Requires-Dist: s3fs==2024.10.0
29
+ Requires-Dist: s3fs==2023.12.1
30
30
  Requires-Dist: scipy==1.14.1
31
+ Requires-Dist: setuptools==75.6.0
31
32
  Requires-Dist: shapely==2.0.3
32
33
  Requires-Dist: typing-extensions==4.10.0
33
- Requires-Dist: xarray==2022.12.0
34
+ Requires-Dist: xarray==2024.10.0
34
35
  Requires-Dist: zarr==2.18.3
35
36
 
36
37
  # Water Column Sonar Processing
@@ -85,6 +86,8 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
85
86
  ```commandline
86
87
  pytest --disable-warnings
87
88
  ```
89
+ or
90
+ > pytest --cache-clear --cov=src tests/ --cov-report=xml
88
91
 
89
92
  # Instructions
90
93
  Following this tutorial:
@@ -122,3 +125,10 @@ https://plugins.jetbrains.com/plugin/20574-ruff
122
125
 
123
126
  # Colab Test
124
127
  https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
128
+
129
+ # Test Coverage
130
+ 20241124
131
+ 8 failed, 32 passed, 3 skipped, 1 warning in 6.92s
132
+ 20241125
133
+ 5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
134
+ 3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
@@ -50,6 +50,8 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
50
50
  ```commandline
51
51
  pytest --disable-warnings
52
52
  ```
53
+ or
54
+ > pytest --cache-clear --cov=src tests/ --cov-report=xml
53
55
 
54
56
  # Instructions
55
57
  Following this tutorial:
@@ -86,4 +88,11 @@ Ruff
86
88
  https://plugins.jetbrains.com/plugin/20574-ruff
87
89
 
88
90
  # Colab Test
89
- https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
91
+ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
92
+
93
+ # Test Coverage
94
+ 20241124
95
+ 8 failed, 32 passed, 3 skipped, 1 warning in 6.92s
96
+ 20241125
97
+ 5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
98
+ 3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
7
7
 
8
8
  [project]
9
9
  name = "water_column_sonar_processing"
10
- version = "0.0.7"
10
+ version = "0.0.9"
11
11
  authors = [
12
12
  { name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
13
13
  ]
@@ -25,19 +25,29 @@ dynamic = ["dependencies"]
25
25
  Homepage = "https://github.com/CI-CMG/water-column-sonar-processing"
26
26
  Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
27
27
 
28
- #[pytest]
28
+ #[tool.pytest.ini_options]
29
+ #filterwarnings = [
30
+ # "error",
31
+ # "ignore::UserWarning",
32
+ # # note the use of single quote below to denote "raw" strings in TOML
33
+ # 'ignore:function ham\(\) is deprecated:DeprecationWarning',
34
+ #]
29
35
  #pythonpath = "src"
30
36
  #testpaths = "tests"
37
+ #[pytest]
38
+ #addopts = [
39
+ # "-p no:warnings"
40
+ #]
31
41
 
32
- [tool.pytest.ini_options]
33
- minversion = "6.0"
34
- #addopts = "-ra -q"
35
- testpaths = [
36
- "tests",
37
- ]
42
+ #[tool.pytest.ini_options]
43
+ #minversion = "6.0"
44
+ ##addopts = "-ra -q"
45
+ #testpaths = [
46
+ # "tests",
47
+ #]
38
48
 
39
49
  [tool.setuptools.dynamic]
40
50
  dependencies = {file = ["requirements.txt"]}
41
- optional-dependencies = {dev = { file = ["requirements-dev.txt"] }}
51
+ optional-dependencies = {dev = { file = ["requirements_dev.txt"] }}
42
52
 
43
53
  # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
@@ -2,26 +2,27 @@
2
2
  # defined for Python 3.12
3
3
  # Note: be careful with conversions for pandas >=2.0.0, timestamps will have a lot of problems
4
4
 
5
- aiobotocore~=2.9.0
6
- aiohttp~=3.10.10
7
- boto3==1.33.13
8
- botocore~=1.33.13
5
+ aiobotocore==2.15.2
6
+ boto3==1.35.36
7
+ botocore==1.35.36
9
8
  echopype==0.9.0
10
9
  fiona==1.10.1
10
+ # Alternative to geopandas: pyogrio
11
11
  geopandas==1.0.1
12
- mock~=5.1.0
13
- moto~=5.0.18
12
+ mock==5.1.0
13
+ moto[all]==5.0.21
14
+ moto[server]==5.0.21
14
15
  numcodecs==0.13.1
15
16
  numpy==1.26.4
16
17
  pandas==2.2.3
17
- pytest~=8.3.3
18
+ pyarrow==18.1.0
18
19
  python-dotenv==1.0.0
19
20
  requests==2.32.3
20
- #s3fs==2023.12.2
21
- s3fs==2024.10.0
21
+ #s3fs==2024.10.0 # this version creates problems
22
+ s3fs==2023.12.1
22
23
  scipy==1.14.1
24
+ setuptools==75.6.0
23
25
  shapely==2.0.3
24
26
  typing-extensions==4.10.0
25
- xarray==2022.12.0
26
- #zarr==2.16.1
27
+ xarray==2024.10.0
27
28
  zarr==2.18.3
@@ -4,9 +4,11 @@ from dotenv import find_dotenv, load_dotenv
4
4
  from moto import mock_aws
5
5
 
6
6
  from water_column_sonar_processing.aws import DynamoDBManager, S3Manager
7
- from water_column_sonar_processing.cruise.create_empty_zarr_store import (
8
- CreateEmptyZarrStore,
9
- )
7
+ from water_column_sonar_processing.cruise import CreateEmptyZarrStore
8
+
9
+
10
+ # from water_column_sonar_processing import DynamoDBManager, S3Manager
11
+ # from water_column_sonar_processing import CreateEmptyZarrStore
10
12
 
11
13
 
12
14
  # TEMPDIR = "/tmp"
@@ -25,7 +27,7 @@ def teardown_module():
25
27
 
26
28
  #######################################################
27
29
  @mock_aws
28
- def test_create_empty_zarr_store():
30
+ def test_create_empty_zarr_store(): # PASSING, needs modification at end
29
31
  dynamo_db_manager = DynamoDBManager()
30
32
  s3_manager = S3Manager()
31
33
 
@@ -39,9 +41,11 @@ def test_create_empty_zarr_store():
39
41
  # [0] create bucket with test files
40
42
  input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
41
43
  output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
44
+
42
45
  s3_manager.create_bucket(bucket_name=input_bucket_name)
43
46
  s3_manager.create_bucket(bucket_name=output_bucket_name)
44
47
  print(s3_manager.list_buckets())
48
+
45
49
  # TODO: put at the path where it needs to be deleted
46
50
  s3_manager.put(bucket_name=input_bucket_name, key="the_key", body="the_body")
47
51
 
@@ -80,7 +84,7 @@ def test_create_empty_zarr_store():
80
84
  249.792,
81
85
  249.792,
82
86
  249.792,
83
- 999.744,
87
+ 999.744, # note: different depth
84
88
  249.792,
85
89
  249.792,
86
90
  249.792,
@@ -211,7 +215,7 @@ def test_create_empty_zarr_store():
211
215
  tempdir="/tmp",
212
216
  )
213
217
 
214
- assert os.path.exists(f"/tmp/{cruise_name}.zarr")
218
+ assert os.path.exists(f"/tmp/{cruise_name}.zarr") # TODO: create better tmp directory for testing
215
219
  # TODO: should actually assert in the bucket
216
220
  # mount and verify:
217
221
  # shape
@@ -0,0 +1,376 @@
1
+ import numpy as np
2
+ from dotenv import find_dotenv, load_dotenv
3
+ from moto import mock_aws
4
+
5
+ from water_column_sonar_processing.aws import DynamoDBManager
6
+ from water_column_sonar_processing.utility import PipelineStatus
7
+
8
+
9
+ #######################################################
10
+ def setup_module():
11
+ print("setup")
12
+ env_file = find_dotenv(".env-test")
13
+ load_dotenv(dotenv_path=env_file, override=True)
14
+
15
+
16
+ def teardown_module():
17
+ print("teardown")
18
+
19
+
20
+ #######################################################
21
+ def test_serializer_deserializer():
22
+ # ---Initialize--- #
23
+ low_level_data = {
24
+ "ACTIVE": {"BOOL": True},
25
+ "CRC": {"N": "-1600155180"},
26
+ "ID": {"S": "bewfv43843b"},
27
+ "params": {"M": {"customer": {"S": "TEST"}, "index": {"N": "1"}}},
28
+ "THIS_STATUS": {"N": "10"},
29
+ "TYPE": {"N": "22"},
30
+ }
31
+
32
+ # Lazy-eval the dynamodb attribute (boto3 is dynamic!)
33
+ dynamo_db_manager = DynamoDBManager()
34
+
35
+ # To go from low-level format to python
36
+ deserializer = dynamo_db_manager.type_deserializer
37
+ python_data = {k: deserializer.deserialize(v) for k, v in low_level_data.items()}
38
+
39
+ assert python_data["ACTIVE"]
40
+ assert python_data["CRC"] == -1600155180
41
+ assert python_data["ID"]
42
+ assert python_data["params"] == {"customer": "TEST", "index": 1}
43
+ assert python_data["THIS_STATUS"] == 10
44
+ assert python_data["TYPE"] == 22
45
+
46
+ # To go from python to low-level format
47
+ serializer = dynamo_db_manager.type_serializer
48
+ low_level_copy = {k: serializer.serialize(v) for k, v in python_data.items()}
49
+
50
+ assert low_level_data == low_level_copy
51
+
52
+
53
+ #######################################################
54
+ @mock_aws
55
+ def test_dynamodb_manager(): # PASSING
56
+ # ---Initialize--- #
57
+ table_name = "test_table"
58
+ dynamo_db_manager = DynamoDBManager()
59
+
60
+ # ---Create Table--- #
61
+ dynamo_db_manager.create_water_column_sonar_table(table_name=table_name)
62
+
63
+ # ---Add Items to Table--- #
64
+ test_channels = [
65
+ "GPT 38 kHz 009072055a7f 2 ES38B",
66
+ "GPT 70 kHz 00907203400a 3 ES70-7C",
67
+ "GPT 120 kHz 009072034d52 1 ES120-7",
68
+ "GPT 200 kHz 0090720564e4 4 ES200-7C",
69
+ ]
70
+ test_frequencies = [38_000, 70_000, 120_000, 200_000]
71
+ ship_name = "David_Starr_Jordan"
72
+ cruise_name = "DS0604"
73
+ sensor_name = "EK60"
74
+ file_name = "DSJ0604-D20060419-T184612.raw"
75
+
76
+ dynamo_db_manager.update_item(
77
+ table_name=table_name,
78
+ key={
79
+ "FILE_NAME": {"S": file_name}, # Partition Key
80
+ "CRUISE_NAME": {"S": cruise_name}, # Sort Key
81
+ },
82
+ expression_attribute_names={
83
+ "#CH": "CHANNELS",
84
+ "#ET": "END_TIME",
85
+ "#ED": "ERROR_DETAIL",
86
+ "#FR": "FREQUENCIES",
87
+ "#MA": "MAX_ECHO_RANGE",
88
+ "#MI": "MIN_ECHO_RANGE",
89
+ "#ND": "NUM_PING_TIME_DROPNA",
90
+ "#PS": "PIPELINE_STATUS", # testing this updated
91
+ "#PT": "PIPELINE_TIME", # testing this updated
92
+ "#SE": "SENSOR_NAME",
93
+ "#SH": "SHIP_NAME",
94
+ "#ST": "START_TIME",
95
+ "#ZB": "ZARR_BUCKET",
96
+ "#ZP": "ZARR_PATH",
97
+ },
98
+ expression_attribute_values={
99
+ ":ch": {"L": [{"S": i} for i in test_channels]},
100
+ ":et": {"S": "2006-04-19T20:07:33.623Z"},
101
+ ":ed": {"S": ""},
102
+ ":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
103
+ ":ma": {"N": str(np.round(500.785201, 4))},
104
+ ":mi": {"N": str(np.round(0.25001, 4))},
105
+ ":nd": {"N": str(2428)},
106
+ ":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
107
+ ":pt": {"S": "2023-10-02T08:08:08Z"},
108
+ ":se": {"S": sensor_name},
109
+ ":sh": {"S": ship_name},
110
+ ":st": {"S": "2006-04-19T18:46:12.564Z"},
111
+ ":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
112
+ ":zp": {
113
+ "S": f"level_1/{ship_name}/{cruise_name}/{sensor_name}/DSJ0604-D20060419-T184612.zarr"
114
+ },
115
+ },
116
+ update_expression=(
117
+ "SET "
118
+ "#CH = :ch, "
119
+ "#ET = :et, "
120
+ "#ED = :ed, "
121
+ "#FR = :fr, "
122
+ "#MA = :ma, "
123
+ "#MI = :mi, "
124
+ "#ND = :nd, "
125
+ "#PS = :ps, "
126
+ "#PT = :pt, "
127
+ "#SE = :se, "
128
+ "#SH = :sh, "
129
+ "#ST = :st, "
130
+ "#ZB = :zb, "
131
+ "#ZP = :zp"
132
+ ),
133
+ )
134
+
135
+ # ---Read From Table--- #
136
+ response = dynamo_db_manager.get_table_item(
137
+ table_name=table_name,
138
+ key={"FILE_NAME": "DSJ0604-D20060419-T184612.raw", "CRUISE_NAME": "DS0604"},
139
+ )
140
+ assert (
141
+ response["Item"]["PIPELINE_STATUS"]
142
+ == "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"
143
+ )
144
+
145
+ # ---Change Items in Table (using src)--- #
146
+ dynamo_db_manager.update_item(
147
+ table_name=table_name,
148
+ key={
149
+ "FILE_NAME": {
150
+ "S": "DSJ0604-D20060419-T184612.raw",
151
+ },
152
+ "CRUISE_NAME": {
153
+ "S": "DS0604",
154
+ },
155
+ },
156
+ expression_attribute_names={
157
+ "#PS": "PIPELINE_STATUS",
158
+ "#PT": "PIPELINE_TIME",
159
+ },
160
+ expression_attribute_values={
161
+ ":ps": {"S": PipelineStatus.SUCCESS_CRUISE_PROCESSOR.name},
162
+ ":pt": {"S": "2023-10-02T09:09:09Z"},
163
+ },
164
+ update_expression=("SET " "#PS = :ps, " "#PT = :pt"),
165
+ )
166
+
167
+ # ---Read From Table Again--- #
168
+ response = dynamo_db_manager.get_table_item(
169
+ table_name=table_name,
170
+ key={"FILE_NAME": "DSJ0604-D20060419-T184612.raw", "CRUISE_NAME": "DS0604"},
171
+ )
172
+
173
+ assert response["Item"]["PIPELINE_STATUS"] == "SUCCESS_CRUISE_PROCESSOR"
174
+
175
+ # TODO: get the table as a dataframe
176
+ df = dynamo_db_manager.get_table_as_df(
177
+ table_name=table_name,
178
+ ship_name="David_Starr_Jordan",
179
+ cruise_name="DS0604",
180
+ sensor_name="EK60",
181
+ )
182
+
183
+ assert df.shape[1] == 16
184
+ # TODO: check fields
185
+
186
+
187
+ #######################################################
188
+ def update_dynamodb_item(
189
+ dynamo_db_manager,
190
+ table_name,
191
+ ship_name,
192
+ sensor_name,
193
+ cruise_name,
194
+ file_name,
195
+ test_channels,
196
+ test_frequencies,
197
+ ):
198
+ dynamo_db_manager.update_item(
199
+ table_name=table_name,
200
+ key={
201
+ "FILE_NAME": {"S": file_name}, # Partition Key
202
+ "CRUISE_NAME": {"S": cruise_name}, # Sort Key
203
+ # TODO: should be FILE_NAME & SENSOR_NAME? so they are truly unique for when two sensors are processed within one cruise
204
+ },
205
+ expression_attribute_names={
206
+ "#CH": "CHANNELS",
207
+ "#ET": "END_TIME",
208
+ "#ED": "ERROR_DETAIL",
209
+ "#FR": "FREQUENCIES",
210
+ "#MA": "MAX_ECHO_RANGE",
211
+ "#MI": "MIN_ECHO_RANGE",
212
+ "#ND": "NUM_PING_TIME_DROPNA",
213
+ "#PS": "PIPELINE_STATUS", # testing this updated
214
+ "#PT": "PIPELINE_TIME", # testing this updated
215
+ "#SE": "SENSOR_NAME",
216
+ "#SH": "SHIP_NAME",
217
+ "#ST": "START_TIME",
218
+ "#ZB": "ZARR_BUCKET",
219
+ "#ZP": "ZARR_PATH",
220
+ },
221
+ expression_attribute_values={
222
+ ":ch": {"L": [{"S": i} for i in test_channels]},
223
+ ":et": {"S": "2006-04-19T20:07:33.623Z"},
224
+ ":ed": {"S": ""},
225
+ ":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
226
+ ":ma": {"N": str(np.round(500.785201, 4))},
227
+ ":mi": {"N": str(np.round(0.25001, 4))},
228
+ ":nd": {"N": str(2428)},
229
+ ":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
230
+ ":pt": {"S": "2023-10-02T08:08:080Z"}, # datetime.now().isoformat(timespec="seconds") + "Z"
231
+ ":se": {"S": sensor_name},
232
+ ":sh": {"S": ship_name},
233
+ ":st": {"S": "2006-04-19T18:46:12.564Z"},
234
+ ":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
235
+ ":zp": {
236
+ "S": f"level_1/{ship_name}/{cruise_name}/{sensor_name}/DSJ0604-D20060419-T184612.zarr"
237
+ },
238
+ },
239
+ update_expression=(
240
+ "SET "
241
+ "#CH = :ch, "
242
+ "#ET = :et, "
243
+ "#ED = :ed, "
244
+ "#FR = :fr, "
245
+ "#MA = :ma, "
246
+ "#MI = :mi, "
247
+ "#ND = :nd, "
248
+ "#PS = :ps, "
249
+ "#PT = :pt, "
250
+ "#SE = :se, "
251
+ "#SH = :sh, "
252
+ "#ST = :st, "
253
+ "#ZB = :zb, "
254
+ "#ZP = :zp"
255
+ ),
256
+ )
257
+
258
+ @mock_aws
259
+ def test_delete_item(): # PASSING
260
+ # ---Initialize--- #
261
+ table_name = "test_table"
262
+ dynamo_db_manager = DynamoDBManager()
263
+
264
+ # ---Create Table--- #
265
+ dynamo_db_manager.create_water_column_sonar_table(table_name=table_name)
266
+
267
+ # ---Add Items to Table--- #
268
+ test_channels = [
269
+ "GPT 38 kHz 009072055a7f 2 ES38B",
270
+ "GPT 70 kHz 00907203400a 3 ES70-7C",
271
+ "GPT 120 kHz 009072034d52 1 ES120-7",
272
+ "GPT 200 kHz 0090720564e4 4 ES200-7C",
273
+ ]
274
+ test_frequencies = [38_000, 70_000, 120_000, 200_000]
275
+
276
+ sensor_name = "EK60"
277
+
278
+ # First Cruise
279
+ ship_name1 = "David_Starr_Jordan"
280
+ cruise_name1 = "DS0604"
281
+ file_name1a = "DSJ0604-D20060406-T035914.raw"
282
+ file_name1b = "DSJ0604-D20060406-T050022.raw"
283
+
284
+ # Second Cruise
285
+ ship_name2 = "David_Starr_Jordan"
286
+ cruise_name2 = "DS0704"
287
+ file_name2a = "DSJ270307-D20070327-T225442.raw"
288
+ file_name2b = "DSJ270307-D20070328-T001728.raw"
289
+
290
+ update_dynamodb_item(
291
+ dynamo_db_manager=dynamo_db_manager,
292
+ table_name=table_name,
293
+ ship_name=ship_name1,
294
+ sensor_name=sensor_name,
295
+ cruise_name=cruise_name1,
296
+ file_name=file_name1a,
297
+ test_channels=test_channels,
298
+ test_frequencies=test_frequencies,
299
+ )
300
+ update_dynamodb_item(
301
+ dynamo_db_manager=dynamo_db_manager,
302
+ table_name=table_name,
303
+ ship_name=ship_name1,
304
+ sensor_name=sensor_name,
305
+ cruise_name=cruise_name1,
306
+ file_name=file_name1b,
307
+ test_channels=test_channels,
308
+ test_frequencies=test_frequencies,
309
+ )
310
+ update_dynamodb_item(
311
+ dynamo_db_manager=dynamo_db_manager,
312
+ table_name=table_name,
313
+ ship_name=ship_name2,
314
+ sensor_name=sensor_name,
315
+ cruise_name=cruise_name2,
316
+ file_name=file_name2a,
317
+ test_channels=test_channels,
318
+ test_frequencies=test_frequencies,
319
+ )
320
+ update_dynamodb_item(
321
+ dynamo_db_manager=dynamo_db_manager,
322
+ table_name=table_name,
323
+ ship_name=ship_name2,
324
+ sensor_name=sensor_name,
325
+ cruise_name=cruise_name2,
326
+ file_name=file_name2b,
327
+ test_channels=test_channels,
328
+ test_frequencies=test_frequencies,
329
+ )
330
+
331
+ # ---Read From Table, verify four items--- #
332
+ response = dynamo_db_manager.describe_table(table_name=table_name)
333
+ assert response['Table']['ItemCount'] == 4
334
+ df1 = dynamo_db_manager.get_table_as_df(
335
+ ship_name=ship_name1,
336
+ cruise_name=cruise_name1,
337
+ sensor_name="EK60",
338
+ table_name=table_name,
339
+ )
340
+ df2 = dynamo_db_manager.get_table_as_df(
341
+ ship_name=ship_name2,
342
+ cruise_name=cruise_name2,
343
+ sensor_name="EK60",
344
+ table_name=table_name,
345
+ )
346
+ assert df1.shape[0] == 2 # TODO: check actual filenames
347
+ assert df2.shape[0] == 2
348
+
349
+ # ---Delete cruise From Table Again--- #
350
+ dynamo_db_manager.delete_item(
351
+ table_name=table_name,
352
+ cruise_name=cruise_name2,
353
+ file_name=file_name2a,
354
+ )
355
+ dynamo_db_manager.delete_item(
356
+ table_name=table_name,
357
+ cruise_name=cruise_name2,
358
+ file_name=file_name2b,
359
+ )
360
+
361
+ # ---Get count of items again---#
362
+ response = dynamo_db_manager.describe_table(table_name=table_name)
363
+ assert response['Table']['ItemCount'] == 2
364
+
365
+ df4 = dynamo_db_manager.get_table_as_df(
366
+ table_name=table_name,
367
+ ship_name=ship_name2,
368
+ cruise_name=cruise_name2,
369
+ sensor_name="EK60",
370
+ )
371
+ assert df4.shape == (0, 0)
372
+
373
+ # assert df.shape[1] == 16
374
+ # TODO: check fields
375
+
376
+ #######################################################
@@ -2,7 +2,7 @@ import echopype as ep
2
2
  import numpy as np
3
3
  from dotenv import find_dotenv, load_dotenv
4
4
 
5
- from water_column_sonar_processing.geometry.geometry_manager import GeometryManager
5
+ from water_column_sonar_processing.geometry import GeometryManager
6
6
 
7
7
 
8
8
  #######################################################
@@ -18,10 +18,13 @@ def teardown_module():
18
18
 
19
19
  #######################################################
20
20
 
21
-
22
21
  # @mock_s3
23
22
  def test_geometry_manager(tmp_path):
24
- bucket_name = "noaa-wcsd-pds"
23
+ """
24
+ # TODO: need to find a small file to test with, put into test bucket, read from there into
25
+ """
26
+ input_bucket_name = "noaa-wcsd-pds"
27
+ output_bucket_name = "noaa-wcsd-zarr-pds"
25
28
  # file_name = 'D20070719-T232718.raw' # too big
26
29
  # file_name = 'D20070720-T224031.raw' # has >4 points in dataset
27
30
  file_name = "D20070724-T042400.raw"
@@ -30,7 +33,7 @@ def test_geometry_manager(tmp_path):
30
33
  cruise_name = "HB0706"
31
34
  sensor_name = "EK60"
32
35
 
33
- s3_path = f"s3://{bucket_name}/data/raw/{ship_name}/{cruise_name}/{sensor_name}/{file_name}"
36
+ s3_path = f"s3://{input_bucket_name}/data/raw/{ship_name}/{cruise_name}/{sensor_name}/{file_name}"
34
37
  # s3_path = f"r2d2-testing-level-2-data/level_2/Henry_B._Bigelow/HB0707/EK60/HB0707.model"
35
38
 
36
39
  print(s3_path)
@@ -44,13 +47,14 @@ def test_geometry_manager(tmp_path):
44
47
 
45
48
  geometry_manager = GeometryManager()
46
49
 
47
- time, lat, lon = geometry_manager.read_echodata_gps_data(
50
+ time, lat, lon = geometry_manager.read_echodata_gps_data( # gps_df.index.values, gps_df.latitude.values, gps_df.longitude.values
48
51
  echodata=echodata,
52
+ output_bucket_name=output_bucket_name,
49
53
  ship_name=ship_name,
50
54
  cruise_name=cruise_name,
51
55
  sensor_name=sensor_name,
52
56
  file_name=file_name,
53
- write_geojson=False,
57
+ write_geojson=False
54
58
  )
55
59
  # NOTE CHECK FOR NULL ISLAND ON RETURN
56
60
  null_island_indices = list(