water-column-sonar-processing 25.1.2__tar.gz → 25.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of water-column-sonar-processing might be problematic. Click here for more details.
- water_column_sonar_processing-25.1.4/.github/workflows/test_action.yaml +46 -0
- water_column_sonar_processing-25.1.4/.python-version +1 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/PKG-INFO +24 -8
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/README.md +9 -6
- water_column_sonar_processing-25.1.4/pyproject.toml +73 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/aws/dynamodb_manager.py +63 -12
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/aws/s3fs_manager.py +1 -1
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/cruise/resample_regrid.py +20 -20
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/model/zarr_manager.py +13 -9
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/utility/constants.py +1 -1
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing.egg-info/PKG-INFO +24 -8
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing.egg-info/SOURCES.txt +0 -4
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing.egg-info/requires.txt +15 -0
- water_column_sonar_processing-25.1.2/.github/workflows/test_action.yaml +0 -24
- water_column_sonar_processing-25.1.2/.python-version +0 -2
- water_column_sonar_processing-25.1.2/pyproject.toml +0 -43
- water_column_sonar_processing-25.1.2/pytest.ini +0 -13
- water_column_sonar_processing-25.1.2/requirements.txt +0 -32
- water_column_sonar_processing-25.1.2/requirements_dev.txt +0 -14
- water_column_sonar_processing-25.1.2/tests/test_process.py +0 -472
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/.env-test +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/.gitignore +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/.pre-commit-config.yaml +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/LICENSE +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/open-science-data-federation/ml/autoencoder_example.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/open-science-data-federation/osdf_examples/foo.ipynb +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/open-science-data-federation/osdf_examples/sonar_ai.ipynb +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/setup.cfg +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/tests/conftest.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/tests/test_resources/index/calibrated_cruises.csv +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/tests/test_resources/raw_to_zarr/D20070724-T042400.bot +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/tests/test_resources/raw_to_zarr/D20070724-T042400.idx +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/tests/test_resources/raw_to_zarr/D20070724-T042400.raw +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/__init__.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/aws/__init__.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/aws/s3_manager.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/aws/sns_manager.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/aws/sqs_manager.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/cruise/__init__.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/cruise/create_empty_zarr_store.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/cruise/datatree_manager.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/geometry/__init__.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/geometry/elevation_manager.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/geometry/geometry_manager.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/geometry/geometry_simplification.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/geometry/pmtile_generation.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/index/__init__.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/index/index_manager.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/model/__init__.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/process.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/processing/__init__.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/processing/batch_downloader.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/processing/raw_to_zarr.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/utility/__init__.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/utility/cleaner.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/utility/pipeline_status.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/utility/timestamp.py +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing.egg-info/dependency_links.txt +0 -0
- {water_column_sonar_processing-25.1.2 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
name: Testing
|
|
2
|
+
|
|
3
|
+
on: [push]
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
# build:
|
|
7
|
+
# runs-on: ubuntu-latest
|
|
8
|
+
# steps:
|
|
9
|
+
# - name: Check out
|
|
10
|
+
# uses: actions/checkout@v4
|
|
11
|
+
# - name: Set up Python
|
|
12
|
+
# uses: actions/setup-python@v5
|
|
13
|
+
# with:
|
|
14
|
+
# # Semantic version range syntax or exact version of a Python version
|
|
15
|
+
# python-version: '3.10'
|
|
16
|
+
# # Optional - x64 or x86 architecture, defaults to x64
|
|
17
|
+
## architecture: 'x64'
|
|
18
|
+
# cache: 'pip'
|
|
19
|
+
# - name: Install dependencies
|
|
20
|
+
# run: |
|
|
21
|
+
# python -m pip install --upgrade pip
|
|
22
|
+
# pip install -r requirements_dev.txt
|
|
23
|
+
# - name: Run the tests
|
|
24
|
+
# run: python -m pytest
|
|
25
|
+
|
|
26
|
+
build:
|
|
27
|
+
name: python
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/checkout@v4
|
|
31
|
+
|
|
32
|
+
- name: Install uv
|
|
33
|
+
uses: astral-sh/setup-uv@v5
|
|
34
|
+
with:
|
|
35
|
+
version: "0.5.25"
|
|
36
|
+
|
|
37
|
+
- name: Set up Python
|
|
38
|
+
uses: actions/setup-python@v5
|
|
39
|
+
with:
|
|
40
|
+
python-version-file: ".python-version"
|
|
41
|
+
|
|
42
|
+
- name: Install the project
|
|
43
|
+
run: uv sync --all-extras --dev
|
|
44
|
+
|
|
45
|
+
- name: Run tests
|
|
46
|
+
run: uv run pytest tests
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.10.12
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: water_column_sonar_processing
|
|
3
|
-
Version: 25.1.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 25.1.4
|
|
4
|
+
Summary: Processing tool for water column sonar data.
|
|
5
5
|
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
|
|
7
7
|
Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3.
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: aiobotocore==2.19.0
|
|
@@ -34,6 +34,19 @@ Requires-Dist: typing-extensions==4.10.0
|
|
|
34
34
|
Requires-Dist: xarray==2024.10.0
|
|
35
35
|
Requires-Dist: xbatcher==0.4.0
|
|
36
36
|
Requires-Dist: zarr==2.18.3
|
|
37
|
+
Provides-Extra: dev
|
|
38
|
+
Requires-Dist: bandit[toml]==1.8.0; extra == "dev"
|
|
39
|
+
Requires-Dist: build; extra == "dev"
|
|
40
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
41
|
+
Requires-Dist: pyinstaller; extra == "dev"
|
|
42
|
+
Requires-Dist: twine; extra == "dev"
|
|
43
|
+
Requires-Dist: flake8==7.1.1; extra == "dev"
|
|
44
|
+
Requires-Dist: pooch==1.8.2; extra == "dev"
|
|
45
|
+
Requires-Dist: pytest~=8.3.3; extra == "dev"
|
|
46
|
+
Requires-Dist: tqdm; extra == "dev"
|
|
47
|
+
Requires-Dist: bandit; extra == "dev"
|
|
48
|
+
Provides-Extra: test
|
|
49
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
37
50
|
|
|
38
51
|
# Water Column Sonar Processing
|
|
39
52
|
Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
|
|
@@ -80,14 +93,17 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
80
93
|
3. Set interpreter
|
|
81
94
|
|
|
82
95
|
# Installing Dependencies
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
96
|
+
```
|
|
97
|
+
uv pip install --upgrade pip
|
|
98
|
+
#uv pip install -r requirements_dev.txt
|
|
99
|
+
uv pip install -r pyproject.toml --extra dev
|
|
100
|
+
```
|
|
86
101
|
|
|
87
102
|
|
|
88
103
|
# Pytest
|
|
89
104
|
```commandline
|
|
90
|
-
pytest
|
|
105
|
+
uv run pytest tests
|
|
106
|
+
#pytest --disable-warnings
|
|
91
107
|
```
|
|
92
108
|
or
|
|
93
109
|
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
@@ -120,7 +136,7 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
|
|
|
120
136
|
# Tag a Release
|
|
121
137
|
Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
|
|
122
138
|
```commandline
|
|
123
|
-
git tag -a v25.1.
|
|
139
|
+
git tag -a v25.1.4 -m "Releasing version v25.1.4"
|
|
124
140
|
git push origin --tags
|
|
125
141
|
```
|
|
126
142
|
|
|
@@ -43,14 +43,17 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
43
43
|
3. Set interpreter
|
|
44
44
|
|
|
45
45
|
# Installing Dependencies
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
```
|
|
47
|
+
uv pip install --upgrade pip
|
|
48
|
+
#uv pip install -r requirements_dev.txt
|
|
49
|
+
uv pip install -r pyproject.toml --extra dev
|
|
50
|
+
```
|
|
49
51
|
|
|
50
52
|
|
|
51
53
|
# Pytest
|
|
52
54
|
```commandline
|
|
53
|
-
pytest
|
|
55
|
+
uv run pytest tests
|
|
56
|
+
#pytest --disable-warnings
|
|
54
57
|
```
|
|
55
58
|
or
|
|
56
59
|
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
@@ -83,7 +86,7 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
|
|
|
83
86
|
# Tag a Release
|
|
84
87
|
Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
|
|
85
88
|
```commandline
|
|
86
|
-
git tag -a v25.1.
|
|
89
|
+
git tag -a v25.1.4 -m "Releasing version v25.1.4"
|
|
87
90
|
git push origin --tags
|
|
88
91
|
```
|
|
89
92
|
|
|
@@ -105,4 +108,4 @@ Experimental Plotting in Xarray (hvPlot):
|
|
|
105
108
|
https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
|
|
106
109
|
|
|
107
110
|
HB0707 Cruise zoomable:
|
|
108
|
-
https://hb0707.s3.us-east-1.amazonaws.com/index.html
|
|
111
|
+
https://hb0707.s3.us-east-1.amazonaws.com/index.html
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = [
|
|
3
|
+
"setuptools>=61.0",
|
|
4
|
+
"wheel >= 0.29.0",
|
|
5
|
+
]
|
|
6
|
+
build-backend = "setuptools.build_meta"
|
|
7
|
+
|
|
8
|
+
[project]
|
|
9
|
+
name = "water_column_sonar_processing"
|
|
10
|
+
version = "25.1.4"
|
|
11
|
+
authors = [
|
|
12
|
+
{ name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
|
|
13
|
+
]
|
|
14
|
+
description = "Processing tool for water column sonar data."
|
|
15
|
+
readme = "README.md"
|
|
16
|
+
requires-python = ">=3.10"
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
dependencies = [
|
|
24
|
+
"aiobotocore==2.19.0",
|
|
25
|
+
"boto3==1.36.3",
|
|
26
|
+
"botocore==1.36.3",
|
|
27
|
+
"echopype==0.9.0",
|
|
28
|
+
"fiona==1.10.1",
|
|
29
|
+
"geopandas==1.0.1",
|
|
30
|
+
"mock==5.1.0",
|
|
31
|
+
"moto[all]==5.0.27",
|
|
32
|
+
"moto[server]==5.0.27",
|
|
33
|
+
"numcodecs==0.13.1",
|
|
34
|
+
"numpy==1.26.4",
|
|
35
|
+
"pandas==2.2.3",
|
|
36
|
+
"pyarrow==18.1.0",
|
|
37
|
+
"python-dotenv==1.0.1",
|
|
38
|
+
"requests==2.32.3",
|
|
39
|
+
"s3fs==2024.2.0",
|
|
40
|
+
"scipy==1.14.1",
|
|
41
|
+
"setuptools",
|
|
42
|
+
"shapely==2.0.3",
|
|
43
|
+
"typing-extensions==4.10.0",
|
|
44
|
+
"xarray==2024.10.0",
|
|
45
|
+
"xbatcher==0.4.0",
|
|
46
|
+
"zarr==2.18.3",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
[project.optional-dependencies]
|
|
50
|
+
dev = [
|
|
51
|
+
"bandit[toml]==1.8.0",
|
|
52
|
+
"build",
|
|
53
|
+
"pre-commit",
|
|
54
|
+
"pyinstaller",
|
|
55
|
+
"twine",
|
|
56
|
+
"flake8==7.1.1",
|
|
57
|
+
"pooch==1.8.2",
|
|
58
|
+
"pytest~=8.3.3",
|
|
59
|
+
"tqdm",
|
|
60
|
+
"bandit"
|
|
61
|
+
]
|
|
62
|
+
test = [
|
|
63
|
+
"pytest-cov",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
[project.urls]
|
|
67
|
+
Homepage = "https://github.com/CI-CMG/water-column-sonar-processing"
|
|
68
|
+
Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
|
|
69
|
+
|
|
70
|
+
[tool.bandit]
|
|
71
|
+
exclude_dirs = ["tests"]
|
|
72
|
+
[tool.pre-commit-hooks.bandit]
|
|
73
|
+
exclude = ["*/tests/*"]
|
|
@@ -140,15 +140,7 @@ class DynamoDBManager:
|
|
|
140
140
|
To be used to initialize a cruise, deletes all entries associated with that cruise
|
|
141
141
|
in the database.
|
|
142
142
|
"""
|
|
143
|
-
|
|
144
|
-
":cr": {"S": cruise_name},
|
|
145
|
-
":se": {"S": sensor_name},
|
|
146
|
-
":sh": {"S": ship_name},
|
|
147
|
-
}
|
|
148
|
-
filter_expression = (
|
|
149
|
-
"CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh"
|
|
150
|
-
)
|
|
151
|
-
# filter_expression = "CRUISE_NAME = :cr"
|
|
143
|
+
filter_expression = "CRUISE_NAME = :cr"
|
|
152
144
|
response = self.dynamodb_client.scan(
|
|
153
145
|
TableName=table_name,
|
|
154
146
|
# Limit=1000,
|
|
@@ -156,7 +148,7 @@ class DynamoDBManager:
|
|
|
156
148
|
# ExclusiveStartKey=where to pick up
|
|
157
149
|
#ReturnConsumedCapacity='INDEXES' | 'TOTAL' | 'NONE', ...not sure
|
|
158
150
|
# ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
|
|
159
|
-
FilterExpression=
|
|
151
|
+
FilterExpression=filter_expression,
|
|
160
152
|
# ExpressionAttributeNames={
|
|
161
153
|
# '#SH': 'SHIP_NAME',
|
|
162
154
|
# '#CR': 'CRUISE_NAME',
|
|
@@ -170,7 +162,6 @@ class DynamoDBManager:
|
|
|
170
162
|
ConsistentRead=True
|
|
171
163
|
# ExclusiveStartKey=response["LastEvaluatedKey"],
|
|
172
164
|
)
|
|
173
|
-
print(response)
|
|
174
165
|
# Note: table.scan() has 1 MB limit on results so pagination is used
|
|
175
166
|
|
|
176
167
|
if len(response["Items"]) == 0 and "LastEvaluatedKey" not in response:
|
|
@@ -183,7 +174,7 @@ class DynamoDBManager:
|
|
|
183
174
|
TableName=table_name,
|
|
184
175
|
### Either 'Select' or 'ExpressionAttributeNames'/'ProjectionExpression'
|
|
185
176
|
Select='ALL_ATTRIBUTES', # or 'SPECIFIC_ATTRIBUTES',
|
|
186
|
-
FilterExpression=
|
|
177
|
+
FilterExpression=filter_expression,
|
|
187
178
|
#ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
|
|
188
179
|
# ExpressionAttributeNames={ # would need to specify all cols in df
|
|
189
180
|
# '#SH': 'SHIP_NAME',
|
|
@@ -205,6 +196,66 @@ class DynamoDBManager:
|
|
|
205
196
|
|
|
206
197
|
return df.sort_values(by="START_TIME", ignore_index=True)
|
|
207
198
|
|
|
199
|
+
#####################################################################
|
|
200
|
+
# def get_cruise_list(
|
|
201
|
+
# self,
|
|
202
|
+
# table_name,
|
|
203
|
+
# ) -> list:
|
|
204
|
+
# """
|
|
205
|
+
# Experimental, gets all cruise names as list
|
|
206
|
+
# """
|
|
207
|
+
# filter_expression = "CRUISE_NAME = :cr"
|
|
208
|
+
# response = self.dynamodb_client.scan(
|
|
209
|
+
# TableName=table_name,
|
|
210
|
+
# Select='SPECIFIC_ATTRIBUTES',
|
|
211
|
+
# #ReturnConsumedCapacity='INDEXES' | 'TOTAL' | 'NONE', ...not sure
|
|
212
|
+
# # ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
|
|
213
|
+
# FilterExpression=filter_expression,
|
|
214
|
+
# # ExpressionAttributeNames={
|
|
215
|
+
# # '#SH': 'SHIP_NAME',
|
|
216
|
+
# # '#CR': 'CRUISE_NAME',
|
|
217
|
+
# # '#FN': 'FILE_NAME',
|
|
218
|
+
# # },
|
|
219
|
+
# # ExpressionAttributeValues={ # criteria
|
|
220
|
+
# # ':cr': {
|
|
221
|
+
# # 'S': cruise_name,
|
|
222
|
+
# # },
|
|
223
|
+
# # },
|
|
224
|
+
# )
|
|
225
|
+
# # Note: table.scan() has 1 MB limit on results so pagination is used
|
|
226
|
+
#
|
|
227
|
+
# if len(response["Items"]) == 0 and "LastEvaluatedKey" not in response:
|
|
228
|
+
# return pd.DataFrame() # If no results, return empty dataframe
|
|
229
|
+
#
|
|
230
|
+
# data = response["Items"]
|
|
231
|
+
#
|
|
232
|
+
# while response.get('LastEvaluatedKey'): #"LastEvaluatedKey" in response:
|
|
233
|
+
# response = self.dynamodb_client.scan(
|
|
234
|
+
# TableName=table_name,
|
|
235
|
+
# ### Either 'Select' or 'ExpressionAttributeNames'/'ProjectionExpression'
|
|
236
|
+
# Select='ALL_ATTRIBUTES', # or 'SPECIFIC_ATTRIBUTES',
|
|
237
|
+
# FilterExpression=filter_expression,
|
|
238
|
+
# #ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
|
|
239
|
+
# # ExpressionAttributeNames={ # would need to specify all cols in df
|
|
240
|
+
# # '#SH': 'SHIP_NAME',
|
|
241
|
+
# # '#CR': 'CRUISE_NAME',
|
|
242
|
+
# # '#FN': 'FILE_NAME',
|
|
243
|
+
# # },
|
|
244
|
+
# ExpressionAttributeValues={ # criteria
|
|
245
|
+
# ':cr': {
|
|
246
|
+
# 'S': cruise_name,
|
|
247
|
+
# },
|
|
248
|
+
# },
|
|
249
|
+
# ConsistentRead=True,
|
|
250
|
+
# ExclusiveStartKey=response["LastEvaluatedKey"],
|
|
251
|
+
# )
|
|
252
|
+
# data.extend(response["Items"])
|
|
253
|
+
#
|
|
254
|
+
# deserializer = self.type_deserializer
|
|
255
|
+
# df = pd.DataFrame([deserializer.deserialize({"M": i}) for i in data])
|
|
256
|
+
#
|
|
257
|
+
# return df.sort_values(by="START_TIME", ignore_index=True)
|
|
258
|
+
|
|
208
259
|
#####################################################################
|
|
209
260
|
# TODO: WIP
|
|
210
261
|
def delete_item(
|
|
@@ -16,7 +16,7 @@ class S3FSManager:
|
|
|
16
16
|
# self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
|
|
17
17
|
self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
|
|
18
18
|
self.s3fs = s3fs.S3FileSystem(
|
|
19
|
-
asynchronous=False,
|
|
19
|
+
# asynchronous=False,
|
|
20
20
|
endpoint_url=endpoint_url,
|
|
21
21
|
key=os.environ.get("OUTPUT_BUCKET_ACCESS_KEY"),
|
|
22
22
|
secret=os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY"),
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import gc
|
|
2
|
-
import os
|
|
3
2
|
from pathlib import Path
|
|
4
3
|
|
|
5
4
|
import numcodecs
|
|
@@ -198,9 +197,9 @@ class ResampleRegrid:
|
|
|
198
197
|
# df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
|
|
199
198
|
|
|
200
199
|
# Get index from all cruise files. Note: should be based on which are included in cruise.
|
|
201
|
-
index = cruise_df.index[
|
|
200
|
+
index = int(cruise_df.index[
|
|
202
201
|
cruise_df["FILE_NAME"] == f"{file_name_stem}.raw"
|
|
203
|
-
][0]
|
|
202
|
+
][0])
|
|
204
203
|
|
|
205
204
|
# get input store
|
|
206
205
|
input_xr_zarr_store = zarr_manager.open_s3_zarr_store_with_xarray(
|
|
@@ -227,18 +226,20 @@ class ResampleRegrid:
|
|
|
227
226
|
min_echo_range = np.nanmin(np.float32(cruise_df["MIN_ECHO_RANGE"]))
|
|
228
227
|
max_echo_range = np.nanmax(np.float32(cruise_df["MAX_ECHO_RANGE"]))
|
|
229
228
|
|
|
230
|
-
print(
|
|
231
|
-
|
|
232
|
-
|
|
229
|
+
print("Creating empty ndarray for Sv data.") # Note: cruise dims (depth, time, frequency)
|
|
230
|
+
output_zarr_store_shape = output_zarr_store.Sv.shape
|
|
231
|
+
end_ping_time_index - start_ping_time_index
|
|
232
|
+
output_zarr_store_height = output_zarr_store_shape[0]
|
|
233
|
+
output_zarr_store_width = end_ping_time_index - start_ping_time_index
|
|
234
|
+
output_zarr_store_depth = output_zarr_store_shape[2]
|
|
233
235
|
cruise_sv_subset = np.empty(
|
|
234
|
-
shape=
|
|
235
|
-
:, start_ping_time_index:end_ping_time_index, :
|
|
236
|
-
].shape
|
|
236
|
+
shape=(output_zarr_store_height, output_zarr_store_width, output_zarr_store_depth)
|
|
237
237
|
)
|
|
238
238
|
cruise_sv_subset[:, :, :] = np.nan
|
|
239
239
|
|
|
240
240
|
all_cruise_depth_values = zarr_manager.get_depth_values(
|
|
241
|
-
min_echo_range=min_echo_range,
|
|
241
|
+
min_echo_range=min_echo_range,
|
|
242
|
+
max_echo_range=max_echo_range
|
|
242
243
|
) # (5262,) and
|
|
243
244
|
|
|
244
245
|
print(" ".join(list(input_xr_zarr_store.Sv.dims)))
|
|
@@ -282,16 +283,6 @@ class ResampleRegrid:
|
|
|
282
283
|
#########################################################################
|
|
283
284
|
# write Sv values to cruise-level-model-store
|
|
284
285
|
output_zarr_store.Sv[:, start_ping_time_index:end_ping_time_index, :] = regrid_resample.values
|
|
285
|
-
|
|
286
|
-
#########################################################################
|
|
287
|
-
# [5] write subset of latitude/longitude
|
|
288
|
-
output_zarr_store.latitude[
|
|
289
|
-
start_ping_time_index:end_ping_time_index
|
|
290
|
-
] = geospatial.dropna()["latitude"].values # TODO: get from ds_sv directly, dont need geojson anymore
|
|
291
|
-
output_zarr_store.longitude[
|
|
292
|
-
start_ping_time_index:end_ping_time_index
|
|
293
|
-
] = geospatial.dropna()["longitude"].values
|
|
294
|
-
|
|
295
286
|
#########################################################################
|
|
296
287
|
# TODO: add the "detected_seafloor_depth/" to the
|
|
297
288
|
# L2 cruise dataarrays
|
|
@@ -311,6 +302,15 @@ class ResampleRegrid:
|
|
|
311
302
|
output_zarr_store.bottom[
|
|
312
303
|
start_ping_time_index:end_ping_time_index
|
|
313
304
|
] = detected_seafloor_depths
|
|
305
|
+
#
|
|
306
|
+
#########################################################################
|
|
307
|
+
# [5] write subset of latitude/longitude
|
|
308
|
+
output_zarr_store.latitude[
|
|
309
|
+
start_ping_time_index:end_ping_time_index
|
|
310
|
+
] = geospatial.dropna()["latitude"].values # TODO: get from ds_sv directly, dont need geojson anymore
|
|
311
|
+
output_zarr_store.longitude[
|
|
312
|
+
start_ping_time_index:end_ping_time_index
|
|
313
|
+
] = geospatial.dropna()["longitude"].values
|
|
314
314
|
#########################################################################
|
|
315
315
|
#########################################################################
|
|
316
316
|
except Exception as err:
|
|
@@ -2,6 +2,7 @@ import numcodecs
|
|
|
2
2
|
import numpy as np
|
|
3
3
|
import xarray as xr
|
|
4
4
|
import zarr
|
|
5
|
+
import importlib.metadata
|
|
5
6
|
from numcodecs import Blosc
|
|
6
7
|
|
|
7
8
|
from water_column_sonar_processing.aws import S3FSManager
|
|
@@ -249,9 +250,9 @@ class ZarrManager:
|
|
|
249
250
|
root.attrs["sensor_name"] = sensor_name
|
|
250
251
|
#
|
|
251
252
|
root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
253
|
+
|
|
254
|
+
current_project_version = importlib.metadata.version('water_column_sonar_processing')
|
|
255
|
+
root.attrs["processing_software_version"] = current_project_version
|
|
255
256
|
root.attrs["processing_software_time"] = Timestamp.get_timestamp()
|
|
256
257
|
#
|
|
257
258
|
root.attrs["calibration_status"] = calibration_status
|
|
@@ -290,7 +291,7 @@ class ZarrManager:
|
|
|
290
291
|
# zarr_synchronizer: Union[str, None] = None, # TODO:
|
|
291
292
|
output_bucket_name: str,
|
|
292
293
|
endpoint_url=None,
|
|
293
|
-
):
|
|
294
|
+
) -> zarr.hierarchy.Group:
|
|
294
295
|
# Mounts a Zarr store using pythons Zarr implementation. The mounted store
|
|
295
296
|
# will have read/write privileges so that store can be updated.
|
|
296
297
|
print("Opening L2 Zarr store with Zarr for writing.")
|
|
@@ -316,18 +317,21 @@ class ZarrManager:
|
|
|
316
317
|
input_bucket_name: str,
|
|
317
318
|
endpoint_url=None,
|
|
318
319
|
) -> xr.Dataset:
|
|
319
|
-
print("Opening L1 Zarr store in S3 with Xarray.")
|
|
320
|
+
print("Opening L1 Zarr store in S3 with Xarray.") # TODO: Is this only used for reading from?
|
|
320
321
|
try:
|
|
321
322
|
zarr_path = f"s3://{input_bucket_name}/level_1/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.zarr"
|
|
322
323
|
s3fs_manager = S3FSManager(endpoint_url=endpoint_url)
|
|
323
324
|
store_s3_map = s3fs_manager.s3_map(s3_zarr_store_path=zarr_path)
|
|
324
|
-
ds = xr.
|
|
325
|
-
|
|
326
|
-
|
|
325
|
+
ds = xr.open_dataset(
|
|
326
|
+
filename_or_obj=store_s3_map,
|
|
327
|
+
engine="zarr",
|
|
328
|
+
chunks={}
|
|
329
|
+
)
|
|
327
330
|
except Exception as err:
|
|
328
331
|
print("Problem opening Zarr store in S3 as Xarray.")
|
|
329
332
|
raise err
|
|
330
|
-
|
|
333
|
+
finally:
|
|
334
|
+
print("Exiting opening Zarr store in S3 as Xarray.")
|
|
331
335
|
return ds
|
|
332
336
|
|
|
333
337
|
def open_l2_zarr_store_with_xarray(
|
|
@@ -3,7 +3,7 @@ from enum import Enum, Flag, unique
|
|
|
3
3
|
|
|
4
4
|
@unique
|
|
5
5
|
class Constants(Flag):
|
|
6
|
-
TILE_SIZE =
|
|
6
|
+
TILE_SIZE = 1024
|
|
7
7
|
|
|
8
8
|
# Average https://noaa-wcsd-zarr-pds.s3.us-east-1.amazonaws.com/level_2/Henry_B._Bigelow/HB0902/EK60/HB0902.zarr/time/927
|
|
9
9
|
# chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon data
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: water_column_sonar_processing
|
|
3
|
-
Version: 25.1.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 25.1.4
|
|
4
|
+
Summary: Processing tool for water column sonar data.
|
|
5
5
|
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
|
|
7
7
|
Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3.
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: aiobotocore==2.19.0
|
|
@@ -34,6 +34,19 @@ Requires-Dist: typing-extensions==4.10.0
|
|
|
34
34
|
Requires-Dist: xarray==2024.10.0
|
|
35
35
|
Requires-Dist: xbatcher==0.4.0
|
|
36
36
|
Requires-Dist: zarr==2.18.3
|
|
37
|
+
Provides-Extra: dev
|
|
38
|
+
Requires-Dist: bandit[toml]==1.8.0; extra == "dev"
|
|
39
|
+
Requires-Dist: build; extra == "dev"
|
|
40
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
41
|
+
Requires-Dist: pyinstaller; extra == "dev"
|
|
42
|
+
Requires-Dist: twine; extra == "dev"
|
|
43
|
+
Requires-Dist: flake8==7.1.1; extra == "dev"
|
|
44
|
+
Requires-Dist: pooch==1.8.2; extra == "dev"
|
|
45
|
+
Requires-Dist: pytest~=8.3.3; extra == "dev"
|
|
46
|
+
Requires-Dist: tqdm; extra == "dev"
|
|
47
|
+
Requires-Dist: bandit; extra == "dev"
|
|
48
|
+
Provides-Extra: test
|
|
49
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
37
50
|
|
|
38
51
|
# Water Column Sonar Processing
|
|
39
52
|
Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
|
|
@@ -80,14 +93,17 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
80
93
|
3. Set interpreter
|
|
81
94
|
|
|
82
95
|
# Installing Dependencies
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
96
|
+
```
|
|
97
|
+
uv pip install --upgrade pip
|
|
98
|
+
#uv pip install -r requirements_dev.txt
|
|
99
|
+
uv pip install -r pyproject.toml --extra dev
|
|
100
|
+
```
|
|
86
101
|
|
|
87
102
|
|
|
88
103
|
# Pytest
|
|
89
104
|
```commandline
|
|
90
|
-
pytest
|
|
105
|
+
uv run pytest tests
|
|
106
|
+
#pytest --disable-warnings
|
|
91
107
|
```
|
|
92
108
|
or
|
|
93
109
|
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
@@ -120,7 +136,7 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
|
|
|
120
136
|
# Tag a Release
|
|
121
137
|
Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
|
|
122
138
|
```commandline
|
|
123
|
-
git tag -a v25.1.
|
|
139
|
+
git tag -a v25.1.4 -m "Releasing version v25.1.4"
|
|
124
140
|
git push origin --tags
|
|
125
141
|
```
|
|
126
142
|
|
|
@@ -5,15 +5,11 @@
|
|
|
5
5
|
LICENSE
|
|
6
6
|
README.md
|
|
7
7
|
pyproject.toml
|
|
8
|
-
pytest.ini
|
|
9
|
-
requirements.txt
|
|
10
|
-
requirements_dev.txt
|
|
11
8
|
.github/workflows/test_action.yaml
|
|
12
9
|
open-science-data-federation/ml/autoencoder_example.py
|
|
13
10
|
open-science-data-federation/osdf_examples/foo.ipynb
|
|
14
11
|
open-science-data-federation/osdf_examples/sonar_ai.ipynb
|
|
15
12
|
tests/conftest.py
|
|
16
|
-
tests/test_process.py
|
|
17
13
|
tests/test_resources/index/calibrated_cruises.csv
|
|
18
14
|
tests/test_resources/raw_to_zarr/D20070724-T042400.bot
|
|
19
15
|
tests/test_resources/raw_to_zarr/D20070724-T042400.idx
|
|
@@ -21,3 +21,18 @@ typing-extensions==4.10.0
|
|
|
21
21
|
xarray==2024.10.0
|
|
22
22
|
xbatcher==0.4.0
|
|
23
23
|
zarr==2.18.3
|
|
24
|
+
|
|
25
|
+
[dev]
|
|
26
|
+
bandit[toml]==1.8.0
|
|
27
|
+
build
|
|
28
|
+
pre-commit
|
|
29
|
+
pyinstaller
|
|
30
|
+
twine
|
|
31
|
+
flake8==7.1.1
|
|
32
|
+
pooch==1.8.2
|
|
33
|
+
pytest~=8.3.3
|
|
34
|
+
tqdm
|
|
35
|
+
bandit
|
|
36
|
+
|
|
37
|
+
[test]
|
|
38
|
+
pytest-cov
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
name: Python package
|
|
2
|
-
|
|
3
|
-
on: [push]
|
|
4
|
-
|
|
5
|
-
jobs:
|
|
6
|
-
build:
|
|
7
|
-
runs-on: ubuntu-latest
|
|
8
|
-
steps:
|
|
9
|
-
- name: Check out
|
|
10
|
-
uses: actions/checkout@v4
|
|
11
|
-
- name: Set up Python
|
|
12
|
-
uses: actions/setup-python@v5
|
|
13
|
-
with:
|
|
14
|
-
# Semantic version range syntax or exact version of a Python version
|
|
15
|
-
python-version: '3.10'
|
|
16
|
-
# Optional - x64 or x86 architecture, defaults to x64
|
|
17
|
-
# architecture: 'x64'
|
|
18
|
-
cache: 'pip'
|
|
19
|
-
- name: Install dependencies
|
|
20
|
-
run: |
|
|
21
|
-
python -m pip install --upgrade pip
|
|
22
|
-
pip install -r requirements_dev.txt
|
|
23
|
-
- name: Run the tests
|
|
24
|
-
run: python -m pytest
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = [
|
|
3
|
-
"setuptools>=61.0",
|
|
4
|
-
#"setuptools_scm[toml] >= 4, <6",
|
|
5
|
-
"wheel >= 0.29.0",
|
|
6
|
-
]
|
|
7
|
-
build-backend = "setuptools.build_meta"
|
|
8
|
-
|
|
9
|
-
[project]
|
|
10
|
-
name = "water_column_sonar_processing"
|
|
11
|
-
version = "25.1.2"
|
|
12
|
-
authors = [
|
|
13
|
-
{ name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
|
|
14
|
-
]
|
|
15
|
-
description = "A processing tool for water column sonar data."
|
|
16
|
-
readme = "README.md"
|
|
17
|
-
#requires-python = ">=3.10"
|
|
18
|
-
requires-python = ">=3.8"
|
|
19
|
-
classifiers = [
|
|
20
|
-
"Programming Language :: Python :: 3",
|
|
21
|
-
"License :: OSI Approved :: MIT License",
|
|
22
|
-
"Operating System :: OS Independent",
|
|
23
|
-
]
|
|
24
|
-
dynamic = ["dependencies"]
|
|
25
|
-
|
|
26
|
-
[project.urls]
|
|
27
|
-
Homepage = "https://github.com/CI-CMG/water-column-sonar-processing"
|
|
28
|
-
Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
|
|
29
|
-
|
|
30
|
-
[tool.setuptools.dynamic]
|
|
31
|
-
dependencies = {file = ["requirements.txt"]}
|
|
32
|
-
optional-dependencies = {dev = { file = ["requirements_dev.txt"] }}
|
|
33
|
-
|
|
34
|
-
#[tool.setuptools_scm]
|
|
35
|
-
#fallback_version = "unknown"
|
|
36
|
-
#local_scheme = "node-and-date"
|
|
37
|
-
#write_to = "_water_column_sonar_processing_version.py"
|
|
38
|
-
#write_to_template = 'version = "{version}"'
|
|
39
|
-
|
|
40
|
-
[tool.bandit]
|
|
41
|
-
exclude_dirs = ["tests"]
|
|
42
|
-
[tool.pre-commit-hooks.bandit]
|
|
43
|
-
exclude = ["*/tests/*"]
|