water-column-sonar-processing 0.0.11__tar.gz → 0.0.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of water-column-sonar-processing might be problematic. Click here for more details.
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/.gitignore +4 -8
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/PKG-INFO +18 -16
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/README.md +16 -14
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/pyproject.toml +1 -1
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/requirements.txt +3 -1
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/requirements_dev.txt +4 -2
- water_column_sonar_processing-0.0.12/tests/conftest.py +47 -0
- water_column_sonar_processing-0.0.12/tests/test_process.py +472 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/dynamodb_manager.py +1 -1
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/s3_manager.py +39 -13
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/s3fs_manager.py +42 -35
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/cruise/create_empty_zarr_store.py +15 -8
- water_column_sonar_processing-0.0.12/water_column_sonar_processing/cruise/experiment_datatree.py +13 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/cruise/resample_regrid.py +29 -18
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/geometry/geometry_manager.py +10 -4
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/model/zarr_manager.py +33 -10
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/processing/raw_to_zarr.py +24 -9
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/PKG-INFO +18 -16
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/SOURCES.txt +1 -14
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/requires.txt +1 -1
- water_column_sonar_processing-0.0.11/tests/conftest.py +0 -23
- water_column_sonar_processing-0.0.11/tests/example_input_bucket/example_directory/foo.txt +0 -0
- water_column_sonar_processing-0.0.11/tests/test_create_empty_zarr_store.py +0 -230
- water_column_sonar_processing-0.0.11/tests/test_dynamodb_manager.py +0 -376
- water_column_sonar_processing-0.0.11/tests/test_geometry_manager.py +0 -74
- water_column_sonar_processing-0.0.11/tests/test_geometry_simplification.py +0 -39
- water_column_sonar_processing-0.0.11/tests/test_index.py +0 -189
- water_column_sonar_processing-0.0.11/tests/test_pmtile_generation.py +0 -266
- water_column_sonar_processing-0.0.11/tests/test_process.py +0 -471
- water_column_sonar_processing-0.0.11/tests/test_raw_to_zarr.py +0 -162
- water_column_sonar_processing-0.0.11/tests/test_resample_regrid.py +0 -51
- water_column_sonar_processing-0.0.11/tests/test_s3_manager.py +0 -148
- water_column_sonar_processing-0.0.11/tests/test_s3fs_manager.py +0 -325
- water_column_sonar_processing-0.0.11/tests/test_s3fs_with_moto.py +0 -78
- water_column_sonar_processing-0.0.11/tests/test_sns_sqs_manager.py +0 -72
- water_column_sonar_processing-0.0.11/tests/test_zarr_manager.py +0 -283
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/.env-test +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/.github/workflows/test_action.yaml +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/.pre-commit-config.yaml +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/.python-version +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/LICENSE +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/open-science-data-federation/ml/autoencoder_example.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/open-science-data-federation/osdf_examples/foo.ipynb +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/open-science-data-federation/osdf_examples/sonar_ai.ipynb +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/pytest.ini +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/setup.cfg +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/tests/test_resources/index/calibrated_cruises.csv +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/tests/test_resources/raw_to_zarr/D20070724-T042400.bot +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/tests/test_resources/raw_to_zarr/D20070724-T042400.idx +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/tests/test_resources/raw_to_zarr/D20070724-T042400.raw +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/__init__.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/__init__.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/sns_manager.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/sqs_manager.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/cruise/__init__.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/geometry/__init__.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/geometry/geometry_simplification.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/geometry/pmtile_generation.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/index/__init__.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/index/index_manager.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/model/__init__.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/process.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/processing/__init__.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/processing/cruise_sampler.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/__init__.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/cleaner.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/constants.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/pipeline_status.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/timestamp.py +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/dependency_links.txt +0 -0
- {water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/top_level.txt +0 -0
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
#tests/*.zarr
|
|
2
|
-
#tests/*.bot
|
|
3
|
-
#tests/*.raw
|
|
4
|
-
#tests/*.json
|
|
5
1
|
*.geojson
|
|
6
2
|
|
|
7
3
|
# Byte-compiled / optimized / DLL files
|
|
@@ -172,12 +168,12 @@ __pycache__
|
|
|
172
168
|
.DS_Store
|
|
173
169
|
|
|
174
170
|
*.env-prod*
|
|
175
|
-
|
|
171
|
+
#*.zarr
|
|
176
172
|
*.csv
|
|
177
173
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
174
|
+
#*.raw
|
|
175
|
+
#*.bot
|
|
176
|
+
#*.idx
|
|
181
177
|
|
|
182
178
|
.idea
|
|
183
179
|
.pytest_cache
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: water_column_sonar_processing
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.12
|
|
4
4
|
Summary: A processing tool for water column sonar data.
|
|
5
5
|
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
|
|
@@ -28,7 +28,7 @@ Requires-Dist: python-dotenv==1.0.1
|
|
|
28
28
|
Requires-Dist: requests==2.32.3
|
|
29
29
|
Requires-Dist: s3fs==2023.12.1
|
|
30
30
|
Requires-Dist: scipy==1.14.1
|
|
31
|
-
Requires-Dist: setuptools
|
|
31
|
+
Requires-Dist: setuptools
|
|
32
32
|
Requires-Dist: shapely==2.0.3
|
|
33
33
|
Requires-Dist: typing-extensions==4.10.0
|
|
34
34
|
Requires-Dist: xarray==2024.10.0
|
|
@@ -37,6 +37,16 @@ Requires-Dist: zarr==2.18.3
|
|
|
37
37
|
# Water Column Sonar Processing
|
|
38
38
|
Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
|
|
39
39
|
|
|
40
|
+

|
|
41
|
+
|
|
42
|
+

|
|
43
|
+
|
|
44
|
+

|
|
45
|
+
|
|
46
|
+

|
|
47
|
+
|
|
48
|
+
 
|
|
49
|
+
|
|
40
50
|
# Setting up the Python Environment
|
|
41
51
|
> Python 3.10.12
|
|
42
52
|
|
|
@@ -93,20 +103,6 @@ or
|
|
|
93
103
|
Following this tutorial:
|
|
94
104
|
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
95
105
|
|
|
96
|
-
# To Publish To TEST
|
|
97
|
-
```commandline
|
|
98
|
-
python -m build
|
|
99
|
-
# python -m build --sdist
|
|
100
|
-
# python -m build --wheel
|
|
101
|
-
python -m twine upload --repository testpypi dist/*
|
|
102
|
-
pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
|
|
103
|
-
python
|
|
104
|
-
```
|
|
105
|
-
```
|
|
106
|
-
from water-column-sonar-processing import ZarrManager
|
|
107
|
-
example.add_one(2)
|
|
108
|
-
```
|
|
109
|
-
|
|
110
106
|
# To Publish To PROD
|
|
111
107
|
```commandline
|
|
112
108
|
python -m build
|
|
@@ -134,6 +130,12 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
|
|
|
134
130
|
5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
|
|
135
131
|
3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
|
|
136
132
|
|
|
133
|
+
# Tag a Release
|
|
134
|
+
```commandline
|
|
135
|
+
git tag "v0.0.12" -a
|
|
136
|
+
# enter description
|
|
137
|
+
git push origin --tags
|
|
138
|
+
```
|
|
137
139
|
|
|
138
140
|
# TODO:
|
|
139
141
|
add https://pypi.org/project/setuptools-scm/
|
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
# Water Column Sonar Processing
|
|
2
2
|
Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
|
|
3
3
|
|
|
4
|
+

|
|
5
|
+
|
|
6
|
+

|
|
7
|
+
|
|
8
|
+

|
|
9
|
+
|
|
10
|
+

|
|
11
|
+
|
|
12
|
+
 
|
|
13
|
+
|
|
4
14
|
# Setting up the Python Environment
|
|
5
15
|
> Python 3.10.12
|
|
6
16
|
|
|
@@ -57,20 +67,6 @@ or
|
|
|
57
67
|
Following this tutorial:
|
|
58
68
|
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
59
69
|
|
|
60
|
-
# To Publish To TEST
|
|
61
|
-
```commandline
|
|
62
|
-
python -m build
|
|
63
|
-
# python -m build --sdist
|
|
64
|
-
# python -m build --wheel
|
|
65
|
-
python -m twine upload --repository testpypi dist/*
|
|
66
|
-
pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
|
|
67
|
-
python
|
|
68
|
-
```
|
|
69
|
-
```
|
|
70
|
-
from water-column-sonar-processing import ZarrManager
|
|
71
|
-
example.add_one(2)
|
|
72
|
-
```
|
|
73
|
-
|
|
74
70
|
# To Publish To PROD
|
|
75
71
|
```commandline
|
|
76
72
|
python -m build
|
|
@@ -98,6 +94,12 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
|
|
|
98
94
|
5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
|
|
99
95
|
3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
|
|
100
96
|
|
|
97
|
+
# Tag a Release
|
|
98
|
+
```commandline
|
|
99
|
+
git tag "v0.0.12" -a
|
|
100
|
+
# enter description
|
|
101
|
+
git push origin --tags
|
|
102
|
+
```
|
|
101
103
|
|
|
102
104
|
# TODO:
|
|
103
105
|
add https://pypi.org/project/setuptools-scm/
|
{water_column_sonar_processing-0.0.11 → water_column_sonar_processing-0.0.12}/requirements.txt
RENAMED
|
@@ -18,10 +18,12 @@ pandas==2.2.3
|
|
|
18
18
|
pyarrow==18.1.0
|
|
19
19
|
python-dotenv==1.0.1
|
|
20
20
|
requests==2.32.3
|
|
21
|
+
#s3fs==2024.3.1
|
|
21
22
|
#s3fs==2024.10.0 # this version creates problems
|
|
22
23
|
s3fs==2023.12.1
|
|
23
24
|
scipy==1.14.1
|
|
24
|
-
setuptools==75.6.0
|
|
25
|
+
#setuptools==75.6.0
|
|
26
|
+
setuptools
|
|
25
27
|
shapely==2.0.3
|
|
26
28
|
typing-extensions==4.10.0
|
|
27
29
|
xarray==2024.10.0
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import pooch
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
HERE = Path(__file__).parent.absolute()
|
|
7
|
+
TEST_DATA_FOLDER = HERE / "test_resources"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
HB0707_RAW = pooch.create(
|
|
11
|
+
path=pooch.os_cache("water-column-sonar-processing"),
|
|
12
|
+
base_url="https://noaa-wcsd-pds.s3.amazonaws.com/data/raw/Henry_B._Bigelow/HB0707/EK60/",
|
|
13
|
+
retry_if_failed=1,
|
|
14
|
+
registry={
|
|
15
|
+
#https://noaa-wcsd-zarr-pds.s3.amazonaws.com/level_1/Henry_B._Bigelow/HB0707/EK60/D20070711-T182032.zarr/
|
|
16
|
+
#https://noaa-wcsd-pds.s3.amazonaws.com/data/raw/Henry_B._Bigelow/HB0707/EK60/D20070711-T182032.raw
|
|
17
|
+
# TODO: add bottom files
|
|
18
|
+
"D20070712-T124906.raw": "sha256:44f9b2402a8d6d51c69235d1e33c3e4ab570fc541e9f269009924378bf4d97a2", # 250 m, 158 MB
|
|
19
|
+
"D20070712-T152416.raw": "sha256:94a937eefd6ae5763c27c9ba1e4769b2b76fcc2d840e7db6c2e0edd925d6f70f", # 1000 m, 200 MB
|
|
20
|
+
},
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
def fetch_raw_files():
|
|
24
|
+
HB0707_RAW.fetch(fname="D20070712-T124906.raw", progressbar=True)
|
|
25
|
+
file_name = HB0707_RAW.fetch(fname="D20070712-T152416.raw", progressbar=True)
|
|
26
|
+
return Path(file_name).parent #joinpath(Path(file_path).stem)
|
|
27
|
+
|
|
28
|
+
@pytest.fixture(scope="session")
|
|
29
|
+
def test_path():
|
|
30
|
+
return {
|
|
31
|
+
'RAW_TO_ZARR_TEST_PATH': TEST_DATA_FOLDER / "raw_to_zarr",
|
|
32
|
+
'INDEX_TEST_PATH': TEST_DATA_FOLDER / "index",
|
|
33
|
+
'ZARR_MANAGER_TEST_PATH': TEST_DATA_FOLDER / "zarr_manager",
|
|
34
|
+
'PMTILE_GENERATION_TEST_PATH': TEST_DATA_FOLDER / "pmtile",
|
|
35
|
+
'CREATE_EMPTY_ZARR_TEST_PATH': TEST_DATA_FOLDER / "create_empty_zarr",
|
|
36
|
+
# 'RESAMPLE_REGRID_TEST_PATH': TEST_DATA_FOLDER / "resample_regrid",
|
|
37
|
+
'RESAMPLE_REGRID_TEST_PATH': fetch_raw_files(),
|
|
38
|
+
'S3FS_MANAGER_TEST_PATH': TEST_DATA_FOLDER / "s3fs_manager",
|
|
39
|
+
'S3_MANAGER_TEST_PATH': TEST_DATA_FOLDER / "s3_manager",
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
# """
|
|
43
|
+
# Windows
|
|
44
|
+
# C:\Users\<user>\AppData\Local\echopype\Cache\2024.12.23.10.10
|
|
45
|
+
# MacOS
|
|
46
|
+
# /Users//Library/Caches/echopype/2024.12.23.10.10
|
|
47
|
+
# """
|