water-column-sonar-processing 0.0.10__tar.gz → 0.0.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of water-column-sonar-processing might be problematic. Click here for more details.

Files changed (71) hide show
  1. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/.gitignore +4 -8
  2. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/PKG-INFO +18 -16
  3. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/README.md +16 -14
  4. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/pyproject.toml +1 -1
  5. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/requirements.txt +3 -1
  6. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/requirements_dev.txt +4 -2
  7. water_column_sonar_processing-0.0.12/tests/conftest.py +47 -0
  8. water_column_sonar_processing-0.0.12/tests/test_process.py +472 -0
  9. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/dynamodb_manager.py +15 -11
  10. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/s3_manager.py +39 -13
  11. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/s3fs_manager.py +42 -35
  12. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/cruise/create_empty_zarr_store.py +15 -8
  13. water_column_sonar_processing-0.0.12/water_column_sonar_processing/cruise/experiment_datatree.py +13 -0
  14. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/cruise/resample_regrid.py +29 -18
  15. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/geometry/geometry_manager.py +10 -4
  16. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/model/zarr_manager.py +33 -10
  17. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/processing/raw_to_zarr.py +45 -20
  18. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/PKG-INFO +18 -16
  19. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/SOURCES.txt +1 -14
  20. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/requires.txt +1 -1
  21. water_column_sonar_processing-0.0.10/tests/conftest.py +0 -23
  22. water_column_sonar_processing-0.0.10/tests/example_input_bucket/example_directory/foo.txt +0 -0
  23. water_column_sonar_processing-0.0.10/tests/test_create_empty_zarr_store.py +0 -230
  24. water_column_sonar_processing-0.0.10/tests/test_dynamodb_manager.py +0 -376
  25. water_column_sonar_processing-0.0.10/tests/test_geometry_manager.py +0 -74
  26. water_column_sonar_processing-0.0.10/tests/test_geometry_simplification.py +0 -39
  27. water_column_sonar_processing-0.0.10/tests/test_index.py +0 -189
  28. water_column_sonar_processing-0.0.10/tests/test_pmtile_generation.py +0 -266
  29. water_column_sonar_processing-0.0.10/tests/test_process.py +0 -471
  30. water_column_sonar_processing-0.0.10/tests/test_raw_to_zarr.py +0 -161
  31. water_column_sonar_processing-0.0.10/tests/test_resample_regrid.py +0 -51
  32. water_column_sonar_processing-0.0.10/tests/test_s3_manager.py +0 -148
  33. water_column_sonar_processing-0.0.10/tests/test_s3fs_manager.py +0 -325
  34. water_column_sonar_processing-0.0.10/tests/test_s3fs_with_moto.py +0 -78
  35. water_column_sonar_processing-0.0.10/tests/test_sns_sqs_manager.py +0 -72
  36. water_column_sonar_processing-0.0.10/tests/test_zarr_manager.py +0 -283
  37. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/.env-test +0 -0
  38. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/.github/workflows/test_action.yaml +0 -0
  39. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/.pre-commit-config.yaml +0 -0
  40. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/.python-version +0 -0
  41. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/LICENSE +0 -0
  42. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/open-science-data-federation/ml/autoencoder_example.py +0 -0
  43. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/open-science-data-federation/osdf_examples/foo.ipynb +0 -0
  44. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/open-science-data-federation/osdf_examples/sonar_ai.ipynb +0 -0
  45. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/pytest.ini +0 -0
  46. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/setup.cfg +0 -0
  47. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/tests/test_resources/index/calibrated_cruises.csv +0 -0
  48. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/tests/test_resources/raw_to_zarr/D20070724-T042400.bot +0 -0
  49. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/tests/test_resources/raw_to_zarr/D20070724-T042400.idx +0 -0
  50. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/tests/test_resources/raw_to_zarr/D20070724-T042400.raw +0 -0
  51. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/__init__.py +0 -0
  52. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/__init__.py +0 -0
  53. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/sns_manager.py +0 -0
  54. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/aws/sqs_manager.py +0 -0
  55. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/cruise/__init__.py +0 -0
  56. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/geometry/__init__.py +0 -0
  57. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/geometry/geometry_simplification.py +0 -0
  58. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/geometry/pmtile_generation.py +0 -0
  59. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/index/__init__.py +0 -0
  60. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/index/index_manager.py +0 -0
  61. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/model/__init__.py +0 -0
  62. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/process.py +0 -0
  63. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/processing/__init__.py +0 -0
  64. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/processing/cruise_sampler.py +0 -0
  65. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/__init__.py +0 -0
  66. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/cleaner.py +0 -0
  67. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/constants.py +0 -0
  68. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/pipeline_status.py +0 -0
  69. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing/utility/timestamp.py +0 -0
  70. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/dependency_links.txt +0 -0
  71. {water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.12}/water_column_sonar_processing.egg-info/top_level.txt +0 -0
@@ -1,7 +1,3 @@
1
- #tests/*.zarr
2
- #tests/*.bot
3
- #tests/*.raw
4
- #tests/*.json
5
1
  *.geojson
6
2
 
7
3
  # Byte-compiled / optimized / DLL files
@@ -172,12 +168,12 @@ __pycache__
172
168
  .DS_Store
173
169
 
174
170
  *.env-prod*
175
- *.zarr
171
+ #*.zarr
176
172
  *.csv
177
173
 
178
- *.raw
179
- *.bot
180
- *.idx
174
+ #*.raw
175
+ #*.bot
176
+ #*.idx
181
177
 
182
178
  .idea
183
179
  .pytest_cache
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: water_column_sonar_processing
3
- Version: 0.0.10
3
+ Version: 0.0.12
4
4
  Summary: A processing tool for water column sonar data.
5
5
  Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
6
6
  Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
@@ -28,7 +28,7 @@ Requires-Dist: python-dotenv==1.0.1
28
28
  Requires-Dist: requests==2.32.3
29
29
  Requires-Dist: s3fs==2023.12.1
30
30
  Requires-Dist: scipy==1.14.1
31
- Requires-Dist: setuptools==75.6.0
31
+ Requires-Dist: setuptools
32
32
  Requires-Dist: shapely==2.0.3
33
33
  Requires-Dist: typing-extensions==4.10.0
34
34
  Requires-Dist: xarray==2024.10.0
@@ -37,6 +37,16 @@ Requires-Dist: zarr==2.18.3
37
37
  # Water Column Sonar Processing
38
38
  Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
39
39
 
40
+ ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/CI-CMG/water-column-sonar-processing/test_action.yaml)
41
+
42
+ ![GitHub License](https://img.shields.io/github/license/CI-CMG/water-column-sonar-processing)
43
+
44
+ ![PyPI - Implementation](https://img.shields.io/pypi/v/water-column-sonar-processing?color=black)
45
+
46
+ ![PyPI - Downloads](https://img.shields.io/pypi/dd/water-column-sonar-processing)
47
+
48
+ ![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/CI-CMG/water-column-sonar-processing) ![GitHub repo size](https://img.shields.io/github/repo-size/CI-CMG/water-column-sonar-processing)
49
+
40
50
  # Setting up the Python Environment
41
51
  > Python 3.10.12
42
52
 
@@ -93,20 +103,6 @@ or
93
103
  Following this tutorial:
94
104
  https://packaging.python.org/en/latest/tutorials/packaging-projects/
95
105
 
96
- # To Publish To TEST
97
- ```commandline
98
- python -m build
99
- # python -m build --sdist
100
- # python -m build --wheel
101
- python -m twine upload --repository testpypi dist/*
102
- pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
103
- python
104
- ```
105
- ```
106
- from water-column-sonar-processing import ZarrManager
107
- example.add_one(2)
108
- ```
109
-
110
106
  # To Publish To PROD
111
107
  ```commandline
112
108
  python -m build
@@ -134,6 +130,12 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
134
130
  5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
135
131
  3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
136
132
 
133
+ # Tag a Release
134
+ ```commandline
135
+ git tag "v0.0.12" -a
136
+ # enter description
137
+ git push origin --tags
138
+ ```
137
139
 
138
140
  # TODO:
139
141
  add https://pypi.org/project/setuptools-scm/
@@ -1,6 +1,16 @@
1
1
  # Water Column Sonar Processing
2
2
  Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
3
3
 
4
+ ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/CI-CMG/water-column-sonar-processing/test_action.yaml)
5
+
6
+ ![GitHub License](https://img.shields.io/github/license/CI-CMG/water-column-sonar-processing)
7
+
8
+ ![PyPI - Implementation](https://img.shields.io/pypi/v/water-column-sonar-processing?color=black)
9
+
10
+ ![PyPI - Downloads](https://img.shields.io/pypi/dd/water-column-sonar-processing)
11
+
12
+ ![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/CI-CMG/water-column-sonar-processing) ![GitHub repo size](https://img.shields.io/github/repo-size/CI-CMG/water-column-sonar-processing)
13
+
4
14
  # Setting up the Python Environment
5
15
  > Python 3.10.12
6
16
 
@@ -57,20 +67,6 @@ or
57
67
  Following this tutorial:
58
68
  https://packaging.python.org/en/latest/tutorials/packaging-projects/
59
69
 
60
- # To Publish To TEST
61
- ```commandline
62
- python -m build
63
- # python -m build --sdist
64
- # python -m build --wheel
65
- python -m twine upload --repository testpypi dist/*
66
- pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
67
- python
68
- ```
69
- ```
70
- from water-column-sonar-processing import ZarrManager
71
- example.add_one(2)
72
- ```
73
-
74
70
  # To Publish To PROD
75
71
  ```commandline
76
72
  python -m build
@@ -98,6 +94,12 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
98
94
  5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
99
95
  3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
100
96
 
97
+ # Tag a Release
98
+ ```commandline
99
+ git tag "v0.0.12" -a
100
+ # enter description
101
+ git push origin --tags
102
+ ```
101
103
 
102
104
  # TODO:
103
105
  add https://pypi.org/project/setuptools-scm/
@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
8
8
 
9
9
  [project]
10
10
  name = "water_column_sonar_processing"
11
- version = "0.0.10"
11
+ version = "0.0.12"
12
12
  authors = [
13
13
  { name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
14
14
  ]
@@ -18,10 +18,12 @@ pandas==2.2.3
18
18
  pyarrow==18.1.0
19
19
  python-dotenv==1.0.1
20
20
  requests==2.32.3
21
+ #s3fs==2024.3.1
21
22
  #s3fs==2024.10.0 # this version creates problems
22
23
  s3fs==2023.12.1
23
24
  scipy==1.14.1
24
- setuptools==75.6.0
25
+ #setuptools==75.6.0
26
+ setuptools
25
27
  shapely==2.0.3
26
28
  typing-extensions==4.10.0
27
29
  xarray==2024.10.0
@@ -5,6 +5,8 @@ pre-commit
5
5
  pyinstaller
6
6
  twine
7
7
 
8
- pytest~=8.3.3
9
8
  flake8==7.1.1
10
- pytest-cov==6.0.0
9
+ pooch==1.8.2
10
+ pytest~=8.3.3
11
+ pytest-cov==6.0.0
12
+ tqdm
@@ -0,0 +1,47 @@
1
+ from pathlib import Path
2
+
3
+ import pooch
4
+ import pytest
5
+
6
+ HERE = Path(__file__).parent.absolute()
7
+ TEST_DATA_FOLDER = HERE / "test_resources"
8
+
9
+
10
+ HB0707_RAW = pooch.create(
11
+ path=pooch.os_cache("water-column-sonar-processing"),
12
+ base_url="https://noaa-wcsd-pds.s3.amazonaws.com/data/raw/Henry_B._Bigelow/HB0707/EK60/",
13
+ retry_if_failed=1,
14
+ registry={
15
+ #https://noaa-wcsd-zarr-pds.s3.amazonaws.com/level_1/Henry_B._Bigelow/HB0707/EK60/D20070711-T182032.zarr/
16
+ #https://noaa-wcsd-pds.s3.amazonaws.com/data/raw/Henry_B._Bigelow/HB0707/EK60/D20070711-T182032.raw
17
+ # TODO: add bottom files
18
+ "D20070712-T124906.raw": "sha256:44f9b2402a8d6d51c69235d1e33c3e4ab570fc541e9f269009924378bf4d97a2", # 250 m, 158 MB
19
+ "D20070712-T152416.raw": "sha256:94a937eefd6ae5763c27c9ba1e4769b2b76fcc2d840e7db6c2e0edd925d6f70f", # 1000 m, 200 MB
20
+ },
21
+ )
22
+
23
+ def fetch_raw_files():
24
+ HB0707_RAW.fetch(fname="D20070712-T124906.raw", progressbar=True)
25
+ file_name = HB0707_RAW.fetch(fname="D20070712-T152416.raw", progressbar=True)
26
+ return Path(file_name).parent #joinpath(Path(file_path).stem)
27
+
28
+ @pytest.fixture(scope="session")
29
+ def test_path():
30
+ return {
31
+ 'RAW_TO_ZARR_TEST_PATH': TEST_DATA_FOLDER / "raw_to_zarr",
32
+ 'INDEX_TEST_PATH': TEST_DATA_FOLDER / "index",
33
+ 'ZARR_MANAGER_TEST_PATH': TEST_DATA_FOLDER / "zarr_manager",
34
+ 'PMTILE_GENERATION_TEST_PATH': TEST_DATA_FOLDER / "pmtile",
35
+ 'CREATE_EMPTY_ZARR_TEST_PATH': TEST_DATA_FOLDER / "create_empty_zarr",
36
+ # 'RESAMPLE_REGRID_TEST_PATH': TEST_DATA_FOLDER / "resample_regrid",
37
+ 'RESAMPLE_REGRID_TEST_PATH': fetch_raw_files(),
38
+ 'S3FS_MANAGER_TEST_PATH': TEST_DATA_FOLDER / "s3fs_manager",
39
+ 'S3_MANAGER_TEST_PATH': TEST_DATA_FOLDER / "s3_manager",
40
+ }
41
+
42
+ # """
43
+ # Windows
44
+ # C:\Users\<user>\AppData\Local\echopype\Cache\2024.12.23.10.10
45
+ # MacOS
46
+ # /Users//Library/Caches/echopype/2024.12.23.10.10
47
+ # """