water-column-sonar-processing 25.3.2__py3-none-any.whl → 25.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of water-column-sonar-processing might be problematic. Click here for more details.

Files changed (32) hide show
  1. water_column_sonar_processing/aws/dynamodb_manager.py +6 -6
  2. water_column_sonar_processing/aws/s3_manager.py +95 -90
  3. water_column_sonar_processing/aws/s3fs_manager.py +5 -3
  4. water_column_sonar_processing/aws/sqs_manager.py +1 -1
  5. water_column_sonar_processing/cruise/__init__.py +2 -1
  6. water_column_sonar_processing/cruise/create_empty_zarr_store.py +49 -43
  7. water_column_sonar_processing/cruise/create_empty_zarr_store_level_3.py +161 -0
  8. water_column_sonar_processing/cruise/datatree_manager.py +21 -21
  9. water_column_sonar_processing/cruise/resample_regrid.py +57 -47
  10. water_column_sonar_processing/dataset/__init__.py +3 -0
  11. water_column_sonar_processing/dataset/dataset_manager.py +205 -0
  12. water_column_sonar_processing/dataset/feature_manager.py +32 -0
  13. water_column_sonar_processing/geometry/geometry_manager.py +11 -12
  14. water_column_sonar_processing/geometry/line_simplification.py +26 -1
  15. water_column_sonar_processing/geometry/pmtile_generation.py +211 -247
  16. water_column_sonar_processing/index/index_manager.py +18 -17
  17. water_column_sonar_processing/model/zarr_manager.py +504 -256
  18. water_column_sonar_processing/processing/__init__.py +3 -2
  19. water_column_sonar_processing/processing/batch_downloader.py +11 -11
  20. water_column_sonar_processing/processing/raw_to_netcdf.py +319 -0
  21. water_column_sonar_processing/processing/raw_to_zarr.py +41 -31
  22. water_column_sonar_processing/utility/__init__.py +9 -2
  23. water_column_sonar_processing/utility/cleaner.py +1 -2
  24. water_column_sonar_processing/utility/constants.py +26 -7
  25. water_column_sonar_processing/utility/timestamp.py +1 -0
  26. water_column_sonar_processing-25.8.0.dist-info/METADATA +162 -0
  27. water_column_sonar_processing-25.8.0.dist-info/RECORD +39 -0
  28. {water_column_sonar_processing-25.3.2.dist-info → water_column_sonar_processing-25.8.0.dist-info}/WHEEL +1 -1
  29. water_column_sonar_processing-25.3.2.dist-info/licenses/LICENSE → water_column_sonar_processing-25.8.0.dist-info/licenses/LICENSE-MIT +1 -1
  30. water_column_sonar_processing-25.3.2.dist-info/METADATA +0 -170
  31. water_column_sonar_processing-25.3.2.dist-info/RECORD +0 -34
  32. {water_column_sonar_processing-25.3.2.dist-info → water_column_sonar_processing-25.8.0.dist-info}/top_level.txt +0 -0
water_column_sonar_processing/utility/constants.py CHANGED
@@ -1,24 +1,32 @@
1
- from enum import Enum, Flag, unique
1
+ from enum import Enum, unique
2
2
 
3
3
 
4
4
  @unique
5
- class Constants(Flag):
5
+ class Instruments(Enum):
6
+ # Values are determined using scan of the fist byte of data
7
+ EK60 = "EK60"
8
+ EK80 = "EK80"
9
+
10
+
11
+ # @unique
12
+ class Constants(Enum):
6
13
  TILE_SIZE = 512
7
14
 
8
15
  # Average https://noaa-wcsd-zarr-pds.s3.us-east-1.amazonaws.com/level_2/Henry_B._Bigelow/HB0902/EK60/HB0902.zarr/time/927
9
- # chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon data
10
- # NOTE: larger value here will speed up the TurfJS download of data in the UI
11
- # Problem interpolating the data: cannot reshape array of size 65536 into shape...
16
+ # chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon dataset
17
+ # NOTE: larger value here will speed up the TurfJS download of dataset in the UI
18
+ # Problem interpolating the dataset: cannot reshape array of size 65536 into shape...
12
19
  SPATIOTEMPORAL_CHUNK_SIZE = int(2**16) - 1024 # e.g. int(2**14)
13
20
  # TODO: create test for SPATIOTEMPORAL_CHUNK_SIZE with requirement!
14
21
 
15
22
  LEVEL_0 = "raw"
16
- LEVEL_1 = "level_1"
23
+ LEVEL_1 = "level_1" # from bucket path
17
24
  LEVEL_2 = "level_2"
18
25
  LEVEL_3 = "level_3"
19
26
 
20
27
  EK60 = "EK60" # TODO: use for "instrument"
21
28
  EK80 = "EK80"
29
+ # INSTRUMENT = EK60 | EK80
22
30
 
23
31
 
24
32
  class Coordinates(Enum):
@@ -78,7 +86,18 @@ class Coordinates(Enum):
78
86
  SPEED_STANDARD_NAME = "speed"
79
87
 
80
88
  SV = "Sv"
81
- SV_DTYPE = "float32" # TODO: experiment with dtype of int
89
+ SV_DTYPE = "float32" # int64
82
90
  SV_UNITS = "dB"
83
91
  SV_LONG_NAME = "Volume backscattering strength (Sv re 1 m-1)"
84
92
  SV_STANDARD_NAME = "volume_backscattering_strength"
93
+
94
+
95
+ class BatchShape(Enum):
96
+ """
97
+ The tensor shape of a machine learning sample.
98
+ """
99
+
100
+ DEPTH = 2
101
+ TIME = 3
102
+ FREQUENCY = 4
103
+ BATCH_SIZE = 5
water_column_sonar_processing/utility/timestamp.py CHANGED
@@ -7,6 +7,7 @@ class Timestamp:
7
7
  def get_timestamp():
8
8
  # return timestamp in form:
9
9
  # PIPELINE_TIME = '2024-03-29T19:36:52.433Z'
10
+
10
11
  return f"{datetime.utcnow().isoformat()[:23]}Z"
11
12
 
12
13
 
water_column_sonar_processing-25.8.0.dist-info/METADATA ADDED
@@ -0,0 +1,162 @@
1
+ Metadata-Version: 2.4
2
+ Name: water-column-sonar-processing
3
+ Version: 25.8.0
4
+ Summary: Processing tool for water column sonar data.
5
+ Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
6
+ Maintainer-email: Rudy Klucik <rudy.klucik@noaa.gov>
7
+ License-Expression: MIT
8
+ Project-URL: Homepage, https://echo.fish
9
+ Project-URL: Repository, https://github.com/CI-CMG/water-column-sonar-processing
10
+ Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
11
+ Keywords: ocean,sonar,water column,zarr
12
+ Requires-Python: >=3.12
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE-MIT
15
+ Requires-Dist: aiobotocore==2.23.0
16
+ Requires-Dist: boto3==1.38.23
17
+ Requires-Dist: botocore==1.38.23
18
+ Requires-Dist: echopype==0.10.1
19
+ Requires-Dist: fiona==1.10.1
20
+ Requires-Dist: geopandas==1.1.1
21
+ Requires-Dist: mock==5.2.0
22
+ Requires-Dist: moto[all]==5.1.8
23
+ Requires-Dist: moto[server]==5.1.8
24
+ Requires-Dist: multidict==6.6.3
25
+ Requires-Dist: netcdf4==1.6.5
26
+ Requires-Dist: networkx==3.5.0
27
+ Requires-Dist: numcodecs==0.15.1
28
+ Requires-Dist: numpy==1.26.4
29
+ Requires-Dist: pandas==2.3.0
30
+ Requires-Dist: pooch==1.8.2
31
+ Requires-Dist: pyarrow==20.0.0
32
+ Requires-Dist: pykalman==0.10.1
33
+ Requires-Dist: python-dotenv==1.1.1
34
+ Requires-Dist: requests==2.32.4
35
+ Requires-Dist: s3fs==2025.5.1
36
+ Requires-Dist: scipy==1.15.1
37
+ Requires-Dist: setuptools==80.9.0
38
+ Requires-Dist: shapely==2.1.1
39
+ Requires-Dist: xarray==2025.7.0
40
+ Requires-Dist: xbatcher[tensorflow]==0.4.0
41
+ Requires-Dist: zarr==2.18.7
42
+ Provides-Extra: dev
43
+ Requires-Dist: bandit[toml]==1.8.0; extra == "dev"
44
+ Requires-Dist: bandit; extra == "dev"
45
+ Requires-Dist: black; extra == "dev"
46
+ Requires-Dist: build; extra == "dev"
47
+ Requires-Dist: isort; extra == "dev"
48
+ Requires-Dist: matplotlib; extra == "dev"
49
+ Requires-Dist: pooch==1.8.2; extra == "dev"
50
+ Requires-Dist: pre-commit; extra == "dev"
51
+ Requires-Dist: pyinstaller; extra == "dev"
52
+ Requires-Dist: pytest; extra == "dev"
53
+ Requires-Dist: ruff; extra == "dev"
54
+ Requires-Dist: tqdm; extra == "dev"
55
+ Dynamic: license-file
56
+
57
+ # Water Column Sonar Processing
58
+ Processing tool for converting Level_0 water column sonar data to Level_1 and Level_2 derived data sets as well as generating geospatial information.
59
+
60
+ ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/CI-CMG/water-column-sonar-processing/test_action.yaml)
61
+ ![PyPI - Implementation](https://img.shields.io/pypi/v/ci-cmg-water-column-sonar-processing) ![GitHub License](https://img.shields.io/github/license/CI-CMG/water-column-sonar-processing) ![PyPI - Downloads](https://img.shields.io/pypi/dd/ci-cmg-water-column-sonar-processing) ![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/CI-CMG/water-column-sonar-processing) ![GitHub repo size](https://img.shields.io/github/repo-size/CI-CMG/water-column-sonar-processing)
62
+
63
+ # Setting up the Python Environment
64
+ > Python 3.12.11
65
+
66
+ # Installing Dependencies
67
+ ```
68
+ source .venv/bin/activate
69
+
70
+ uv pip install --upgrade pip
71
+
72
+ uv pip install -r pyproject.toml --all-extras
73
+
74
+ uv run pre-commit install
75
+ ```
76
+
77
+ # Pytest
78
+ ```
79
+ uv run pytest --cache-clear tests -W ignore::DeprecationWarning
80
+ ```
81
+ or
82
+ > pytest --cache-clear --cov=src tests/ --cov-report=xml
83
+
84
+ # Instructions
85
+ Following this tutorial:
86
+ https://packaging.python.org/en/latest/tutorials/packaging-projects/
87
+
88
+ # Pre Commit Hook
89
+ see here for installation: https://pre-commit.com/
90
+ https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
91
+ ```
92
+ uv run pre-commit install --allow-missing-config
93
+ # or
94
+ uv run pre-commit install
95
+ ```
96
+
97
+ # Black
98
+ https://ljvmiranda921.github.io/notebook/2018/06/21/precommits-using-black-and-flake8/
99
+ ```
100
+ Settings > Black
101
+ Execution mode: Package
102
+ Python Interpreter: .../.venv/bin/python
103
+ Use Black Formatter: X On Code reformat, X On Save
104
+ ```
105
+
106
+ # Linting
107
+ Ruff
108
+ https://plugins.jetbrains.com/plugin/20574-ruff
109
+
110
+ # Colab Test
111
+ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
112
+
113
+ # Test Coverage
114
+ TODO
115
+
116
+ # Tag a Release
117
+ Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
118
+ ```commandline
119
+ git tag -a v25.8.0 -m "Releasing v25.8.0"
120
+ git push origin --tags
121
+ ```
122
+
123
+ # To Publish To PROD
124
+ ```
125
+ uv build --no-sources
126
+ uv publish
127
+ ```
128
+
129
+ # TODO:
130
+ add https://pypi.org/project/setuptools-scm/
131
+ for extracting the version
132
+
133
+ # Security scanning
134
+ > bandit -r water_column_sonar_processing/
135
+
136
+ # Data Debugging
137
+ Experimental Plotting in Xarray (hvPlot):
138
+ https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
139
+
140
+ HB0707 Zoomable Cruise:
141
+ https://hb0707.s3.us-east-1.amazonaws.com/index.html
142
+
143
+
144
+ # UV Debugging
145
+ ```
146
+ uv lock --check
147
+ uv lock
148
+ uv sync --extra dev
149
+ ```
150
+
151
+ # Fixing S3FS Problems
152
+ ```commandline
153
+ To enable/disa asyncio for the debugger, follow the steps:
154
+ Open PyCharm
155
+ Use Shift + Shift (Search Everywhere)
156
+ In the popup type: Registry and press Enter
157
+ Find "Registry" in the list of results and click on it.
158
+ In the new popup find python.debug.asyncio.repl line and check the respective checkbox
159
+ Press Close.
160
+ Restart the IDE.
161
+ The asyncio support will be enabled in the debugger.
162
+ ```
water_column_sonar_processing-25.8.0.dist-info/RECORD ADDED
@@ -0,0 +1,39 @@
1
+ water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
2
+ water_column_sonar_processing/process.py,sha256=eatdHHc7_mPapTjxsOyD24OlEWeqj56MmVTMPjxF2QM,5290
3
+ water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
4
+ water_column_sonar_processing/aws/dynamodb_manager.py,sha256=EwSc4UTVQTR6tXsrx2eNdbhvexxfrz3GkkvO97F_9fM,14017
5
+ water_column_sonar_processing/aws/s3_manager.py,sha256=ZuShUfg4hx7XUWRiSywHJ3m0Szl699Bd8AfPualbuhs,16299
6
+ water_column_sonar_processing/aws/s3fs_manager.py,sha256=vl_NbekHoAMv81wtx3QaNOFmv0erxAtOyD5t_RLUqCA,2546
7
+ water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
8
+ water_column_sonar_processing/aws/sqs_manager.py,sha256=j_DGOKble3i-DlHT_uGxCFEmHVbYhFrpbhXdJKLtBSo,1600
9
+ water_column_sonar_processing/cruise/__init__.py,sha256=vTGiP1c0idwgxgXTrxKapvQCYwrhwW4g40KksflEWBM,258
10
+ water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=--SaVbXS7vPcMCDycNRhLEOpBqbqK1dvmizd44IwIgQ,8239
11
+ water_column_sonar_processing/cruise/create_empty_zarr_store_level_3.py,sha256=anOfYL0Jw-TqDNYiI-RinmDPE9FXp0GmLlj7f_ejmhU,6406
12
+ water_column_sonar_processing/cruise/datatree_manager.py,sha256=ed_lY04MyaXKADl7EaqHyEZ8JQMiyIAXVhvtE7YnWcg,662
13
+ water_column_sonar_processing/cruise/resample_regrid.py,sha256=piaWsEUKm3LIY4xHjuGn4uRv-GKVw6Em4DhTdM722qM,15437
14
+ water_column_sonar_processing/dataset/__init__.py,sha256=o8PyHq5NyvL6CANQMPxZDosrXza1hqXfWSvYGZ7L_MI,74
15
+ water_column_sonar_processing/dataset/dataset_manager.py,sha256=r8IZtjsV2RChFX2nz3oOIFH7f2LeB5Fp3Uj2H5yGS7o,7411
16
+ water_column_sonar_processing/dataset/feature_manager.py,sha256=MjOYNWxFODcXSJPB-X_R7cPKjlwdTAetYr8IOpiQvrk,793
17
+ water_column_sonar_processing/geometry/__init__.py,sha256=KNhgfKimxuMtyZwMQlIc_OTtEkYXNXl1WyvfNJThlWY,306
18
+ water_column_sonar_processing/geometry/elevation_manager.py,sha256=Wi7xl6LFsxqUPGbrquqtPEKnOMpl3dMLJtEb7DR5o1c,4249
19
+ water_column_sonar_processing/geometry/geometry_manager.py,sha256=dmPdFjeGXfh-bhdRSNVr1pBjnSlcjFUGwXIwXmq4Fe0,10986
20
+ water_column_sonar_processing/geometry/line_simplification.py,sha256=WliMhpvrIfPX8qK7N8EVT59SPPiKkIBgxBbw8twvPNA,9278
21
+ water_column_sonar_processing/geometry/pmtile_generation.py,sha256=kZPG5oskCqsVyCr97c9nlhgNBkLtu_-gOlB2GKvtQhA,9462
22
+ water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
23
+ water_column_sonar_processing/index/index_manager.py,sha256=7qODHuYsyWoMhqot2-P1Pl7yzZQpVPLSf6Ad-2hjtRY,16154
24
+ water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
25
+ water_column_sonar_processing/model/zarr_manager.py,sha256=Tx3UkAIEVg1PiPvlpYtOuWGxYu6D0_sPK9JLCdOTOyA,28959
26
+ water_column_sonar_processing/processing/__init__.py,sha256=Mq4J2ikGIreyIDaery4PXjGV_6VrYdjHA4M4KPFa7fs,259
27
+ water_column_sonar_processing/processing/batch_downloader.py,sha256=iiCXS2mM7jBrl5UByAgLLDLHlR2wRogYJgSnufsRQ-U,6683
28
+ water_column_sonar_processing/processing/raw_to_netcdf.py,sha256=f2N8jgNmtpB2R892HwBb5hzYs2dScmKZI-MAy5TfmZ8,13091
29
+ water_column_sonar_processing/processing/raw_to_zarr.py,sha256=fhR5qZ4KTwSKCWzcSgpePir0B8IeIPwOHWIL4tHrIMo,18620
30
+ water_column_sonar_processing/utility/__init__.py,sha256=5gV8sJcnQGrV2AecwkBFZljOC-fQwLuDX70DxX59x2o,289
31
+ water_column_sonar_processing/utility/cleaner.py,sha256=q9xx4NeVhaxQgGCCYyZv29PxvMHfD9txbnMahOccF8U,607
32
+ water_column_sonar_processing/utility/constants.py,sha256=8ETn-4NyiquDS4Mr-GaG76aP6yOCB48H_Y6Y7ei_FTI,2959
33
+ water_column_sonar_processing/utility/pipeline_status.py,sha256=xbl6-4ePq1Krfo18Mgr0jzWz9C_g2Kbey_QZNJuwdkI,4406
34
+ water_column_sonar_processing/utility/timestamp.py,sha256=7DJW4_5Szn4Z4B51hlNSdXfSXZosrK69oIbkeJoqqW0,362
35
+ water_column_sonar_processing-25.8.0.dist-info/licenses/LICENSE-MIT,sha256=TosqaZpJgYvhgXIyYBti-ggJaO8rxRg3FtThY08s9Aw,1110
36
+ water_column_sonar_processing-25.8.0.dist-info/METADATA,sha256=6lgAQth4pNquX6N9UV3doVDTNJMcN7QR7xgRqi0Cl44,5076
37
+ water_column_sonar_processing-25.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
38
+ water_column_sonar_processing-25.8.0.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
39
+ water_column_sonar_processing-25.8.0.dist-info/RECORD,,
{water_column_sonar_processing-25.3.2.dist-info → water_column_sonar_processing-25.8.0.dist-info}/WHEEL RENAMED
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (77.0.3)
2
+ Generator: setuptools (80.9.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
water_column_sonar_processing-25.3.2.dist-info/licenses/LICENSE → water_column_sonar_processing-25.8.0.dist-info/licenses/LICENSE-MIT RENAMED
@@ -1,6 +1,6 @@
1
1
  MIT License
2
2
 
3
- Copyright (c) 2024 Cooperative Institutes, Coastal and Marine Geophysics
3
+ Copyright (c) 2025 Cooperative Institutes, Coastal and Marine Geophysics
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
water_column_sonar_processing-25.3.2.dist-info/METADATA DELETED
@@ -1,170 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: water_column_sonar_processing
3
- Version: 25.3.2
4
- Summary: Processing tool for water column sonar data.
5
- Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
6
- Maintainer-email: Rudy Klucik <rudy.klucik@noaa.gov>
7
- Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
8
- Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
9
- Keywords: ocean,sonar,water column
10
- Classifier: Programming Language :: Python :: 3
11
- Classifier: License :: OSI Approved :: MIT License
12
- Classifier: Operating System :: OS Independent
13
- Requires-Python: >=3.10
14
- Description-Content-Type: text/markdown
15
- License-File: LICENSE
16
- Requires-Dist: aiobotocore==2.19.0
17
- Requires-Dist: boto3==1.36.3
18
- Requires-Dist: botocore==1.36.3
19
- Requires-Dist: echopype==0.9.0
20
- Requires-Dist: fiona==1.10.1
21
- Requires-Dist: geopandas==1.0.1
22
- Requires-Dist: mock==5.1.0
23
- Requires-Dist: moto[all]==5.0.27
24
- Requires-Dist: moto[server]==5.0.27
25
- Requires-Dist: networkx==3.4.2
26
- Requires-Dist: numcodecs==0.13.1
27
- Requires-Dist: numpy==1.26.4
28
- Requires-Dist: pandas==2.2.3
29
- Requires-Dist: pyarrow==18.1.0
30
- Requires-Dist: pykalman==0.10.1
31
- Requires-Dist: python-dotenv==1.0.1
32
- Requires-Dist: requests==2.32.3
33
- Requires-Dist: s3fs==2024.2.0
34
- Requires-Dist: safety==3.2.14
35
- Requires-Dist: scipy==1.14.1
36
- Requires-Dist: setuptools
37
- Requires-Dist: shapely==2.0.3
38
- Requires-Dist: typing-extensions==4.10.0
39
- Requires-Dist: xarray==2024.10.0
40
- Requires-Dist: xbatcher==0.4.0
41
- Requires-Dist: zarr==2.18.3
42
- Provides-Extra: dev
43
- Requires-Dist: bandit[toml]==1.8.0; extra == "dev"
44
- Requires-Dist: bandit; extra == "dev"
45
- Requires-Dist: build; extra == "dev"
46
- Requires-Dist: flake8==7.1.1; extra == "dev"
47
- Requires-Dist: isort; extra == "dev"
48
- Requires-Dist: matplotlib==3.10.1; extra == "dev"
49
- Requires-Dist: pooch==1.8.2; extra == "dev"
50
- Requires-Dist: pre-commit; extra == "dev"
51
- Requires-Dist: pyinstaller; extra == "dev"
52
- Requires-Dist: pytest~=8.3.4; extra == "dev"
53
- Requires-Dist: ruff; extra == "dev"
54
- Requires-Dist: tqdm; extra == "dev"
55
- Requires-Dist: twine; extra == "dev"
56
- Provides-Extra: test
57
- Requires-Dist: pytest-cov; extra == "test"
58
- Dynamic: license-file
59
-
60
- # Water Column Sonar Processing
61
- Processing tool for converting Level_0 water column sonar data to Level_1 and Level_2 derived data sets as well as generating geospatial information.
62
-
63
- ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/CI-CMG/water-column-sonar-processing/test_action.yaml) ![PyPI - Implementation](https://img.shields.io/pypi/v/water-column-sonar-processing) ![GitHub License](https://img.shields.io/github/license/CI-CMG/water-column-sonar-processing) ![PyPI - Downloads](https://img.shields.io/pypi/dd/water-column-sonar-processing) ![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/CI-CMG/water-column-sonar-processing) ![GitHub repo size](https://img.shields.io/github/repo-size/CI-CMG/water-column-sonar-processing)
64
-
65
- # Setting up the Python Environment
66
- > Python 3.10.12
67
-
68
- # MacOS Pyenv Installation Instructions
69
- 1. Install pyenv (https://github.com/pyenv/pyenv#set-up-your-shell-environment-for-pyenv)
70
- 1. ```brew update```
71
- 2. ```arch -arm64 brew install pyenv```
72
- 3. In ~/.bashrc add
73
- 1. ```export PYENV_ROOT="$HOME/.pyenv"```
74
- 2. ```export PATH="$PYENV_ROOT/bin:$PATH"```
75
- 3. ```eval "$(pyenv init -)"```
76
- 4. ```arch -arm64 brew install openssl readline sqlite3 xz zlib tcl-tk```
77
- 2. Install pyenv-virtualenv (https://github.com/pyenv/pyenv-virtualenv)
78
- 1. ```arch -arm64 brew install pyenv-virtualenv```
79
- 2. In ~/.bashrc add
80
- 1. ```eval "$(pyenv virtualenv-init -)"```
81
- 3. Open a new terminal
82
- 4. Install Python version
83
- 1. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
84
- 5. Create virtual env (to delete 'pyenv uninstall 3.10.12/water-column-sonar-processing')
85
- 1. ```pyenv virtualenv 3.10.12 water-column-sonar-processing```
86
- 6. Set local version of python (if not done already)
87
- 1. change directory to root of project
88
- 2. ```pyenv local 3.10.12 water-column-sonar-processing```
89
- 3. ```pyenv activate water-column-sonar-processing```
90
-
91
- # Setting up IntelliJ
92
-
93
- 1. Install the IntelliJ Python plugin
94
- 2. Set up pyenv
95
- 1. File -> Project Structure or CMD + ;
96
- 2. SDKs -> + -> Add Python SDK -> Virtual Environment
97
- 3. Select Existing Environment
98
- 4. Choose ~/.pyenv/versions/mocking_aws/bin/python
99
- 3. Set up Python Facet (not sure if this is required)
100
- 1. File -> Project Structure or CMD + ;
101
- 2. Facets -> + -> Python
102
- 3. Set interpreter
103
-
104
- # Installing Dependencies
105
- ```
106
- uv pip install --upgrade pip
107
- #uv pip install -r requirements_dev.txt
108
- uv pip install -r pyproject.toml --extra dev
109
- ```
110
-
111
-
112
- # Pytest
113
- ```commandline
114
- uv run pytest tests
115
- #pytest --disable-warnings
116
- ```
117
- or
118
- > pytest --cache-clear --cov=src tests/ --cov-report=xml
119
-
120
- # Instructions
121
- Following this tutorial:
122
- https://packaging.python.org/en/latest/tutorials/packaging-projects/
123
-
124
- # Pre Commit Hook
125
- see here for installation: https://pre-commit.com/
126
- https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
127
- ```
128
- pre-commit install --allow-missing-config
129
- ```
130
-
131
- # Linting
132
- Ruff
133
- https://plugins.jetbrains.com/plugin/20574-ruff
134
-
135
- # Colab Test
136
- https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
137
-
138
- # Test Coverage
139
- 20241124
140
- 8 failed, 32 passed, 3 skipped, 1 warning in 6.92s
141
- 20241125
142
- 5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
143
- 3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
144
-
145
- # Tag a Release
146
- Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
147
- ```commandline
148
- git tag -a v25.1.8 -m "Releasing version v25.1.8"
149
- git push origin --tags
150
- ```
151
-
152
- # To Publish To PROD
153
- ```commandline
154
- uv build
155
- python -m twine upload --repository pypi dist/*
156
- ```
157
-
158
- # TODO:
159
- add https://pypi.org/project/setuptools-scm/
160
- for extracting the version
161
-
162
- # Security scanning
163
- > bandit -r water_column_sonar_processing/
164
-
165
- # Data Debugging
166
- Experimental Plotting in Xarray (hvPlot):
167
- https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
168
-
169
- HB0707 Cruise zoomable:
170
- https://hb0707.s3.us-east-1.amazonaws.com/index.html
water_column_sonar_processing-25.3.2.dist-info/RECORD DELETED
@@ -1,34 +0,0 @@
1
- water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
2
- water_column_sonar_processing/process.py,sha256=eatdHHc7_mPapTjxsOyD24OlEWeqj56MmVTMPjxF2QM,5290
3
- water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
4
- water_column_sonar_processing/aws/dynamodb_manager.py,sha256=zD2VkI1tTt6HgxXPjCp5im3_jl12wANaP_vz37FmHNo,13957
5
- water_column_sonar_processing/aws/s3_manager.py,sha256=JT9sCqlE1vm0rKo7bb_-T_uWd5ob21UzU58EcmC8454,16146
6
- water_column_sonar_processing/aws/s3fs_manager.py,sha256=U5AO9-V2VdBpVZtQyIfMORz7hSyJMiATuevfe5z6VdA,2497
7
- water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
8
- water_column_sonar_processing/aws/sqs_manager.py,sha256=NSUrWmnSC8h8Gf7gT0U8zFaQQ-yX89h0Q0mDLKGqp2Y,1597
9
- water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
10
- water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=UcUN5COMvHsxI8cNL1htUivnkI8zF9pQEnGxgVQhcDk,7855
11
- water_column_sonar_processing/cruise/datatree_manager.py,sha256=V0gVlqIN7qZh1X8e9DSH745j6eX6rmCM3OZAxCDpiXo,620
12
- water_column_sonar_processing/cruise/resample_regrid.py,sha256=fGJX2_ts4K_ZLcEGxo6x-nBQQeltp-FrWRCh30928Ac,14860
13
- water_column_sonar_processing/geometry/__init__.py,sha256=KNhgfKimxuMtyZwMQlIc_OTtEkYXNXl1WyvfNJThlWY,306
14
- water_column_sonar_processing/geometry/elevation_manager.py,sha256=Wi7xl6LFsxqUPGbrquqtPEKnOMpl3dMLJtEb7DR5o1c,4249
15
- water_column_sonar_processing/geometry/geometry_manager.py,sha256=noPx6Es87jeE5ApQJ5ZhVUHhpJAaiFTlh5zJSxdCyys,10988
16
- water_column_sonar_processing/geometry/line_simplification.py,sha256=b-rqPnb3zl0LNFaFoEiNCrZ-wyagIHWKWUDBDBImKwk,8353
17
- water_column_sonar_processing/geometry/pmtile_generation.py,sha256=jgCZgmG-K-hi0O0MRUlnbx50_XM5UTP9BewE9tDpY4M,13016
18
- water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
19
- water_column_sonar_processing/index/index_manager.py,sha256=wvG0rzyR-osqpkF_ufuz2JheZAnpE_eYr-Z088rJPAc,16073
20
- water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
21
- water_column_sonar_processing/model/zarr_manager.py,sha256=WMSdbhefAANyg71pu-8LDeiyDrBZu0TLERZXZwuy8Jg,16964
22
- water_column_sonar_processing/processing/__init__.py,sha256=9zLaJZlI9ylGwCWderusidz9d9wPQoadMstwyKfnao8,169
23
- water_column_sonar_processing/processing/batch_downloader.py,sha256=PJvm8NKF0bygqnL9yYwoK5ydntUNrCEC4xPyjSrj_ZE,6650
24
- water_column_sonar_processing/processing/raw_to_zarr.py,sha256=9Kl8Tl4pZTJ-nVTGAk5ALunPjF--jlVAtc-_dE874jI,18280
25
- water_column_sonar_processing/utility/__init__.py,sha256=yDObMOL0_OxKWet5wffK2-XVJgoE9iwiY2q04GZrtBQ,234
26
- water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
27
- water_column_sonar_processing/utility/constants.py,sha256=Gk--g8yFLOc6KTOi-YvOo9zvBhaa8PEtjdDIU3XH3b4,2639
28
- water_column_sonar_processing/utility/pipeline_status.py,sha256=xbl6-4ePq1Krfo18Mgr0jzWz9C_g2Kbey_QZNJuwdkI,4406
29
- water_column_sonar_processing/utility/timestamp.py,sha256=bO0oir7KxxoEHPGRkz9FCBfOligkocUyRiWRzAq8fnU,361
30
- water_column_sonar_processing-25.3.2.dist-info/licenses/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
31
- water_column_sonar_processing-25.3.2.dist-info/METADATA,sha256=dMrpWsWRHjoYb56zGgB_WoHQtx0hWafWRb1CXpiDQwE,6257
32
- water_column_sonar_processing-25.3.2.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
33
- water_column_sonar_processing-25.3.2.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
34
- water_column_sonar_processing-25.3.2.dist-info/RECORD,,