water-column-sonar-processing 0.0.1__py3-none-any.whl → 26.1.14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of water-column-sonar-processing might be problematic. Click here for more details.

Files changed (60) hide show
  1. water_column_sonar_processing/__init__.py +13 -0
  2. water_column_sonar_processing/aws/__init__.py +7 -0
  3. water_column_sonar_processing/aws/dynamodb_manager.py +355 -0
  4. water_column_sonar_processing/aws/s3_manager.py +418 -0
  5. water_column_sonar_processing/aws/s3fs_manager.py +64 -0
  6. {model → water_column_sonar_processing}/aws/sns_manager.py +10 -21
  7. {model → water_column_sonar_processing}/aws/sqs_manager.py +11 -19
  8. water_column_sonar_processing/cruise/__init__.py +4 -0
  9. water_column_sonar_processing/cruise/create_empty_zarr_store.py +129 -0
  10. water_column_sonar_processing/cruise/datatree_manager.py +21 -0
  11. water_column_sonar_processing/cruise/resample_regrid.py +323 -0
  12. water_column_sonar_processing/geometry/__init__.py +13 -0
  13. water_column_sonar_processing/geometry/elevation_manager.py +111 -0
  14. water_column_sonar_processing/geometry/geometry_manager.py +241 -0
  15. water_column_sonar_processing/geometry/line_simplification.py +176 -0
  16. water_column_sonar_processing/geometry/pmtile_generation.py +266 -0
  17. water_column_sonar_processing/geometry/spatiotemporal.py +106 -0
  18. water_column_sonar_processing/index/__init__.py +3 -0
  19. water_column_sonar_processing/index/index_manager.py +381 -0
  20. water_column_sonar_processing/model/__init__.py +3 -0
  21. water_column_sonar_processing/model/zarr_manager.py +741 -0
  22. water_column_sonar_processing/processing/__init__.py +4 -0
  23. water_column_sonar_processing/processing/raw_to_netcdf.py +320 -0
  24. water_column_sonar_processing/processing/raw_to_zarr.py +331 -0
  25. water_column_sonar_processing/utility/__init__.py +13 -0
  26. {model → water_column_sonar_processing}/utility/cleaner.py +7 -7
  27. water_column_sonar_processing/utility/constants.py +118 -0
  28. {model → water_column_sonar_processing}/utility/pipeline_status.py +47 -24
  29. water_column_sonar_processing/utility/timestamp.py +12 -0
  30. water_column_sonar_processing-26.1.14.dist-info/METADATA +240 -0
  31. water_column_sonar_processing-26.1.14.dist-info/RECORD +34 -0
  32. {water_column_sonar_processing-0.0.1.dist-info → water_column_sonar_processing-26.1.14.dist-info}/WHEEL +1 -1
  33. {water_column_sonar_processing-0.0.1.dist-info → water_column_sonar_processing-26.1.14.dist-info/licenses}/LICENSE +1 -1
  34. water_column_sonar_processing-26.1.14.dist-info/top_level.txt +1 -0
  35. __init__.py +0 -0
  36. model/__init__.py +0 -0
  37. model/aws/__init__.py +0 -0
  38. model/aws/dynamodb_manager.py +0 -149
  39. model/aws/s3_manager.py +0 -356
  40. model/aws/s3fs_manager.py +0 -74
  41. model/cruise/__init__.py +0 -0
  42. model/cruise/create_empty_zarr_store.py +0 -166
  43. model/cruise/resample_regrid.py +0 -248
  44. model/geospatial/__init__.py +0 -0
  45. model/geospatial/geometry_manager.py +0 -194
  46. model/geospatial/geometry_simplification.py +0 -81
  47. model/geospatial/pmtile_generation.py +0 -74
  48. model/index/__init__.py +0 -0
  49. model/index/index.py +0 -228
  50. model/model.py +0 -138
  51. model/utility/__init__.py +0 -0
  52. model/utility/constants.py +0 -56
  53. model/utility/timestamp.py +0 -12
  54. model/zarr/__init__.py +0 -0
  55. model/zarr/bar.py +0 -28
  56. model/zarr/foo.py +0 -11
  57. model/zarr/zarr_manager.py +0 -298
  58. water_column_sonar_processing-0.0.1.dist-info/METADATA +0 -89
  59. water_column_sonar_processing-0.0.1.dist-info/RECORD +0 -32
  60. water_column_sonar_processing-0.0.1.dist-info/top_level.txt +0 -2
@@ -1,21 +1,21 @@
1
- import os
2
1
  import glob
2
+ import os
3
3
  import shutil
4
4
 
5
5
 
6
6
  ###########################################################
7
7
  class Cleaner:
8
8
  @staticmethod
9
- def delete_local_files(
10
- file_types=['*.raw*', '*.zarr'] # '*.json'
11
- ):
12
- print('Deleting all local raw and zarr files')
9
+ def delete_local_files(file_types=["*.raw*", "*.model"]): # '*.json'
10
+ # TODO: add .zarr to this
11
+ print("Deleting all local raw and model files")
13
12
  for i in file_types:
14
13
  for j in glob.glob(i):
15
14
  if os.path.isdir(j):
16
15
  shutil.rmtree(j, ignore_errors=True)
17
16
  elif os.path.isfile(j):
18
17
  os.remove(j)
19
- print('done deleting')
18
+ print("done deleting")
20
19
 
21
- ###########################################################
20
+
21
+ ###########################################################
@@ -0,0 +1,118 @@
1
+ from enum import Enum, unique
2
+
3
+ import numpy as np
4
+
5
+
6
+ @unique
7
+ class Instruments(Enum):
8
+ # Values are determined using scan of the fist byte of data
9
+ EK60 = "EK60"
10
+ EK80 = "EK80"
11
+
12
+
13
+ # @unique
14
+ class Constants(Enum):
15
+ """
16
+ See here for data type support: https://github.com/zarr-developers/zarr-extensions/tree/main/data-types
17
+ """
18
+
19
+ TILE_SIZE = 512
20
+
21
+ # Average https://noaa-wcsd-zarr-pds.s3.us-east-1.amazonaws.com/level_2/Henry_B._Bigelow/HB0902/EK60/HB0902.zarr/time/927
22
+ # chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon dataset
23
+ # NOTE: larger value here will speed up the TurfJS download of dataset in the UI
24
+ # Problem interpolating the dataset: cannot reshape array of size 65536 into shape...
25
+ # TODO: needs to be enum
26
+ SPATIOTEMPORAL_CHUNK_SIZE = int(1e6) # int(2 ** 16) - 1024
27
+ # TODO: create test for SPATIOTEMPORAL_CHUNK_SIZE with requirement!
28
+
29
+ LEVEL_0 = "raw"
30
+ LEVEL_1 = "level_1" # from bucket path
31
+ LEVEL_2 = "level_2a" # updating zarr store path for zarr v3
32
+ LEVEL_3 = "level_3"
33
+
34
+ EK60 = "EK60" # TODO: use for "instrument"
35
+ EK80 = "EK80"
36
+ # INSTRUMENT = EK60 | EK80
37
+
38
+
39
+ class Coordinates(Enum):
40
+ """
41
+ dtype: data type
42
+ units: netcdf defined units
43
+ long_name: most readable description of variable
44
+ standard_name: name in lowercase and snake_case
45
+ """
46
+
47
+ PROJECT_NAME = "echofish"
48
+
49
+ DEPTH = "depth"
50
+ DEPTH_DTYPE = "float32"
51
+ DEPTH_UNITS = "m" # TODO: Pint? <https://pint.readthedocs.io/en/stable/>
52
+ DEPTH_LONG_NAME = "Depth below surface"
53
+ DEPTH_STANDARD_NAME = "depth"
54
+
55
+ # https://cfconventions.org/Data/cf-conventions/cf-conventions-1.8/cf-conventions.html#table-supported-units
56
+ TIME = "time"
57
+ TIME_DTYPE = "datetime64[ns]"
58
+ # Note: units and calendar are used downstream by Xarray
59
+ TIME_UNITS = "nanoseconds since 1970-01-01"
60
+ TIME_LONG_NAME = "Timestamp of each ping"
61
+ TIME_STANDARD_NAME = "time"
62
+ TIME_CALENDAR = "proleptic_gregorian"
63
+ # TODO: create test for reading out timestamps in Xarray
64
+
65
+ FREQUENCY = "frequency"
66
+ FREQUENCY_DTYPE = np.uint64
67
+ FREQUENCY_UNITS = "Hz"
68
+ FREQUENCY_LONG_NAME = "Transducer frequency"
69
+ FREQUENCY_STANDARD_NAME = "sound_frequency"
70
+
71
+ LATITUDE = "latitude"
72
+ LATITUDE_DTYPE = np.float32
73
+ LATITUDE_UNITS = "degrees_north"
74
+ LATITUDE_LONG_NAME = "Latitude"
75
+ LATITUDE_STANDARD_NAME = "latitude"
76
+
77
+ LONGITUDE = "longitude"
78
+ LONGITUDE_DTYPE = np.float32
79
+ LONGITUDE_UNITS = "degrees_east"
80
+ LONGITUDE_LONG_NAME = "Longitude"
81
+ LONGITUDE_STANDARD_NAME = "longitude"
82
+
83
+ BOTTOM = "bottom"
84
+ BOTTOM_DTYPE = np.float32
85
+ BOTTOM_UNITS = "m"
86
+ BOTTOM_LONG_NAME = "Detected sea floor depth"
87
+ BOTTOM_STANDARD_NAME = "bottom"
88
+
89
+ SPEED = "speed"
90
+ SPEED_DTYPE = np.float32
91
+ SPEED_UNITS = "Knots"
92
+ SPEED_LONG_NAME = "Nautical miles per hour"
93
+ SPEED_STANDARD_NAME = "speed"
94
+
95
+ # This is the width of each 'pixel' of the water columns
96
+ DISTANCE = "distance"
97
+ DISTANCE_DTYPE = np.float32
98
+ DISTANCE_UNITS = "m"
99
+ DISTANCE_LONG_NAME = "GPS distance"
100
+ DISTANCE_STANDARD_NAME = "distance"
101
+
102
+ SV = "Sv"
103
+ SV_DTYPE = np.float32
104
+ SV_UNITS = "dB"
105
+ SV_LONG_NAME = "Volume backscattering strength (Sv re 1 m-1)"
106
+ SV_STANDARD_NAME = "volume_backscattering_strength"
107
+
108
+
109
+ # TODO: delete this
110
+ # class BatchShape(Enum):
111
+ # """
112
+ # The tensor shape of a machine learning sample.
113
+ # """
114
+ #
115
+ # DEPTH = 2
116
+ # TIME = 3
117
+ # FREQUENCY = 4
118
+ # BATCH_SIZE = 5
@@ -16,13 +16,17 @@ class PipelineStatus(Flag):
16
16
  PROCESSING_RAW_PROCESSOR = auto()
17
17
  FAILURE_RAW_PROCESSOR = auto()
18
18
  SUCCESS_RAW_PROCESSOR = auto()
19
- RAW_PROCESSOR = PROCESSING_RAW_PROCESSOR | FAILURE_RAW_PROCESSOR | SUCCESS_RAW_PROCESSOR
19
+ RAW_PROCESSOR = (
20
+ PROCESSING_RAW_PROCESSOR | FAILURE_RAW_PROCESSOR | SUCCESS_RAW_PROCESSOR
21
+ )
20
22
  #
21
23
  # RAW_AGGREGATOR --> AGGREGATOR (Scatter-Gather EIP)
22
24
  PROCESSING_RAW_AGGREGATOR = auto()
23
25
  FAILURE_RAW_AGGREGATOR = auto()
24
26
  SUCCESS_RAW_AGGREGATOR = auto()
25
- RAW_AGGREGATOR = PROCESSING_RAW_AGGREGATOR | FAILURE_RAW_AGGREGATOR | SUCCESS_RAW_AGGREGATOR
27
+ RAW_AGGREGATOR = (
28
+ PROCESSING_RAW_AGGREGATOR | FAILURE_RAW_AGGREGATOR | SUCCESS_RAW_AGGREGATOR
29
+ )
26
30
  #
27
31
  LEVEL_1_PROCESSING = RAW_SPLITTER | RAW_PROCESSOR | RAW_AGGREGATOR
28
32
  #
@@ -35,27 +39,43 @@ class PipelineStatus(Flag):
35
39
  PROCESSING_CRUISE_INITIALIZER = auto()
36
40
  FAILURE_CRUISE_INITIALIZER = auto()
37
41
  SUCCESS_CRUISE_INITIALIZER = auto()
38
- CRUISE_INITIALIZER = PROCESSING_CRUISE_INITIALIZER | FAILURE_CRUISE_INITIALIZER | SUCCESS_CRUISE_INITIALIZER
42
+ CRUISE_INITIALIZER = (
43
+ PROCESSING_CRUISE_INITIALIZER
44
+ | FAILURE_CRUISE_INITIALIZER
45
+ | SUCCESS_CRUISE_INITIALIZER
46
+ )
39
47
  #
40
48
  # CRUISE_SPLITTER --> SPLITTER
41
49
  PROCESSING_CRUISE_SPLITTER = auto()
42
50
  FAILURE_CRUISE_SPLITTER = auto()
43
51
  SUCCESS_CRUISE_SPLITTER = auto()
44
- CRUISE_SPLITTER = PROCESSING_CRUISE_SPLITTER | FAILURE_CRUISE_SPLITTER | SUCCESS_CRUISE_SPLITTER
52
+ CRUISE_SPLITTER = (
53
+ PROCESSING_CRUISE_SPLITTER | FAILURE_CRUISE_SPLITTER | SUCCESS_CRUISE_SPLITTER
54
+ )
45
55
  #
46
56
  # CRUISE_PROCESSOR --> PROCESSOR <-- Note: these need to run sequentially now
47
57
  PROCESSING_CRUISE_PROCESSOR = auto()
48
58
  FAILURE_CRUISE_PROCESSOR = auto()
49
59
  SUCCESS_CRUISE_PROCESSOR = auto()
50
- CRUISE_PROCESSOR = PROCESSING_CRUISE_PROCESSOR | FAILURE_CRUISE_PROCESSOR | SUCCESS_CRUISE_PROCESSOR
60
+ CRUISE_PROCESSOR = (
61
+ PROCESSING_CRUISE_PROCESSOR
62
+ | FAILURE_CRUISE_PROCESSOR
63
+ | SUCCESS_CRUISE_PROCESSOR
64
+ )
51
65
  #
52
66
  # CRUISE_AGGREGATOR --> AGGREGATOR
53
67
  PROCESSING_CRUISE_AGGREGATOR = auto()
54
68
  FAILURE_CRUISE_AGGREGATOR = auto()
55
69
  SUCCESS_CRUISE_AGGREGATOR = auto()
56
- CRUISE_AGGREGATOR = PROCESSING_CRUISE_AGGREGATOR | FAILURE_CRUISE_AGGREGATOR | SUCCESS_CRUISE_AGGREGATOR
70
+ CRUISE_AGGREGATOR = (
71
+ PROCESSING_CRUISE_AGGREGATOR
72
+ | FAILURE_CRUISE_AGGREGATOR
73
+ | SUCCESS_CRUISE_AGGREGATOR
74
+ )
57
75
  #
58
- LEVEL_2_PROCESSING = CRUISE_INITIALIZER | CRUISE_SPLITTER | CRUISE_PROCESSOR | CRUISE_AGGREGATOR
76
+ LEVEL_2_PROCESSING = (
77
+ CRUISE_INITIALIZER | CRUISE_SPLITTER | CRUISE_PROCESSOR | CRUISE_AGGREGATOR
78
+ )
59
79
  #
60
80
  # --------------------------------------------------- #
61
81
  # --- Level 3 Data ---------------------------------- #
@@ -64,35 +84,38 @@ class PipelineStatus(Flag):
64
84
  PROCESSING_TILE_PROCESSOR = auto()
65
85
  FAILURE_TILE_PROCESSOR = auto()
66
86
  SUCCESS_TILE_PROCESSOR = auto()
67
- TILE_PROCESSOR = PROCESSING_TILE_PROCESSOR | FAILURE_TILE_PROCESSOR | SUCCESS_TILE_PROCESSOR
87
+ TILE_PROCESSOR = (
88
+ PROCESSING_TILE_PROCESSOR | FAILURE_TILE_PROCESSOR | SUCCESS_TILE_PROCESSOR
89
+ )
68
90
  #
69
91
  # GEOHASH_PROCESSOR
70
92
  PROCESSING_GEOHASH_PROCESSOR = auto()
71
93
  FAILURE_GEOHASH_PROCESSOR = auto()
72
94
  SUCCESS_GEOHASH_PROCESSOR = auto()
73
- GEOHASH_PROCESSOR = PROCESSING_GEOHASH_PROCESSOR | FAILURE_GEOHASH_PROCESSOR | SUCCESS_GEOHASH_PROCESSOR
95
+ GEOHASH_PROCESSOR = (
96
+ PROCESSING_GEOHASH_PROCESSOR
97
+ | FAILURE_GEOHASH_PROCESSOR
98
+ | SUCCESS_GEOHASH_PROCESSOR
99
+ )
74
100
  #
75
101
  LEVEL_3_PROCESSING = TILE_PROCESSOR | GEOHASH_PROCESSOR
76
102
  # --------------------------------------------------- #
77
103
  # --------------------------------------------------- #
78
104
 
105
+
79
106
  # Status.PROCESSING_RAW_AGGREGATOR in Status.LEVEL_1_PROCESSING
80
107
  # Status.LEVEL_1_PROCESSING.value < Status.LEVEL_2_PROCESSING.value
81
108
 
82
109
  # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cloudformation-stack.html
83
- """
84
- CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
85
- ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
86
- DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
87
- UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
88
- UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
89
- UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
90
- REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
91
- IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
92
-
93
- failure - noun -
94
- failed - verb - "verbs should be avoided"
95
-
96
- success - noun
97
110
 
98
- """
111
+ # CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
112
+ # ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
113
+ # DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
114
+ # UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
115
+ # UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
116
+ # UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
117
+ # REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
118
+ # IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
119
+ # failure - noun -
120
+ # failed - verb - "verbs should be avoided"
121
+ # success - noun
@@ -0,0 +1,12 @@
1
+ import datetime
2
+
3
+
4
+ ###########################################################
5
+ class Timestamp:
6
+ @staticmethod
7
+ def get_timestamp():
8
+ # return timestamp in form: '2024-03-29T19:36:52.433Z'
9
+ return f"{datetime.datetime.now(datetime.UTC).isoformat()[:23]}Z"
10
+
11
+
12
+ ###########################################################
@@ -0,0 +1,240 @@
1
+ Metadata-Version: 2.4
2
+ Name: water-column-sonar-processing
3
+ Version: 26.1.14
4
+ Summary: Processing tool for water column sonar data.
5
+ Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
6
+ Maintainer-email: Rudy Klucik <rudy.klucik@noaa.gov>
7
+ License-Expression: MIT
8
+ Project-URL: Homepage, https://echo.fish
9
+ Project-URL: Repository, https://github.com/CI-CMG/water-column-sonar-processing
10
+ Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
11
+ Keywords: ocean,sonar,water column,zarr
12
+ Requires-Python: >=3.12
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+ Requires-Dist: boto3==1.41.5
16
+ Requires-Dist: boto3-stubs
17
+ Requires-Dist: botocore==1.41.5
18
+ Requires-Dist: build
19
+ Requires-Dist: echopype==0.11.0
20
+ Requires-Dist: fiona==1.10.1
21
+ Requires-Dist: geopandas==1.1.2
22
+ Requires-Dist: importlib==1.0.4
23
+ Requires-Dist: mock
24
+ Requires-Dist: moto[all]==5.1.19
25
+ Requires-Dist: moto[server]==5.1.19
26
+ Requires-Dist: multidict==6.7.0
27
+ Requires-Dist: netcdf4==1.7.3
28
+ Requires-Dist: numpy==2.4.1
29
+ Requires-Dist: pandas==2.3.3
30
+ Requires-Dist: pandas-stubs
31
+ Requires-Dist: pooch==1.8.2
32
+ Requires-Dist: pyarrow==22.0.0
33
+ Requires-Dist: pykalman==0.11.1
34
+ Requires-Dist: python-dotenv
35
+ Requires-Dist: requests==2.32.5
36
+ Requires-Dist: ruff>=0.14.13
37
+ Requires-Dist: s3fs==2026.1.0
38
+ Requires-Dist: scipy==1.17.0
39
+ Requires-Dist: setuptools==80.9.0
40
+ Requires-Dist: shapely==2.1.2
41
+ Requires-Dist: xarray==2025.12.0
42
+ Requires-Dist: zarr==3.1.5
43
+ Dynamic: license-file
44
+
45
+ # Water Column Sonar Processing
46
+
47
+ Processing tool for converting Level_0 water column sonar data to Level_1 and Level_2 derived data sets as well as
48
+ generating geospatial information.
49
+
50
+ ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/CI-CMG/water-column-sonar-processing/test_action.yaml)
51
+ ![PyPI - Implementation](https://img.shields.io/pypi/v/water-column-sonar-processing) ![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/CI-CMG/water-column-sonar-processing) ![GitHub repo size](https://img.shields.io/github/repo-size/CI-CMG/water-column-sonar-processing)
52
+
53
+ # Setting up the Python Environment
54
+
55
+ > uv python install --reinstall
56
+ > uv venv
57
+ > Python 3.10.12 # 3.13.1
58
+
59
+ # Installing Dependencies
60
+
61
+ ```
62
+ source .venv/bin/activate
63
+ # or ".venv\Scripts\activate" in windows
64
+
65
+ uv sync --all-groups
66
+
67
+ uv pip install --upgrade pip
68
+
69
+ uv run pre-commit install
70
+ ```
71
+
72
+ # Pytest
73
+
74
+ ```
75
+ uv run pytest --cache-clear tests # -W ignore::DeprecationWarning
76
+ ```
77
+
78
+ or
79
+ > pytest --cache-clear --cov=src tests/ --cov-report=xml
80
+
81
+ # Test Coverage
82
+
83
+ ```commandline
84
+ uv run pytest --cov=water_column_sonar_processing
85
+ ```
86
+
87
+ With line numbers
88
+
89
+ ```commandline
90
+ uv run pytest tests/geometry --cov=water_column_sonar_processing --cov-report term-missing
91
+ ```
92
+
93
+ Current status:
94
+
95
+ ```commandline
96
+ ======================================================================================= tests coverage =======================================================================================
97
+ _____________________________________________________________________ coverage: platform darwin, python 3.12.12-final-0 ______________________________________________________________________
98
+
99
+ Name Stmts Miss Cover
100
+ -------------------------------------------------------------------------------------
101
+ water_column_sonar_processing/__init__.py 3 0 100%
102
+ water_column_sonar_processing/aws/__init__.py 6 0 100%
103
+ water_column_sonar_processing/aws/dynamodb_manager.py 44 4 91%
104
+ water_column_sonar_processing/aws/s3_manager.py 161 27 83%
105
+ water_column_sonar_processing/aws/s3fs_manager.py 18 0 100%
106
+ water_column_sonar_processing/aws/sns_manager.py 18 1 94%
107
+ water_column_sonar_processing/aws/sqs_manager.py 17 2 88%
108
+ water_column_sonar_processing/cruise/__init__.py 3 0 100%
109
+ water_column_sonar_processing/cruise/create_empty_zarr_store.py 38 2 95%
110
+ water_column_sonar_processing/cruise/datatree_manager.py 0 0 100%
111
+ water_column_sonar_processing/cruise/resample_regrid.py 87 6 93%
112
+ water_column_sonar_processing/geometry/__init__.py 6 0 100%
113
+ water_column_sonar_processing/geometry/elevation_manager.py 29 1 97%
114
+ water_column_sonar_processing/geometry/geometry_manager.py 72 33 54%
115
+ water_column_sonar_processing/geometry/line_simplification.py 38 4 89%
116
+ water_column_sonar_processing/geometry/pmtile_generation.py 80 58 28%
117
+ water_column_sonar_processing/geometry/spatiotemporal.py 42 2 95%
118
+ water_column_sonar_processing/index/__init__.py 2 0 100%
119
+ water_column_sonar_processing/index/index_manager.py 118 91 23%
120
+ water_column_sonar_processing/model/__init__.py 2 0 100%
121
+ water_column_sonar_processing/model/zarr_manager.py 103 8 92%
122
+ water_column_sonar_processing/processing/__init__.py 3 0 100%
123
+ water_column_sonar_processing/processing/raw_to_netcdf.py 85 24 72%
124
+ water_column_sonar_processing/processing/raw_to_zarr.py 88 5 94%
125
+ water_column_sonar_processing/utility/__init__.py 5 0 100%
126
+ water_column_sonar_processing/utility/cleaner.py 14 0 100%
127
+ water_column_sonar_processing/utility/constants.py 62 0 100%
128
+ water_column_sonar_processing/utility/pipeline_status.py 42 0 100%
129
+ water_column_sonar_processing/utility/timestamp.py 5 0 100%
130
+ -------------------------------------------------------------------------------------
131
+ TOTAL 1191 268 77%
132
+ =================================================================== 47 passed, 4 skipped, 21 warnings in 337.77s (0:05:37)
133
+ ```
134
+
135
+ # Instructions
136
+
137
+ Following this tutorial:
138
+ https://packaging.python.org/en/latest/tutorials/packaging-projects/
139
+
140
+ # Pre Commit Hook
141
+
142
+ see here for installation: https://pre-commit.com/
143
+ https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
144
+
145
+ ```
146
+ uv run pre-commit install --allow-missing-config
147
+ # or
148
+ uv run pre-commit install
149
+ ```
150
+
151
+ # Black
152
+
153
+ https://ljvmiranda921.github.io/notebook/2018/06/21/precommits-using-black-and-flake8/
154
+
155
+ ```
156
+ Settings > Black
157
+ Execution mode: Package
158
+ Python Interpreter: .../.venv/bin/python
159
+ Use Black Formatter: X On Code reformat, X On Save
160
+ ```
161
+
162
+ # Linting
163
+
164
+ Ruff
165
+ https://plugins.jetbrains.com/plugin/20574-ruff
166
+
167
+ # Colab Test
168
+
169
+ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
170
+
171
+ # Tag a Release
172
+
173
+ Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
174
+
175
+ ```commandline
176
+ git tag -a v26.1.13 -m "Releasing v26.1.13"
177
+ git push origin --tags
178
+ #gh release create v26.1.12
179
+ ```
180
+
181
+ # To Publish To PROD
182
+
183
+ ```
184
+ uv build --no-sources
185
+ uv publish
186
+ ```
187
+
188
+ # TODO:
189
+
190
+ add https://pypi.org/project/setuptools-scm/
191
+ for extracting the version
192
+
193
+ # Security scanning
194
+
195
+ > bandit -r water_column_sonar_processing/
196
+
197
+ # Data Debugging
198
+
199
+ Experimental Plotting in Xarray (hvPlot):
200
+ https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
201
+
202
+ HB0707 Zoomable Cruise:
203
+ https://hb0707.s3.us-east-1.amazonaws.com/index.html
204
+
205
+ # UV Debugging
206
+
207
+ ```
208
+ uv pip install --upgrade pip
209
+ #uv sync --all-groups
210
+ uv run pre-commit install
211
+ uv lock --check
212
+ uv lock
213
+ uv sync --all-groups
214
+ uv run pytest --cache-clear tests
215
+ ```
216
+
217
+ # Fixing S3FS Problems
218
+
219
+ ```commandline
220
+ To enable/disa asyncio for the debugger, follow the steps:
221
+ Open PyCharm
222
+ Use Shift + Shift (Search Everywhere)
223
+ In the popup type: Registry and press Enter
224
+ Find "Registry" in the list of results and click on it.
225
+ In the new popup find python.debug.asyncio.repl line and check the respective checkbox
226
+ Press Close.
227
+ Restart the IDE.
228
+ The asyncio support will be enabled in the debugger.
229
+ ```
230
+
231
+ Another useful trick is to turn off "gevent" to speed up debugging:
232
+
233
+ ```commandline
234
+ Python > Debugger > "Gevent Compatible"
235
+ ```
236
+
237
+ # Fixing windows/wsl/ubuntu/mac git compatability
238
+
239
+ > git config --global core.filemode false
240
+ > git config --global core.autocrlf true
@@ -0,0 +1,34 @@
1
+ water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
2
+ water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
3
+ water_column_sonar_processing/aws/dynamodb_manager.py,sha256=EwSc4UTVQTR6tXsrx2eNdbhvexxfrz3GkkvO97F_9fM,14017
4
+ water_column_sonar_processing/aws/s3_manager.py,sha256=SI1X1ju7ueYJbtEn6DVjUuXNN6so1EKy9xQwCW3rqp4,16183
5
+ water_column_sonar_processing/aws/s3fs_manager.py,sha256=NJoky3uvMgt7m04LBZhrRkR-w5y18MIPP22EOWQos9E,2275
6
+ water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
7
+ water_column_sonar_processing/aws/sqs_manager.py,sha256=j_DGOKble3i-DlHT_uGxCFEmHVbYhFrpbhXdJKLtBSo,1600
8
+ water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
9
+ water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=4sZ_6ThlFs7pfnbHsbdpnlkpnGr9lBDM4-S4hRtYi0Q,5125
10
+ water_column_sonar_processing/cruise/datatree_manager.py,sha256=ed_lY04MyaXKADl7EaqHyEZ8JQMiyIAXVhvtE7YnWcg,662
11
+ water_column_sonar_processing/cruise/resample_regrid.py,sha256=R1l5XjksaH5xqD3x3jdD-sJpe7xuXu4YKh3uCzfCumM,14541
12
+ water_column_sonar_processing/geometry/__init__.py,sha256=VfUXTH_OnjQy6EWCw1Hn2ua7OYGI-6EITeH37b4Wjxc,371
13
+ water_column_sonar_processing/geometry/elevation_manager.py,sha256=2TBiDB3qNkM5ZRUEWhQCjrqRYagur0z8hT102Sts0p8,4251
14
+ water_column_sonar_processing/geometry/geometry_manager.py,sha256=cQhLGMwHZskvuKU83gdLunQbM62s57k8qEKkuDWRlPE,10929
15
+ water_column_sonar_processing/geometry/line_simplification.py,sha256=tsUPu3XeMqBHZ8zVbtXI-l_leosj2ENd7JWJSzz_2ws,9288
16
+ water_column_sonar_processing/geometry/pmtile_generation.py,sha256=6dKLbpyZEEj8KfYmqjzUe8rrtNKAFhUK94BhnepFbSg,9577
17
+ water_column_sonar_processing/geometry/spatiotemporal.py,sha256=KnNqQiuon15EDafUMxO80Grnu_yTxq1aYEKE1u1pXdU,4095
18
+ water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
19
+ water_column_sonar_processing/index/index_manager.py,sha256=Dino5EO58Aq2tDhoz8sya3YFXUpqOQMsSoKEnJZMgdI,16170
20
+ water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
21
+ water_column_sonar_processing/model/zarr_manager.py,sha256=Ghxsd4lI01khbOP0Pl1T5IJNZUxgWdxXHQiYzTZck50,32420
22
+ water_column_sonar_processing/processing/__init__.py,sha256=TDnA4_Nyb8MnU8HaixaCzEdTbFkXsUXA0ekvxtYm1-U,150
23
+ water_column_sonar_processing/processing/raw_to_netcdf.py,sha256=3UdglrWXFLxa_jHkhvC9P9NUkp-iKmp6MvnXFnXi4ns,13098
24
+ water_column_sonar_processing/processing/raw_to_zarr.py,sha256=XhNCQe3YXUhlhQAzGTcktpJVX7t6Gf_yi29Vpsp4L4k,13558
25
+ water_column_sonar_processing/utility/__init__.py,sha256=5gV8sJcnQGrV2AecwkBFZljOC-fQwLuDX70DxX59x2o,289
26
+ water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
27
+ water_column_sonar_processing/utility/constants.py,sha256=atCf0MAJJFt2QdS-8mN0NLgmVOMTmyaryWKXPmDTimY,3530
28
+ water_column_sonar_processing/utility/pipeline_status.py,sha256=xbl6-4ePq1Krfo18Mgr0jzWz9C_g2Kbey_QZNJuwdkI,4406
29
+ water_column_sonar_processing/utility/timestamp.py,sha256=rn8SDbGYjRvanDUOrvKpNjp3-AK6-KlU2NaCk6Ok8rc,337
30
+ water_column_sonar_processing-26.1.14.dist-info/licenses/LICENSE,sha256=TosqaZpJgYvhgXIyYBti-ggJaO8rxRg3FtThY08s9Aw,1110
31
+ water_column_sonar_processing-26.1.14.dist-info/METADATA,sha256=3Huf0Mv53tOFWZitSynNCvBG03zmVIsv3hyLcMx7RI4,8432
32
+ water_column_sonar_processing-26.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
33
+ water_column_sonar_processing-26.1.14.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
34
+ water_column_sonar_processing-26.1.14.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.3.0)
2
+ Generator: setuptools (80.9.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -1,6 +1,6 @@
1
1
  MIT License
2
2
 
3
- Copyright (c) 2024 Cooperative Institutes, Coastal and Marine Geophysics
3
+ Copyright (c) 2025 Cooperative Institutes, Coastal and Marine Geophysics
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
@@ -0,0 +1 @@
1
+ water_column_sonar_processing
__init__.py DELETED
File without changes
model/__init__.py DELETED
File without changes
model/aws/__init__.py DELETED
File without changes