water-column-sonar-processing 0.0.1__tar.gz → 0.0.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of water-column-sonar-processing might be problematic. Click here for more details.

Files changed (50) hide show
  1. {water_column_sonar_processing-0.0.1/src/water_column_sonar_processing.egg-info → water_column_sonar_processing-0.0.3}/PKG-INFO +16 -12
  2. {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.3}/README.md +16 -12
  3. {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.3}/pyproject.toml +14 -3
  4. water_column_sonar_processing-0.0.3/src/aws_manager/__init__.py +4 -0
  5. {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.3/src/aws_manager}/dynamodb_manager.py +1 -1
  6. {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.3/src/aws_manager}/s3_manager.py +12 -12
  7. {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.3/src/aws_manager}/sns_manager.py +1 -1
  8. {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.3/src/aws_manager}/sqs_manager.py +1 -1
  9. {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.3/src}/cruise/create_empty_zarr_store.py +15 -14
  10. {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.3/src}/cruise/resample_regrid.py +11 -10
  11. {water_column_sonar_processing-0.0.1/src/model/geospatial → water_column_sonar_processing-0.0.3/src/geometry_manager}/geometry_manager.py +3 -3
  12. {water_column_sonar_processing-0.0.1/src/model/geospatial → water_column_sonar_processing-0.0.3/src/geometry_manager}/pmtile_generation.py +1 -1
  13. water_column_sonar_processing-0.0.1/src/model/index/index.py → water_column_sonar_processing-0.0.3/src/index_manager/index_manager.py +2 -3
  14. {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.3/src}/model.py +7 -5
  15. {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.3/src}/utility/cleaner.py +2 -2
  16. {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.3/src/water_column_sonar_processing.egg-info}/PKG-INFO +16 -12
  17. water_column_sonar_processing-0.0.3/src/water_column_sonar_processing.egg-info/SOURCES.txt +42 -0
  18. water_column_sonar_processing-0.0.3/src/water_column_sonar_processing.egg-info/top_level.txt +8 -0
  19. {water_column_sonar_processing-0.0.1/src/model/zarr → water_column_sonar_processing-0.0.3/src/zarr_manager}/zarr_manager.py +10 -9
  20. water_column_sonar_processing-0.0.3/tests/test_create_empty_zarr_store.py +97 -0
  21. water_column_sonar_processing-0.0.3/tests/test_dynamodb_manager.py +233 -0
  22. water_column_sonar_processing-0.0.3/tests/test_geometry_manager.py +65 -0
  23. water_column_sonar_processing-0.0.3/tests/test_geometry_simplification.py +39 -0
  24. water_column_sonar_processing-0.0.3/tests/test_index.py +167 -0
  25. water_column_sonar_processing-0.0.3/tests/test_model.py +488 -0
  26. water_column_sonar_processing-0.0.3/tests/test_resample_regrid.py +50 -0
  27. water_column_sonar_processing-0.0.3/tests/test_s3_manager.py +121 -0
  28. water_column_sonar_processing-0.0.3/tests/test_s3fs_manager.py +327 -0
  29. water_column_sonar_processing-0.0.3/tests/test_sns_sqs_manager.py +78 -0
  30. water_column_sonar_processing-0.0.3/tests/test_zarr_manager.py +267 -0
  31. water_column_sonar_processing-0.0.1/src/model/utility/__init__.py +0 -0
  32. water_column_sonar_processing-0.0.1/src/model/zarr/__init__.py +0 -0
  33. water_column_sonar_processing-0.0.1/src/model/zarr/bar.py +0 -28
  34. water_column_sonar_processing-0.0.1/src/model/zarr/foo.py +0 -11
  35. water_column_sonar_processing-0.0.1/src/water_column_sonar_processing.egg-info/SOURCES.txt +0 -34
  36. water_column_sonar_processing-0.0.1/src/water_column_sonar_processing.egg-info/top_level.txt +0 -2
  37. {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.3}/LICENSE +0 -0
  38. {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.3}/setup.cfg +0 -0
  39. {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.3}/src/__init__.py +0 -0
  40. {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.3/src/aws_manager}/s3fs_manager.py +0 -0
  41. {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.3/src/cruise}/__init__.py +0 -0
  42. {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.3/src/geometry_manager}/__init__.py +0 -0
  43. {water_column_sonar_processing-0.0.1/src/model/geospatial → water_column_sonar_processing-0.0.3/src/geometry_manager}/geometry_simplification.py +0 -0
  44. {water_column_sonar_processing-0.0.1/src/model/cruise → water_column_sonar_processing-0.0.3/src/index_manager}/__init__.py +0 -0
  45. {water_column_sonar_processing-0.0.1/src/model/geospatial → water_column_sonar_processing-0.0.3/src/utility}/__init__.py +0 -0
  46. {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.3/src}/utility/constants.py +0 -0
  47. {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.3/src}/utility/pipeline_status.py +0 -0
  48. {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.3/src}/utility/timestamp.py +0 -0
  49. {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.3}/src/water_column_sonar_processing.egg-info/dependency_links.txt +0 -0
  50. {water_column_sonar_processing-0.0.1/src/model/index → water_column_sonar_processing-0.0.3/src/zarr_manager}/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: water-column-sonar-processing
3
- Version: 0.0.1
3
+ Version: 0.0.3
4
4
  Summary: A processing tool for water column sonar data.
5
5
  Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
6
6
  Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
@@ -12,10 +12,10 @@ Requires-Python: >=3.10
12
12
  Description-Content-Type: text/markdown
13
13
  License-File: LICENSE
14
14
 
15
- # water-column-sonar-processing
15
+ # Water Column Sonar Processing
16
16
  Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
17
17
 
18
- ## Setting up the Python Environment
18
+ # Setting up the Python Environment
19
19
  > Python 3.10.12
20
20
 
21
21
  # MacOS Pyenv Installation Instructions
@@ -33,8 +33,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
33
33
  1. ```eval "$(pyenv virtualenv-init -)"```
34
34
  3. Open a new terminal
35
35
  4. Install Python version
36
- 1. ```env PYTHON_CONFIGURE_OPTS="--enable-shared"```
37
- 2. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
36
+ 1. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
38
37
  5. Create virtual env (to delete 'pyenv uninstall 3.10.12/water-column-sonar-processing')
39
38
  1. ```pyenv virtualenv 3.10.12 water-column-sonar-processing```
40
39
  6. Set local version of python (if not done already)
@@ -42,7 +41,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
42
41
  2. ```pyenv local 3.10.12 water-column-sonar-processing```
43
42
  3. ```pyenv activate water-column-sonar-processing```
44
43
 
45
- ## Setting up IntelliJ
44
+ # Setting up IntelliJ
46
45
 
47
46
  1. Install the IntelliJ Python plugin
48
47
  2. Set up pyenv
@@ -55,35 +54,40 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
55
54
  2. Facets -> + -> Python
56
55
  3. Set interpreter
57
56
 
58
- ## Installing Dependencies
57
+ # Installing Dependencies
59
58
 
60
59
  1. Add dependencies with versions to requirements.txt
61
60
  2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
62
61
 
63
62
 
64
- ## Pytest
63
+ # Pytest
65
64
  ```commandline
66
65
  pytest --disable-warnings
67
66
  ```
68
67
 
69
- ## Instructions
68
+ # Instructions
70
69
  Following this tutorial:
71
70
  https://packaging.python.org/en/latest/tutorials/packaging-projects/
72
71
 
73
- ## To Publish To TEST
72
+ # To Publish To TEST
74
73
  ```commandline
75
74
  python -m build
76
75
  python -m twine upload --repository testpypi dist/*
77
76
  pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
78
77
  python
79
78
  ```
80
- ```python
79
+ ```
81
80
  from water-column-sonar-processing import ZarrManager
82
81
  example.add_one(2)
83
82
  ```
84
83
 
85
- ## To Publish To PROD
84
+ # To Publish To PROD
86
85
  ```commandline
87
86
  python -m build
88
87
  python -m twine upload --repository pypi dist/*
89
88
  ```
89
+
90
+
91
+ # Linting
92
+ Ruff
93
+ https://plugins.jetbrains.com/plugin/20574-ruff
@@ -1,7 +1,7 @@
1
- # water-column-sonar-processing
1
+ # Water Column Sonar Processing
2
2
  Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
3
3
 
4
- ## Setting up the Python Environment
4
+ # Setting up the Python Environment
5
5
  > Python 3.10.12
6
6
 
7
7
  # MacOS Pyenv Installation Instructions
@@ -19,8 +19,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
19
19
  1. ```eval "$(pyenv virtualenv-init -)"```
20
20
  3. Open a new terminal
21
21
  4. Install Python version
22
- 1. ```env PYTHON_CONFIGURE_OPTS="--enable-shared"```
23
- 2. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
22
+ 1. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
24
23
  5. Create virtual env (to delete 'pyenv uninstall 3.10.12/water-column-sonar-processing')
25
24
  1. ```pyenv virtualenv 3.10.12 water-column-sonar-processing```
26
25
  6. Set local version of python (if not done already)
@@ -28,7 +27,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
28
27
  2. ```pyenv local 3.10.12 water-column-sonar-processing```
29
28
  3. ```pyenv activate water-column-sonar-processing```
30
29
 
31
- ## Setting up IntelliJ
30
+ # Setting up IntelliJ
32
31
 
33
32
  1. Install the IntelliJ Python plugin
34
33
  2. Set up pyenv
@@ -41,35 +40,40 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
41
40
  2. Facets -> + -> Python
42
41
  3. Set interpreter
43
42
 
44
- ## Installing Dependencies
43
+ # Installing Dependencies
45
44
 
46
45
  1. Add dependencies with versions to requirements.txt
47
46
  2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
48
47
 
49
48
 
50
- ## Pytest
49
+ # Pytest
51
50
  ```commandline
52
51
  pytest --disable-warnings
53
52
  ```
54
53
 
55
- ## Instructions
54
+ # Instructions
56
55
  Following this tutorial:
57
56
  https://packaging.python.org/en/latest/tutorials/packaging-projects/
58
57
 
59
- ## To Publish To TEST
58
+ # To Publish To TEST
60
59
  ```commandline
61
60
  python -m build
62
61
  python -m twine upload --repository testpypi dist/*
63
62
  pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
64
63
  python
65
64
  ```
66
- ```python
65
+ ```
67
66
  from water-column-sonar-processing import ZarrManager
68
67
  example.add_one(2)
69
68
  ```
70
69
 
71
- ## To Publish To PROD
70
+ # To Publish To PROD
72
71
  ```commandline
73
72
  python -m build
74
73
  python -m twine upload --repository pypi dist/*
75
- ```
74
+ ```
75
+
76
+
77
+ # Linting
78
+ Ruff
79
+ https://plugins.jetbrains.com/plugin/20574-ruff
@@ -1,10 +1,10 @@
1
1
  [build-system]
2
- requires = ["setuptools>=61.0"]
2
+ requires = ["setuptools", "wheel"]
3
3
  build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "water-column-sonar-processing"
7
- version = "0.0.1"
7
+ version = "0.0.3"
8
8
  authors = [
9
9
  { name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
10
10
  ]
@@ -19,4 +19,15 @@ classifiers = [
19
19
 
20
20
  [project.urls]
21
21
  Homepage = "https://github.com/CI-CMG/water-column-sonar-processing"
22
- Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
22
+ Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
23
+
24
+ #[pytest]
25
+ #pythonpath = "src"
26
+ #testpaths = "tests"
27
+
28
+ [tool.pytest.ini_options]
29
+ minversion = "6.0"
30
+ addopts = "-ra -q"
31
+ testpaths = [
32
+ "tests",
33
+ ]
@@ -0,0 +1,4 @@
1
+ # from .dynamodb_manager import DynamoDBManager
2
+ # from .s3_manager import S3Manager
3
+ # from .s3fs_manager import S3FSManager
4
+ # from .sns_manager import SNSManager
@@ -23,7 +23,7 @@ class DynamoDBManager:
23
23
  self.type_deserializer = TypeDeserializer()
24
24
 
25
25
  #####################################################################
26
- ### defined in raw-to-zarr, not used
26
+ ### defined in raw-to-zarr_manager, not used
27
27
  # def put_item(
28
28
  # self,
29
29
  # table_name,
@@ -1,10 +1,10 @@
1
1
  import json
2
2
  import os
3
3
  import boto3
4
- import pandas as pd
4
+ # import pandas as pd
5
5
  from collections.abc import Generator
6
6
 
7
- import geopandas
7
+ # import geopandas
8
8
  from botocore.config import Config
9
9
  from boto3.s3.transfer import TransferConfig
10
10
  from botocore.exceptions import ClientError
@@ -139,12 +139,12 @@ class S3Manager:
139
139
  return all_uploads
140
140
 
141
141
  #####################################################################
142
- def upload_zarr_files_to_bucket( # noaa-wcsd-zarr-pds
142
+ def upload_zarr_files_to_bucket( # noaa-wcsd-zarr_manager-pds
143
143
  self,
144
144
  local_directory,
145
145
  remote_directory,
146
146
  ):
147
- # Right now this is just for uploading a zarr store to s3
147
+ # Right now this is just for uploading a zarr_manager store to s3
148
148
  print('Uploading files to output bucket.')
149
149
  store_name = os.path.basename(local_directory)
150
150
  all_files = []
@@ -162,8 +162,8 @@ class S3Manager:
162
162
  return all_uploads
163
163
 
164
164
  #####################################################################
165
- # used: raw-to-zarr
166
- def list_objects( # noaa-wcsd-pds and noaa-wcsd-zarr-pds
165
+ # used: raw-to-zarr_manager
166
+ def list_objects( # noaa-wcsd-pds and noaa-wcsd-zarr_manager-pds
167
167
  self,
168
168
  bucket_name,
169
169
  prefix
@@ -250,8 +250,8 @@ class S3Manager:
250
250
  return raw_files
251
251
 
252
252
  #####################################################################
253
- def get_object( # TODO: Move this to index.py
254
- # noaa-wcsd-pds or noaa-wcsd-zarr-pds
253
+ def get_object( # TODO: Move this to index_manager.py
254
+ # noaa-wcsd-pds or noaa-wcsd-zarr_manager-pds
255
255
  self,
256
256
  bucket_name,
257
257
  key_name,
@@ -272,9 +272,9 @@ class S3Manager:
272
272
  return response
273
273
 
274
274
  #####################################################################
275
- # used raw-to-zarr
275
+ # used raw-to-zarr_manager
276
276
  def download_file( # TODO: change to download_object
277
- # noaa-wcsd-pds or noaa-wcsd-zarr-pds
277
+ # noaa-wcsd-pds or noaa-wcsd-zarr_manager-pds
278
278
  self,
279
279
  bucket_name,
280
280
  key,
@@ -289,7 +289,7 @@ class S3Manager:
289
289
 
290
290
  #####################################################################
291
291
  # not used
292
- # def delete_nodd_object( # noaa-wcsd-zarr-pds
292
+ # def delete_nodd_object( # noaa-wcsd-zarr_manager-pds
293
293
  # self,
294
294
  # bucket_name,
295
295
  # key
@@ -319,7 +319,7 @@ class S3Manager:
319
319
 
320
320
  #####################################################################
321
321
  # not used TODO: remove
322
- def put( # noaa-wcsd-zarr-pds
322
+ def put( # noaa-wcsd-zarr_manager-pds
323
323
  self,
324
324
  bucket_name,
325
325
  key,
@@ -1,5 +1,5 @@
1
1
  import os
2
- import json
2
+ # import json
3
3
  import boto3
4
4
 
5
5
 
@@ -1,6 +1,6 @@
1
1
  import os
2
2
  import boto3
3
- import time
3
+ # import time
4
4
 
5
5
 
6
6
  ###########################################################
@@ -1,17 +1,18 @@
1
1
  import os
2
2
  import numcodecs
3
3
  import numpy as np
4
- from ..utility.cleaner import Cleaner
5
- from ..aws.dynamodb_manager import DynamoDBManager
6
- from ..aws.s3_manager import S3Manager
7
- from ..zarr.zarr_manager import ZarrManager
4
+
5
+ from utility.cleaner import Cleaner
6
+ from aws_manager.dynamodb_manager import DynamoDBManager
7
+ from aws_manager.s3_manager import S3Manager
8
+ from zarr_manager.zarr_manager import ZarrManager
8
9
 
9
10
  numcodecs.blosc.use_threads = False
10
11
  numcodecs.blosc.set_nthreads(1)
11
12
 
12
13
  TEMPDIR = "/tmp"
13
14
 
14
- # TODO: when ready switch to version 3 of zarr spec
15
+ # TODO: when ready switch to version 3 of zarr_manager spec
15
16
  # ZARR_V3_EXPERIMENTAL_API = 1
16
17
  # creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
17
18
 
@@ -33,17 +34,17 @@ class CreateEmptyZarrStore:
33
34
  object_prefix: str,
34
35
  cruise_name: str,
35
36
  ) -> None:
36
- print('uploading zarr store to s3')
37
+ print('uploading zarr_manager store to s3')
37
38
  s3_manager = S3Manager()
38
39
  #
39
40
  print('Starting upload with thread pool executor.')
40
41
  # # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
41
42
  all_files = []
42
- for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.zarr"):
43
+ for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.zarr_manager"):
43
44
  for file in files:
44
45
  local_path = os.path.join(subdir, file)
45
- # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.zarr/.zattrs'
46
- s3_key = f'{object_prefix}/{cruise_name}.zarr{local_path.split(f"{cruise_name}.zarr")[-1]}'
46
+ # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.zarr_manager/.zattrs'
47
+ s3_key = f'{object_prefix}/{cruise_name}.zarr_manager{local_path.split(f"{cruise_name}.zarr_manager")[-1]}'
47
48
  all_files.append([local_path, s3_key])
48
49
  #
49
50
  # print(all_files)
@@ -97,10 +98,10 @@ class CreateEmptyZarrStore:
97
98
  new_width = int(consolidated_zarr_width)
98
99
  print(f"new_width: {new_width}")
99
100
  #################################################################
100
- store_name = f"{cruise_name}.zarr"
101
+ store_name = f"{cruise_name}.zarr_manager"
101
102
  print(store_name)
102
103
  ################################################################
103
- # Delete existing zarr store if it exists
104
+ # Delete existing zarr_manager store if it exists
104
105
  s3_manager = S3Manager()
105
106
  zarr_prefix = os.path.join("level_2", ship_name, cruise_name, sensor_name)
106
107
  child_objects = s3_manager.get_child_objects(
@@ -112,7 +113,7 @@ class CreateEmptyZarrStore:
112
113
  objects=child_objects,
113
114
  )
114
115
  ################################################################
115
- # Create new zarr store
116
+ # Create new zarr_manager store
116
117
  zarr_manager = ZarrManager()
117
118
  new_height = len(zarr_manager.get_depth_values(
118
119
  min_echo_range=cruise_min_echo_range,
@@ -156,11 +157,11 @@ class CreateEmptyZarrStore:
156
157
  # TODO: update enum in dynamodb
157
158
  #################################################################
158
159
  except Exception as err:
159
- print(f"Problem trying to create new cruise zarr store: {err}")
160
+ print(f"Problem trying to create new cruise zarr_manager store: {err}")
160
161
  finally:
161
162
  cleaner = Cleaner()
162
163
  cleaner.delete_local_files()
163
- print("Done creating cruise level zarr store")
164
+ print("Done creating cruise level zarr_manager store")
164
165
 
165
166
 
166
167
  ###########################################################
@@ -6,15 +6,16 @@ import numpy as np
6
6
  import xarray as xr
7
7
  import pandas as pd
8
8
 
9
- from ..geospatial.geometry_manager import GeoManager
10
- from ..aws.dynamodb_manager import DynamoDBManager
11
- from ..zarr.zarr_manager import ZarrManager
9
+ from aws_manager.dynamodb_manager import DynamoDBManager
10
+ from zarr_manager.zarr_manager import ZarrManager
11
+ from geometry_manager.geometry_manager import GeometryManager
12
+
12
13
 
13
14
  numcodecs.blosc.use_threads = False
14
15
  numcodecs.blosc.set_nthreads(1)
15
16
 
16
17
 
17
- # TODO: when ready switch to version 3 of zarr spec
18
+ # TODO: when ready switch to version 3 of zarr_manager spec
18
19
  # ZARR_V3_EXPERIMENTAL_API = 1
19
20
  # creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
20
21
 
@@ -114,7 +115,7 @@ class ResampleRegrid:
114
115
  ) -> None:
115
116
  """
116
117
  The goal here is to interpolate the data against the depth values already populated
117
- in the existing file level zarr stores. We open the cruise-level store with zarr for
118
+ in the existing file level zarr_manager stores. We open the cruise-level store with zarr_manager for
118
119
  read/write operations. We open the file-level store with Xarray to leverage tools for
119
120
  resampling and subsetting the data.
120
121
  """
@@ -122,8 +123,8 @@ class ResampleRegrid:
122
123
  try:
123
124
  zarr_manager = ZarrManager()
124
125
  # s3_manager = S3Manager()
125
- geo_manager = GeoManager()
126
- # get zarr store
126
+ geo_manager = GeometryManager()
127
+ # get zarr_manager store
127
128
  output_zarr_store = zarr_manager.open_s3_zarr_store_with_zarr(
128
129
  ship_name=ship_name,
129
130
  cruise_name=cruise_name,
@@ -156,7 +157,7 @@ class ResampleRegrid:
156
157
  # TODO: filter rows by enum success, filter the dataframe just for enums >= LEVEL_1_PROCESSING
157
158
  # df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
158
159
 
159
- # Get index from all cruise files. Note: should be based on which are included in cruise.
160
+ # Get index_manager from all cruise files. Note: should be based on which are included in cruise.
160
161
  index = cruise_df.index[cruise_df['FILE_NAME'] == f"{file_name_stem}.raw"][0]
161
162
 
162
163
  # get input store
@@ -168,7 +169,7 @@ class ResampleRegrid:
168
169
  )
169
170
  #########################################################################
170
171
  # [3] Get needed indices
171
- # Offset from start index to insert new data. Note that missing values are excluded.
172
+ # Offset from start index_manager to insert new data. Note that missing values are excluded.
172
173
  ping_time_cumsum = np.insert(
173
174
  np.cumsum(cruise_df['NUM_PING_TIME_DROPNA'].dropna().to_numpy(dtype=int)),
174
175
  obj=0,
@@ -222,7 +223,7 @@ class ResampleRegrid:
222
223
  print(f"start_ping_time_index: {start_ping_time_index}, end_ping_time_index: {end_ping_time_index}")
223
224
 
224
225
  #########################################################################
225
- # write Sv values to cruise-level-zarr-store
226
+ # write Sv values to cruise-level-zarr_manager-store
226
227
  for channel in range(len(input_xr.channel.values)): # doesn't like being written in one fell swoop :(
227
228
  output_zarr_store.Sv[
228
229
  :,
@@ -3,8 +3,8 @@ import numpy as np
3
3
  import geopandas
4
4
  import pandas as pd
5
5
 
6
- from ..utility.cleaner import Cleaner
7
- from ..aws.s3_manager import S3Manager
6
+ from utility.cleaner import Cleaner
7
+ from aws_manager.s3_manager import S3Manager
8
8
 
9
9
  """
10
10
  // [Decimal / Places / Degrees / Object that can be recognized at scale / N/S or E/W at equator, E/W at 23N/S, E/W at 45N/S, E/W at 67N/S]
@@ -19,7 +19,7 @@ from ..aws.s3_manager import S3Manager
19
19
  """
20
20
 
21
21
 
22
- class GeoManager:
22
+ class GeometryManager:
23
23
  #######################################################
24
24
  def __init__(
25
25
  self,
@@ -23,7 +23,7 @@ class PMTileGeneration(object):
23
23
 
24
24
  #######################################################
25
25
  def generate_geojson_feature_collection(self):
26
- # This was used to read from noaa-wcsd-zarr-pds bucket geojson files and then to
26
+ # This was used to read from noaa-wcsd-zarr_manager-pds bucket geojson files and then to
27
27
  # generate the geopandas dataframe which could be exported to another comprehensive
28
28
  # geojson file. That
29
29
  result = list(Path("/Users/r2d2/Documents/echofish/geojson").rglob("*.json"))
@@ -4,8 +4,7 @@ import pandas as pd
4
4
  from datetime import datetime
5
5
  from concurrent.futures import ThreadPoolExecutor
6
6
  from concurrent.futures import as_completed
7
- from ..aws.s3_manager import S3Manager
8
-
7
+ from aws_manager.s3_manager import S3Manager
9
8
 
10
9
  class IndexManager:
11
10
 
@@ -194,7 +193,7 @@ class IndexManager:
194
193
  return calibration_statuses
195
194
 
196
195
  #################################################################
197
- # def index( # TODO: get rid of this?
196
+ # def index_manager( # TODO: get rid of this?
198
197
  # self
199
198
  # ):
200
199
  # start_time = datetime.now() # used for benchmarking
@@ -1,10 +1,10 @@
1
1
  import os
2
2
  import json
3
3
  import numpy as np
4
- from src.model.aws.s3_manager import S3Manager
5
- from src.model.aws.sns_manager import SNSManager
6
- from src.model.aws.dynamodb_manager import DynamoDBManager
7
-
4
+ from aws_manager.s3_manager import S3Manager
5
+ from aws_manager.s3fs_manager import S3FSManager
6
+ from aws_manager.sns_manager import SNSManager
7
+ from aws_manager.dynamodb_manager import DynamoDBManager
8
8
 
9
9
  ###########################################################
10
10
  class Model:
@@ -21,6 +21,8 @@ class Model:
21
21
 
22
22
  def execute(self):
23
23
  input_s3_manager = S3Manager() # TODO: Need to allow passing in of credentials when writing to protected bucket
24
+ s3fs_manager = S3FSManager() # TODO: delete this
25
+ print(s3fs_manager) # TODO: delete this
24
26
  output_s3_manager = S3Manager()
25
27
  # TODO: s3fs?
26
28
  sns_manager = SNSManager()
@@ -85,7 +87,7 @@ class Model:
85
87
  ':sh': {'S': ship_name},
86
88
  ':st': {'S': '2006-04-06T11:34:07.288Z'},
87
89
  ':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
88
- ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.zarr'},
90
+ ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.zarr_manager'},
89
91
  },
90
92
  update_expression=(
91
93
  'SET '
@@ -7,9 +7,9 @@ import shutil
7
7
  class Cleaner:
8
8
  @staticmethod
9
9
  def delete_local_files(
10
- file_types=['*.raw*', '*.zarr'] # '*.json'
10
+ file_types=['*.raw*', '*.zarr_manager'] # '*.json'
11
11
  ):
12
- print('Deleting all local raw and zarr files')
12
+ print('Deleting all local raw and zarr_manager files')
13
13
  for i in file_types:
14
14
  for j in glob.glob(i):
15
15
  if os.path.isdir(j):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: water-column-sonar-processing
3
- Version: 0.0.1
3
+ Version: 0.0.3
4
4
  Summary: A processing tool for water column sonar data.
5
5
  Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
6
6
  Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
@@ -12,10 +12,10 @@ Requires-Python: >=3.10
12
12
  Description-Content-Type: text/markdown
13
13
  License-File: LICENSE
14
14
 
15
- # water-column-sonar-processing
15
+ # Water Column Sonar Processing
16
16
  Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
17
17
 
18
- ## Setting up the Python Environment
18
+ # Setting up the Python Environment
19
19
  > Python 3.10.12
20
20
 
21
21
  # MacOS Pyenv Installation Instructions
@@ -33,8 +33,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
33
33
  1. ```eval "$(pyenv virtualenv-init -)"```
34
34
  3. Open a new terminal
35
35
  4. Install Python version
36
- 1. ```env PYTHON_CONFIGURE_OPTS="--enable-shared"```
37
- 2. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
36
+ 1. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
38
37
  5. Create virtual env (to delete 'pyenv uninstall 3.10.12/water-column-sonar-processing')
39
38
  1. ```pyenv virtualenv 3.10.12 water-column-sonar-processing```
40
39
  6. Set local version of python (if not done already)
@@ -42,7 +41,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
42
41
  2. ```pyenv local 3.10.12 water-column-sonar-processing```
43
42
  3. ```pyenv activate water-column-sonar-processing```
44
43
 
45
- ## Setting up IntelliJ
44
+ # Setting up IntelliJ
46
45
 
47
46
  1. Install the IntelliJ Python plugin
48
47
  2. Set up pyenv
@@ -55,35 +54,40 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
55
54
  2. Facets -> + -> Python
56
55
  3. Set interpreter
57
56
 
58
- ## Installing Dependencies
57
+ # Installing Dependencies
59
58
 
60
59
  1. Add dependencies with versions to requirements.txt
61
60
  2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
62
61
 
63
62
 
64
- ## Pytest
63
+ # Pytest
65
64
  ```commandline
66
65
  pytest --disable-warnings
67
66
  ```
68
67
 
69
- ## Instructions
68
+ # Instructions
70
69
  Following this tutorial:
71
70
  https://packaging.python.org/en/latest/tutorials/packaging-projects/
72
71
 
73
- ## To Publish To TEST
72
+ # To Publish To TEST
74
73
  ```commandline
75
74
  python -m build
76
75
  python -m twine upload --repository testpypi dist/*
77
76
  pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
78
77
  python
79
78
  ```
80
- ```python
79
+ ```
81
80
  from water-column-sonar-processing import ZarrManager
82
81
  example.add_one(2)
83
82
  ```
84
83
 
85
- ## To Publish To PROD
84
+ # To Publish To PROD
86
85
  ```commandline
87
86
  python -m build
88
87
  python -m twine upload --repository pypi dist/*
89
88
  ```
89
+
90
+
91
+ # Linting
92
+ Ruff
93
+ https://plugins.jetbrains.com/plugin/20574-ruff