water-column-sonar-processing 0.0.1__tar.gz → 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of water-column-sonar-processing might be problematic. Click here for more details.
- {water_column_sonar_processing-0.0.1/src/water_column_sonar_processing.egg-info → water_column_sonar_processing-0.0.2}/PKG-INFO +16 -12
- {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.2}/README.md +16 -12
- {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.2}/pyproject.toml +13 -2
- water_column_sonar_processing-0.0.2/src/aws_manager/__init__.py +4 -0
- {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.2/src/aws_manager}/dynamodb_manager.py +1 -1
- {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.2/src/aws_manager}/s3_manager.py +12 -12
- {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.2/src/aws_manager}/sns_manager.py +1 -1
- {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.2/src/aws_manager}/sqs_manager.py +1 -1
- {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.2/src}/cruise/create_empty_zarr_store.py +15 -14
- {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.2/src}/cruise/resample_regrid.py +11 -10
- {water_column_sonar_processing-0.0.1/src/model/geospatial → water_column_sonar_processing-0.0.2/src/geometry_manager}/geometry_manager.py +3 -3
- {water_column_sonar_processing-0.0.1/src/model/geospatial → water_column_sonar_processing-0.0.2/src/geometry_manager}/pmtile_generation.py +1 -1
- water_column_sonar_processing-0.0.1/src/model/index/index.py → water_column_sonar_processing-0.0.2/src/index_manager/index_manager.py +2 -3
- {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.2/src}/model.py +7 -5
- {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.2/src}/utility/cleaner.py +2 -2
- {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.2/src/water_column_sonar_processing.egg-info}/PKG-INFO +16 -12
- water_column_sonar_processing-0.0.2/src/water_column_sonar_processing.egg-info/SOURCES.txt +42 -0
- water_column_sonar_processing-0.0.2/src/water_column_sonar_processing.egg-info/top_level.txt +8 -0
- {water_column_sonar_processing-0.0.1/src/model/zarr → water_column_sonar_processing-0.0.2/src/zarr_manager}/zarr_manager.py +10 -9
- water_column_sonar_processing-0.0.2/tests/test_create_empty_zarr_store.py +97 -0
- water_column_sonar_processing-0.0.2/tests/test_dynamodb_manager.py +233 -0
- water_column_sonar_processing-0.0.2/tests/test_geometry_manager.py +65 -0
- water_column_sonar_processing-0.0.2/tests/test_geometry_simplification.py +39 -0
- water_column_sonar_processing-0.0.2/tests/test_index.py +167 -0
- water_column_sonar_processing-0.0.2/tests/test_model.py +488 -0
- water_column_sonar_processing-0.0.2/tests/test_resample_regrid.py +50 -0
- water_column_sonar_processing-0.0.2/tests/test_s3_manager.py +121 -0
- water_column_sonar_processing-0.0.2/tests/test_s3fs_manager.py +327 -0
- water_column_sonar_processing-0.0.2/tests/test_sns_sqs_manager.py +78 -0
- water_column_sonar_processing-0.0.2/tests/test_zarr_manager.py +267 -0
- water_column_sonar_processing-0.0.1/src/model/utility/__init__.py +0 -0
- water_column_sonar_processing-0.0.1/src/model/zarr/__init__.py +0 -0
- water_column_sonar_processing-0.0.1/src/model/zarr/bar.py +0 -28
- water_column_sonar_processing-0.0.1/src/model/zarr/foo.py +0 -11
- water_column_sonar_processing-0.0.1/src/water_column_sonar_processing.egg-info/SOURCES.txt +0 -34
- water_column_sonar_processing-0.0.1/src/water_column_sonar_processing.egg-info/top_level.txt +0 -2
- {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.2}/LICENSE +0 -0
- {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.2}/setup.cfg +0 -0
- {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.2}/src/__init__.py +0 -0
- {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.2/src/aws_manager}/s3fs_manager.py +0 -0
- {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.2/src/cruise}/__init__.py +0 -0
- {water_column_sonar_processing-0.0.1/src/model/aws → water_column_sonar_processing-0.0.2/src/geometry_manager}/__init__.py +0 -0
- {water_column_sonar_processing-0.0.1/src/model/geospatial → water_column_sonar_processing-0.0.2/src/geometry_manager}/geometry_simplification.py +0 -0
- {water_column_sonar_processing-0.0.1/src/model/cruise → water_column_sonar_processing-0.0.2/src/index_manager}/__init__.py +0 -0
- {water_column_sonar_processing-0.0.1/src/model/geospatial → water_column_sonar_processing-0.0.2/src/utility}/__init__.py +0 -0
- {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.2/src}/utility/constants.py +0 -0
- {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.2/src}/utility/pipeline_status.py +0 -0
- {water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.2/src}/utility/timestamp.py +0 -0
- {water_column_sonar_processing-0.0.1 → water_column_sonar_processing-0.0.2}/src/water_column_sonar_processing.egg-info/dependency_links.txt +0 -0
- {water_column_sonar_processing-0.0.1/src/model/index → water_column_sonar_processing-0.0.2/src/zarr_manager}/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: water-column-sonar-processing
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.2
|
|
4
4
|
Summary: A processing tool for water column sonar data.
|
|
5
5
|
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
|
|
@@ -12,10 +12,10 @@ Requires-Python: >=3.10
|
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
|
|
15
|
-
#
|
|
15
|
+
# Water Column Sonar Processing
|
|
16
16
|
Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
# Setting up the Python Environment
|
|
19
19
|
> Python 3.10.12
|
|
20
20
|
|
|
21
21
|
# MacOS Pyenv Installation Instructions
|
|
@@ -33,8 +33,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
33
33
|
1. ```eval "$(pyenv virtualenv-init -)"```
|
|
34
34
|
3. Open a new terminal
|
|
35
35
|
4. Install Python version
|
|
36
|
-
1. ```env
|
|
37
|
-
2. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
|
|
36
|
+
1. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
|
|
38
37
|
5. Create virtual env (to delete 'pyenv uninstall 3.10.12/water-column-sonar-processing')
|
|
39
38
|
1. ```pyenv virtualenv 3.10.12 water-column-sonar-processing```
|
|
40
39
|
6. Set local version of python (if not done already)
|
|
@@ -42,7 +41,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
42
41
|
2. ```pyenv local 3.10.12 water-column-sonar-processing```
|
|
43
42
|
3. ```pyenv activate water-column-sonar-processing```
|
|
44
43
|
|
|
45
|
-
|
|
44
|
+
# Setting up IntelliJ
|
|
46
45
|
|
|
47
46
|
1. Install the IntelliJ Python plugin
|
|
48
47
|
2. Set up pyenv
|
|
@@ -55,35 +54,40 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
55
54
|
2. Facets -> + -> Python
|
|
56
55
|
3. Set interpreter
|
|
57
56
|
|
|
58
|
-
|
|
57
|
+
# Installing Dependencies
|
|
59
58
|
|
|
60
59
|
1. Add dependencies with versions to requirements.txt
|
|
61
60
|
2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
|
|
62
61
|
|
|
63
62
|
|
|
64
|
-
|
|
63
|
+
# Pytest
|
|
65
64
|
```commandline
|
|
66
65
|
pytest --disable-warnings
|
|
67
66
|
```
|
|
68
67
|
|
|
69
|
-
|
|
68
|
+
# Instructions
|
|
70
69
|
Following this tutorial:
|
|
71
70
|
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
72
71
|
|
|
73
|
-
|
|
72
|
+
# To Publish To TEST
|
|
74
73
|
```commandline
|
|
75
74
|
python -m build
|
|
76
75
|
python -m twine upload --repository testpypi dist/*
|
|
77
76
|
pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
|
|
78
77
|
python
|
|
79
78
|
```
|
|
80
|
-
```
|
|
79
|
+
```
|
|
81
80
|
from water-column-sonar-processing import ZarrManager
|
|
82
81
|
example.add_one(2)
|
|
83
82
|
```
|
|
84
83
|
|
|
85
|
-
|
|
84
|
+
# To Publish To PROD
|
|
86
85
|
```commandline
|
|
87
86
|
python -m build
|
|
88
87
|
python -m twine upload --repository pypi dist/*
|
|
89
88
|
```
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# Linting
|
|
92
|
+
Ruff
|
|
93
|
+
https://plugins.jetbrains.com/plugin/20574-ruff
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Water Column Sonar Processing
|
|
2
2
|
Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
# Setting up the Python Environment
|
|
5
5
|
> Python 3.10.12
|
|
6
6
|
|
|
7
7
|
# MacOS Pyenv Installation Instructions
|
|
@@ -19,8 +19,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
19
19
|
1. ```eval "$(pyenv virtualenv-init -)"```
|
|
20
20
|
3. Open a new terminal
|
|
21
21
|
4. Install Python version
|
|
22
|
-
1. ```env
|
|
23
|
-
2. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
|
|
22
|
+
1. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
|
|
24
23
|
5. Create virtual env (to delete 'pyenv uninstall 3.10.12/water-column-sonar-processing')
|
|
25
24
|
1. ```pyenv virtualenv 3.10.12 water-column-sonar-processing```
|
|
26
25
|
6. Set local version of python (if not done already)
|
|
@@ -28,7 +27,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
28
27
|
2. ```pyenv local 3.10.12 water-column-sonar-processing```
|
|
29
28
|
3. ```pyenv activate water-column-sonar-processing```
|
|
30
29
|
|
|
31
|
-
|
|
30
|
+
# Setting up IntelliJ
|
|
32
31
|
|
|
33
32
|
1. Install the IntelliJ Python plugin
|
|
34
33
|
2. Set up pyenv
|
|
@@ -41,35 +40,40 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
41
40
|
2. Facets -> + -> Python
|
|
42
41
|
3. Set interpreter
|
|
43
42
|
|
|
44
|
-
|
|
43
|
+
# Installing Dependencies
|
|
45
44
|
|
|
46
45
|
1. Add dependencies with versions to requirements.txt
|
|
47
46
|
2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
|
|
48
47
|
|
|
49
48
|
|
|
50
|
-
|
|
49
|
+
# Pytest
|
|
51
50
|
```commandline
|
|
52
51
|
pytest --disable-warnings
|
|
53
52
|
```
|
|
54
53
|
|
|
55
|
-
|
|
54
|
+
# Instructions
|
|
56
55
|
Following this tutorial:
|
|
57
56
|
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
58
57
|
|
|
59
|
-
|
|
58
|
+
# To Publish To TEST
|
|
60
59
|
```commandline
|
|
61
60
|
python -m build
|
|
62
61
|
python -m twine upload --repository testpypi dist/*
|
|
63
62
|
pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
|
|
64
63
|
python
|
|
65
64
|
```
|
|
66
|
-
```
|
|
65
|
+
```
|
|
67
66
|
from water-column-sonar-processing import ZarrManager
|
|
68
67
|
example.add_one(2)
|
|
69
68
|
```
|
|
70
69
|
|
|
71
|
-
|
|
70
|
+
# To Publish To PROD
|
|
72
71
|
```commandline
|
|
73
72
|
python -m build
|
|
74
73
|
python -m twine upload --repository pypi dist/*
|
|
75
|
-
```
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# Linting
|
|
78
|
+
Ruff
|
|
79
|
+
https://plugins.jetbrains.com/plugin/20574-ruff
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "water-column-sonar-processing"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.2"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
|
|
10
10
|
]
|
|
@@ -19,4 +19,15 @@ classifiers = [
|
|
|
19
19
|
|
|
20
20
|
[project.urls]
|
|
21
21
|
Homepage = "https://github.com/CI-CMG/water-column-sonar-processing"
|
|
22
|
-
Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
|
|
22
|
+
Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
|
|
23
|
+
|
|
24
|
+
#[pytest]
|
|
25
|
+
#pythonpath = "src"
|
|
26
|
+
#testpaths = "tests"
|
|
27
|
+
|
|
28
|
+
[tool.pytest.ini_options]
|
|
29
|
+
minversion = "6.0"
|
|
30
|
+
addopts = "-ra -q"
|
|
31
|
+
testpaths = [
|
|
32
|
+
"tests",
|
|
33
|
+
]
|
|
@@ -23,7 +23,7 @@ class DynamoDBManager:
|
|
|
23
23
|
self.type_deserializer = TypeDeserializer()
|
|
24
24
|
|
|
25
25
|
#####################################################################
|
|
26
|
-
### defined in raw-to-
|
|
26
|
+
### defined in raw-to-zarr_manager, not used
|
|
27
27
|
# def put_item(
|
|
28
28
|
# self,
|
|
29
29
|
# table_name,
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
3
|
import boto3
|
|
4
|
-
import pandas as pd
|
|
4
|
+
# import pandas as pd
|
|
5
5
|
from collections.abc import Generator
|
|
6
6
|
|
|
7
|
-
import geopandas
|
|
7
|
+
# import geopandas
|
|
8
8
|
from botocore.config import Config
|
|
9
9
|
from boto3.s3.transfer import TransferConfig
|
|
10
10
|
from botocore.exceptions import ClientError
|
|
@@ -139,12 +139,12 @@ class S3Manager:
|
|
|
139
139
|
return all_uploads
|
|
140
140
|
|
|
141
141
|
#####################################################################
|
|
142
|
-
def upload_zarr_files_to_bucket( # noaa-wcsd-
|
|
142
|
+
def upload_zarr_files_to_bucket( # noaa-wcsd-zarr_manager-pds
|
|
143
143
|
self,
|
|
144
144
|
local_directory,
|
|
145
145
|
remote_directory,
|
|
146
146
|
):
|
|
147
|
-
# Right now this is just for uploading a
|
|
147
|
+
# Right now this is just for uploading a zarr_manager store to s3
|
|
148
148
|
print('Uploading files to output bucket.')
|
|
149
149
|
store_name = os.path.basename(local_directory)
|
|
150
150
|
all_files = []
|
|
@@ -162,8 +162,8 @@ class S3Manager:
|
|
|
162
162
|
return all_uploads
|
|
163
163
|
|
|
164
164
|
#####################################################################
|
|
165
|
-
# used: raw-to-
|
|
166
|
-
def list_objects( # noaa-wcsd-pds and noaa-wcsd-
|
|
165
|
+
# used: raw-to-zarr_manager
|
|
166
|
+
def list_objects( # noaa-wcsd-pds and noaa-wcsd-zarr_manager-pds
|
|
167
167
|
self,
|
|
168
168
|
bucket_name,
|
|
169
169
|
prefix
|
|
@@ -250,8 +250,8 @@ class S3Manager:
|
|
|
250
250
|
return raw_files
|
|
251
251
|
|
|
252
252
|
#####################################################################
|
|
253
|
-
def get_object( # TODO: Move this to
|
|
254
|
-
# noaa-wcsd-pds or noaa-wcsd-
|
|
253
|
+
def get_object( # TODO: Move this to index_manager.py
|
|
254
|
+
# noaa-wcsd-pds or noaa-wcsd-zarr_manager-pds
|
|
255
255
|
self,
|
|
256
256
|
bucket_name,
|
|
257
257
|
key_name,
|
|
@@ -272,9 +272,9 @@ class S3Manager:
|
|
|
272
272
|
return response
|
|
273
273
|
|
|
274
274
|
#####################################################################
|
|
275
|
-
# used raw-to-
|
|
275
|
+
# used raw-to-zarr_manager
|
|
276
276
|
def download_file( # TODO: change to download_object
|
|
277
|
-
# noaa-wcsd-pds or noaa-wcsd-
|
|
277
|
+
# noaa-wcsd-pds or noaa-wcsd-zarr_manager-pds
|
|
278
278
|
self,
|
|
279
279
|
bucket_name,
|
|
280
280
|
key,
|
|
@@ -289,7 +289,7 @@ class S3Manager:
|
|
|
289
289
|
|
|
290
290
|
#####################################################################
|
|
291
291
|
# not used
|
|
292
|
-
# def delete_nodd_object( # noaa-wcsd-
|
|
292
|
+
# def delete_nodd_object( # noaa-wcsd-zarr_manager-pds
|
|
293
293
|
# self,
|
|
294
294
|
# bucket_name,
|
|
295
295
|
# key
|
|
@@ -319,7 +319,7 @@ class S3Manager:
|
|
|
319
319
|
|
|
320
320
|
#####################################################################
|
|
321
321
|
# not used TODO: remove
|
|
322
|
-
def put( # noaa-wcsd-
|
|
322
|
+
def put( # noaa-wcsd-zarr_manager-pds
|
|
323
323
|
self,
|
|
324
324
|
bucket_name,
|
|
325
325
|
key,
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import numcodecs
|
|
3
3
|
import numpy as np
|
|
4
|
-
|
|
5
|
-
from
|
|
6
|
-
from
|
|
7
|
-
from
|
|
4
|
+
|
|
5
|
+
from utility.cleaner import Cleaner
|
|
6
|
+
from aws_manager.dynamodb_manager import DynamoDBManager
|
|
7
|
+
from aws_manager.s3_manager import S3Manager
|
|
8
|
+
from zarr_manager.zarr_manager import ZarrManager
|
|
8
9
|
|
|
9
10
|
numcodecs.blosc.use_threads = False
|
|
10
11
|
numcodecs.blosc.set_nthreads(1)
|
|
11
12
|
|
|
12
13
|
TEMPDIR = "/tmp"
|
|
13
14
|
|
|
14
|
-
# TODO: when ready switch to version 3 of
|
|
15
|
+
# TODO: when ready switch to version 3 of zarr_manager spec
|
|
15
16
|
# ZARR_V3_EXPERIMENTAL_API = 1
|
|
16
17
|
# creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
|
|
17
18
|
|
|
@@ -33,17 +34,17 @@ class CreateEmptyZarrStore:
|
|
|
33
34
|
object_prefix: str,
|
|
34
35
|
cruise_name: str,
|
|
35
36
|
) -> None:
|
|
36
|
-
print('uploading
|
|
37
|
+
print('uploading zarr_manager store to s3')
|
|
37
38
|
s3_manager = S3Manager()
|
|
38
39
|
#
|
|
39
40
|
print('Starting upload with thread pool executor.')
|
|
40
41
|
# # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
|
|
41
42
|
all_files = []
|
|
42
|
-
for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.
|
|
43
|
+
for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.zarr_manager"):
|
|
43
44
|
for file in files:
|
|
44
45
|
local_path = os.path.join(subdir, file)
|
|
45
|
-
# 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.
|
|
46
|
-
s3_key = f'{object_prefix}/{cruise_name}.
|
|
46
|
+
# 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.zarr_manager/.zattrs'
|
|
47
|
+
s3_key = f'{object_prefix}/{cruise_name}.zarr_manager{local_path.split(f"{cruise_name}.zarr_manager")[-1]}'
|
|
47
48
|
all_files.append([local_path, s3_key])
|
|
48
49
|
#
|
|
49
50
|
# print(all_files)
|
|
@@ -97,10 +98,10 @@ class CreateEmptyZarrStore:
|
|
|
97
98
|
new_width = int(consolidated_zarr_width)
|
|
98
99
|
print(f"new_width: {new_width}")
|
|
99
100
|
#################################################################
|
|
100
|
-
store_name = f"{cruise_name}.
|
|
101
|
+
store_name = f"{cruise_name}.zarr_manager"
|
|
101
102
|
print(store_name)
|
|
102
103
|
################################################################
|
|
103
|
-
# Delete existing
|
|
104
|
+
# Delete existing zarr_manager store if it exists
|
|
104
105
|
s3_manager = S3Manager()
|
|
105
106
|
zarr_prefix = os.path.join("level_2", ship_name, cruise_name, sensor_name)
|
|
106
107
|
child_objects = s3_manager.get_child_objects(
|
|
@@ -112,7 +113,7 @@ class CreateEmptyZarrStore:
|
|
|
112
113
|
objects=child_objects,
|
|
113
114
|
)
|
|
114
115
|
################################################################
|
|
115
|
-
# Create new
|
|
116
|
+
# Create new zarr_manager store
|
|
116
117
|
zarr_manager = ZarrManager()
|
|
117
118
|
new_height = len(zarr_manager.get_depth_values(
|
|
118
119
|
min_echo_range=cruise_min_echo_range,
|
|
@@ -156,11 +157,11 @@ class CreateEmptyZarrStore:
|
|
|
156
157
|
# TODO: update enum in dynamodb
|
|
157
158
|
#################################################################
|
|
158
159
|
except Exception as err:
|
|
159
|
-
print(f"Problem trying to create new cruise
|
|
160
|
+
print(f"Problem trying to create new cruise zarr_manager store: {err}")
|
|
160
161
|
finally:
|
|
161
162
|
cleaner = Cleaner()
|
|
162
163
|
cleaner.delete_local_files()
|
|
163
|
-
print("Done creating cruise level
|
|
164
|
+
print("Done creating cruise level zarr_manager store")
|
|
164
165
|
|
|
165
166
|
|
|
166
167
|
###########################################################
|
|
@@ -6,15 +6,16 @@ import numpy as np
|
|
|
6
6
|
import xarray as xr
|
|
7
7
|
import pandas as pd
|
|
8
8
|
|
|
9
|
-
from
|
|
10
|
-
from
|
|
11
|
-
from
|
|
9
|
+
from aws_manager.dynamodb_manager import DynamoDBManager
|
|
10
|
+
from zarr_manager.zarr_manager import ZarrManager
|
|
11
|
+
from geometry_manager.geometry_manager import GeometryManager
|
|
12
|
+
|
|
12
13
|
|
|
13
14
|
numcodecs.blosc.use_threads = False
|
|
14
15
|
numcodecs.blosc.set_nthreads(1)
|
|
15
16
|
|
|
16
17
|
|
|
17
|
-
# TODO: when ready switch to version 3 of
|
|
18
|
+
# TODO: when ready switch to version 3 of zarr_manager spec
|
|
18
19
|
# ZARR_V3_EXPERIMENTAL_API = 1
|
|
19
20
|
# creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
|
|
20
21
|
|
|
@@ -114,7 +115,7 @@ class ResampleRegrid:
|
|
|
114
115
|
) -> None:
|
|
115
116
|
"""
|
|
116
117
|
The goal here is to interpolate the data against the depth values already populated
|
|
117
|
-
in the existing file level
|
|
118
|
+
in the existing file level zarr_manager stores. We open the cruise-level store with zarr_manager for
|
|
118
119
|
read/write operations. We open the file-level store with Xarray to leverage tools for
|
|
119
120
|
resampling and subsetting the data.
|
|
120
121
|
"""
|
|
@@ -122,8 +123,8 @@ class ResampleRegrid:
|
|
|
122
123
|
try:
|
|
123
124
|
zarr_manager = ZarrManager()
|
|
124
125
|
# s3_manager = S3Manager()
|
|
125
|
-
geo_manager =
|
|
126
|
-
# get
|
|
126
|
+
geo_manager = GeometryManager()
|
|
127
|
+
# get zarr_manager store
|
|
127
128
|
output_zarr_store = zarr_manager.open_s3_zarr_store_with_zarr(
|
|
128
129
|
ship_name=ship_name,
|
|
129
130
|
cruise_name=cruise_name,
|
|
@@ -156,7 +157,7 @@ class ResampleRegrid:
|
|
|
156
157
|
# TODO: filter rows by enum success, filter the dataframe just for enums >= LEVEL_1_PROCESSING
|
|
157
158
|
# df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
|
|
158
159
|
|
|
159
|
-
# Get
|
|
160
|
+
# Get index_manager from all cruise files. Note: should be based on which are included in cruise.
|
|
160
161
|
index = cruise_df.index[cruise_df['FILE_NAME'] == f"{file_name_stem}.raw"][0]
|
|
161
162
|
|
|
162
163
|
# get input store
|
|
@@ -168,7 +169,7 @@ class ResampleRegrid:
|
|
|
168
169
|
)
|
|
169
170
|
#########################################################################
|
|
170
171
|
# [3] Get needed indices
|
|
171
|
-
# Offset from start
|
|
172
|
+
# Offset from start index_manager to insert new data. Note that missing values are excluded.
|
|
172
173
|
ping_time_cumsum = np.insert(
|
|
173
174
|
np.cumsum(cruise_df['NUM_PING_TIME_DROPNA'].dropna().to_numpy(dtype=int)),
|
|
174
175
|
obj=0,
|
|
@@ -222,7 +223,7 @@ class ResampleRegrid:
|
|
|
222
223
|
print(f"start_ping_time_index: {start_ping_time_index}, end_ping_time_index: {end_ping_time_index}")
|
|
223
224
|
|
|
224
225
|
#########################################################################
|
|
225
|
-
# write Sv values to cruise-level-
|
|
226
|
+
# write Sv values to cruise-level-zarr_manager-store
|
|
226
227
|
for channel in range(len(input_xr.channel.values)): # doesn't like being written in one fell swoop :(
|
|
227
228
|
output_zarr_store.Sv[
|
|
228
229
|
:,
|
|
@@ -3,8 +3,8 @@ import numpy as np
|
|
|
3
3
|
import geopandas
|
|
4
4
|
import pandas as pd
|
|
5
5
|
|
|
6
|
-
from
|
|
7
|
-
from
|
|
6
|
+
from utility.cleaner import Cleaner
|
|
7
|
+
from aws_manager.s3_manager import S3Manager
|
|
8
8
|
|
|
9
9
|
"""
|
|
10
10
|
// [Decimal / Places / Degrees / Object that can be recognized at scale / N/S or E/W at equator, E/W at 23N/S, E/W at 45N/S, E/W at 67N/S]
|
|
@@ -19,7 +19,7 @@ from ..aws.s3_manager import S3Manager
|
|
|
19
19
|
"""
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
class
|
|
22
|
+
class GeometryManager:
|
|
23
23
|
#######################################################
|
|
24
24
|
def __init__(
|
|
25
25
|
self,
|
|
@@ -23,7 +23,7 @@ class PMTileGeneration(object):
|
|
|
23
23
|
|
|
24
24
|
#######################################################
|
|
25
25
|
def generate_geojson_feature_collection(self):
|
|
26
|
-
# This was used to read from noaa-wcsd-
|
|
26
|
+
# This was used to read from noaa-wcsd-zarr_manager-pds bucket geojson files and then to
|
|
27
27
|
# generate the geopandas dataframe which could be exported to another comprehensive
|
|
28
28
|
# geojson file. That
|
|
29
29
|
result = list(Path("/Users/r2d2/Documents/echofish/geojson").rglob("*.json"))
|
|
@@ -4,8 +4,7 @@ import pandas as pd
|
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from concurrent.futures import ThreadPoolExecutor
|
|
6
6
|
from concurrent.futures import as_completed
|
|
7
|
-
from
|
|
8
|
-
|
|
7
|
+
from aws_manager.s3_manager import S3Manager
|
|
9
8
|
|
|
10
9
|
class IndexManager:
|
|
11
10
|
|
|
@@ -194,7 +193,7 @@ class IndexManager:
|
|
|
194
193
|
return calibration_statuses
|
|
195
194
|
|
|
196
195
|
#################################################################
|
|
197
|
-
# def
|
|
196
|
+
# def index_manager( # TODO: get rid of this?
|
|
198
197
|
# self
|
|
199
198
|
# ):
|
|
200
199
|
# start_time = datetime.now() # used for benchmarking
|
{water_column_sonar_processing-0.0.1/src/model → water_column_sonar_processing-0.0.2/src}/model.py
RENAMED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import json
|
|
3
3
|
import numpy as np
|
|
4
|
-
from
|
|
5
|
-
from
|
|
6
|
-
from
|
|
7
|
-
|
|
4
|
+
from aws_manager.s3_manager import S3Manager
|
|
5
|
+
from aws_manager.s3fs_manager import S3FSManager
|
|
6
|
+
from aws_manager.sns_manager import SNSManager
|
|
7
|
+
from aws_manager.dynamodb_manager import DynamoDBManager
|
|
8
8
|
|
|
9
9
|
###########################################################
|
|
10
10
|
class Model:
|
|
@@ -21,6 +21,8 @@ class Model:
|
|
|
21
21
|
|
|
22
22
|
def execute(self):
|
|
23
23
|
input_s3_manager = S3Manager() # TODO: Need to allow passing in of credentials when writing to protected bucket
|
|
24
|
+
s3fs_manager = S3FSManager() # TODO: delete this
|
|
25
|
+
print(s3fs_manager) # TODO: delete this
|
|
24
26
|
output_s3_manager = S3Manager()
|
|
25
27
|
# TODO: s3fs?
|
|
26
28
|
sns_manager = SNSManager()
|
|
@@ -85,7 +87,7 @@ class Model:
|
|
|
85
87
|
':sh': {'S': ship_name},
|
|
86
88
|
':st': {'S': '2006-04-06T11:34:07.288Z'},
|
|
87
89
|
':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
|
|
88
|
-
':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.
|
|
90
|
+
':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.zarr_manager'},
|
|
89
91
|
},
|
|
90
92
|
update_expression=(
|
|
91
93
|
'SET '
|
|
@@ -7,9 +7,9 @@ import shutil
|
|
|
7
7
|
class Cleaner:
|
|
8
8
|
@staticmethod
|
|
9
9
|
def delete_local_files(
|
|
10
|
-
file_types=['*.raw*', '*.
|
|
10
|
+
file_types=['*.raw*', '*.zarr_manager'] # '*.json'
|
|
11
11
|
):
|
|
12
|
-
print('Deleting all local raw and
|
|
12
|
+
print('Deleting all local raw and zarr_manager files')
|
|
13
13
|
for i in file_types:
|
|
14
14
|
for j in glob.glob(i):
|
|
15
15
|
if os.path.isdir(j):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: water-column-sonar-processing
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.2
|
|
4
4
|
Summary: A processing tool for water column sonar data.
|
|
5
5
|
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
|
|
@@ -12,10 +12,10 @@ Requires-Python: >=3.10
|
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
|
|
15
|
-
#
|
|
15
|
+
# Water Column Sonar Processing
|
|
16
16
|
Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
# Setting up the Python Environment
|
|
19
19
|
> Python 3.10.12
|
|
20
20
|
|
|
21
21
|
# MacOS Pyenv Installation Instructions
|
|
@@ -33,8 +33,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
33
33
|
1. ```eval "$(pyenv virtualenv-init -)"```
|
|
34
34
|
3. Open a new terminal
|
|
35
35
|
4. Install Python version
|
|
36
|
-
1. ```env
|
|
37
|
-
2. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
|
|
36
|
+
1. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
|
|
38
37
|
5. Create virtual env (to delete 'pyenv uninstall 3.10.12/water-column-sonar-processing')
|
|
39
38
|
1. ```pyenv virtualenv 3.10.12 water-column-sonar-processing```
|
|
40
39
|
6. Set local version of python (if not done already)
|
|
@@ -42,7 +41,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
42
41
|
2. ```pyenv local 3.10.12 water-column-sonar-processing```
|
|
43
42
|
3. ```pyenv activate water-column-sonar-processing```
|
|
44
43
|
|
|
45
|
-
|
|
44
|
+
# Setting up IntelliJ
|
|
46
45
|
|
|
47
46
|
1. Install the IntelliJ Python plugin
|
|
48
47
|
2. Set up pyenv
|
|
@@ -55,35 +54,40 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
55
54
|
2. Facets -> + -> Python
|
|
56
55
|
3. Set interpreter
|
|
57
56
|
|
|
58
|
-
|
|
57
|
+
# Installing Dependencies
|
|
59
58
|
|
|
60
59
|
1. Add dependencies with versions to requirements.txt
|
|
61
60
|
2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
|
|
62
61
|
|
|
63
62
|
|
|
64
|
-
|
|
63
|
+
# Pytest
|
|
65
64
|
```commandline
|
|
66
65
|
pytest --disable-warnings
|
|
67
66
|
```
|
|
68
67
|
|
|
69
|
-
|
|
68
|
+
# Instructions
|
|
70
69
|
Following this tutorial:
|
|
71
70
|
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
72
71
|
|
|
73
|
-
|
|
72
|
+
# To Publish To TEST
|
|
74
73
|
```commandline
|
|
75
74
|
python -m build
|
|
76
75
|
python -m twine upload --repository testpypi dist/*
|
|
77
76
|
pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
|
|
78
77
|
python
|
|
79
78
|
```
|
|
80
|
-
```
|
|
79
|
+
```
|
|
81
80
|
from water-column-sonar-processing import ZarrManager
|
|
82
81
|
example.add_one(2)
|
|
83
82
|
```
|
|
84
83
|
|
|
85
|
-
|
|
84
|
+
# To Publish To PROD
|
|
86
85
|
```commandline
|
|
87
86
|
python -m build
|
|
88
87
|
python -m twine upload --repository pypi dist/*
|
|
89
88
|
```
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# Linting
|
|
92
|
+
Ruff
|
|
93
|
+
https://plugins.jetbrains.com/plugin/20574-ruff
|