water-column-sonar-processing 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of water-column-sonar-processing might be problematic. Click here for more details.
- aws_manager/__init__.py +4 -0
- {model/aws → aws_manager}/dynamodb_manager.py +1 -1
- {model/aws → aws_manager}/s3_manager.py +12 -12
- {model/aws → aws_manager}/sns_manager.py +1 -1
- {model/aws → aws_manager}/sqs_manager.py +1 -1
- {model/cruise → cruise}/create_empty_zarr_store.py +15 -14
- {model/cruise → cruise}/resample_regrid.py +11 -10
- {model/geospatial → geometry_manager}/geometry_manager.py +3 -3
- {model/geospatial → geometry_manager}/pmtile_generation.py +1 -1
- model/index/index.py → index_manager/index_manager.py +2 -3
- model/model.py → model.py +7 -5
- {model/utility → utility}/cleaner.py +2 -2
- {water_column_sonar_processing-0.0.1.dist-info → water_column_sonar_processing-0.0.2.dist-info}/METADATA +16 -12
- water_column_sonar_processing-0.0.2.dist-info/RECORD +29 -0
- water_column_sonar_processing-0.0.2.dist-info/top_level.txt +8 -0
- {model/zarr → zarr_manager}/zarr_manager.py +10 -9
- model/utility/__init__.py +0 -0
- model/zarr/__init__.py +0 -0
- model/zarr/bar.py +0 -28
- model/zarr/foo.py +0 -11
- water_column_sonar_processing-0.0.1.dist-info/RECORD +0 -32
- water_column_sonar_processing-0.0.1.dist-info/top_level.txt +0 -2
- {model/aws → aws_manager}/s3fs_manager.py +0 -0
- {model → cruise}/__init__.py +0 -0
- {model/aws → geometry_manager}/__init__.py +0 -0
- {model/geospatial → geometry_manager}/geometry_simplification.py +0 -0
- {model/cruise → index_manager}/__init__.py +0 -0
- {model/geospatial → utility}/__init__.py +0 -0
- {model/utility → utility}/constants.py +0 -0
- {model/utility → utility}/pipeline_status.py +0 -0
- {model/utility → utility}/timestamp.py +0 -0
- {water_column_sonar_processing-0.0.1.dist-info → water_column_sonar_processing-0.0.2.dist-info}/LICENSE +0 -0
- {water_column_sonar_processing-0.0.1.dist-info → water_column_sonar_processing-0.0.2.dist-info}/WHEEL +0 -0
- {model/index → zarr_manager}/__init__.py +0 -0
aws_manager/__init__.py
ADDED
|
@@ -23,7 +23,7 @@ class DynamoDBManager:
|
|
|
23
23
|
self.type_deserializer = TypeDeserializer()
|
|
24
24
|
|
|
25
25
|
#####################################################################
|
|
26
|
-
### defined in raw-to-
|
|
26
|
+
### defined in raw-to-zarr_manager, not used
|
|
27
27
|
# def put_item(
|
|
28
28
|
# self,
|
|
29
29
|
# table_name,
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
3
|
import boto3
|
|
4
|
-
import pandas as pd
|
|
4
|
+
# import pandas as pd
|
|
5
5
|
from collections.abc import Generator
|
|
6
6
|
|
|
7
|
-
import geopandas
|
|
7
|
+
# import geopandas
|
|
8
8
|
from botocore.config import Config
|
|
9
9
|
from boto3.s3.transfer import TransferConfig
|
|
10
10
|
from botocore.exceptions import ClientError
|
|
@@ -139,12 +139,12 @@ class S3Manager:
|
|
|
139
139
|
return all_uploads
|
|
140
140
|
|
|
141
141
|
#####################################################################
|
|
142
|
-
def upload_zarr_files_to_bucket( # noaa-wcsd-
|
|
142
|
+
def upload_zarr_files_to_bucket( # noaa-wcsd-zarr_manager-pds
|
|
143
143
|
self,
|
|
144
144
|
local_directory,
|
|
145
145
|
remote_directory,
|
|
146
146
|
):
|
|
147
|
-
# Right now this is just for uploading a
|
|
147
|
+
# Right now this is just for uploading a zarr_manager store to s3
|
|
148
148
|
print('Uploading files to output bucket.')
|
|
149
149
|
store_name = os.path.basename(local_directory)
|
|
150
150
|
all_files = []
|
|
@@ -162,8 +162,8 @@ class S3Manager:
|
|
|
162
162
|
return all_uploads
|
|
163
163
|
|
|
164
164
|
#####################################################################
|
|
165
|
-
# used: raw-to-
|
|
166
|
-
def list_objects( # noaa-wcsd-pds and noaa-wcsd-
|
|
165
|
+
# used: raw-to-zarr_manager
|
|
166
|
+
def list_objects( # noaa-wcsd-pds and noaa-wcsd-zarr_manager-pds
|
|
167
167
|
self,
|
|
168
168
|
bucket_name,
|
|
169
169
|
prefix
|
|
@@ -250,8 +250,8 @@ class S3Manager:
|
|
|
250
250
|
return raw_files
|
|
251
251
|
|
|
252
252
|
#####################################################################
|
|
253
|
-
def get_object( # TODO: Move this to
|
|
254
|
-
# noaa-wcsd-pds or noaa-wcsd-
|
|
253
|
+
def get_object( # TODO: Move this to index_manager.py
|
|
254
|
+
# noaa-wcsd-pds or noaa-wcsd-zarr_manager-pds
|
|
255
255
|
self,
|
|
256
256
|
bucket_name,
|
|
257
257
|
key_name,
|
|
@@ -272,9 +272,9 @@ class S3Manager:
|
|
|
272
272
|
return response
|
|
273
273
|
|
|
274
274
|
#####################################################################
|
|
275
|
-
# used raw-to-
|
|
275
|
+
# used raw-to-zarr_manager
|
|
276
276
|
def download_file( # TODO: change to download_object
|
|
277
|
-
# noaa-wcsd-pds or noaa-wcsd-
|
|
277
|
+
# noaa-wcsd-pds or noaa-wcsd-zarr_manager-pds
|
|
278
278
|
self,
|
|
279
279
|
bucket_name,
|
|
280
280
|
key,
|
|
@@ -289,7 +289,7 @@ class S3Manager:
|
|
|
289
289
|
|
|
290
290
|
#####################################################################
|
|
291
291
|
# not used
|
|
292
|
-
# def delete_nodd_object( # noaa-wcsd-
|
|
292
|
+
# def delete_nodd_object( # noaa-wcsd-zarr_manager-pds
|
|
293
293
|
# self,
|
|
294
294
|
# bucket_name,
|
|
295
295
|
# key
|
|
@@ -319,7 +319,7 @@ class S3Manager:
|
|
|
319
319
|
|
|
320
320
|
#####################################################################
|
|
321
321
|
# not used TODO: remove
|
|
322
|
-
def put( # noaa-wcsd-
|
|
322
|
+
def put( # noaa-wcsd-zarr_manager-pds
|
|
323
323
|
self,
|
|
324
324
|
bucket_name,
|
|
325
325
|
key,
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import numcodecs
|
|
3
3
|
import numpy as np
|
|
4
|
-
|
|
5
|
-
from
|
|
6
|
-
from
|
|
7
|
-
from
|
|
4
|
+
|
|
5
|
+
from utility.cleaner import Cleaner
|
|
6
|
+
from aws_manager.dynamodb_manager import DynamoDBManager
|
|
7
|
+
from aws_manager.s3_manager import S3Manager
|
|
8
|
+
from zarr_manager.zarr_manager import ZarrManager
|
|
8
9
|
|
|
9
10
|
numcodecs.blosc.use_threads = False
|
|
10
11
|
numcodecs.blosc.set_nthreads(1)
|
|
11
12
|
|
|
12
13
|
TEMPDIR = "/tmp"
|
|
13
14
|
|
|
14
|
-
# TODO: when ready switch to version 3 of
|
|
15
|
+
# TODO: when ready switch to version 3 of zarr_manager spec
|
|
15
16
|
# ZARR_V3_EXPERIMENTAL_API = 1
|
|
16
17
|
# creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
|
|
17
18
|
|
|
@@ -33,17 +34,17 @@ class CreateEmptyZarrStore:
|
|
|
33
34
|
object_prefix: str,
|
|
34
35
|
cruise_name: str,
|
|
35
36
|
) -> None:
|
|
36
|
-
print('uploading
|
|
37
|
+
print('uploading zarr_manager store to s3')
|
|
37
38
|
s3_manager = S3Manager()
|
|
38
39
|
#
|
|
39
40
|
print('Starting upload with thread pool executor.')
|
|
40
41
|
# # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
|
|
41
42
|
all_files = []
|
|
42
|
-
for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.
|
|
43
|
+
for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.zarr_manager"):
|
|
43
44
|
for file in files:
|
|
44
45
|
local_path = os.path.join(subdir, file)
|
|
45
|
-
# 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.
|
|
46
|
-
s3_key = f'{object_prefix}/{cruise_name}.
|
|
46
|
+
# 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.zarr_manager/.zattrs'
|
|
47
|
+
s3_key = f'{object_prefix}/{cruise_name}.zarr_manager{local_path.split(f"{cruise_name}.zarr_manager")[-1]}'
|
|
47
48
|
all_files.append([local_path, s3_key])
|
|
48
49
|
#
|
|
49
50
|
# print(all_files)
|
|
@@ -97,10 +98,10 @@ class CreateEmptyZarrStore:
|
|
|
97
98
|
new_width = int(consolidated_zarr_width)
|
|
98
99
|
print(f"new_width: {new_width}")
|
|
99
100
|
#################################################################
|
|
100
|
-
store_name = f"{cruise_name}.
|
|
101
|
+
store_name = f"{cruise_name}.zarr_manager"
|
|
101
102
|
print(store_name)
|
|
102
103
|
################################################################
|
|
103
|
-
# Delete existing
|
|
104
|
+
# Delete existing zarr_manager store if it exists
|
|
104
105
|
s3_manager = S3Manager()
|
|
105
106
|
zarr_prefix = os.path.join("level_2", ship_name, cruise_name, sensor_name)
|
|
106
107
|
child_objects = s3_manager.get_child_objects(
|
|
@@ -112,7 +113,7 @@ class CreateEmptyZarrStore:
|
|
|
112
113
|
objects=child_objects,
|
|
113
114
|
)
|
|
114
115
|
################################################################
|
|
115
|
-
# Create new
|
|
116
|
+
# Create new zarr_manager store
|
|
116
117
|
zarr_manager = ZarrManager()
|
|
117
118
|
new_height = len(zarr_manager.get_depth_values(
|
|
118
119
|
min_echo_range=cruise_min_echo_range,
|
|
@@ -156,11 +157,11 @@ class CreateEmptyZarrStore:
|
|
|
156
157
|
# TODO: update enum in dynamodb
|
|
157
158
|
#################################################################
|
|
158
159
|
except Exception as err:
|
|
159
|
-
print(f"Problem trying to create new cruise
|
|
160
|
+
print(f"Problem trying to create new cruise zarr_manager store: {err}")
|
|
160
161
|
finally:
|
|
161
162
|
cleaner = Cleaner()
|
|
162
163
|
cleaner.delete_local_files()
|
|
163
|
-
print("Done creating cruise level
|
|
164
|
+
print("Done creating cruise level zarr_manager store")
|
|
164
165
|
|
|
165
166
|
|
|
166
167
|
###########################################################
|
|
@@ -6,15 +6,16 @@ import numpy as np
|
|
|
6
6
|
import xarray as xr
|
|
7
7
|
import pandas as pd
|
|
8
8
|
|
|
9
|
-
from
|
|
10
|
-
from
|
|
11
|
-
from
|
|
9
|
+
from aws_manager.dynamodb_manager import DynamoDBManager
|
|
10
|
+
from zarr_manager.zarr_manager import ZarrManager
|
|
11
|
+
from geometry_manager.geometry_manager import GeometryManager
|
|
12
|
+
|
|
12
13
|
|
|
13
14
|
numcodecs.blosc.use_threads = False
|
|
14
15
|
numcodecs.blosc.set_nthreads(1)
|
|
15
16
|
|
|
16
17
|
|
|
17
|
-
# TODO: when ready switch to version 3 of
|
|
18
|
+
# TODO: when ready switch to version 3 of zarr_manager spec
|
|
18
19
|
# ZARR_V3_EXPERIMENTAL_API = 1
|
|
19
20
|
# creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
|
|
20
21
|
|
|
@@ -114,7 +115,7 @@ class ResampleRegrid:
|
|
|
114
115
|
) -> None:
|
|
115
116
|
"""
|
|
116
117
|
The goal here is to interpolate the data against the depth values already populated
|
|
117
|
-
in the existing file level
|
|
118
|
+
in the existing file level zarr_manager stores. We open the cruise-level store with zarr_manager for
|
|
118
119
|
read/write operations. We open the file-level store with Xarray to leverage tools for
|
|
119
120
|
resampling and subsetting the data.
|
|
120
121
|
"""
|
|
@@ -122,8 +123,8 @@ class ResampleRegrid:
|
|
|
122
123
|
try:
|
|
123
124
|
zarr_manager = ZarrManager()
|
|
124
125
|
# s3_manager = S3Manager()
|
|
125
|
-
geo_manager =
|
|
126
|
-
# get
|
|
126
|
+
geo_manager = GeometryManager()
|
|
127
|
+
# get zarr_manager store
|
|
127
128
|
output_zarr_store = zarr_manager.open_s3_zarr_store_with_zarr(
|
|
128
129
|
ship_name=ship_name,
|
|
129
130
|
cruise_name=cruise_name,
|
|
@@ -156,7 +157,7 @@ class ResampleRegrid:
|
|
|
156
157
|
# TODO: filter rows by enum success, filter the dataframe just for enums >= LEVEL_1_PROCESSING
|
|
157
158
|
# df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
|
|
158
159
|
|
|
159
|
-
# Get
|
|
160
|
+
# Get index_manager from all cruise files. Note: should be based on which are included in cruise.
|
|
160
161
|
index = cruise_df.index[cruise_df['FILE_NAME'] == f"{file_name_stem}.raw"][0]
|
|
161
162
|
|
|
162
163
|
# get input store
|
|
@@ -168,7 +169,7 @@ class ResampleRegrid:
|
|
|
168
169
|
)
|
|
169
170
|
#########################################################################
|
|
170
171
|
# [3] Get needed indices
|
|
171
|
-
# Offset from start
|
|
172
|
+
# Offset from start index_manager to insert new data. Note that missing values are excluded.
|
|
172
173
|
ping_time_cumsum = np.insert(
|
|
173
174
|
np.cumsum(cruise_df['NUM_PING_TIME_DROPNA'].dropna().to_numpy(dtype=int)),
|
|
174
175
|
obj=0,
|
|
@@ -222,7 +223,7 @@ class ResampleRegrid:
|
|
|
222
223
|
print(f"start_ping_time_index: {start_ping_time_index}, end_ping_time_index: {end_ping_time_index}")
|
|
223
224
|
|
|
224
225
|
#########################################################################
|
|
225
|
-
# write Sv values to cruise-level-
|
|
226
|
+
# write Sv values to cruise-level-zarr_manager-store
|
|
226
227
|
for channel in range(len(input_xr.channel.values)): # doesn't like being written in one fell swoop :(
|
|
227
228
|
output_zarr_store.Sv[
|
|
228
229
|
:,
|
|
@@ -3,8 +3,8 @@ import numpy as np
|
|
|
3
3
|
import geopandas
|
|
4
4
|
import pandas as pd
|
|
5
5
|
|
|
6
|
-
from
|
|
7
|
-
from
|
|
6
|
+
from utility.cleaner import Cleaner
|
|
7
|
+
from aws_manager.s3_manager import S3Manager
|
|
8
8
|
|
|
9
9
|
"""
|
|
10
10
|
// [Decimal / Places / Degrees / Object that can be recognized at scale / N/S or E/W at equator, E/W at 23N/S, E/W at 45N/S, E/W at 67N/S]
|
|
@@ -19,7 +19,7 @@ from ..aws.s3_manager import S3Manager
|
|
|
19
19
|
"""
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
class
|
|
22
|
+
class GeometryManager:
|
|
23
23
|
#######################################################
|
|
24
24
|
def __init__(
|
|
25
25
|
self,
|
|
@@ -23,7 +23,7 @@ class PMTileGeneration(object):
|
|
|
23
23
|
|
|
24
24
|
#######################################################
|
|
25
25
|
def generate_geojson_feature_collection(self):
|
|
26
|
-
# This was used to read from noaa-wcsd-
|
|
26
|
+
# This was used to read from noaa-wcsd-zarr_manager-pds bucket geojson files and then to
|
|
27
27
|
# generate the geopandas dataframe which could be exported to another comprehensive
|
|
28
28
|
# geojson file. That
|
|
29
29
|
result = list(Path("/Users/r2d2/Documents/echofish/geojson").rglob("*.json"))
|
|
@@ -4,8 +4,7 @@ import pandas as pd
|
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from concurrent.futures import ThreadPoolExecutor
|
|
6
6
|
from concurrent.futures import as_completed
|
|
7
|
-
from
|
|
8
|
-
|
|
7
|
+
from aws_manager.s3_manager import S3Manager
|
|
9
8
|
|
|
10
9
|
class IndexManager:
|
|
11
10
|
|
|
@@ -194,7 +193,7 @@ class IndexManager:
|
|
|
194
193
|
return calibration_statuses
|
|
195
194
|
|
|
196
195
|
#################################################################
|
|
197
|
-
# def
|
|
196
|
+
# def index_manager( # TODO: get rid of this?
|
|
198
197
|
# self
|
|
199
198
|
# ):
|
|
200
199
|
# start_time = datetime.now() # used for benchmarking
|
model/model.py → model.py
RENAMED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import json
|
|
3
3
|
import numpy as np
|
|
4
|
-
from
|
|
5
|
-
from
|
|
6
|
-
from
|
|
7
|
-
|
|
4
|
+
from aws_manager.s3_manager import S3Manager
|
|
5
|
+
from aws_manager.s3fs_manager import S3FSManager
|
|
6
|
+
from aws_manager.sns_manager import SNSManager
|
|
7
|
+
from aws_manager.dynamodb_manager import DynamoDBManager
|
|
8
8
|
|
|
9
9
|
###########################################################
|
|
10
10
|
class Model:
|
|
@@ -21,6 +21,8 @@ class Model:
|
|
|
21
21
|
|
|
22
22
|
def execute(self):
|
|
23
23
|
input_s3_manager = S3Manager() # TODO: Need to allow passing in of credentials when writing to protected bucket
|
|
24
|
+
s3fs_manager = S3FSManager() # TODO: delete this
|
|
25
|
+
print(s3fs_manager) # TODO: delete this
|
|
24
26
|
output_s3_manager = S3Manager()
|
|
25
27
|
# TODO: s3fs?
|
|
26
28
|
sns_manager = SNSManager()
|
|
@@ -85,7 +87,7 @@ class Model:
|
|
|
85
87
|
':sh': {'S': ship_name},
|
|
86
88
|
':st': {'S': '2006-04-06T11:34:07.288Z'},
|
|
87
89
|
':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
|
|
88
|
-
':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.
|
|
90
|
+
':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.zarr_manager'},
|
|
89
91
|
},
|
|
90
92
|
update_expression=(
|
|
91
93
|
'SET '
|
|
@@ -7,9 +7,9 @@ import shutil
|
|
|
7
7
|
class Cleaner:
|
|
8
8
|
@staticmethod
|
|
9
9
|
def delete_local_files(
|
|
10
|
-
file_types=['*.raw*', '*.
|
|
10
|
+
file_types=['*.raw*', '*.zarr_manager'] # '*.json'
|
|
11
11
|
):
|
|
12
|
-
print('Deleting all local raw and
|
|
12
|
+
print('Deleting all local raw and zarr_manager files')
|
|
13
13
|
for i in file_types:
|
|
14
14
|
for j in glob.glob(i):
|
|
15
15
|
if os.path.isdir(j):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: water-column-sonar-processing
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.2
|
|
4
4
|
Summary: A processing tool for water column sonar data.
|
|
5
5
|
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
|
|
@@ -12,10 +12,10 @@ Requires-Python: >=3.10
|
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
|
|
15
|
-
#
|
|
15
|
+
# Water Column Sonar Processing
|
|
16
16
|
Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
# Setting up the Python Environment
|
|
19
19
|
> Python 3.10.12
|
|
20
20
|
|
|
21
21
|
# MacOS Pyenv Installation Instructions
|
|
@@ -33,8 +33,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
33
33
|
1. ```eval "$(pyenv virtualenv-init -)"```
|
|
34
34
|
3. Open a new terminal
|
|
35
35
|
4. Install Python version
|
|
36
|
-
1. ```env
|
|
37
|
-
2. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
|
|
36
|
+
1. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
|
|
38
37
|
5. Create virtual env (to delete 'pyenv uninstall 3.10.12/water-column-sonar-processing')
|
|
39
38
|
1. ```pyenv virtualenv 3.10.12 water-column-sonar-processing```
|
|
40
39
|
6. Set local version of python (if not done already)
|
|
@@ -42,7 +41,7 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
42
41
|
2. ```pyenv local 3.10.12 water-column-sonar-processing```
|
|
43
42
|
3. ```pyenv activate water-column-sonar-processing```
|
|
44
43
|
|
|
45
|
-
|
|
44
|
+
# Setting up IntelliJ
|
|
46
45
|
|
|
47
46
|
1. Install the IntelliJ Python plugin
|
|
48
47
|
2. Set up pyenv
|
|
@@ -55,35 +54,40 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
55
54
|
2. Facets -> + -> Python
|
|
56
55
|
3. Set interpreter
|
|
57
56
|
|
|
58
|
-
|
|
57
|
+
# Installing Dependencies
|
|
59
58
|
|
|
60
59
|
1. Add dependencies with versions to requirements.txt
|
|
61
60
|
2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
|
|
62
61
|
|
|
63
62
|
|
|
64
|
-
|
|
63
|
+
# Pytest
|
|
65
64
|
```commandline
|
|
66
65
|
pytest --disable-warnings
|
|
67
66
|
```
|
|
68
67
|
|
|
69
|
-
|
|
68
|
+
# Instructions
|
|
70
69
|
Following this tutorial:
|
|
71
70
|
https://packaging.python.org/en/latest/tutorials/packaging-projects/
|
|
72
71
|
|
|
73
|
-
|
|
72
|
+
# To Publish To TEST
|
|
74
73
|
```commandline
|
|
75
74
|
python -m build
|
|
76
75
|
python -m twine upload --repository testpypi dist/*
|
|
77
76
|
pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
|
|
78
77
|
python
|
|
79
78
|
```
|
|
80
|
-
```
|
|
79
|
+
```
|
|
81
80
|
from water-column-sonar-processing import ZarrManager
|
|
82
81
|
example.add_one(2)
|
|
83
82
|
```
|
|
84
83
|
|
|
85
|
-
|
|
84
|
+
# To Publish To PROD
|
|
86
85
|
```commandline
|
|
87
86
|
python -m build
|
|
88
87
|
python -m twine upload --repository pypi dist/*
|
|
89
88
|
```
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# Linting
|
|
92
|
+
Ruff
|
|
93
|
+
https://plugins.jetbrains.com/plugin/20574-ruff
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
model.py,sha256=r5S-ZZEV1lJdpEU7yzGKD9WIWuRRjaw1d7gdGQoBaTo,5254
|
|
3
|
+
aws_manager/__init__.py,sha256=pYvL00VjrwCe9k5jfof2k-nc9IG1N-hy1k95W7VlKOs,162
|
|
4
|
+
aws_manager/dynamodb_manager.py,sha256=dy2IUqQMlyXbnSGgJWt4ggxUnu2heb_Z2jgueaUkCzQ,5346
|
|
5
|
+
aws_manager/s3_manager.py,sha256=r5CJ0XIvA_i4azFvLF4xKdcCuhKBJRNFOL2luvpIBnM,13481
|
|
6
|
+
aws_manager/s3fs_manager.py,sha256=k4nW48E0reQwj1t4pe_ybK5kX0H1SHeG8urddHelGZ4,2463
|
|
7
|
+
aws_manager/sns_manager.py,sha256=IR5Y3MvwYFjd-eSg4tEjoNyABGxdJb85wOj0I-vckX0,2204
|
|
8
|
+
aws_manager/sqs_manager.py,sha256=VQSkxYtdhH4zfZqQl5FiNE2_svczN0hU-UBgbviwvtc,1696
|
|
9
|
+
cruise/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
cruise/create_empty_zarr_store.py,sha256=K_Jfm_7Rt8snBXLQOJOjYilPH3pUXHveql9hay8HvCE,7012
|
|
11
|
+
cruise/resample_regrid.py,sha256=Do6j-OhU8t4n10EkHRSyPC5MfqbAwA829q9S_cd4W1U,11311
|
|
12
|
+
geometry_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
geometry_manager/geometry_manager.py,sha256=tCNatSleMCJhGxLp9H_WYr1MquBU9sBCFWsZ1o0rTL4,9409
|
|
14
|
+
geometry_manager/geometry_simplification.py,sha256=_TCrlxqWlhO05Od3ZB4Sfgob9p3tQr3r1vjseG0qMNQ,3071
|
|
15
|
+
geometry_manager/pmtile_generation.py,sha256=7v1XZKprlNV_fFWXHWD6mn4z0xf21rpUMqj3RLIAgy8,2798
|
|
16
|
+
index_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
index_manager/index_manager.py,sha256=spw3OHAgacr2WsTgCg4VpFbCg-brg6S29fd44xi1Lqc,10718
|
|
18
|
+
utility/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
+
utility/cleaner.py,sha256=QaB1-y7cBYWgZn_H9whYQSpguGcLKN5LZVtfIwMNRvw,615
|
|
20
|
+
utility/constants.py,sha256=uTuSyH9YGG8bDd5xNbT4zA5O_lu874C-8m1MRvVppPw,1631
|
|
21
|
+
utility/pipeline_status.py,sha256=iRLyzI3gsesyz0I4NukbJwqiRmXgSZBOdB3C1x2eMSg,4185
|
|
22
|
+
utility/timestamp.py,sha256=i4ub7ob24edHU2yt4aOmvle_Nmlf1F_9HVGa9TJ5HgM,359
|
|
23
|
+
zarr_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
+
zarr_manager/zarr_manager.py,sha256=QK6M7YJ-wxcxRp3lxAc8mhdUtTplKgE-9WZRyyFYwdw,12408
|
|
25
|
+
water_column_sonar_processing-0.0.2.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
|
|
26
|
+
water_column_sonar_processing-0.0.2.dist-info/METADATA,sha256=no6JUcXQrILs8FLWlj0AYDYM_V0AQbew4mCzphaNTBM,3157
|
|
27
|
+
water_column_sonar_processing-0.0.2.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
28
|
+
water_column_sonar_processing-0.0.2.dist-info/top_level.txt,sha256=l_orjhw-0yuQ3IW_ZgpmaFmAC6ngnS20SYXWqneatVI,86
|
|
29
|
+
water_column_sonar_processing-0.0.2.dist-info/RECORD,,
|
|
@@ -4,15 +4,16 @@ import numcodecs
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
import xarray as xr
|
|
6
6
|
from numcodecs import Blosc
|
|
7
|
-
|
|
8
|
-
from
|
|
9
|
-
from
|
|
7
|
+
|
|
8
|
+
from utility.constants import Constants, Coordinates
|
|
9
|
+
from utility.timestamp import Timestamp
|
|
10
|
+
from aws_manager.s3fs_manager import S3FSManager
|
|
10
11
|
|
|
11
12
|
numcodecs.blosc.use_threads = False
|
|
12
13
|
numcodecs.blosc.set_nthreads(1)
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
# TODO: when ready switch to version 3 of
|
|
16
|
+
# TODO: when ready switch to version 3 of zarr_manager spec
|
|
16
17
|
# ZARR_V3_EXPERIMENTAL_API = 1
|
|
17
18
|
|
|
18
19
|
# creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
|
|
@@ -36,7 +37,7 @@ class ZarrManager:
|
|
|
36
37
|
max_echo_range: float = 100., # maximum depth measured from whole cruise
|
|
37
38
|
):
|
|
38
39
|
# Gets the set of depth values that will be used when resampling and
|
|
39
|
-
# regridding the data to a cruise level
|
|
40
|
+
# regridding the data to a cruise level zarr_manager store.
|
|
40
41
|
# Note: returned values do not start at zero.
|
|
41
42
|
print('Getting depth values.')
|
|
42
43
|
all_cruise_depth_values = np.linspace(
|
|
@@ -62,7 +63,7 @@ class ZarrManager:
|
|
|
62
63
|
max_echo_range: float,
|
|
63
64
|
calibration_status: bool = False # Assume uncalibrated
|
|
64
65
|
) -> str:
|
|
65
|
-
print(f'Creating local
|
|
66
|
+
print(f'Creating local zarr_manager store at {cruise_name}.zarr for ship {ship_name}')
|
|
66
67
|
|
|
67
68
|
# There should be no repeated frequencies
|
|
68
69
|
assert len(frequencies) == len(set(frequencies))
|
|
@@ -251,7 +252,7 @@ class ZarrManager:
|
|
|
251
252
|
s3fs_manager = S3FSManager()
|
|
252
253
|
root = f'{self.output_bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr'
|
|
253
254
|
store = s3fs_manager.s3_map(s3_zarr_store_path=root)
|
|
254
|
-
# synchronizer =
|
|
255
|
+
# synchronizer = zarr_manager.ProcessSynchronizer(f"/tmp/{ship_name}_{cruise_name}.sync")
|
|
255
256
|
cruise_zarr = zarr.open(store=store, mode="r+")
|
|
256
257
|
except Exception as err: # Failure
|
|
257
258
|
print(f'Exception encountered opening Zarr store with Zarr.: {err}')
|
|
@@ -283,12 +284,12 @@ class ZarrManager:
|
|
|
283
284
|
|
|
284
285
|
#######################################################
|
|
285
286
|
# def create_process_synchronizer(self):
|
|
286
|
-
# # TODO: explore
|
|
287
|
+
# # TODO: explore aws_manager redis options
|
|
287
288
|
# pass
|
|
288
289
|
|
|
289
290
|
#######################################################
|
|
290
291
|
# def verify_cruise_store_data(self):
|
|
291
|
-
# # TODO: run a check on a finished
|
|
292
|
+
# # TODO: run a check on a finished zarr_manager store to ensure that
|
|
292
293
|
# # none of the time, latitude, longitude, or depth values
|
|
293
294
|
# # are NaN.
|
|
294
295
|
# pass
|
model/utility/__init__.py
DELETED
|
File without changes
|
model/zarr/__init__.py
DELETED
|
File without changes
|
model/zarr/bar.py
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import requests
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
# class Bar(object):
|
|
5
|
-
#
|
|
6
|
-
# def biz(self):
|
|
7
|
-
# pass
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
# class Bar(object):
|
|
11
|
-
#
|
|
12
|
-
# def sync(self, id, query_first):
|
|
13
|
-
# if query_first:
|
|
14
|
-
# requests.get('/remote/api/{id}'.format(id=id))
|
|
15
|
-
#
|
|
16
|
-
# requests.put(
|
|
17
|
-
# '/remote/other/api/{id}'.format(id=id),
|
|
18
|
-
# data=123 # current_data()
|
|
19
|
-
# )
|
|
20
|
-
|
|
21
|
-
# class Bar(object):
|
|
22
|
-
# def biz(self, url, method, data, headers):
|
|
23
|
-
# pass
|
|
24
|
-
|
|
25
|
-
class Bar(object):
|
|
26
|
-
|
|
27
|
-
def biz(self):
|
|
28
|
-
return 1
|
model/zarr/foo.py
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
model/model.py,sha256=JjhqZM2t8evZrFjbFHBOdHH0tlG_twOsysRMoIh2Zdw,5099
|
|
4
|
-
model/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
-
model/aws/dynamodb_manager.py,sha256=fppu4Kvl-22psu8MrF-NGKJwymwVf5CyvO7-Ye8zeFc,5338
|
|
6
|
-
model/aws/s3_manager.py,sha256=_DcBOYRcuyRTcmlTQ1qzMCtE0GRcGU8LaZ-JXKGKaOo,13397
|
|
7
|
-
model/aws/s3fs_manager.py,sha256=k4nW48E0reQwj1t4pe_ybK5kX0H1SHeG8urddHelGZ4,2463
|
|
8
|
-
model/aws/sns_manager.py,sha256=5ZW0bi7BQHldxBbmAxN4tT1KVfDAF_eYPJwpa_W1C7Q,2202
|
|
9
|
-
model/aws/sqs_manager.py,sha256=01y3LB5B_R0rmxB00SWLwL2rBUSDy_8OgjpZImEuxYQ,1694
|
|
10
|
-
model/cruise/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
-
model/cruise/create_empty_zarr_store.py,sha256=gbQWvMsmmGPl359aakYF--x09OMOzlOmk8T1aIOe-Bs,6907
|
|
12
|
-
model/cruise/resample_regrid.py,sha256=Nhb1yMRG7dCYEgsS4fPxtlJ_iYKMcdgcOHPy_Xxx9kw,11228
|
|
13
|
-
model/geospatial/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
-
model/geospatial/geometry_manager.py,sha256=ele0bd8sDp2iyOjx9scZay8Iy3D-IhGO8L3DlPOonWs,9400
|
|
15
|
-
model/geospatial/geometry_simplification.py,sha256=_TCrlxqWlhO05Od3ZB4Sfgob9p3tQr3r1vjseG0qMNQ,3071
|
|
16
|
-
model/geospatial/pmtile_generation.py,sha256=ND9rqAH4ow6FkRr_6yFF0o0DBby8u0zRVj6cqci2cPw,2790
|
|
17
|
-
model/index/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
-
model/index/index.py,sha256=s9GhEJHKF5urHWqjehHeral-lQlmTS1Zx1tvlQ7vcEM,10705
|
|
19
|
-
model/utility/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
-
model/utility/cleaner.py,sha256=l4G3iRso1xOTGH8hBXRRVPEn1ZTOJJT9_9JA0yRpcyA,599
|
|
21
|
-
model/utility/constants.py,sha256=uTuSyH9YGG8bDd5xNbT4zA5O_lu874C-8m1MRvVppPw,1631
|
|
22
|
-
model/utility/pipeline_status.py,sha256=iRLyzI3gsesyz0I4NukbJwqiRmXgSZBOdB3C1x2eMSg,4185
|
|
23
|
-
model/utility/timestamp.py,sha256=i4ub7ob24edHU2yt4aOmvle_Nmlf1F_9HVGa9TJ5HgM,359
|
|
24
|
-
model/zarr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
25
|
-
model/zarr/bar.py,sha256=0mBMTw_QmZZdEWCscRsCk0Axw6B6Yy1ujvuG1QuHcQo,504
|
|
26
|
-
model/zarr/foo.py,sha256=KpXmnl2_Tp2Ig1A_WPnoWuMVhi85CvC-JIaPe6LtbTg,215
|
|
27
|
-
model/zarr/zarr_manager.py,sha256=DsQsBe-lN4e2MExKRCg7PIFyra5GG_vTGKqA9hHwKS0,12357
|
|
28
|
-
water_column_sonar_processing-0.0.1.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
|
|
29
|
-
water_column_sonar_processing-0.0.1.dist-info/METADATA,sha256=tY2NjTDmibt3kJBvEhqAzA1xU1i-UpeZSElMBgdXvBg,3163
|
|
30
|
-
water_column_sonar_processing-0.0.1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
31
|
-
water_column_sonar_processing-0.0.1.dist-info/top_level.txt,sha256=Uq9bL3hpULwyr0JXiGOnwhgNE1h68gVXxofcjXVZpPo,15
|
|
32
|
-
water_column_sonar_processing-0.0.1.dist-info/RECORD,,
|
|
File without changes
|
{model → cruise}/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|