water-column-sonar-processing 0.0.7__py3-none-any.whl → 0.0.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of water-column-sonar-processing might be problematic. Click here for more details.
- water_column_sonar_processing/__init__.py +4 -5
- water_column_sonar_processing/aws/dynamodb_manager.py +149 -43
- water_column_sonar_processing/aws/s3_manager.py +71 -37
- water_column_sonar_processing/cruise/create_empty_zarr_store.py +6 -4
- water_column_sonar_processing/cruise/resample_regrid.py +3 -3
- water_column_sonar_processing/geometry/geometry_manager.py +21 -6
- water_column_sonar_processing/geometry/pmtile_generation.py +202 -13
- water_column_sonar_processing/index/index_manager.py +25 -13
- water_column_sonar_processing/model/zarr_manager.py +26 -25
- water_column_sonar_processing/process.py +4 -4
- water_column_sonar_processing/processing/__init__.py +4 -0
- water_column_sonar_processing/processing/cruise_sampler.py +342 -0
- water_column_sonar_processing/processing/raw_to_zarr.py +349 -0
- water_column_sonar_processing/utility/cleaner.py +1 -0
- water_column_sonar_processing/utility/constants.py +6 -2
- {water_column_sonar_processing-0.0.7.dist-info → water_column_sonar_processing-0.0.8.dist-info}/METADATA +20 -10
- water_column_sonar_processing-0.0.8.dist-info/RECORD +32 -0
- {water_column_sonar_processing-0.0.7.dist-info → water_column_sonar_processing-0.0.8.dist-info}/WHEEL +1 -1
- water_column_sonar_processing-0.0.7.dist-info/RECORD +0 -29
- {water_column_sonar_processing-0.0.7.dist-info → water_column_sonar_processing-0.0.8.dist-info}/LICENSE +0 -0
- {water_column_sonar_processing-0.0.7.dist-info → water_column_sonar_processing-0.0.8.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
import os
|
|
3
|
+
import echopype as ep
|
|
4
|
+
import numcodecs
|
|
5
|
+
import numpy as np
|
|
6
|
+
from numcodecs import Blosc
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from pathlib import Path # , PurePath
|
|
9
|
+
|
|
10
|
+
from src.water_column_sonar_processing.aws import DynamoDBManager, S3Manager
|
|
11
|
+
from src.water_column_sonar_processing.geometry.geometry_manager import GeometryManager
|
|
12
|
+
from src.water_column_sonar_processing.utility import Cleaner
|
|
13
|
+
|
|
14
|
+
TEMPDIR = "/tmp"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# This code is getting copied from echofish-aws-raw-to-zarr-lambda
|
|
18
|
+
class RawToZarr:
|
|
19
|
+
#######################################################
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
# output_bucket_access_key,
|
|
23
|
+
# output_bucket_secret_access_key,
|
|
24
|
+
# # overwrite_existing_zarr_store,
|
|
25
|
+
):
|
|
26
|
+
# TODO: revert to Blosc.BITSHUFFLE, troubleshooting misc error
|
|
27
|
+
self.__compressor = Blosc(cname="zstd", clevel=2) # shuffle=Blosc.NOSHUFFLE
|
|
28
|
+
self.__overwrite = True
|
|
29
|
+
self.__num_threads = numcodecs.blosc.get_nthreads()
|
|
30
|
+
# self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
|
|
31
|
+
# self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
|
|
32
|
+
# self.__table_name = table_name
|
|
33
|
+
# # self.__overwrite_existing_zarr_store = overwrite_existing_zarr_store
|
|
34
|
+
|
|
35
|
+
############################################################################
|
|
36
|
+
############################################################################
|
|
37
|
+
def __zarr_info_to_table(
|
|
38
|
+
self,
|
|
39
|
+
output_bucket_name,
|
|
40
|
+
table_name,
|
|
41
|
+
ship_name,
|
|
42
|
+
cruise_name,
|
|
43
|
+
sensor_name,
|
|
44
|
+
file_name,
|
|
45
|
+
zarr_path,
|
|
46
|
+
min_echo_range,
|
|
47
|
+
max_echo_range,
|
|
48
|
+
num_ping_time_dropna,
|
|
49
|
+
start_time,
|
|
50
|
+
end_time,
|
|
51
|
+
frequencies,
|
|
52
|
+
channels
|
|
53
|
+
):
|
|
54
|
+
print('Writing Zarr information to DynamoDB table.')
|
|
55
|
+
dynamodb_manager = DynamoDBManager()
|
|
56
|
+
|
|
57
|
+
# The problem is that these values were never populated
|
|
58
|
+
# and so when the query looks for values that aren't there
|
|
59
|
+
# they fail
|
|
60
|
+
dynamodb_manager.update_item(
|
|
61
|
+
table_name=table_name,
|
|
62
|
+
key={
|
|
63
|
+
'FILE_NAME': {'S': file_name}, # Partition Key
|
|
64
|
+
'CRUISE_NAME': {'S': cruise_name}, # Sort Key
|
|
65
|
+
},
|
|
66
|
+
expression_attribute_names={
|
|
67
|
+
'#CH': 'CHANNELS',
|
|
68
|
+
'#ET': 'END_TIME',
|
|
69
|
+
# "#ED": "ERROR_DETAIL",
|
|
70
|
+
'#FR': 'FREQUENCIES',
|
|
71
|
+
'#MA': 'MAX_ECHO_RANGE',
|
|
72
|
+
'#MI': 'MIN_ECHO_RANGE',
|
|
73
|
+
'#ND': 'NUM_PING_TIME_DROPNA',
|
|
74
|
+
"#PS": "PIPELINE_STATUS",
|
|
75
|
+
"#PT": "PIPELINE_TIME",
|
|
76
|
+
"#SE": "SENSOR_NAME",
|
|
77
|
+
"#SH": "SHIP_NAME",
|
|
78
|
+
'#ST': 'START_TIME',
|
|
79
|
+
'#ZB': 'ZARR_BUCKET',
|
|
80
|
+
'#ZP': 'ZARR_PATH',
|
|
81
|
+
},
|
|
82
|
+
expression_attribute_values={
|
|
83
|
+
":ch": {"L": [{"S": i} for i in channels]},
|
|
84
|
+
":et": {"S": end_time},
|
|
85
|
+
# ":ed": {"S": ""},
|
|
86
|
+
":fr": {"L": [{"N": str(i)} for i in frequencies]},
|
|
87
|
+
":ma": {"N": str(np.round(max_echo_range, 4))},
|
|
88
|
+
":mi": {"N": str(np.round(min_echo_range, 4))},
|
|
89
|
+
":nd": {"N": str(num_ping_time_dropna)},
|
|
90
|
+
":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
|
|
91
|
+
":pt": {"S": datetime.now().isoformat(timespec="seconds") + "Z"},
|
|
92
|
+
":se": {"S": sensor_name},
|
|
93
|
+
":sh": {"S": ship_name},
|
|
94
|
+
":st": {"S": start_time},
|
|
95
|
+
":zb": {"S": output_bucket_name},
|
|
96
|
+
":zp": { "S": zarr_path },
|
|
97
|
+
},
|
|
98
|
+
update_expression=(
|
|
99
|
+
"SET "
|
|
100
|
+
"#CH = :ch, "
|
|
101
|
+
"#ET = :et, "
|
|
102
|
+
# "#ED = :ed, "
|
|
103
|
+
"#FR = :fr, "
|
|
104
|
+
"#MA = :ma, "
|
|
105
|
+
"#MI = :mi, "
|
|
106
|
+
"#ND = :nd, "
|
|
107
|
+
"#PS = :ps, "
|
|
108
|
+
"#PT = :pt, "
|
|
109
|
+
"#SE = :se, "
|
|
110
|
+
"#SH = :sh, "
|
|
111
|
+
"#ST = :st, "
|
|
112
|
+
"#ZB = :zb, "
|
|
113
|
+
"#ZP = :zp"
|
|
114
|
+
),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
############################################################################
|
|
118
|
+
############################################################################
|
|
119
|
+
############################################################################
|
|
120
|
+
def __upload_files_to_output_bucket(
|
|
121
|
+
self,
|
|
122
|
+
output_bucket_name,
|
|
123
|
+
local_directory,
|
|
124
|
+
object_prefix,
|
|
125
|
+
):
|
|
126
|
+
# Note: this will be passed credentials if using NODD
|
|
127
|
+
s3_manager = S3Manager()
|
|
128
|
+
print('Uploading files using thread pool executor.')
|
|
129
|
+
all_files = []
|
|
130
|
+
for subdir, dirs, files in os.walk(local_directory):
|
|
131
|
+
for file in files:
|
|
132
|
+
local_path = os.path.join(subdir, file)
|
|
133
|
+
s3_key = os.path.join(object_prefix, local_path)
|
|
134
|
+
all_files.append([local_path, s3_key])
|
|
135
|
+
# all_files
|
|
136
|
+
all_uploads = s3_manager.upload_files_with_thread_pool_executor(
|
|
137
|
+
output_bucket_name=output_bucket_name,
|
|
138
|
+
all_files=all_files,
|
|
139
|
+
)
|
|
140
|
+
return all_uploads
|
|
141
|
+
|
|
142
|
+
############################################################################
|
|
143
|
+
def raw_to_zarr(
|
|
144
|
+
self,
|
|
145
|
+
table_name,
|
|
146
|
+
output_bucket_name,
|
|
147
|
+
ship_name,
|
|
148
|
+
cruise_name,
|
|
149
|
+
sensor_name,
|
|
150
|
+
raw_file_name,
|
|
151
|
+
):
|
|
152
|
+
print(f'Opening raw: {raw_file_name} and creating zarr store.')
|
|
153
|
+
geometry_manager = GeometryManager()
|
|
154
|
+
cleaner = Cleaner()
|
|
155
|
+
cleaner.delete_local_files(file_types=["*.zarr", "*.json"]) # TODO: include bot and raw?ß
|
|
156
|
+
try:
|
|
157
|
+
gc.collect()
|
|
158
|
+
print('Opening raw file with echopype.')
|
|
159
|
+
# s3_file_path = f"s3://{bucket_name}/data/raw/{ship_name}/{cruise_name}/{sensor_name}/{file_name}"
|
|
160
|
+
# s3_file_path = Path(f"s3://noaa-wcsd-pds/data/raw/{ship_name}/{cruise_name}/{sensor_name}/{file_name}")
|
|
161
|
+
echodata = ep.open_raw(
|
|
162
|
+
raw_file=raw_file_name,
|
|
163
|
+
sonar_model=sensor_name,
|
|
164
|
+
include_bot=True,
|
|
165
|
+
use_swap=True,
|
|
166
|
+
# max_chunk_size=100,
|
|
167
|
+
# storage_options={'anon': True } # 'endpoint_url': self.endpoint_url} # this was creating problems
|
|
168
|
+
)
|
|
169
|
+
print('Compute volume backscattering strength (Sv) from raw data.')
|
|
170
|
+
ds_sv = ep.calibrate.compute_Sv(echodata)
|
|
171
|
+
print('Done computing volume backscattering strength (Sv) from raw data.')
|
|
172
|
+
frequencies = echodata.environment.frequency_nominal.values
|
|
173
|
+
#################################################################
|
|
174
|
+
# Get GPS coordinates
|
|
175
|
+
gps_data, lat, lon = geometry_manager.read_echodata_gps_data(
|
|
176
|
+
echodata=echodata,
|
|
177
|
+
output_bucket_name=output_bucket_name,
|
|
178
|
+
ship_name=ship_name,
|
|
179
|
+
cruise_name=cruise_name,
|
|
180
|
+
sensor_name=sensor_name,
|
|
181
|
+
file_name=raw_file_name,
|
|
182
|
+
write_geojson=True
|
|
183
|
+
)
|
|
184
|
+
# gps_data, lat, lon = self.__get_gps_data(echodata=echodata)
|
|
185
|
+
#################################################################
|
|
186
|
+
# Technically the min_echo_range would be 0 m.
|
|
187
|
+
# TODO: this var name is supposed to represent minimum resolution of depth measurements
|
|
188
|
+
# TODO revert this so that smaller diffs can be used
|
|
189
|
+
# The most minimum the resolution can be is as small as 0.25 meters
|
|
190
|
+
min_echo_range = np.maximum(
|
|
191
|
+
0.25,
|
|
192
|
+
np.nanmin(np.diff(ds_sv.echo_range.values))
|
|
193
|
+
)
|
|
194
|
+
max_echo_range = float(np.nanmax(ds_sv.echo_range))
|
|
195
|
+
#
|
|
196
|
+
num_ping_time_dropna = lat[~np.isnan(lat)].shape[0] # symmetric to lon
|
|
197
|
+
#
|
|
198
|
+
start_time = np.datetime_as_string(ds_sv.ping_time.values[0], unit='ms') + "Z"
|
|
199
|
+
end_time = np.datetime_as_string(ds_sv.ping_time.values[-1], unit='ms') + "Z"
|
|
200
|
+
channels = list(ds_sv.channel.values)
|
|
201
|
+
#
|
|
202
|
+
#################################################################
|
|
203
|
+
# Create the zarr store
|
|
204
|
+
store_name = f"{Path(raw_file_name).stem}.zarr"
|
|
205
|
+
ds_sv.to_zarr(store=store_name)
|
|
206
|
+
#################################################################
|
|
207
|
+
# TODO: do i still need this?
|
|
208
|
+
# print('Note: Adding GeoJSON inside Zarr store')
|
|
209
|
+
# self.__write_geojson_to_file( # Was trying to write geojson to the L1 zarr store
|
|
210
|
+
# store_name=store_name,
|
|
211
|
+
# data=gps_data
|
|
212
|
+
# )
|
|
213
|
+
#################################################################
|
|
214
|
+
output_zarr_prefix = f"level_1/{ship_name}/{cruise_name}/{sensor_name}/"
|
|
215
|
+
self.__zarr_info_to_table(
|
|
216
|
+
output_bucket_name=output_bucket_name,
|
|
217
|
+
table_name=table_name,
|
|
218
|
+
ship_name=ship_name,
|
|
219
|
+
cruise_name=cruise_name,
|
|
220
|
+
sensor_name=sensor_name,
|
|
221
|
+
file_name=raw_file_name,
|
|
222
|
+
zarr_path=os.path.join(output_zarr_prefix, store_name),
|
|
223
|
+
min_echo_range=min_echo_range,
|
|
224
|
+
max_echo_range=max_echo_range,
|
|
225
|
+
num_ping_time_dropna=num_ping_time_dropna,
|
|
226
|
+
start_time=start_time,
|
|
227
|
+
end_time=end_time,
|
|
228
|
+
frequencies=frequencies,
|
|
229
|
+
channels=channels
|
|
230
|
+
)
|
|
231
|
+
###################################################################
|
|
232
|
+
#######################################################################
|
|
233
|
+
self.__upload_files_to_output_bucket(
|
|
234
|
+
output_bucket_name=output_bucket_name,
|
|
235
|
+
local_directory=store_name,
|
|
236
|
+
object_prefix=output_zarr_prefix
|
|
237
|
+
)
|
|
238
|
+
#######################################################################
|
|
239
|
+
# # TODO: verify count of objects matches
|
|
240
|
+
# s3_objects = self.__s3.list_objects(
|
|
241
|
+
# bucket_name=self.__output_bucket,
|
|
242
|
+
# prefix=f"{zarr_prefix}/{os.path.splitext(input_file_name)[0]}.zarr/",
|
|
243
|
+
# access_key_id=self.__output_bucket_access_key,
|
|
244
|
+
# secret_access_key=self.__output_bucket_secret_access_key
|
|
245
|
+
# )
|
|
246
|
+
#######################################################################
|
|
247
|
+
# self.__update_processing_status(
|
|
248
|
+
# file_name=input_file_name,
|
|
249
|
+
# cruise_name=cruise_name,
|
|
250
|
+
# pipeline_status='SUCCESS_RAW_TO_ZARR'
|
|
251
|
+
# )
|
|
252
|
+
#######################################################################
|
|
253
|
+
# self.__publish_done_message(input_message)
|
|
254
|
+
print('here')
|
|
255
|
+
except Exception as err:
|
|
256
|
+
print(f'Exception encountered creating local Zarr store with echopype: {err}')
|
|
257
|
+
raise RuntimeError(f"Problem creating local Zarr store, {err}")
|
|
258
|
+
finally:
|
|
259
|
+
cleaner.delete_local_files(file_types=["*.raw", "*.bot", "*.zarr", "*.json"])
|
|
260
|
+
print('Done creating local zarr store.')
|
|
261
|
+
|
|
262
|
+
############################################################################
|
|
263
|
+
# TODO: does this get called?
|
|
264
|
+
# def execute(self, input_message):
|
|
265
|
+
# ship_name = input_message['shipName']
|
|
266
|
+
# cruise_name = input_message['cruiseName']
|
|
267
|
+
# sensor_name = input_message['sensorName']
|
|
268
|
+
# input_file_name = input_message['fileName']
|
|
269
|
+
# #
|
|
270
|
+
# try:
|
|
271
|
+
# self.__update_processing_status(
|
|
272
|
+
# file_name=input_file_name,
|
|
273
|
+
# cruise_name=cruise_name,
|
|
274
|
+
# pipeline_status="PROCESSING_RAW_TO_ZARR"
|
|
275
|
+
# )
|
|
276
|
+
# #######################################################################
|
|
277
|
+
# store_name = f"{os.path.splitext(input_file_name)[0]}.zarr"
|
|
278
|
+
# output_zarr_prefix = f"level_1/{ship_name}/{cruise_name}/{sensor_name}"
|
|
279
|
+
# bucket_key = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{input_file_name}"
|
|
280
|
+
# zarr_prefix = os.path.join("level_1", ship_name, cruise_name, sensor_name)
|
|
281
|
+
# #
|
|
282
|
+
# os.chdir(TEMPDIR) # Lambdas require use of temp directory
|
|
283
|
+
# #######################################################################
|
|
284
|
+
# #######################################################################
|
|
285
|
+
# # Check if zarr store already exists
|
|
286
|
+
# s3_objects = self.__s3.list_objects(
|
|
287
|
+
# bucket_name=self.__output_bucket,
|
|
288
|
+
# prefix=f"{zarr_prefix}/{os.path.splitext(input_file_name)[0]}.zarr/",
|
|
289
|
+
# access_key_id=self.__output_bucket_access_key,
|
|
290
|
+
# secret_access_key=self.__output_bucket_secret_access_key
|
|
291
|
+
# )
|
|
292
|
+
# if len(s3_objects) > 0:
|
|
293
|
+
# print('Zarr store data already exists in s3, deleting existing and continuing.')
|
|
294
|
+
# self.__s3.delete_objects(
|
|
295
|
+
# bucket_name=self.__output_bucket,
|
|
296
|
+
# objects=s3_objects,
|
|
297
|
+
# access_key_id=self.__output_bucket_access_key,
|
|
298
|
+
# secret_access_key=self.__output_bucket_secret_access_key
|
|
299
|
+
# )
|
|
300
|
+
# #######################################################################
|
|
301
|
+
# # self.__delete_all_local_raw_and_zarr_files()
|
|
302
|
+
# Cleaner.delete_local_files(file_types=["*.raw*", "*.zarr"])
|
|
303
|
+
# self.__s3.download_file(
|
|
304
|
+
# bucket_name=self.__input_bucket,
|
|
305
|
+
# key=bucket_key,
|
|
306
|
+
# file_name=input_file_name
|
|
307
|
+
# )
|
|
308
|
+
# self.__create_local_zarr_store(
|
|
309
|
+
# raw_file_name=input_file_name,
|
|
310
|
+
# cruise_name=cruise_name,
|
|
311
|
+
# sensor_name=sensor_name,
|
|
312
|
+
# output_zarr_prefix=output_zarr_prefix,
|
|
313
|
+
# store_name=store_name
|
|
314
|
+
# )
|
|
315
|
+
# #######################################################################
|
|
316
|
+
# self.__upload_files_to_output_bucket(store_name, output_zarr_prefix)
|
|
317
|
+
# #######################################################################
|
|
318
|
+
# # # TODO: verify count of objects matches
|
|
319
|
+
# # s3_objects = self.__s3.list_objects(
|
|
320
|
+
# # bucket_name=self.__output_bucket,
|
|
321
|
+
# # prefix=f"{zarr_prefix}/{os.path.splitext(input_file_name)[0]}.zarr/",
|
|
322
|
+
# # access_key_id=self.__output_bucket_access_key,
|
|
323
|
+
# # secret_access_key=self.__output_bucket_secret_access_key
|
|
324
|
+
# # )
|
|
325
|
+
# #######################################################################
|
|
326
|
+
# self.__update_processing_status(
|
|
327
|
+
# file_name=input_file_name,
|
|
328
|
+
# cruise_name=cruise_name,
|
|
329
|
+
# pipeline_status='SUCCESS_RAW_TO_ZARR'
|
|
330
|
+
# )
|
|
331
|
+
# #######################################################################
|
|
332
|
+
# self.__publish_done_message(input_message)
|
|
333
|
+
# #######################################################################
|
|
334
|
+
# # except Exception as err:
|
|
335
|
+
# # print(f'Exception encountered: {err}')
|
|
336
|
+
# # self.__update_processing_status(
|
|
337
|
+
# # file_name=input_file_name,
|
|
338
|
+
# # cruise_name=cruise_name,
|
|
339
|
+
# # pipeline_status='FAILURE_RAW_TO_ZARR',
|
|
340
|
+
# # error_message=str(err),
|
|
341
|
+
# # )
|
|
342
|
+
# finally:
|
|
343
|
+
# self.__delete_all_local_raw_and_zarr_files()
|
|
344
|
+
#######################################################################
|
|
345
|
+
|
|
346
|
+
############################################################################
|
|
347
|
+
|
|
348
|
+
################################################################################
|
|
349
|
+
############################################################################
|
|
@@ -3,7 +3,11 @@ from enum import Enum, Flag, unique
|
|
|
3
3
|
|
|
4
4
|
@unique
|
|
5
5
|
class Constants(Flag):
|
|
6
|
-
TILE_SIZE =
|
|
6
|
+
TILE_SIZE = 2048
|
|
7
|
+
# Average https://noaa-wcsd-zarr-pds.s3.us-east-1.amazonaws.com/level_2/Henry_B._Bigelow/HB0902/EK60/HB0902.zarr/time/927
|
|
8
|
+
# chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon data
|
|
9
|
+
# NOTE: larger value here will speed up the TurfJS download of data in the UI
|
|
10
|
+
SPATIOTEMPORAL_CHUNK_SIZE = int(1e5) # 2**17
|
|
7
11
|
|
|
8
12
|
|
|
9
13
|
class Coordinates(Enum):
|
|
@@ -33,7 +37,7 @@ class Coordinates(Enum):
|
|
|
33
37
|
# TODO: create test for reading out timestamps in Xarray
|
|
34
38
|
|
|
35
39
|
FREQUENCY = "frequency"
|
|
36
|
-
FREQUENCY_DTYPE = "
|
|
40
|
+
FREQUENCY_DTYPE = "uint"
|
|
37
41
|
FREQUENCY_UNITS = "Hz"
|
|
38
42
|
FREQUENCY_LONG_NAME = "Transducer frequency"
|
|
39
43
|
FREQUENCY_STANDARD_NAME = "sound_frequency"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: water_column_sonar_processing
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.8
|
|
4
4
|
Summary: A processing tool for water column sonar data.
|
|
5
5
|
Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
|
|
@@ -11,26 +11,27 @@ Classifier: Operating System :: OS Independent
|
|
|
11
11
|
Requires-Python: >=3.10
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
|
-
Requires-Dist: aiobotocore
|
|
15
|
-
Requires-Dist:
|
|
16
|
-
Requires-Dist:
|
|
17
|
-
Requires-Dist: botocore~=1.33.13
|
|
14
|
+
Requires-Dist: aiobotocore==2.15.2
|
|
15
|
+
Requires-Dist: boto3==1.35.36
|
|
16
|
+
Requires-Dist: botocore==1.35.36
|
|
18
17
|
Requires-Dist: echopype==0.9.0
|
|
19
18
|
Requires-Dist: fiona==1.10.1
|
|
20
19
|
Requires-Dist: geopandas==1.0.1
|
|
21
|
-
Requires-Dist: mock
|
|
22
|
-
Requires-Dist: moto
|
|
20
|
+
Requires-Dist: mock==5.1.0
|
|
21
|
+
Requires-Dist: moto[all]==5.0.21
|
|
22
|
+
Requires-Dist: moto[server]==5.0.21
|
|
23
23
|
Requires-Dist: numcodecs==0.13.1
|
|
24
24
|
Requires-Dist: numpy==1.26.4
|
|
25
25
|
Requires-Dist: pandas==2.2.3
|
|
26
|
-
Requires-Dist:
|
|
26
|
+
Requires-Dist: pyarrow==18.1.0
|
|
27
27
|
Requires-Dist: python-dotenv==1.0.0
|
|
28
28
|
Requires-Dist: requests==2.32.3
|
|
29
|
-
Requires-Dist: s3fs==
|
|
29
|
+
Requires-Dist: s3fs==2023.12.1
|
|
30
30
|
Requires-Dist: scipy==1.14.1
|
|
31
|
+
Requires-Dist: setuptools==75.6.0
|
|
31
32
|
Requires-Dist: shapely==2.0.3
|
|
32
33
|
Requires-Dist: typing-extensions==4.10.0
|
|
33
|
-
Requires-Dist: xarray==
|
|
34
|
+
Requires-Dist: xarray==2024.10.0
|
|
34
35
|
Requires-Dist: zarr==2.18.3
|
|
35
36
|
|
|
36
37
|
# Water Column Sonar Processing
|
|
@@ -85,6 +86,8 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
|
|
|
85
86
|
```commandline
|
|
86
87
|
pytest --disable-warnings
|
|
87
88
|
```
|
|
89
|
+
or
|
|
90
|
+
> pytest --cache-clear --cov=src tests/ --cov-report=xml
|
|
88
91
|
|
|
89
92
|
# Instructions
|
|
90
93
|
Following this tutorial:
|
|
@@ -122,3 +125,10 @@ https://plugins.jetbrains.com/plugin/20574-ruff
|
|
|
122
125
|
|
|
123
126
|
# Colab Test
|
|
124
127
|
https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
|
|
128
|
+
|
|
129
|
+
# Test Coverage
|
|
130
|
+
20241124
|
|
131
|
+
8 failed, 32 passed, 3 skipped, 1 warning in 6.92s
|
|
132
|
+
20241125
|
|
133
|
+
5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
|
|
134
|
+
3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
water_column_sonar_processing/__init__.py,sha256=Ipl74g4btRHNYqgJ4Ro957HQC4YXBoIrGhgMmBPP0Uw,290
|
|
2
|
+
water_column_sonar_processing/process.py,sha256=mb8_UpcOTy7RAAMlv-nF5hZG5zFoMsFxlT4OyvrpObk,5455
|
|
3
|
+
water_column_sonar_processing/aws/__init__.py,sha256=u5J-TOEVgAQsMdc5LMo1igUESRclzV8gf-b0jUaJ9Gg,277
|
|
4
|
+
water_column_sonar_processing/aws/dynamodb_manager.py,sha256=sZHn-hgCt3K3w0x5BcXfF5jLMt_F11dAtQHJToij9nU,10008
|
|
5
|
+
water_column_sonar_processing/aws/s3_manager.py,sha256=ctNWMkgqMlwbwmXHgwKEV8otLwIjr-dHX6bQ2rOw1ug,14718
|
|
6
|
+
water_column_sonar_processing/aws/s3fs_manager.py,sha256=thVJPQKhbvF1g-Ue3BYgwazFOFDYOICIEJx4zkXBQ1E,2381
|
|
7
|
+
water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
|
|
8
|
+
water_column_sonar_processing/aws/sqs_manager.py,sha256=NSUrWmnSC8h8Gf7gT0U8zFaQQ-yX89h0Q0mDLKGqp2Y,1597
|
|
9
|
+
water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
|
|
10
|
+
water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=ev5jMZiwLyY1zrIAQhAw9X55eD3rxMrW5PpotjlOxDE,7382
|
|
11
|
+
water_column_sonar_processing/cruise/resample_regrid.py,sha256=aCbY84wtQma7GK9QooMqXzrJuIpC8IPUp1cRwN8ZOoA,12368
|
|
12
|
+
water_column_sonar_processing/geometry/__init__.py,sha256=_ol5nI8AL30pYXeAh5rtP7YmQggitPC6LA_kuTfPJ0Q,231
|
|
13
|
+
water_column_sonar_processing/geometry/geometry_manager.py,sha256=xN1zyD_4Apry69BNcm5GtBKPpbHaPc_TYRYYFXjtwMc,10577
|
|
14
|
+
water_column_sonar_processing/geometry/geometry_simplification.py,sha256=im1HG9nfYIerQv3w-PUHzphw2B7aGgnsA3Zcdy2oTmA,3016
|
|
15
|
+
water_column_sonar_processing/geometry/pmtile_generation.py,sha256=Hh7UP8_mSC4_vw0F-LEv0DCT0FBYzbuNWZ0bITdg3gI,12458
|
|
16
|
+
water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
|
|
17
|
+
water_column_sonar_processing/index/index_manager.py,sha256=k_OOpU8FNQ5pz4HZkr1xGLakLbuyeBm8LZcFVqjU2cQ,11254
|
|
18
|
+
water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
|
|
19
|
+
water_column_sonar_processing/model/zarr_manager.py,sha256=hKikypzqqxGPUpjkSrf_RweHihaBuAxIQyOichf4Vhs,13637
|
|
20
|
+
water_column_sonar_processing/processing/__init__.py,sha256=UwdB3BnoUxy4q3k9-ZjBF6KzmCWVDcqbcArTeHgmvGA,118
|
|
21
|
+
water_column_sonar_processing/processing/cruise_sampler.py,sha256=d5_rwk9ucCGxNnL4gjDoY12fN6t9jOwJJenftGV4dGE,15931
|
|
22
|
+
water_column_sonar_processing/processing/raw_to_zarr.py,sha256=W5BswuiHSnHj6jhVY_31qHo6OQHbYEvP2cqBliIqVZQ,16154
|
|
23
|
+
water_column_sonar_processing/utility/__init__.py,sha256=nyqPobcvwftr6T4MNxNtQtfbWzW9Kgpbp6JO7Gr5IZI,206
|
|
24
|
+
water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
|
|
25
|
+
water_column_sonar_processing/utility/constants.py,sha256=EbzsorvYKadsPjuutRjQKKByGibhFm0Gw6D-Sp2ZD3I,2143
|
|
26
|
+
water_column_sonar_processing/utility/pipeline_status.py,sha256=O-0SySqdRGJ6bs3zQe1NV9vkOpmsRM7zj5QoHgzYioY,4395
|
|
27
|
+
water_column_sonar_processing/utility/timestamp.py,sha256=bO0oir7KxxoEHPGRkz9FCBfOligkocUyRiWRzAq8fnU,361
|
|
28
|
+
water_column_sonar_processing-0.0.8.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
|
|
29
|
+
water_column_sonar_processing-0.0.8.dist-info/METADATA,sha256=N2RBgvC6W-KnKzS4CoijfHeQ5PVIVhhBNOZkIRH4Kuc,4432
|
|
30
|
+
water_column_sonar_processing-0.0.8.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
|
31
|
+
water_column_sonar_processing-0.0.8.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
|
|
32
|
+
water_column_sonar_processing-0.0.8.dist-info/RECORD,,
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
water_column_sonar_processing/__init__.py,sha256=S11loSDNOmZRyiTMbTwiQ1PP59SSPq2q2P61e5gPrW4,295
|
|
2
|
-
water_column_sonar_processing/process.py,sha256=AHoCehc95XqGI1IZkvfcbnYpjr8J-OQfCBGs0-1O9_w,5439
|
|
3
|
-
water_column_sonar_processing/aws/__init__.py,sha256=u5J-TOEVgAQsMdc5LMo1igUESRclzV8gf-b0jUaJ9Gg,277
|
|
4
|
-
water_column_sonar_processing/aws/dynamodb_manager.py,sha256=sPa_5RlTNjcFavZ74LF0UINKSyrbgJNROXbe16hCz7U,5936
|
|
5
|
-
water_column_sonar_processing/aws/s3_manager.py,sha256=P-0ZFsa6D6obIy-iNnWF1BrcoHOfAvYM-Qn9cA0e5H0,13142
|
|
6
|
-
water_column_sonar_processing/aws/s3fs_manager.py,sha256=thVJPQKhbvF1g-Ue3BYgwazFOFDYOICIEJx4zkXBQ1E,2381
|
|
7
|
-
water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
|
|
8
|
-
water_column_sonar_processing/aws/sqs_manager.py,sha256=NSUrWmnSC8h8Gf7gT0U8zFaQQ-yX89h0Q0mDLKGqp2Y,1597
|
|
9
|
-
water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
|
|
10
|
-
water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=w2lxfPS2HRVlNCr6OWBRxIyGaPUnxAnWrYgmu0EgOL0,7275
|
|
11
|
-
water_column_sonar_processing/cruise/resample_regrid.py,sha256=HeRN4EOZch-HhiK-s5LjX0bz4sKXKpea5YVIMEZOAGg,12356
|
|
12
|
-
water_column_sonar_processing/geometry/__init__.py,sha256=_ol5nI8AL30pYXeAh5rtP7YmQggitPC6LA_kuTfPJ0Q,231
|
|
13
|
-
water_column_sonar_processing/geometry/geometry_manager.py,sha256=RJ5pljMLqjNdJbc8Q0s-HGphMy6B7e_7R7ZZfdxEY-k,9913
|
|
14
|
-
water_column_sonar_processing/geometry/geometry_simplification.py,sha256=im1HG9nfYIerQv3w-PUHzphw2B7aGgnsA3Zcdy2oTmA,3016
|
|
15
|
-
water_column_sonar_processing/geometry/pmtile_generation.py,sha256=KI5y3CVBXSrxYoenQPHBUhPPAvcw1bkqCg1LZ9xvlzY,2772
|
|
16
|
-
water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
|
|
17
|
-
water_column_sonar_processing/index/index_manager.py,sha256=2D4mbWK2D3p0wsZqeoxieCtz9HsDkm5Mvt9u2t12BFI,11107
|
|
18
|
-
water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
|
|
19
|
-
water_column_sonar_processing/model/zarr_manager.py,sha256=IvA2mTZ6YrfTKL3U6NVgmjeqYPUXMSI3V-HI5ESgAwg,13408
|
|
20
|
-
water_column_sonar_processing/utility/__init__.py,sha256=nyqPobcvwftr6T4MNxNtQtfbWzW9Kgpbp6JO7Gr5IZI,206
|
|
21
|
-
water_column_sonar_processing/utility/cleaner.py,sha256=jaNDHbZxqQv2I8VfFkbwlQoCtIHi1pQonqt3XtioVq0,585
|
|
22
|
-
water_column_sonar_processing/utility/constants.py,sha256=qSuxPg4dnyI4DDXWc1OYqTBTChxZ9iWdUrFfqLtH21c,1793
|
|
23
|
-
water_column_sonar_processing/utility/pipeline_status.py,sha256=O-0SySqdRGJ6bs3zQe1NV9vkOpmsRM7zj5QoHgzYioY,4395
|
|
24
|
-
water_column_sonar_processing/utility/timestamp.py,sha256=bO0oir7KxxoEHPGRkz9FCBfOligkocUyRiWRzAq8fnU,361
|
|
25
|
-
water_column_sonar_processing-0.0.7.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
|
|
26
|
-
water_column_sonar_processing-0.0.7.dist-info/METADATA,sha256=lRFXVLJlxfxchtc5G0-FXEq23t1UGpt0YWG9ExZ9Trs,4138
|
|
27
|
-
water_column_sonar_processing-0.0.7.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
28
|
-
water_column_sonar_processing-0.0.7.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
|
|
29
|
-
water_column_sonar_processing-0.0.7.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|