oceanstream 0.0.6.dev0__tar.gz → 0.0.10.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/PKG-INFO +3 -2
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/cli/main.py +29 -18
- oceanstream-0.0.10.dev0/oceanstream/cli/requirements.txt +2 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/core.py +18 -6
- oceanstream-0.0.10.dev0/oceanstream/denoise/requirements.txt +0 -0
- oceanstream-0.0.10.dev0/oceanstream/echodata/requirements.txt +1 -0
- oceanstream-0.0.10.dev0/oceanstream/exports/requirements.txt +4 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/plot/echogram.py +45 -14
- oceanstream-0.0.10.dev0/oceanstream/plot/requirements.txt +2 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/combine_zarr.py +2 -3
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/file_processor.py +58 -16
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/folder_processor.py +51 -25
- oceanstream-0.0.10.dev0/oceanstream/process/requirements.txt +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/settings/defaults.config.json +2 -2
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/PKG-INFO +3 -2
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/SOURCES.txt +6 -0
- oceanstream-0.0.10.dev0/pyproject.toml +8 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/setup.cfg +3 -2
- oceanstream-0.0.6.dev0/pyproject.toml +0 -55
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/.flake8 +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/.gitattributes +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/.gitignore +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/CODE_OF_CONDUCT.md +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/LICENSE +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/MANIFEST.in +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/README.md +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/notebooks/README.md +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/__init__.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/cli/__init__.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/__init__.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/apply_masks.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/applying_masks_handler.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/background_noise_remover.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/create_masks.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/noise_masks.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/types.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/__init__.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/ensure_time_continuity.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/mvbs_computation.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/raw_handler.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/sv_computation.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/sv_dataset_extension.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/sv_interpolation.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/target_strength_computation.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/__init__.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/csv/csv_export_from_Sv.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/csv/csv_export_from_raw.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/csv/csv_export_nasc.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/frequency_differencing_handler.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/nasc.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/nasc_computation.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/shoals/shoal_detection_handler.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/shoals/shoal_process.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/shoals/shoals_handler.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/plot/__init__.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/__init__.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/process.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/process/processed_data_io.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/settings/__init__.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/settings/config.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/utils.py +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/dependency_links.txt +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/entry_points.txt +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/requires.txt +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/top_level.txt +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/requirements-dev.txt +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/requirements-local.txt +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/requirements.txt +0 -0
- {oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: oceanstream
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.10.dev0
|
|
4
4
|
Summary: OceanStream: process raw sonar data at scale
|
|
5
5
|
Home-page: https://oceanstream.io
|
|
6
6
|
Author: Pineview Labs
|
|
@@ -9,10 +9,11 @@ Maintainer: Andrei Rusu
|
|
|
9
9
|
Maintainer-email: hello@pineview.io
|
|
10
10
|
License: MIT
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
14
|
Classifier: License :: OSI Approved :: MIT License
|
|
14
15
|
Classifier: Operating System :: OS Independent
|
|
15
|
-
Requires-Python: >=3.
|
|
16
|
+
Requires-Python: >=3.10
|
|
16
17
|
Description-Content-Type: text/markdown
|
|
17
18
|
Provides-Extra: cli
|
|
18
19
|
Provides-Extra: echodata
|
|
@@ -11,7 +11,7 @@ from pathlib import Path
|
|
|
11
11
|
from rich import print
|
|
12
12
|
from rich.traceback import install, Traceback
|
|
13
13
|
from oceanstream.settings import load_config
|
|
14
|
-
from dask.distributed import LocalCluster, Client
|
|
14
|
+
from dask.distributed import LocalCluster, Client, Variable
|
|
15
15
|
from rich.console import Console
|
|
16
16
|
|
|
17
17
|
|
|
@@ -113,16 +113,21 @@ def convert(
|
|
|
113
113
|
filePath = Path(source)
|
|
114
114
|
configData = initialize(settings, filePath, log_level=log_level)
|
|
115
115
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
116
|
+
try:
|
|
117
|
+
if filePath.is_file():
|
|
118
|
+
from oceanstream.process import convert_raw_file
|
|
119
|
+
convert_raw_file(filePath, configData)
|
|
120
|
+
print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
|
|
121
|
+
elif filePath.is_dir():
|
|
122
|
+
from oceanstream.process import convert_raw_files
|
|
123
|
+
convert_raw_files(configData, workers_count=workers_count)
|
|
124
|
+
else:
|
|
125
|
+
print(f"[red]❌ The provided path '{source}' is not a valid file/folder.[/red]")
|
|
126
|
+
except KeyboardInterrupt:
|
|
127
|
+
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
128
|
+
except Exception as e:
|
|
129
|
+
logging.exception("Error processing folder %s", configData['raw_path'])
|
|
130
|
+
print(Traceback())
|
|
126
131
|
|
|
127
132
|
|
|
128
133
|
@app.command()
|
|
@@ -171,7 +176,7 @@ def combine(
|
|
|
171
176
|
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
172
177
|
except Exception as e:
|
|
173
178
|
logging.exception("Error processing folder %s", configData['raw_path'])
|
|
174
|
-
|
|
179
|
+
print(Traceback())
|
|
175
180
|
finally:
|
|
176
181
|
status.stop()
|
|
177
182
|
client.close()
|
|
@@ -190,6 +195,7 @@ def compute_sv(
|
|
|
190
195
|
sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
|
|
191
196
|
show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
|
|
192
197
|
plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
|
|
198
|
+
use_dask: bool = typer.Option(False, help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
|
|
193
199
|
depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
|
|
194
200
|
waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
|
|
195
201
|
show_choices=["CW", "BB"]),
|
|
@@ -208,11 +214,14 @@ def compute_sv(
|
|
|
208
214
|
file_path = Path(source)
|
|
209
215
|
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
210
216
|
|
|
217
|
+
client = None
|
|
211
218
|
console = Console()
|
|
219
|
+
single_file = file_path.is_dir() and source.endswith(".zarr")
|
|
212
220
|
with console.status("Processing...", spinner="dots") as status:
|
|
213
221
|
status.start()
|
|
214
|
-
|
|
215
|
-
|
|
222
|
+
if use_dask or not single_file:
|
|
223
|
+
cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
|
|
224
|
+
client = Client(cluster)
|
|
216
225
|
|
|
217
226
|
try:
|
|
218
227
|
if file_path.is_dir() and source.endswith(".zarr"):
|
|
@@ -230,13 +239,14 @@ def compute_sv(
|
|
|
230
239
|
f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
|
|
231
240
|
f"http://localhost:8787/status for progress")
|
|
232
241
|
from oceanstream.process import process_zarr_files
|
|
233
|
-
|
|
242
|
+
processed_count_var = Variable('processed_count', client)
|
|
234
243
|
process_zarr_files(config_data,
|
|
235
244
|
workers_count=workers_count,
|
|
236
245
|
status=status,
|
|
237
246
|
chunks=config_data.get('base_chunk_sizes'),
|
|
238
247
|
plot_echogram=plot_echogram,
|
|
239
248
|
waveform_mode=waveform_mode,
|
|
249
|
+
processed_count_var=processed_count_var,
|
|
240
250
|
depth_offset=depth_offset)
|
|
241
251
|
else:
|
|
242
252
|
print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
|
|
@@ -245,10 +255,11 @@ def compute_sv(
|
|
|
245
255
|
logging.info("KeyboardInterrupt received, terminating processes...")
|
|
246
256
|
except Exception as e:
|
|
247
257
|
logging.exception("Error while processing %s", config_data['raw_path'])
|
|
248
|
-
|
|
258
|
+
print(Traceback())
|
|
249
259
|
finally:
|
|
250
|
-
|
|
251
|
-
|
|
260
|
+
if use_dask:
|
|
261
|
+
client.close()
|
|
262
|
+
cluster.close()
|
|
252
263
|
status.stop()
|
|
253
264
|
|
|
254
265
|
|
|
@@ -14,6 +14,9 @@ logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(messa
|
|
|
14
14
|
|
|
15
15
|
def initialize(settings, file_path, log_level=None):
|
|
16
16
|
logging.debug(f"Initializing with settings: {settings}, file path: {file_path}, log level: {log_level}")
|
|
17
|
+
if "config" not in settings:
|
|
18
|
+
settings["config"] = ""
|
|
19
|
+
|
|
17
20
|
config_data = load_config(settings["config"])
|
|
18
21
|
config_data["raw_path"] = file_path
|
|
19
22
|
|
|
@@ -27,6 +30,9 @@ def initialize(settings, file_path, log_level=None):
|
|
|
27
30
|
if settings["output_folder"] is not None:
|
|
28
31
|
config_data["output_folder"] = settings["output_folder"]
|
|
29
32
|
|
|
33
|
+
if settings['cloud_storage'] is not None:
|
|
34
|
+
config_data['cloud_storage'] = settings['cloud_storage']
|
|
35
|
+
|
|
30
36
|
return config_data
|
|
31
37
|
|
|
32
38
|
|
|
@@ -103,8 +109,7 @@ def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
|
|
|
103
109
|
|
|
104
110
|
file_name = f"{Path(dir_path).stem}-combined.zarr"
|
|
105
111
|
zarr_output_file = os.path.join(config_data['output_folder'], file_name)
|
|
106
|
-
logging.info(
|
|
107
|
-
f"Combining Zarr files to {zarr_output_file}; navigate to http://localhost:8787/status for progress")
|
|
112
|
+
logging.info(f"Combining Zarr files to {zarr_output_file}")
|
|
108
113
|
|
|
109
114
|
combine_zarr_files(dir_path, zarr_output_file=zarr_output_file, chunks=chunks)
|
|
110
115
|
logging.info("Zarr files have been combined successfully.")
|
|
@@ -113,18 +118,24 @@ def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
|
|
|
113
118
|
|
|
114
119
|
|
|
115
120
|
def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONAR_MODEL, plot_echogram=False,
|
|
116
|
-
depth_offset=0.0, waveform_mode="CW",
|
|
117
|
-
|
|
121
|
+
depth_offset=0.0, waveform_mode="CW", log_level="WARNING", chunks=None, config=None,
|
|
122
|
+
processed_count_var=None):
|
|
118
123
|
settings_dict = {
|
|
119
|
-
"config": config,
|
|
120
124
|
"sonar_model": sonar_model,
|
|
121
125
|
"output_folder": output or DEFAULT_OUTPUT_FOLDER
|
|
122
126
|
}
|
|
127
|
+
|
|
128
|
+
if config is not None:
|
|
129
|
+
settings_dict.update(config)
|
|
130
|
+
# settings_dict["config"] = ''
|
|
131
|
+
|
|
123
132
|
file_path = Path(source)
|
|
124
133
|
config_data = initialize(settings_dict, file_path, log_level=log_level)
|
|
125
134
|
|
|
126
135
|
if chunks:
|
|
127
136
|
config_data['chunks'] = chunks
|
|
137
|
+
else:
|
|
138
|
+
config_data['chunks'] = config_data.get('base_chunk_sizes', None)
|
|
128
139
|
|
|
129
140
|
if file_path.is_dir() and source.endswith(".zarr"):
|
|
130
141
|
logging.debug(f"Computing Sv for Zarr root file: {file_path}")
|
|
@@ -136,7 +147,8 @@ def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONA
|
|
|
136
147
|
logging.debug(f"Processing Zarr files in directory: {file_path}")
|
|
137
148
|
from oceanstream.process import process_zarr_files
|
|
138
149
|
|
|
139
|
-
process_zarr_files(config_data, workers_count=workers_count, chunks=chunks,
|
|
150
|
+
process_zarr_files(config_data, workers_count=workers_count, chunks=chunks,
|
|
151
|
+
processed_count_var=processed_count_var, plot_echogram=plot_echogram,
|
|
140
152
|
waveform_mode=waveform_mode, depth_offset=depth_offset)
|
|
141
153
|
else:
|
|
142
154
|
logging.error(f"The provided path '{source}' is not a valid Zarr root.")
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pydantic
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import os
|
|
3
|
+
import tempfile
|
|
3
4
|
import xarray as xr
|
|
4
5
|
import numpy as np
|
|
6
|
+
import logging
|
|
7
|
+
|
|
5
8
|
from matplotlib.colors import LinearSegmentedColormap, Colormap
|
|
6
9
|
import matplotlib.pyplot as plt
|
|
7
10
|
import matplotlib.dates as mdates
|
|
@@ -287,18 +290,13 @@ def plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_na
|
|
|
287
290
|
plt.close()
|
|
288
291
|
|
|
289
292
|
|
|
290
|
-
def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name,
|
|
291
|
-
cmap='
|
|
292
|
-
regions2d=None,
|
|
293
|
-
region_ids=None, region_class=None):
|
|
293
|
+
def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, echogram_path=None,
|
|
294
|
+
config_data=None, cmap='ocean_r'):
|
|
294
295
|
"""Plot and save echogram for a single channel with optional regions and enhancements."""
|
|
295
296
|
full_channel_name = ds_Sv.channel.values[channel]
|
|
296
297
|
channel_name = "_".join(full_channel_name.split()[:3])
|
|
297
298
|
|
|
298
299
|
plt.figure(figsize=(30, 18))
|
|
299
|
-
echogram_output_path = os.path.join(output_path, f"{file_base_name}_{channel_name}.png")
|
|
300
|
-
|
|
301
|
-
# Apply the same preprocessing steps from _plot_echogram
|
|
302
300
|
ds = ds_Sv
|
|
303
301
|
|
|
304
302
|
filtered_ds = ds['Sv']
|
|
@@ -332,7 +330,7 @@ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_na
|
|
|
332
330
|
yincrease=False,
|
|
333
331
|
vmin=-80,
|
|
334
332
|
vmax=-50,
|
|
335
|
-
cmap=
|
|
333
|
+
cmap=cmap,
|
|
336
334
|
cbar_kwargs={'label': 'Volume backscattering strength (Sv re 1 m-1)'}
|
|
337
335
|
)
|
|
338
336
|
|
|
@@ -340,8 +338,20 @@ def plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_na
|
|
|
340
338
|
plt.xlabel('Ping time', fontsize=14)
|
|
341
339
|
plt.ylabel('Depth', fontsize=14)
|
|
342
340
|
plt.title(f'Echogram for Channel {channel_name}', fontsize=16, fontweight='bold')
|
|
343
|
-
|
|
344
|
-
|
|
341
|
+
|
|
342
|
+
echogram_file_name = f"{file_base_name}_{channel_name}.png"
|
|
343
|
+
|
|
344
|
+
if config_data and 'cloud_storage' in config_data:
|
|
345
|
+
with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as temp_file:
|
|
346
|
+
plt.savefig(temp_file.name, dpi=300, bbox_inches='tight')
|
|
347
|
+
plt.close()
|
|
348
|
+
echogram_output_path = os.path.join(echogram_path, echogram_file_name)
|
|
349
|
+
upload_to_cloud_storage(temp_file.name, echogram_output_path, config_data['cloud_storage'])
|
|
350
|
+
os.remove(temp_file.name)
|
|
351
|
+
else:
|
|
352
|
+
echogram_output_path = os.path.join(output_path, echogram_file_name)
|
|
353
|
+
plt.savefig(echogram_output_path, dpi=300, bbox_inches='tight')
|
|
354
|
+
plt.close()
|
|
345
355
|
|
|
346
356
|
|
|
347
357
|
def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=None, client=None):
|
|
@@ -360,15 +370,15 @@ def plot_sv_data_parallel(ds_Sv, file_base_name=None, output_path=None, cmap=Non
|
|
|
360
370
|
wait(futures)
|
|
361
371
|
|
|
362
372
|
|
|
363
|
-
def plot_sv_data(ds_Sv, file_base_name=None, output_path=None,
|
|
364
|
-
region_class=None):
|
|
373
|
+
def plot_sv_data(ds_Sv, file_base_name=None, output_path=None, echogram_path=None, config_data=None, cmap=None):
|
|
365
374
|
"""Plot the echogram data and the regions."""
|
|
366
375
|
if not plt.isinteractive():
|
|
367
376
|
plt.switch_backend('Agg')
|
|
368
377
|
|
|
369
378
|
for channel in range(ds_Sv.dims['channel']):
|
|
370
|
-
plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name,
|
|
371
|
-
|
|
379
|
+
plot_individual_channel_simplified(ds_Sv, channel, output_path, file_base_name, echogram_path=echogram_path,
|
|
380
|
+
config_data=config_data,
|
|
381
|
+
cmap='ocean_r')
|
|
372
382
|
# plot_individual_channel_image_only(ds_Sv, channel, output_path, file_base_name, cmap)
|
|
373
383
|
# plot_individual_channel_shaders(ds_Sv=ds_Sv, channel=channel, output_path=output_path,
|
|
374
384
|
# file_base_name=file_base_name, cmap='ocean_r')
|
|
@@ -540,3 +550,24 @@ def _plot_region_ids(colors, ds_Sv, idx, labels_added, region_ids, regions2d):
|
|
|
540
550
|
labels_added.add(label)
|
|
541
551
|
idx += 1
|
|
542
552
|
return idx
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def upload_to_cloud_storage(local_path, remote_path, cloud_storage_config):
|
|
556
|
+
storage_type = cloud_storage_config['storage_type']
|
|
557
|
+
container_name = cloud_storage_config['container_name']
|
|
558
|
+
storage_options = cloud_storage_config['storage_options']
|
|
559
|
+
|
|
560
|
+
if storage_type == 'azure':
|
|
561
|
+
upload_to_azure_blob(local_path, remote_path, container_name, storage_options)
|
|
562
|
+
else:
|
|
563
|
+
raise ValueError(f"Unsupported storage type: {storage_type}")
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def upload_to_azure_blob(local_path, remote_path, container_name, storage_options):
|
|
567
|
+
from azure.storage.blob import BlobServiceClient
|
|
568
|
+
blob_service_client = BlobServiceClient.from_connection_string(storage_options['connection_string'])
|
|
569
|
+
blob_client = blob_service_client.get_blob_client(container=container_name, blob=remote_path)
|
|
570
|
+
|
|
571
|
+
with open(local_path, "rb") as data:
|
|
572
|
+
blob_client.upload_blob(data, overwrite=True)
|
|
573
|
+
logging.info(f"Uploaded {local_path} to Azure Blob Storage as {remote_path}")
|
|
@@ -5,7 +5,6 @@ import time
|
|
|
5
5
|
from datetime import datetime
|
|
6
6
|
from rich import print
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
import psutil
|
|
9
8
|
from oceanstream.echodata import check_reversed_time, fix_time_reversions
|
|
10
9
|
|
|
11
10
|
|
|
@@ -21,7 +20,7 @@ def read_zarr_files(input_folder):
|
|
|
21
20
|
logging.error("Input folder does not exist: %s", input_folder)
|
|
22
21
|
return
|
|
23
22
|
|
|
24
|
-
zarr_files = list(input_path.
|
|
23
|
+
zarr_files = list(input_path.rglob("*.zarr"))
|
|
25
24
|
if not zarr_files:
|
|
26
25
|
logging.error("No .zarr files found in directory: %s", input_folder)
|
|
27
26
|
return
|
|
@@ -47,7 +46,7 @@ def fix_time(ed):
|
|
|
47
46
|
return ed
|
|
48
47
|
|
|
49
48
|
|
|
50
|
-
def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None
|
|
49
|
+
def combine_zarr_files(input_folder, zarr_output_file=None, chunks=None):
|
|
51
50
|
start_time = time.time()
|
|
52
51
|
|
|
53
52
|
logging.debug("Starting to combine Zarr files from folder: %s", input_folder)
|
|
@@ -15,7 +15,6 @@ from oceanstream.plot import plot_sv_data_with_progress, plot_sv_data
|
|
|
15
15
|
from oceanstream.echodata import get_campaign_metadata
|
|
16
16
|
from .process import compute_sv, process_file_with_progress, read_file_with_progress
|
|
17
17
|
|
|
18
|
-
|
|
19
18
|
install(show_locals=True, width=120)
|
|
20
19
|
|
|
21
20
|
|
|
@@ -23,13 +22,14 @@ def get_chunk_sizes(var_dims, chunk_sizes):
|
|
|
23
22
|
return {dim: chunk_sizes[dim] for dim in var_dims if dim in chunk_sizes}
|
|
24
23
|
|
|
25
24
|
|
|
26
|
-
def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False, **kwargs):
|
|
25
|
+
def compute_Sv_to_zarr(echodata, config_data, base_path=None, chunks=None, plot_echogram=False, **kwargs):
|
|
27
26
|
"""
|
|
28
27
|
Compute Sv from echodata and save to zarr file.
|
|
29
28
|
|
|
30
29
|
Args:
|
|
31
30
|
echodata:
|
|
32
31
|
config_data:
|
|
32
|
+
base_path:
|
|
33
33
|
chunks:
|
|
34
34
|
plot_echogram:
|
|
35
35
|
**kwargs:
|
|
@@ -42,11 +42,21 @@ def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False,
|
|
|
42
42
|
encode_mode = waveform_mode == "CW" and "power" or "complex"
|
|
43
43
|
Sv = compute_sv(echodata, encode_mode=encode_mode, **kwargs)
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
45
|
+
if base_path:
|
|
46
|
+
relative_path = file_path.relative_to(base_path)
|
|
47
|
+
|
|
48
|
+
if relative_path.parent != ".":
|
|
49
|
+
zarr_path = Path(relative_path.parent) / file_path.stem
|
|
50
|
+
else:
|
|
51
|
+
zarr_path = relative_path.stem
|
|
52
|
+
else:
|
|
53
|
+
zarr_path = file_path.stem
|
|
48
54
|
|
|
49
|
-
output_path =
|
|
55
|
+
output_path = Path(config_data["output_folder"]) / zarr_path
|
|
56
|
+
output_path.mkdir(parents=True, exist_ok=True)
|
|
57
|
+
|
|
58
|
+
echogram_path = zarr_path
|
|
59
|
+
zarr_file_name = f"{file_path.stem}_Sv.zarr"
|
|
50
60
|
|
|
51
61
|
if chunks is not None:
|
|
52
62
|
for var in Sv.data_vars:
|
|
@@ -56,14 +66,15 @@ def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False,
|
|
|
56
66
|
if 'chunks' in Sv[var].encoding:
|
|
57
67
|
del Sv[var].encoding['chunks']
|
|
58
68
|
|
|
59
|
-
print("Removing background noise...")
|
|
60
69
|
ds_processed = Sv
|
|
70
|
+
|
|
61
71
|
# ds_processed = apply_background_noise_removal(Sv, config=config_data)
|
|
62
|
-
ds_processed
|
|
72
|
+
write_zarr_file(zarr_path, zarr_file_name, ds_processed, config_data, output_path)
|
|
63
73
|
|
|
64
74
|
if plot_echogram:
|
|
65
75
|
try:
|
|
66
|
-
plot_sv_data(ds_processed, file_base_name=file_path.stem, output_path=
|
|
76
|
+
plot_sv_data(ds_processed, file_base_name=file_path.stem, output_path=output_path,
|
|
77
|
+
echogram_path=echogram_path, config_data=config_data)
|
|
67
78
|
except Exception as e:
|
|
68
79
|
logging.exception(f"Error plotting echogram for {file_path}:")
|
|
69
80
|
raise e
|
|
@@ -71,13 +82,22 @@ def compute_Sv_to_zarr(echodata, config_data, chunks=None, plot_echogram=False,
|
|
|
71
82
|
return output_path
|
|
72
83
|
|
|
73
84
|
|
|
85
|
+
def write_zarr_file(zarr_path, zarr_file_name, ds_processed, config_data=None, output_path=None):
|
|
86
|
+
if 'cloud_storage' in config_data:
|
|
87
|
+
store = get_chunk_store(config_data['cloud_storage'], Path(zarr_path) / zarr_file_name)
|
|
88
|
+
else:
|
|
89
|
+
store = os.path.join(output_path, zarr_file_name)
|
|
90
|
+
|
|
91
|
+
ds_processed.to_zarr(store, mode='w')
|
|
92
|
+
|
|
93
|
+
|
|
74
94
|
async def process_raw_file_with_progress(config_data, plot_echogram, waveform_mode="CW", depth_offset=0):
|
|
75
95
|
try:
|
|
76
96
|
with Progress(
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
97
|
+
TextColumn("[progress.description]{task.description}"),
|
|
98
|
+
BarColumn(),
|
|
99
|
+
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
|
100
|
+
TimeElapsedColumn()
|
|
81
101
|
) as progress:
|
|
82
102
|
print(f"[green] Processing file: {config_data['raw_path']}[/green]")
|
|
83
103
|
read_task = progress.add_task("[cyan]Reading raw file data...", total=100)
|
|
@@ -117,14 +137,25 @@ async def process_raw_file_with_progress(config_data, plot_echogram, waveform_mo
|
|
|
117
137
|
logging.exception(f"Error processing file {config_data['raw_path']}: {e}")
|
|
118
138
|
|
|
119
139
|
|
|
120
|
-
def convert_raw_file(file_path, config_data, progress_queue=None):
|
|
140
|
+
def convert_raw_file(file_path, config_data, base_path=None, progress_queue=None):
|
|
121
141
|
logging.debug("Starting processing of file: %s", file_path)
|
|
122
142
|
from oceanstream.echodata import read_file
|
|
123
143
|
|
|
124
144
|
try:
|
|
125
|
-
|
|
145
|
+
file_path_obj = Path(file_path)
|
|
146
|
+
file_config_data = {**config_data, 'raw_path': file_path_obj}
|
|
147
|
+
|
|
148
|
+
if base_path:
|
|
149
|
+
relative_path = file_path_obj.relative_to(base_path)
|
|
150
|
+
relative_path = relative_path.parent
|
|
151
|
+
else:
|
|
152
|
+
relative_path = file_path_obj.name
|
|
153
|
+
|
|
154
|
+
output_path = Path(config_data["output_folder"]) / relative_path
|
|
155
|
+
output_path.mkdir(parents=True, exist_ok=True)
|
|
156
|
+
|
|
126
157
|
echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
|
|
127
|
-
echodata.to_zarr(save_path=
|
|
158
|
+
echodata.to_zarr(save_path=output_path, overwrite=True, parallel=False)
|
|
128
159
|
|
|
129
160
|
if progress_queue:
|
|
130
161
|
progress_queue.put(file_path)
|
|
@@ -133,6 +164,17 @@ def convert_raw_file(file_path, config_data, progress_queue=None):
|
|
|
133
164
|
print(Traceback())
|
|
134
165
|
|
|
135
166
|
|
|
167
|
+
def get_chunk_store(storage_config, path):
|
|
168
|
+
if storage_config['storage_type'] == 'azure':
|
|
169
|
+
from adlfs import AzureBlobFileSystem
|
|
170
|
+
azfs = AzureBlobFileSystem(**storage_config['storage_options'])
|
|
171
|
+
|
|
172
|
+
return azfs.get_mapper(f"{storage_config['container_name']}/{path}")
|
|
173
|
+
|
|
174
|
+
else:
|
|
175
|
+
raise ValueError(f"Unsupported storage type: {storage_config['storage_type']}")
|
|
176
|
+
|
|
177
|
+
|
|
136
178
|
def compute_single_file(config_data, **kwargs):
|
|
137
179
|
file_path = config_data["raw_path"]
|
|
138
180
|
start_time = time.time()
|
|
@@ -7,6 +7,7 @@ import sys
|
|
|
7
7
|
import traceback
|
|
8
8
|
import warnings
|
|
9
9
|
|
|
10
|
+
from dask import delayed, compute
|
|
10
11
|
from pathlib import Path
|
|
11
12
|
from datetime import datetime
|
|
12
13
|
from rich import print
|
|
@@ -21,8 +22,10 @@ from .process import compute_sv
|
|
|
21
22
|
from .processed_data_io import write_processed
|
|
22
23
|
from .file_processor import convert_raw_file, compute_single_file
|
|
23
24
|
|
|
24
|
-
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
25
|
+
# logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
25
26
|
warnings.filterwarnings("ignore", module="echopype")
|
|
27
|
+
warnings.filterwarnings("ignore", category=UserWarning)
|
|
28
|
+
warnings.filterwarnings("ignore", category=FutureWarning)
|
|
26
29
|
pool = None
|
|
27
30
|
|
|
28
31
|
|
|
@@ -123,6 +126,15 @@ def signal_handler(sig, frame):
|
|
|
123
126
|
sys.exit(0)
|
|
124
127
|
|
|
125
128
|
|
|
129
|
+
def find_raw_files(base_dir):
|
|
130
|
+
raw_files = []
|
|
131
|
+
for root, dirs, files in os.walk(base_dir):
|
|
132
|
+
for file in files:
|
|
133
|
+
if file.endswith('.raw'):
|
|
134
|
+
raw_files.append(os.path.join(root, file))
|
|
135
|
+
return raw_files
|
|
136
|
+
|
|
137
|
+
|
|
126
138
|
def convert_raw_files(config_data, workers_count=os.cpu_count()):
|
|
127
139
|
global pool
|
|
128
140
|
|
|
@@ -132,8 +144,7 @@ def convert_raw_files(config_data, workers_count=os.cpu_count()):
|
|
|
132
144
|
configure_logging(log_level)
|
|
133
145
|
|
|
134
146
|
print(f"Starting to convert folder: {dir_path} to Zarr using {workers_count} parallel processes...")
|
|
135
|
-
raw_files =
|
|
136
|
-
os.path.isfile(os.path.join(dir_path, f)) and f.endswith('.raw')]
|
|
147
|
+
raw_files = find_raw_files(dir_path)
|
|
137
148
|
file_info = []
|
|
138
149
|
|
|
139
150
|
if not raw_files:
|
|
@@ -166,7 +177,8 @@ def convert_raw_files(config_data, workers_count=os.cpu_count()):
|
|
|
166
177
|
pool = Pool(processes=workers_count)
|
|
167
178
|
|
|
168
179
|
# Partial function with config_data, progress_queue and other arguments
|
|
169
|
-
process_func = partial(convert_raw_file, config_data=config_data, progress_queue=progress_queue
|
|
180
|
+
process_func = partial(convert_raw_file, config_data=config_data, progress_queue=progress_queue,
|
|
181
|
+
base_path=dir_path)
|
|
170
182
|
|
|
171
183
|
# Run the progress updater in a separate process
|
|
172
184
|
progress_updater = Process(target=update_progress, args=(progress_queue, len(sorted_files), log_level))
|
|
@@ -192,14 +204,25 @@ def convert_raw_files(config_data, workers_count=os.cpu_count()):
|
|
|
192
204
|
logging.exception("Error processing folder %s: %s", config_data['raw_path'], e)
|
|
193
205
|
|
|
194
206
|
|
|
195
|
-
def process_single_zarr_file(file_path, config_data, chunks=None, plot_echogram=False, waveform_mode="CW",
|
|
196
|
-
depth_offset=0):
|
|
207
|
+
def process_single_zarr_file(file_path, config_data, base_path=None, chunks=None, plot_echogram=False, waveform_mode="CW",
|
|
208
|
+
depth_offset=0, total_files=1, processed_count_var=None):
|
|
197
209
|
file_config_data = {**config_data, 'raw_path': Path(file_path)}
|
|
198
|
-
|
|
210
|
+
|
|
211
|
+
processed_count = None
|
|
212
|
+
|
|
213
|
+
if processed_count_var:
|
|
214
|
+
processed_count = processed_count_var.get() + 1
|
|
215
|
+
processed_count_var.set(processed_count)
|
|
216
|
+
|
|
217
|
+
compute_single_file(file_config_data, base_path=base_path, chunks=chunks, plot_echogram=plot_echogram,
|
|
199
218
|
waveform_mode=waveform_mode, depth_offset=depth_offset)
|
|
200
219
|
|
|
220
|
+
if processed_count:
|
|
221
|
+
print(f"Processed file: {file_path}. {processed_count}/{total_files}")
|
|
222
|
+
|
|
201
223
|
|
|
202
|
-
def process_zarr_files(config_data, workers_count=os.cpu_count(), status=None, chunks=None,
|
|
224
|
+
def process_zarr_files(config_data, workers_count=os.cpu_count(), status=None, chunks=None, processed_count_var=None,
|
|
225
|
+
plot_echogram=False,
|
|
203
226
|
waveform_mode="CW", depth_offset=0):
|
|
204
227
|
dir_path = config_data['raw_path']
|
|
205
228
|
zarr_files = read_zarr_files(dir_path)
|
|
@@ -208,24 +231,26 @@ def process_zarr_files(config_data, workers_count=os.cpu_count(), status=None, c
|
|
|
208
231
|
logging.error("No valid .zarr files with creation time found in directory: %s", dir_path)
|
|
209
232
|
return
|
|
210
233
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
# bag = db.from_sequence(zarr_files, npartitions=workers_count)
|
|
221
|
-
# bag = bag.map(wrapped_process_single_zarr_file)
|
|
222
|
-
#
|
|
223
|
-
# # Compute the results
|
|
224
|
-
# results = bag.compute()
|
|
234
|
+
if status:
|
|
235
|
+
status.update(f"Found {len(zarr_files)} Zarr files in directory: {dir_path}\n")
|
|
236
|
+
|
|
237
|
+
tasks = []
|
|
238
|
+
total_files = len(zarr_files)
|
|
239
|
+
|
|
240
|
+
if processed_count_var:
|
|
241
|
+
processed_count_var.set(0)
|
|
242
|
+
|
|
225
243
|
for file_path in zarr_files:
|
|
226
|
-
status
|
|
227
|
-
|
|
228
|
-
|
|
244
|
+
if status:
|
|
245
|
+
status.update(f"Computing Sv for {file_path}...\n")
|
|
246
|
+
task = delayed(process_single_zarr_file)(file_path, config_data, chunks=chunks, base_path=dir_path,
|
|
247
|
+
plot_echogram=plot_echogram,
|
|
248
|
+
waveform_mode=waveform_mode, depth_offset=depth_offset,
|
|
249
|
+
total_files=total_files, processed_count_var=processed_count_var)
|
|
250
|
+
tasks.append(task)
|
|
251
|
+
|
|
252
|
+
# Execute all tasks in parallel
|
|
253
|
+
compute(*tasks)
|
|
229
254
|
|
|
230
255
|
logging.info("✅ All files have been processed")
|
|
231
256
|
|
|
@@ -243,4 +268,5 @@ def from_filename(file_name):
|
|
|
243
268
|
return None
|
|
244
269
|
|
|
245
270
|
|
|
271
|
+
|
|
246
272
|
signal.signal(signal.SIGINT, signal_handler)
|
|
File without changes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: oceanstream
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.10.dev0
|
|
4
4
|
Summary: OceanStream: process raw sonar data at scale
|
|
5
5
|
Home-page: https://oceanstream.io
|
|
6
6
|
Author: Pineview Labs
|
|
@@ -9,10 +9,11 @@ Maintainer: Andrei Rusu
|
|
|
9
9
|
Maintainer-email: hello@pineview.io
|
|
10
10
|
License: MIT
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
14
|
Classifier: License :: OSI Approved :: MIT License
|
|
14
15
|
Classifier: Operating System :: OS Independent
|
|
15
|
-
Requires-Python: >=3.
|
|
16
|
+
Requires-Python: >=3.10
|
|
16
17
|
Description-Content-Type: text/markdown
|
|
17
18
|
Provides-Extra: cli
|
|
18
19
|
Provides-Extra: echodata
|
|
@@ -23,17 +23,20 @@ oceanstream.egg-info/requires.txt
|
|
|
23
23
|
oceanstream.egg-info/top_level.txt
|
|
24
24
|
oceanstream/cli/__init__.py
|
|
25
25
|
oceanstream/cli/main.py
|
|
26
|
+
oceanstream/cli/requirements.txt
|
|
26
27
|
oceanstream/denoise/__init__.py
|
|
27
28
|
oceanstream/denoise/apply_masks.py
|
|
28
29
|
oceanstream/denoise/applying_masks_handler.py
|
|
29
30
|
oceanstream/denoise/background_noise_remover.py
|
|
30
31
|
oceanstream/denoise/create_masks.py
|
|
31
32
|
oceanstream/denoise/noise_masks.py
|
|
33
|
+
oceanstream/denoise/requirements.txt
|
|
32
34
|
oceanstream/denoise/types.py
|
|
33
35
|
oceanstream/echodata/__init__.py
|
|
34
36
|
oceanstream/echodata/ensure_time_continuity.py
|
|
35
37
|
oceanstream/echodata/mvbs_computation.py
|
|
36
38
|
oceanstream/echodata/raw_handler.py
|
|
39
|
+
oceanstream/echodata/requirements.txt
|
|
37
40
|
oceanstream/echodata/sv_computation.py
|
|
38
41
|
oceanstream/echodata/sv_dataset_extension.py
|
|
39
42
|
oceanstream/echodata/sv_interpolation.py
|
|
@@ -42,6 +45,7 @@ oceanstream/exports/__init__.py
|
|
|
42
45
|
oceanstream/exports/frequency_differencing_handler.py
|
|
43
46
|
oceanstream/exports/nasc.py
|
|
44
47
|
oceanstream/exports/nasc_computation.py
|
|
48
|
+
oceanstream/exports/requirements.txt
|
|
45
49
|
oceanstream/exports/csv/csv_export_from_Sv.py
|
|
46
50
|
oceanstream/exports/csv/csv_export_from_raw.py
|
|
47
51
|
oceanstream/exports/csv/csv_export_nasc.py
|
|
@@ -50,12 +54,14 @@ oceanstream/exports/shoals/shoal_process.py
|
|
|
50
54
|
oceanstream/exports/shoals/shoals_handler.py
|
|
51
55
|
oceanstream/plot/__init__.py
|
|
52
56
|
oceanstream/plot/echogram.py
|
|
57
|
+
oceanstream/plot/requirements.txt
|
|
53
58
|
oceanstream/process/__init__.py
|
|
54
59
|
oceanstream/process/combine_zarr.py
|
|
55
60
|
oceanstream/process/file_processor.py
|
|
56
61
|
oceanstream/process/folder_processor.py
|
|
57
62
|
oceanstream/process/process.py
|
|
58
63
|
oceanstream/process/processed_data_io.py
|
|
64
|
+
oceanstream/process/requirements.txt
|
|
59
65
|
oceanstream/settings/__init__.py
|
|
60
66
|
oceanstream/settings/config.py
|
|
61
67
|
oceanstream/settings/defaults.config.json
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[metadata]
|
|
2
2
|
name = oceanstream
|
|
3
|
-
version = 0.0.
|
|
3
|
+
version = 0.0.10-dev
|
|
4
4
|
description = OceanStream: process raw sonar data at scale
|
|
5
5
|
long_description = file: README.md
|
|
6
6
|
long_description_content_type = text/markdown
|
|
@@ -13,6 +13,7 @@ license_files = LICENSE
|
|
|
13
13
|
url = https://oceanstream.io
|
|
14
14
|
classifiers =
|
|
15
15
|
Programming Language :: Python :: 3
|
|
16
|
+
Programming Language :: Python :: 3.10
|
|
16
17
|
Programming Language :: Python :: 3.11
|
|
17
18
|
License :: OSI Approved :: MIT License
|
|
18
19
|
Operating System :: OS Independent
|
|
@@ -20,7 +21,7 @@ classifiers =
|
|
|
20
21
|
[options]
|
|
21
22
|
packages = find:
|
|
22
23
|
platforms = any
|
|
23
|
-
python_requires = >=3.
|
|
24
|
+
python_requires = >=3.10
|
|
24
25
|
include_package_data = True
|
|
25
26
|
setup_requires =
|
|
26
27
|
setuptools_scm
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = [
|
|
3
|
-
"setuptools >= 48",
|
|
4
|
-
"setuptools_scm[toml] >= 4, <6",
|
|
5
|
-
"setuptools_scm_git_archive",
|
|
6
|
-
"wheel >= 0.29.0"
|
|
7
|
-
]
|
|
8
|
-
build-backend = "setuptools.build_meta"
|
|
9
|
-
|
|
10
|
-
#[project]
|
|
11
|
-
#name = "oceanstream"
|
|
12
|
-
#version = "0.0.5-dev"
|
|
13
|
-
#description = "OceanStream: process raw sonar data at scale"
|
|
14
|
-
#readme = "README.md"
|
|
15
|
-
#requires-python = ">=3.11"
|
|
16
|
-
#authors = [
|
|
17
|
-
# { name="Pineview Labs", email="hello@pineview.io" }
|
|
18
|
-
#]
|
|
19
|
-
#license = { file="LICENSE" }
|
|
20
|
-
#classifiers = [
|
|
21
|
-
# "Programming Language :: Python :: 3",
|
|
22
|
-
# "Programming Language :: Python :: 3.10",
|
|
23
|
-
# "Programming Language :: Python :: 3.11",
|
|
24
|
-
# "License :: OSI Approved :: MIT License",
|
|
25
|
-
# "Operating System :: OS Independent"
|
|
26
|
-
#]
|
|
27
|
-
#
|
|
28
|
-
#dependencies = [
|
|
29
|
-
# "git+https://github.com/OceanStreamIO/echopype-dev.git@oceanstream#egg=echopype"
|
|
30
|
-
#]
|
|
31
|
-
#
|
|
32
|
-
#[project.optional-dependencies]
|
|
33
|
-
#cli = [
|
|
34
|
-
# "typer",
|
|
35
|
-
# "rich"
|
|
36
|
-
#]
|
|
37
|
-
#plot = [
|
|
38
|
-
# "matplotlib"
|
|
39
|
-
#]
|
|
40
|
-
#echodata = [
|
|
41
|
-
# "pydantic"
|
|
42
|
-
#]
|
|
43
|
-
#
|
|
44
|
-
#denoise = []
|
|
45
|
-
#exports = []
|
|
46
|
-
#process = []
|
|
47
|
-
#complete = [
|
|
48
|
-
# "typer",
|
|
49
|
-
# "rich",
|
|
50
|
-
# "matplotlib",
|
|
51
|
-
# "pydantic"
|
|
52
|
-
#]
|
|
53
|
-
#
|
|
54
|
-
#[project.scripts]
|
|
55
|
-
#oceanstream = "oceanstream.cli.main:main"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/applying_masks_handler.py
RENAMED
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/denoise/background_noise_remover.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/ensure_time_continuity.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/echodata/sv_dataset_extension.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/csv/csv_export_from_Sv.py
RENAMED
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/csv/csv_export_from_raw.py
RENAMED
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/csv/csv_export_nasc.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/shoals/shoal_process.py
RENAMED
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream/exports/shoals/shoals_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{oceanstream-0.0.6.dev0 → oceanstream-0.0.10.dev0}/oceanstream.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|