oceanstream 0.0.10.dev0__tar.gz → 0.0.14.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/PKG-INFO +2 -1
  2. oceanstream-0.0.14.dev0/oceanstream/__init__.py +5 -0
  3. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/cli/main.py +113 -28
  4. oceanstream-0.0.14.dev0/oceanstream/cli/requirements.txt +1 -0
  5. oceanstream-0.0.14.dev0/oceanstream/convert/__init__.py +5 -0
  6. oceanstream-0.0.14.dev0/oceanstream/convert/convert_raw_file.py +58 -0
  7. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/core.py +47 -31
  8. oceanstream-0.0.14.dev0/oceanstream/echodata/dataset.py +586 -0
  9. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/sv_computation.py +8 -1
  10. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/sv_dataset_extension.py +18 -19
  11. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/sv_interpolation.py +9 -10
  12. oceanstream-0.0.14.dev0/oceanstream/exports/__init__.py +6 -0
  13. oceanstream-0.0.14.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +183 -0
  14. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/plot/echogram.py +16 -10
  15. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/__init__.py +5 -3
  16. oceanstream-0.0.14.dev0/oceanstream/process/aws/__init__.py +9 -0
  17. oceanstream-0.0.14.dev0/oceanstream/process/aws/s3.py +94 -0
  18. oceanstream-0.0.14.dev0/oceanstream/process/azure/__init__.py +8 -0
  19. oceanstream-0.0.14.dev0/oceanstream/process/azure/azure_processor.py +66 -0
  20. oceanstream-0.0.14.dev0/oceanstream/process/azure/blob_storage.py +85 -0
  21. oceanstream-0.0.14.dev0/oceanstream/process/cloud.py +21 -0
  22. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/combine_zarr.py +1 -1
  23. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/file_processor.py +128 -84
  24. oceanstream-0.0.14.dev0/oceanstream/process/file_processor_class.py +217 -0
  25. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/folder_processor.py +175 -52
  26. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/process.py +5 -1
  27. oceanstream-0.0.14.dev0/oceanstream/process/requirements.txt +0 -0
  28. oceanstream-0.0.14.dev0/oceanstream/process/utils.py +226 -0
  29. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/PKG-INFO +2 -1
  30. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/SOURCES.txt +11 -0
  31. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/requires.txt +8 -4
  32. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/requirements.txt +6 -0
  33. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/setup.cfg +2 -3
  34. oceanstream-0.0.10.dev0/oceanstream/__init__.py +0 -3
  35. oceanstream-0.0.10.dev0/oceanstream/cli/requirements.txt +0 -2
  36. oceanstream-0.0.10.dev0/oceanstream/echodata/requirements.txt +0 -1
  37. oceanstream-0.0.10.dev0/oceanstream/exports/__init__.py +0 -2
  38. oceanstream-0.0.10.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +0 -89
  39. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/.flake8 +0 -0
  40. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/.gitattributes +0 -0
  41. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/.gitignore +0 -0
  42. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/CODE_OF_CONDUCT.md +0 -0
  43. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/LICENSE +0 -0
  44. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/MANIFEST.in +0 -0
  45. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/README.md +0 -0
  46. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/notebooks/README.md +0 -0
  47. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/cli/__init__.py +0 -0
  48. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/__init__.py +0 -0
  49. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/apply_masks.py +0 -0
  50. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/applying_masks_handler.py +0 -0
  51. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/background_noise_remover.py +0 -0
  52. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/create_masks.py +0 -0
  53. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/noise_masks.py +0 -0
  54. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/requirements.txt +0 -0
  55. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/types.py +0 -0
  56. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/__init__.py +0 -0
  57. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/ensure_time_continuity.py +0 -0
  58. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/mvbs_computation.py +0 -0
  59. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/raw_handler.py +0 -0
  60. {oceanstream-0.0.10.dev0/oceanstream/process → oceanstream-0.0.14.dev0/oceanstream/echodata}/requirements.txt +0 -0
  61. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/target_strength_computation.py +0 -0
  62. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/csv/csv_export_from_raw.py +0 -0
  63. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/csv/csv_export_nasc.py +0 -0
  64. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/frequency_differencing_handler.py +0 -0
  65. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/nasc.py +0 -0
  66. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/nasc_computation.py +0 -0
  67. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/requirements.txt +0 -0
  68. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/shoals/shoal_detection_handler.py +0 -0
  69. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/shoals/shoal_process.py +0 -0
  70. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/shoals/shoals_handler.py +0 -0
  71. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/plot/__init__.py +0 -0
  72. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/plot/requirements.txt +0 -0
  73. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/processed_data_io.py +0 -0
  74. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/settings/__init__.py +0 -0
  75. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/settings/config.py +0 -0
  76. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/settings/defaults.config.json +0 -0
  77. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream/utils.py +0 -0
  78. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/dependency_links.txt +0 -0
  79. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/entry_points.txt +0 -0
  80. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/top_level.txt +0 -0
  81. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/pyproject.toml +0 -0
  82. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/requirements-dev.txt +0 -0
  83. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/requirements-local.txt +0 -0
  84. {oceanstream-0.0.10.dev0 → oceanstream-0.0.14.dev0}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oceanstream
3
- Version: 0.0.10.dev0
3
+ Version: 0.0.14.dev0
4
4
  Summary: OceanStream: process raw sonar data at scale
5
5
  Home-page: https://oceanstream.io
6
6
  Author: Pineview Labs
@@ -20,6 +20,7 @@ Provides-Extra: echodata
20
20
  Provides-Extra: plot
21
21
  Provides-Extra: denoise
22
22
  Provides-Extra: process
23
+ Provides-Extra: convert
23
24
  Provides-Extra: exports
24
25
  Provides-Extra: complete
25
26
  License-File: LICENSE
@@ -0,0 +1,5 @@
1
+ from .core import initialize, compute_sv, combine, process_raw_file
2
+
3
+ __all__ = [
4
+ "compute_sv", "combine", "process_raw_file", "initialize"
5
+ ]
@@ -1,4 +1,5 @@
1
1
  import traceback
2
+ from asyncio import CancelledError
2
3
 
3
4
  import typer
4
5
  import asyncio
@@ -6,6 +7,7 @@ import os
6
7
  import logging
7
8
  import sys
8
9
  import warnings
10
+ import dask
9
11
 
10
12
  from pathlib import Path
11
13
  from rich import print
@@ -13,7 +15,7 @@ from rich.traceback import install, Traceback
13
15
  from oceanstream.settings import load_config
14
16
  from dask.distributed import LocalCluster, Client, Variable
15
17
  from rich.console import Console
16
-
18
+ from oceanstream.process import compute_and_export_single_file, process_zarr_files
17
19
 
18
20
  install(show_locals=False, width=120)
19
21
 
@@ -32,6 +34,12 @@ BANNER = """
32
34
 
33
35
  logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
34
36
 
37
+ dask.config.set({
38
+ 'distributed.comm.timeouts.connect': '60s', # Increase the connection timeout
39
+ 'distributed.comm.timeouts.tcp': '120s', # Increase the TCP timeout
40
+ 'distributed.comm.retry.count': 0
41
+ })
42
+
35
43
 
36
44
  def initialize(settings, file_path, log_level=None):
37
45
  config_data = load_config(settings["config"])
@@ -97,6 +105,8 @@ def convert(
97
105
  output: str = typer.Option(None,
98
106
  help="Destination path for saving Zarr converted data. Defaults to a predefined "
99
107
  "directory if not specified."),
108
+ sonar_model: str = typer.Option(None, help="Sonar model used to collect the data",
109
+ show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
100
110
  workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
101
111
  config: str = typer.Option(None, help="Path to a configuration file"),
102
112
  log_level: str = typer.Option("WARNING", help="Set the logging level",
@@ -107,6 +117,7 @@ def convert(
107
117
  """
108
118
  settings = {
109
119
  "config": config,
120
+ "sonar_model": sonar_model,
110
121
  "output_folder": output or DEFAULT_OUTPUT_FOLDER
111
122
  }
112
123
 
@@ -115,9 +126,10 @@ def convert(
115
126
 
116
127
  try:
117
128
  if filePath.is_file():
118
- from oceanstream.process import convert_raw_file
129
+ from oceanstream.convert import convert_raw_file
130
+ print(f"[blue]Converting raw file {source} to Zarr...[/blue]")
119
131
  convert_raw_file(filePath, configData)
120
- print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
132
+ print("✅ The file has been converted successfully.")
121
133
  elif filePath.is_dir():
122
134
  from oceanstream.process import convert_raw_files
123
135
  convert_raw_files(configData, workers_count=workers_count)
@@ -126,8 +138,12 @@ def convert(
126
138
  except KeyboardInterrupt:
127
139
  logging.info("KeyboardInterrupt received, terminating processes...")
128
140
  except Exception as e:
129
- logging.exception("Error processing folder %s", configData['raw_path'])
130
- print(Traceback())
141
+ if filePath.is_file():
142
+ logging.error("Error processing file %s", filePath)
143
+ else:
144
+ logging.exception("Error processing folder %s", configData['raw_path'])
145
+
146
+ logging.error(traceback.format_exc())
131
147
 
132
148
 
133
149
  @app.command()
@@ -195,10 +211,12 @@ def compute_sv(
195
211
  sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
196
212
  show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
197
213
  plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
198
- use_dask: bool = typer.Option(False, help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
214
+ use_dask: bool = typer.Option(False,
215
+ help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
199
216
  depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
200
217
  waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
201
218
  show_choices=["CW", "BB"]),
219
+ encode_mode: str = typer.Option("power", help="Encode mode, can be either power or complex", show_choices=["power", "complex"]),
202
220
  config: str = typer.Option(None, help="Path to a configuration file"),
203
221
  log_level: str = typer.Option("WARNING", help="Set the logging level",
204
222
  show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
@@ -211,6 +229,88 @@ def compute_sv(
211
229
  "sonar_model": sonar_model,
212
230
  "output_folder": output or DEFAULT_OUTPUT_FOLDER
213
231
  }
232
+ dask.config.set({'distributed.comm.retry.count': 1})
233
+
234
+ file_path = Path(source)
235
+ config_data = initialize(settings_dict, file_path, log_level=log_level)
236
+ single_file = file_path.is_dir() and source.endswith(".zarr")
237
+
238
+ client = None
239
+ cluster = None
240
+
241
+ if use_dask or not single_file:
242
+ cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
243
+ client = Client(cluster)
244
+
245
+ try:
246
+ if file_path.is_dir() and source.endswith(".zarr"):
247
+ console = Console()
248
+ with console.status("Processing...", spinner="dots") as status:
249
+ status.start()
250
+ status.update(
251
+ f"[blue] Computing Sv for {file_path}...[/blue]" + use_dask * "– navigate to "
252
+ "http://localhost:8787/status for "
253
+ "progress")
254
+
255
+ chunks = None
256
+ if use_dask:
257
+ chunks = config_data.get('base_chunk_sizes')
258
+
259
+ compute_and_export_single_file(config_data,
260
+ chunks=chunks,
261
+ plot_echogram=plot_echogram,
262
+ waveform_mode=waveform_mode,
263
+ encode_mode=encode_mode,
264
+ depth_offset=depth_offset)
265
+
266
+ status.stop()
267
+ print("✅ The file have been processed successfully.")
268
+ elif file_path.is_dir():
269
+ print(f"Dashboard available at {client.dashboard_link}")
270
+ process_zarr_files(config_data,
271
+ client,
272
+ workers_count=workers_count,
273
+ chunks=config_data.get('base_chunk_sizes'),
274
+ plot_echogram=plot_echogram,
275
+ waveform_mode=waveform_mode,
276
+ depth_offset=depth_offset)
277
+ else:
278
+ print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
279
+ sys.exit(1)
280
+ except KeyboardInterrupt:
281
+ logging.info("KeyboardInterrupt received, terminating processes...")
282
+ except Exception as e:
283
+ logging.exception("Error while processing %s", config_data['raw_path'])
284
+ print(Traceback())
285
+ finally:
286
+ if client is not None:
287
+ client.close()
288
+
289
+ if cluster is not None:
290
+ cluster.close()
291
+
292
+
293
+ @app.command()
294
+ def export_location(
295
+ source: str = typer.Option(..., help="Path to a Zarr root file or a directory containing Zarr files"),
296
+ output: str = typer.Option(None,
297
+ help="Destination path for saving the exported data. Defaults to a predefined "
298
+ "directory if not specified."),
299
+ workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for parallel processing"),
300
+ use_dask: bool = typer.Option(False,
301
+ help="Start a Local Dask cluster for parallel processing (always enabled for "
302
+ "multiple files)"),
303
+ config: str = typer.Option(None, help="Path to a configuration file"),
304
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
305
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
306
+ ):
307
+ """
308
+ Given a Zarr dataset containing Sv data, exports the GPS location data to a JSON file.
309
+ """
310
+ settings_dict = {
311
+ "config": config,
312
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
313
+ }
214
314
  file_path = Path(source)
215
315
  config_data = initialize(settings_dict, file_path, log_level=log_level)
216
316
 
@@ -227,27 +327,17 @@ def compute_sv(
227
327
  if file_path.is_dir() and source.endswith(".zarr"):
228
328
  status.update(
229
329
  f"[blue] Computing Sv for {file_path}...[/blue] – navigate to http://localhost:8787/status for progress")
230
- from oceanstream.process import compute_single_file
231
-
232
- compute_single_file(config_data,
233
- chunks=config_data.get('base_chunk_sizes'),
234
- plot_echogram=plot_echogram,
235
- waveform_mode=waveform_mode,
236
- depth_offset=depth_offset)
330
+ # TODO: Implement export_location_json
237
331
  elif file_path.is_dir():
238
332
  status.update(
239
333
  f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
240
334
  f"http://localhost:8787/status for progress")
241
- from oceanstream.process import process_zarr_files
242
- processed_count_var = Variable('processed_count', client)
243
- process_zarr_files(config_data,
244
- workers_count=workers_count,
245
- status=status,
246
- chunks=config_data.get('base_chunk_sizes'),
247
- plot_echogram=plot_echogram,
248
- waveform_mode=waveform_mode,
249
- processed_count_var=processed_count_var,
250
- depth_offset=depth_offset)
335
+ from oceanstream.process import export_location_from_zarr_files
336
+
337
+ export_location_from_zarr_files(config_data,
338
+ workers_count=workers_count,
339
+ client=client,
340
+ chunks=config_data.get('base_chunk_sizes'))
251
341
  else:
252
342
  print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
253
343
  sys.exit(1)
@@ -263,11 +353,6 @@ def compute_sv(
263
353
  status.stop()
264
354
 
265
355
 
266
- @app.command()
267
- def export():
268
- typer.echo("Export data...")
269
-
270
-
271
356
  def main():
272
357
  print(BANNER)
273
358
  warnings.filterwarnings("ignore", category=UserWarning)
@@ -0,0 +1,5 @@
1
+ from .convert_raw_file import convert_raw_file
2
+
3
+ __all__ = [
4
+ "convert_raw_file"
5
+ ]
@@ -0,0 +1,58 @@
1
+ import json
2
+ import logging
3
+ import os
4
+ import sys
5
+ import time
6
+ import traceback
7
+ import shutil
8
+ from asyncio import CancelledError
9
+ import echopype as ep
10
+
11
+ import xarray as xr
12
+ from pathlib import Path
13
+ from oceanstream.echodata import read_file
14
+
15
+
16
+ def convert_raw_file(file_path, config_data, base_path=None, progress_counter=None, counter_lock=None):
17
+ logging.debug("Starting processing of file: %s", file_path)
18
+
19
+ file_path_obj = Path(file_path)
20
+ file_config_data = {**config_data, 'raw_path': file_path_obj}
21
+
22
+ if base_path:
23
+ relative_path = file_path_obj.relative_to(base_path)
24
+ relative_path = relative_path.parent
25
+ else:
26
+ relative_path = None
27
+
28
+ echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
29
+ file_name = file_path_obj.stem + ".zarr"
30
+
31
+ if 'cloud_storage' in config_data:
32
+ from oceanstream.process.cloud import get_chunk_store
33
+
34
+ if relative_path:
35
+ file_location = Path(relative_path) / file_name
36
+ else:
37
+ file_location = file_name
38
+ store = get_chunk_store(config_data['cloud_storage'], file_location)
39
+ echodata.to_zarr(save_path=store, overwrite=True, parallel=False)
40
+
41
+ output_dest = config_data['cloud_storage']['container_name'] + "/" + file_location
42
+ else:
43
+ if relative_path:
44
+ output_path = Path(config_data["output_folder"]) / relative_path
45
+ output_path.mkdir(parents=True, exist_ok=True)
46
+ else:
47
+ output_path = Path(config_data["output_folder"])
48
+
49
+ output_dest = output_path / file_name
50
+ echodata.to_zarr(save_path=output_dest, overwrite=True, parallel=False)
51
+
52
+ if counter_lock is not None:
53
+ with counter_lock:
54
+ progress_counter.value += 1
55
+
56
+ logging.debug("Finished processing of file: %s", file_path)
57
+
58
+ return output_dest
@@ -5,6 +5,7 @@ import sys
5
5
 
6
6
  from pathlib import Path
7
7
  from oceanstream.settings import load_config
8
+ from oceanstream.process import convert_raw_files, compute_single_file
8
9
 
9
10
  DEFAULT_OUTPUT_FOLDER = "output"
10
11
  DEFAULT_SONAR_MODEL = "EK60"
@@ -12,13 +13,20 @@ DEFAULT_SONAR_MODEL = "EK60"
12
13
  logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
13
14
 
14
15
 
15
- def initialize(settings, file_path, log_level=None):
16
+ def initialize(settings, file_path=None, log_level=None, chunks=None):
16
17
  logging.debug(f"Initializing with settings: {settings}, file path: {file_path}, log level: {log_level}")
17
18
  if "config" not in settings:
18
19
  settings["config"] = ""
19
20
 
20
21
  config_data = load_config(settings["config"])
21
- config_data["raw_path"] = file_path
22
+
23
+ if chunks:
24
+ config_data['chunks'] = chunks
25
+ else:
26
+ config_data['chunks'] = config_data.get('base_chunk_sizes', None)
27
+
28
+ if file_path is not None:
29
+ config_data["raw_path"] = file_path
22
30
 
23
31
  if log_level is not None:
24
32
  logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s', force=True)
@@ -27,10 +35,19 @@ def initialize(settings, file_path, log_level=None):
27
35
  if 'sonar_model' in settings and settings["sonar_model"] is not None:
28
36
  config_data["sonar_model"] = settings["sonar_model"]
29
37
 
30
- if settings["output_folder"] is not None:
38
+ if settings.get('plot_echogram', None) is not None:
39
+ config_data["plot_echogram"] = settings["plot_echogram"]
40
+
41
+ if settings.get('waveform_mode', None) is not None:
42
+ config_data["waveform_mode"] = settings["waveform_mode"]
43
+
44
+ if settings.get('depth_offset', None) is not None:
45
+ config_data["depth_offset"] = settings["depth_offset"]
46
+
47
+ if settings.get("output_folder", None) is not None:
31
48
  config_data["output_folder"] = settings["output_folder"]
32
49
 
33
- if settings['cloud_storage'] is not None:
50
+ if settings.get('cloud_storage', None) is not None:
34
51
  config_data['cloud_storage'] = settings['cloud_storage']
35
52
 
36
53
  return config_data
@@ -63,30 +80,32 @@ def process_raw_file(source, output=None, sonar_model=None, plot_echogram=False,
63
80
  sys.exit(1)
64
81
 
65
82
 
66
- def convert(source, output=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
67
- logging.debug("Starting convert function")
68
- settings = {
69
- "config": config,
70
- "output_folder": output or DEFAULT_OUTPUT_FOLDER
71
- }
72
-
73
- file_path = Path(source)
74
- config_data = initialize(settings, file_path, log_level=log_level)
75
-
76
- if chunks:
77
- config_data['chunks'] = chunks
78
-
79
- if file_path.is_file():
80
- logging.debug(f"Converting raw file: {file_path}")
81
- from oceanstream.process import convert_raw_file
82
- convert_raw_file(file_path, config_data)
83
- logging.info(f"Converted raw file {source} to Zarr and wrote output to: {config_data['output_folder']}")
84
- elif file_path.is_dir():
85
- logging.debug(f"Converting raw files in directory: {file_path}")
86
- from oceanstream.process import convert_raw_files
87
- convert_raw_files(config_data, workers_count=workers_count)
88
- else:
89
- logging.error(f"The provided path '{source}' is not a valid file/folder.")
83
+ # def convert(source, output=None, base_path=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
84
+ # logging.debug("Starting convert function")
85
+ # settings = {
86
+ # "output_folder": output or DEFAULT_OUTPUT_FOLDER
87
+ # }
88
+ #
89
+ # if config is not None:
90
+ # settings.update(config)
91
+ #
92
+ # file_path = Path(source)
93
+ # config_data = initialize(settings, file_path, log_level=log_level)
94
+ #
95
+ # if chunks:
96
+ # config_data['chunks'] = chunks
97
+ # else:
98
+ # config_data['chunks'] = config_data.get('base_chunk_sizes', None)
99
+ #
100
+ # if file_path.is_file():
101
+ # logging.debug(f"Converting raw file: {file_path}")
102
+ # convert_raw_file(file_path, config_data, base_path=base_path)
103
+ # logging.info(f"Converted raw file {source} to Zarr and wrote output to: {config_data['output_folder']}")
104
+ # elif file_path.is_dir():
105
+ # logging.debug(f"Converting raw files in directory: {file_path}")
106
+ # convert_raw_files(config_data, workers_count=workers_count)
107
+ # else:
108
+ # logging.error(f"The provided path '{source}' is not a valid file/folder.")
90
109
 
91
110
 
92
111
  def combine(source, output=None, config=None, log_level="WARNING", chunks=None):
@@ -127,7 +146,6 @@ def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONA
127
146
 
128
147
  if config is not None:
129
148
  settings_dict.update(config)
130
- # settings_dict["config"] = ''
131
149
 
132
150
  file_path = Path(source)
133
151
  config_data = initialize(settings_dict, file_path, log_level=log_level)
@@ -139,8 +157,6 @@ def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONA
139
157
 
140
158
  if file_path.is_dir() and source.endswith(".zarr"):
141
159
  logging.debug(f"Computing Sv for Zarr root file: {file_path}")
142
- from oceanstream.process import compute_single_file
143
-
144
160
  compute_single_file(config_data, chunks=chunks, plot_echogram=plot_echogram, waveform_mode=waveform_mode,
145
161
  depth_offset=depth_offset)
146
162
  elif file_path.is_dir():