oceanstream 0.0.12.dev0__tar.gz → 0.0.14.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/PKG-INFO +2 -1
  2. oceanstream-0.0.14.dev0/oceanstream/__init__.py +5 -0
  3. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/cli/main.py +115 -28
  4. oceanstream-0.0.14.dev0/oceanstream/convert/__init__.py +5 -0
  5. oceanstream-0.0.14.dev0/oceanstream/convert/convert_raw_file.py +58 -0
  6. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/core.py +27 -27
  7. oceanstream-0.0.14.dev0/oceanstream/echodata/dataset.py +586 -0
  8. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/sv_computation.py +8 -1
  9. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/sv_dataset_extension.py +18 -17
  10. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/sv_interpolation.py +9 -10
  11. oceanstream-0.0.14.dev0/oceanstream/exports/__init__.py +6 -0
  12. oceanstream-0.0.14.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +183 -0
  13. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/plot/echogram.py +16 -10
  14. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/__init__.py +5 -3
  15. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/aws/s3.py +3 -2
  16. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/azure/__init__.py +1 -1
  17. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/azure/blob_storage.py +34 -3
  18. oceanstream-0.0.14.dev0/oceanstream/process/cloud.py +21 -0
  19. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/file_processor.py +110 -79
  20. oceanstream-0.0.14.dev0/oceanstream/process/file_processor_class.py +217 -0
  21. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/folder_processor.py +175 -52
  22. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/process.py +5 -1
  23. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/utils.py +68 -1
  24. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/PKG-INFO +2 -1
  25. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/SOURCES.txt +5 -0
  26. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/requires.txt +5 -0
  27. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/requirements.txt +3 -0
  28. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/setup.cfg +2 -1
  29. oceanstream-0.0.12.dev0/oceanstream/__init__.py +0 -5
  30. oceanstream-0.0.12.dev0/oceanstream/exports/__init__.py +0 -2
  31. oceanstream-0.0.12.dev0/oceanstream/exports/csv/csv_export_from_Sv.py +0 -89
  32. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/.flake8 +0 -0
  33. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/.gitattributes +0 -0
  34. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/.gitignore +0 -0
  35. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/CODE_OF_CONDUCT.md +0 -0
  36. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/LICENSE +0 -0
  37. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/MANIFEST.in +0 -0
  38. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/README.md +0 -0
  39. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/notebooks/README.md +0 -0
  40. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/cli/__init__.py +0 -0
  41. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/cli/requirements.txt +0 -0
  42. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/__init__.py +0 -0
  43. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/apply_masks.py +0 -0
  44. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/applying_masks_handler.py +0 -0
  45. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/background_noise_remover.py +0 -0
  46. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/create_masks.py +0 -0
  47. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/noise_masks.py +0 -0
  48. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/requirements.txt +0 -0
  49. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/denoise/types.py +0 -0
  50. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/__init__.py +0 -0
  51. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/ensure_time_continuity.py +0 -0
  52. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/mvbs_computation.py +0 -0
  53. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/raw_handler.py +0 -0
  54. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/requirements.txt +0 -0
  55. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/echodata/target_strength_computation.py +0 -0
  56. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/csv/csv_export_from_raw.py +0 -0
  57. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/csv/csv_export_nasc.py +0 -0
  58. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/frequency_differencing_handler.py +0 -0
  59. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/nasc.py +0 -0
  60. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/nasc_computation.py +0 -0
  61. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/requirements.txt +0 -0
  62. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/shoals/shoal_detection_handler.py +0 -0
  63. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/shoals/shoal_process.py +0 -0
  64. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/exports/shoals/shoals_handler.py +0 -0
  65. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/plot/__init__.py +0 -0
  66. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/plot/requirements.txt +0 -0
  67. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/aws/__init__.py +0 -0
  68. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/azure/azure_processor.py +0 -0
  69. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/combine_zarr.py +0 -0
  70. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/processed_data_io.py +0 -0
  71. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/process/requirements.txt +0 -0
  72. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/settings/__init__.py +0 -0
  73. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/settings/config.py +0 -0
  74. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/settings/defaults.config.json +0 -0
  75. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream/utils.py +0 -0
  76. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/dependency_links.txt +0 -0
  77. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/entry_points.txt +0 -0
  78. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/oceanstream.egg-info/top_level.txt +0 -0
  79. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/pyproject.toml +0 -0
  80. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/requirements-dev.txt +0 -0
  81. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/requirements-local.txt +0 -0
  82. {oceanstream-0.0.12.dev0 → oceanstream-0.0.14.dev0}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oceanstream
3
- Version: 0.0.12.dev0
3
+ Version: 0.0.14.dev0
4
4
  Summary: OceanStream: process raw sonar data at scale
5
5
  Home-page: https://oceanstream.io
6
6
  Author: Pineview Labs
@@ -20,6 +20,7 @@ Provides-Extra: echodata
20
20
  Provides-Extra: plot
21
21
  Provides-Extra: denoise
22
22
  Provides-Extra: process
23
+ Provides-Extra: convert
23
24
  Provides-Extra: exports
24
25
  Provides-Extra: complete
25
26
  License-File: LICENSE
@@ -0,0 +1,5 @@
1
+ from .core import initialize, compute_sv, combine, process_raw_file
2
+
3
+ __all__ = [
4
+ "compute_sv", "combine", "process_raw_file", "initialize"
5
+ ]
@@ -1,9 +1,13 @@
1
+ import traceback
2
+ from asyncio import CancelledError
3
+
1
4
  import typer
2
5
  import asyncio
3
6
  import os
4
7
  import logging
5
8
  import sys
6
9
  import warnings
10
+ import dask
7
11
 
8
12
  from pathlib import Path
9
13
  from rich import print
@@ -11,7 +15,7 @@ from rich.traceback import install, Traceback
11
15
  from oceanstream.settings import load_config
12
16
  from dask.distributed import LocalCluster, Client, Variable
13
17
  from rich.console import Console
14
-
18
+ from oceanstream.process import compute_and_export_single_file, process_zarr_files
15
19
 
16
20
  install(show_locals=False, width=120)
17
21
 
@@ -30,6 +34,12 @@ BANNER = """
30
34
 
31
35
  logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
32
36
 
37
+ dask.config.set({
38
+ 'distributed.comm.timeouts.connect': '60s', # Increase the connection timeout
39
+ 'distributed.comm.timeouts.tcp': '120s', # Increase the TCP timeout
40
+ 'distributed.comm.retry.count': 0
41
+ })
42
+
33
43
 
34
44
  def initialize(settings, file_path, log_level=None):
35
45
  config_data = load_config(settings["config"])
@@ -95,6 +105,8 @@ def convert(
95
105
  output: str = typer.Option(None,
96
106
  help="Destination path for saving Zarr converted data. Defaults to a predefined "
97
107
  "directory if not specified."),
108
+ sonar_model: str = typer.Option(None, help="Sonar model used to collect the data",
109
+ show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
98
110
  workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for processing"),
99
111
  config: str = typer.Option(None, help="Path to a configuration file"),
100
112
  log_level: str = typer.Option("WARNING", help="Set the logging level",
@@ -105,6 +117,7 @@ def convert(
105
117
  """
106
118
  settings = {
107
119
  "config": config,
120
+ "sonar_model": sonar_model,
108
121
  "output_folder": output or DEFAULT_OUTPUT_FOLDER
109
122
  }
110
123
 
@@ -113,9 +126,10 @@ def convert(
113
126
 
114
127
  try:
115
128
  if filePath.is_file():
116
- from oceanstream.process import convert_raw_file
129
+ from oceanstream.convert import convert_raw_file
130
+ print(f"[blue]Converting raw file {source} to Zarr...[/blue]")
117
131
  convert_raw_file(filePath, configData)
118
- print(f"[blue]✅ Converted raw file {source} to Zarr and wrote output to: {configData['output_folder']} [/blue]")
132
+ print("✅ The file has been converted successfully.")
119
133
  elif filePath.is_dir():
120
134
  from oceanstream.process import convert_raw_files
121
135
  convert_raw_files(configData, workers_count=workers_count)
@@ -124,8 +138,12 @@ def convert(
124
138
  except KeyboardInterrupt:
125
139
  logging.info("KeyboardInterrupt received, terminating processes...")
126
140
  except Exception as e:
127
- logging.exception("Error processing folder %s", configData['raw_path'])
128
- print(Traceback())
141
+ if filePath.is_file():
142
+ logging.error("Error processing file %s", filePath)
143
+ else:
144
+ logging.exception("Error processing folder %s", configData['raw_path'])
145
+
146
+ logging.error(traceback.format_exc())
129
147
 
130
148
 
131
149
  @app.command()
@@ -193,10 +211,12 @@ def compute_sv(
193
211
  sonar_model: str = typer.Option(DEFAULT_SONAR_MODEL, help="Sonar model used to collect the data",
194
212
  show_choices=["AZFP", "EK60", "ES70", "EK80", "ES80", "EA640", "AD2CP"]),
195
213
  plot_echogram: bool = typer.Option(False, help="Plot the echogram after processing"),
196
- use_dask: bool = typer.Option(False, help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
214
+ use_dask: bool = typer.Option(False,
215
+ help="Start a Local Dask cluster for parallel processing (always enabled for multiple files)"),
197
216
  depth_offset: float = typer.Option(0.0, help="Depth offset for the echogram plot"),
198
217
  waveform_mode: str = typer.Option("CW", help="Waveform mode, can be either CW or BB",
199
218
  show_choices=["CW", "BB"]),
219
+ encode_mode: str = typer.Option("power", help="Encode mode, can be either power or complex", show_choices=["power", "complex"]),
200
220
  config: str = typer.Option(None, help="Path to a configuration file"),
201
221
  log_level: str = typer.Option("WARNING", help="Set the logging level",
202
222
  show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
@@ -209,6 +229,88 @@ def compute_sv(
209
229
  "sonar_model": sonar_model,
210
230
  "output_folder": output or DEFAULT_OUTPUT_FOLDER
211
231
  }
232
+ dask.config.set({'distributed.comm.retry.count': 1})
233
+
234
+ file_path = Path(source)
235
+ config_data = initialize(settings_dict, file_path, log_level=log_level)
236
+ single_file = file_path.is_dir() and source.endswith(".zarr")
237
+
238
+ client = None
239
+ cluster = None
240
+
241
+ if use_dask or not single_file:
242
+ cluster = LocalCluster(n_workers=workers_count, threads_per_worker=1)
243
+ client = Client(cluster)
244
+
245
+ try:
246
+ if file_path.is_dir() and source.endswith(".zarr"):
247
+ console = Console()
248
+ with console.status("Processing...", spinner="dots") as status:
249
+ status.start()
250
+ status.update(
251
+ f"[blue] Computing Sv for {file_path}...[/blue]" + use_dask * "– navigate to "
252
+ "http://localhost:8787/status for "
253
+ "progress")
254
+
255
+ chunks = None
256
+ if use_dask:
257
+ chunks = config_data.get('base_chunk_sizes')
258
+
259
+ compute_and_export_single_file(config_data,
260
+ chunks=chunks,
261
+ plot_echogram=plot_echogram,
262
+ waveform_mode=waveform_mode,
263
+ encode_mode=encode_mode,
264
+ depth_offset=depth_offset)
265
+
266
+ status.stop()
267
+ print("✅ The file have been processed successfully.")
268
+ elif file_path.is_dir():
269
+ print(f"Dashboard available at {client.dashboard_link}")
270
+ process_zarr_files(config_data,
271
+ client,
272
+ workers_count=workers_count,
273
+ chunks=config_data.get('base_chunk_sizes'),
274
+ plot_echogram=plot_echogram,
275
+ waveform_mode=waveform_mode,
276
+ depth_offset=depth_offset)
277
+ else:
278
+ print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
279
+ sys.exit(1)
280
+ except KeyboardInterrupt:
281
+ logging.info("KeyboardInterrupt received, terminating processes...")
282
+ except Exception as e:
283
+ logging.exception("Error while processing %s", config_data['raw_path'])
284
+ print(Traceback())
285
+ finally:
286
+ if client is not None:
287
+ client.close()
288
+
289
+ if cluster is not None:
290
+ cluster.close()
291
+
292
+
293
+ @app.command()
294
+ def export_location(
295
+ source: str = typer.Option(..., help="Path to a Zarr root file or a directory containing Zarr files"),
296
+ output: str = typer.Option(None,
297
+ help="Destination path for saving the exported data. Defaults to a predefined "
298
+ "directory if not specified."),
299
+ workers_count: int = typer.Option(os.cpu_count(), help="Number of CPU workers to use for parallel processing"),
300
+ use_dask: bool = typer.Option(False,
301
+ help="Start a Local Dask cluster for parallel processing (always enabled for "
302
+ "multiple files)"),
303
+ config: str = typer.Option(None, help="Path to a configuration file"),
304
+ log_level: str = typer.Option("WARNING", help="Set the logging level",
305
+ show_choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"])
306
+ ):
307
+ """
308
+ Given a Zarr dataset containing Sv data, exports the GPS location data to a JSON file.
309
+ """
310
+ settings_dict = {
311
+ "config": config,
312
+ "output_folder": output or DEFAULT_OUTPUT_FOLDER
313
+ }
212
314
  file_path = Path(source)
213
315
  config_data = initialize(settings_dict, file_path, log_level=log_level)
214
316
 
@@ -225,27 +327,17 @@ def compute_sv(
225
327
  if file_path.is_dir() and source.endswith(".zarr"):
226
328
  status.update(
227
329
  f"[blue] Computing Sv for {file_path}...[/blue] – navigate to http://localhost:8787/status for progress")
228
- from oceanstream.process import compute_single_file
229
-
230
- compute_single_file(config_data,
231
- chunks=config_data.get('base_chunk_sizes'),
232
- plot_echogram=plot_echogram,
233
- waveform_mode=waveform_mode,
234
- depth_offset=depth_offset)
330
+ # TODO: Implement export_location_json
235
331
  elif file_path.is_dir():
236
332
  status.update(
237
333
  f"[blue] Processing zarr files in {file_path}...[/blue] – navigate to "
238
334
  f"http://localhost:8787/status for progress")
239
- from oceanstream.process import process_zarr_files
240
- processed_count_var = Variable('processed_count', client)
241
- process_zarr_files(config_data,
242
- workers_count=workers_count,
243
- status=status,
244
- chunks=config_data.get('base_chunk_sizes'),
245
- plot_echogram=plot_echogram,
246
- waveform_mode=waveform_mode,
247
- processed_count_var=processed_count_var,
248
- depth_offset=depth_offset)
335
+ from oceanstream.process import export_location_from_zarr_files
336
+
337
+ export_location_from_zarr_files(config_data,
338
+ workers_count=workers_count,
339
+ client=client,
340
+ chunks=config_data.get('base_chunk_sizes'))
249
341
  else:
250
342
  print(f"[red]❌ The provided path '{source}' is not a valid Zarr root.[/red]")
251
343
  sys.exit(1)
@@ -261,11 +353,6 @@ def compute_sv(
261
353
  status.stop()
262
354
 
263
355
 
264
- @app.command()
265
- def export():
266
- typer.echo("Export data...")
267
-
268
-
269
356
  def main():
270
357
  print(BANNER)
271
358
  warnings.filterwarnings("ignore", category=UserWarning)
@@ -0,0 +1,5 @@
1
+ from .convert_raw_file import convert_raw_file
2
+
3
+ __all__ = [
4
+ "convert_raw_file"
5
+ ]
@@ -0,0 +1,58 @@
1
+ import json
2
+ import logging
3
+ import os
4
+ import sys
5
+ import time
6
+ import traceback
7
+ import shutil
8
+ from asyncio import CancelledError
9
+ import echopype as ep
10
+
11
+ import xarray as xr
12
+ from pathlib import Path
13
+ from oceanstream.echodata import read_file
14
+
15
+
16
+ def convert_raw_file(file_path, config_data, base_path=None, progress_counter=None, counter_lock=None):
17
+ logging.debug("Starting processing of file: %s", file_path)
18
+
19
+ file_path_obj = Path(file_path)
20
+ file_config_data = {**config_data, 'raw_path': file_path_obj}
21
+
22
+ if base_path:
23
+ relative_path = file_path_obj.relative_to(base_path)
24
+ relative_path = relative_path.parent
25
+ else:
26
+ relative_path = None
27
+
28
+ echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
29
+ file_name = file_path_obj.stem + ".zarr"
30
+
31
+ if 'cloud_storage' in config_data:
32
+ from oceanstream.process.cloud import get_chunk_store
33
+
34
+ if relative_path:
35
+ file_location = Path(relative_path) / file_name
36
+ else:
37
+ file_location = file_name
38
+ store = get_chunk_store(config_data['cloud_storage'], file_location)
39
+ echodata.to_zarr(save_path=store, overwrite=True, parallel=False)
40
+
41
+ output_dest = config_data['cloud_storage']['container_name'] + "/" + file_location
42
+ else:
43
+ if relative_path:
44
+ output_path = Path(config_data["output_folder"]) / relative_path
45
+ output_path.mkdir(parents=True, exist_ok=True)
46
+ else:
47
+ output_path = Path(config_data["output_folder"])
48
+
49
+ output_dest = output_path / file_name
50
+ echodata.to_zarr(save_path=output_dest, overwrite=True, parallel=False)
51
+
52
+ if counter_lock is not None:
53
+ with counter_lock:
54
+ progress_counter.value += 1
55
+
56
+ logging.debug("Finished processing of file: %s", file_path)
57
+
58
+ return output_dest
@@ -5,7 +5,7 @@ import sys
5
5
 
6
6
  from pathlib import Path
7
7
  from oceanstream.settings import load_config
8
- from oceanstream.process import convert_raw_file, convert_raw_files, compute_single_file
8
+ from oceanstream.process import convert_raw_files, compute_single_file
9
9
 
10
10
  DEFAULT_OUTPUT_FOLDER = "output"
11
11
  DEFAULT_SONAR_MODEL = "EK60"
@@ -80,32 +80,32 @@ def process_raw_file(source, output=None, sonar_model=None, plot_echogram=False,
80
80
  sys.exit(1)
81
81
 
82
82
 
83
- def convert(source, output=None, base_path=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
84
- logging.debug("Starting convert function")
85
- settings = {
86
- "output_folder": output or DEFAULT_OUTPUT_FOLDER
87
- }
88
-
89
- if config is not None:
90
- settings.update(config)
91
-
92
- file_path = Path(source)
93
- config_data = initialize(settings, file_path, log_level=log_level)
94
-
95
- if chunks:
96
- config_data['chunks'] = chunks
97
- else:
98
- config_data['chunks'] = config_data.get('base_chunk_sizes', None)
99
-
100
- if file_path.is_file():
101
- logging.debug(f"Converting raw file: {file_path}")
102
- convert_raw_file(file_path, config_data, base_path=base_path)
103
- logging.info(f"Converted raw file {source} to Zarr and wrote output to: {config_data['output_folder']}")
104
- elif file_path.is_dir():
105
- logging.debug(f"Converting raw files in directory: {file_path}")
106
- convert_raw_files(config_data, workers_count=workers_count)
107
- else:
108
- logging.error(f"The provided path '{source}' is not a valid file/folder.")
83
+ # def convert(source, output=None, base_path=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
84
+ # logging.debug("Starting convert function")
85
+ # settings = {
86
+ # "output_folder": output or DEFAULT_OUTPUT_FOLDER
87
+ # }
88
+ #
89
+ # if config is not None:
90
+ # settings.update(config)
91
+ #
92
+ # file_path = Path(source)
93
+ # config_data = initialize(settings, file_path, log_level=log_level)
94
+ #
95
+ # if chunks:
96
+ # config_data['chunks'] = chunks
97
+ # else:
98
+ # config_data['chunks'] = config_data.get('base_chunk_sizes', None)
99
+ #
100
+ # if file_path.is_file():
101
+ # logging.debug(f"Converting raw file: {file_path}")
102
+ # convert_raw_file(file_path, config_data, base_path=base_path)
103
+ # logging.info(f"Converted raw file {source} to Zarr and wrote output to: {config_data['output_folder']}")
104
+ # elif file_path.is_dir():
105
+ # logging.debug(f"Converting raw files in directory: {file_path}")
106
+ # convert_raw_files(config_data, workers_count=workers_count)
107
+ # else:
108
+ # logging.error(f"The provided path '{source}' is not a valid file/folder.")
109
109
 
110
110
 
111
111
  def combine(source, output=None, config=None, log_level="WARNING", chunks=None):