oceanstream 0.0.10.dev0__tar.gz → 0.0.12.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/PKG-INFO +1 -1
  2. oceanstream-0.0.12.dev0/oceanstream/__init__.py +5 -0
  3. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/cli/main.py +0 -2
  4. oceanstream-0.0.12.dev0/oceanstream/cli/requirements.txt +1 -0
  5. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/core.py +28 -12
  6. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_dataset_extension.py +0 -2
  7. oceanstream-0.0.12.dev0/oceanstream/process/aws/__init__.py +9 -0
  8. oceanstream-0.0.12.dev0/oceanstream/process/aws/s3.py +93 -0
  9. oceanstream-0.0.12.dev0/oceanstream/process/azure/__init__.py +8 -0
  10. oceanstream-0.0.12.dev0/oceanstream/process/azure/azure_processor.py +66 -0
  11. oceanstream-0.0.12.dev0/oceanstream/process/azure/blob_storage.py +54 -0
  12. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/combine_zarr.py +1 -1
  13. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/file_processor.py +43 -30
  14. oceanstream-0.0.12.dev0/oceanstream/process/requirements.txt +0 -0
  15. oceanstream-0.0.12.dev0/oceanstream/process/utils.py +159 -0
  16. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/PKG-INFO +1 -1
  17. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/SOURCES.txt +6 -0
  18. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/requires.txt +3 -4
  19. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/requirements.txt +3 -0
  20. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/setup.cfg +1 -3
  21. oceanstream-0.0.10.dev0/oceanstream/__init__.py +0 -3
  22. oceanstream-0.0.10.dev0/oceanstream/cli/requirements.txt +0 -2
  23. oceanstream-0.0.10.dev0/oceanstream/echodata/requirements.txt +0 -1
  24. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/.flake8 +0 -0
  25. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/.gitattributes +0 -0
  26. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/.gitignore +0 -0
  27. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/CODE_OF_CONDUCT.md +0 -0
  28. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/LICENSE +0 -0
  29. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/MANIFEST.in +0 -0
  30. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/README.md +0 -0
  31. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/notebooks/README.md +0 -0
  32. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/cli/__init__.py +0 -0
  33. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/__init__.py +0 -0
  34. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/apply_masks.py +0 -0
  35. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/applying_masks_handler.py +0 -0
  36. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/background_noise_remover.py +0 -0
  37. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/create_masks.py +0 -0
  38. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/noise_masks.py +0 -0
  39. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/requirements.txt +0 -0
  40. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/denoise/types.py +0 -0
  41. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/__init__.py +0 -0
  42. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/ensure_time_continuity.py +0 -0
  43. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/mvbs_computation.py +0 -0
  44. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/raw_handler.py +0 -0
  45. {oceanstream-0.0.10.dev0/oceanstream/process → oceanstream-0.0.12.dev0/oceanstream/echodata}/requirements.txt +0 -0
  46. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_computation.py +0 -0
  47. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/sv_interpolation.py +0 -0
  48. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/echodata/target_strength_computation.py +0 -0
  49. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/__init__.py +0 -0
  50. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/csv/csv_export_from_Sv.py +0 -0
  51. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/csv/csv_export_from_raw.py +0 -0
  52. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/csv/csv_export_nasc.py +0 -0
  53. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/frequency_differencing_handler.py +0 -0
  54. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/nasc.py +0 -0
  55. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/nasc_computation.py +0 -0
  56. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/requirements.txt +0 -0
  57. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/shoals/shoal_detection_handler.py +0 -0
  58. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/shoals/shoal_process.py +0 -0
  59. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/exports/shoals/shoals_handler.py +0 -0
  60. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/plot/__init__.py +0 -0
  61. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/plot/echogram.py +0 -0
  62. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/plot/requirements.txt +0 -0
  63. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/__init__.py +0 -0
  64. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/folder_processor.py +0 -0
  65. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/process.py +0 -0
  66. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/process/processed_data_io.py +0 -0
  67. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/settings/__init__.py +0 -0
  68. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/settings/config.py +0 -0
  69. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/settings/defaults.config.json +0 -0
  70. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream/utils.py +0 -0
  71. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/dependency_links.txt +0 -0
  72. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/entry_points.txt +0 -0
  73. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/oceanstream.egg-info/top_level.txt +0 -0
  74. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/pyproject.toml +0 -0
  75. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/requirements-dev.txt +0 -0
  76. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/requirements-local.txt +0 -0
  77. {oceanstream-0.0.10.dev0 → oceanstream-0.0.12.dev0}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oceanstream
3
- Version: 0.0.10.dev0
3
+ Version: 0.0.12.dev0
4
4
  Summary: OceanStream: process raw sonar data at scale
5
5
  Home-page: https://oceanstream.io
6
6
  Author: Pineview Labs
@@ -0,0 +1,5 @@
1
+ from .core import initialize, compute_sv, combine, convert, process_raw_file
2
+
3
+ __all__ = [
4
+ "compute_sv", "combine", "convert", "process_raw_file", "initialize"
5
+ ]
@@ -1,5 +1,3 @@
1
- import traceback
2
-
3
1
  import typer
4
2
  import asyncio
5
3
  import os
@@ -5,6 +5,7 @@ import sys
5
5
 
6
6
  from pathlib import Path
7
7
  from oceanstream.settings import load_config
8
+ from oceanstream.process import convert_raw_file, convert_raw_files, compute_single_file
8
9
 
9
10
  DEFAULT_OUTPUT_FOLDER = "output"
10
11
  DEFAULT_SONAR_MODEL = "EK60"
@@ -12,13 +13,20 @@ DEFAULT_SONAR_MODEL = "EK60"
12
13
  logging.basicConfig(level="ERROR", format='%(asctime)s - %(levelname)s - %(message)s')
13
14
 
14
15
 
15
- def initialize(settings, file_path, log_level=None):
16
+ def initialize(settings, file_path=None, log_level=None, chunks=None):
16
17
  logging.debug(f"Initializing with settings: {settings}, file path: {file_path}, log level: {log_level}")
17
18
  if "config" not in settings:
18
19
  settings["config"] = ""
19
20
 
20
21
  config_data = load_config(settings["config"])
21
- config_data["raw_path"] = file_path
22
+
23
+ if chunks:
24
+ config_data['chunks'] = chunks
25
+ else:
26
+ config_data['chunks'] = config_data.get('base_chunk_sizes', None)
27
+
28
+ if file_path is not None:
29
+ config_data["raw_path"] = file_path
22
30
 
23
31
  if log_level is not None:
24
32
  logging.basicConfig(level=log_level, format='%(asctime)s - %(levelname)s - %(message)s', force=True)
@@ -27,10 +35,19 @@ def initialize(settings, file_path, log_level=None):
27
35
  if 'sonar_model' in settings and settings["sonar_model"] is not None:
28
36
  config_data["sonar_model"] = settings["sonar_model"]
29
37
 
30
- if settings["output_folder"] is not None:
38
+ if settings.get('plot_echogram', None) is not None:
39
+ config_data["plot_echogram"] = settings["plot_echogram"]
40
+
41
+ if settings.get('waveform_mode', None) is not None:
42
+ config_data["waveform_mode"] = settings["waveform_mode"]
43
+
44
+ if settings.get('depth_offset', None) is not None:
45
+ config_data["depth_offset"] = settings["depth_offset"]
46
+
47
+ if settings.get("output_folder", None) is not None:
31
48
  config_data["output_folder"] = settings["output_folder"]
32
49
 
33
- if settings['cloud_storage'] is not None:
50
+ if settings.get('cloud_storage', None) is not None:
34
51
  config_data['cloud_storage'] = settings['cloud_storage']
35
52
 
36
53
  return config_data
@@ -63,27 +80,29 @@ def process_raw_file(source, output=None, sonar_model=None, plot_echogram=False,
63
80
  sys.exit(1)
64
81
 
65
82
 
66
- def convert(source, output=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
83
+ def convert(source, output=None, base_path=None, workers_count=None, config=None, log_level="WARNING", chunks=None):
67
84
  logging.debug("Starting convert function")
68
85
  settings = {
69
- "config": config,
70
86
  "output_folder": output or DEFAULT_OUTPUT_FOLDER
71
87
  }
72
88
 
89
+ if config is not None:
90
+ settings.update(config)
91
+
73
92
  file_path = Path(source)
74
93
  config_data = initialize(settings, file_path, log_level=log_level)
75
94
 
76
95
  if chunks:
77
96
  config_data['chunks'] = chunks
97
+ else:
98
+ config_data['chunks'] = config_data.get('base_chunk_sizes', None)
78
99
 
79
100
  if file_path.is_file():
80
101
  logging.debug(f"Converting raw file: {file_path}")
81
- from oceanstream.process import convert_raw_file
82
- convert_raw_file(file_path, config_data)
102
+ convert_raw_file(file_path, config_data, base_path=base_path)
83
103
  logging.info(f"Converted raw file {source} to Zarr and wrote output to: {config_data['output_folder']}")
84
104
  elif file_path.is_dir():
85
105
  logging.debug(f"Converting raw files in directory: {file_path}")
86
- from oceanstream.process import convert_raw_files
87
106
  convert_raw_files(config_data, workers_count=workers_count)
88
107
  else:
89
108
  logging.error(f"The provided path '{source}' is not a valid file/folder.")
@@ -127,7 +146,6 @@ def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONA
127
146
 
128
147
  if config is not None:
129
148
  settings_dict.update(config)
130
- # settings_dict["config"] = ''
131
149
 
132
150
  file_path = Path(source)
133
151
  config_data = initialize(settings_dict, file_path, log_level=log_level)
@@ -139,8 +157,6 @@ def compute_sv(source, output=None, workers_count=None, sonar_model=DEFAULT_SONA
139
157
 
140
158
  if file_path.is_dir() and source.endswith(".zarr"):
141
159
  logging.debug(f"Computing Sv for Zarr root file: {file_path}")
142
- from oceanstream.process import compute_single_file
143
-
144
160
  compute_single_file(config_data, chunks=chunks, plot_echogram=plot_echogram, waveform_mode=waveform_mode,
145
161
  depth_offset=depth_offset)
146
162
  elif file_path.is_dir():
@@ -96,8 +96,6 @@ def add_depth(Sv: Dataset, depth_offset: float = 0, tilt: float = 0, downward: b
96
96
  first_channel = Sv["channel"].values[0]
97
97
  first_ping_time = Sv["ping_time"].values[0]
98
98
 
99
- original_echo_range = Sv["echo_range"].sel(channel=first_channel, ping_time=first_ping_time).values
100
-
101
99
  # Slice the echo_range to get the desired range of values
102
100
  selected_echo_range = Sv["echo_range"].sel(channel=first_channel, ping_time=first_ping_time)
103
101
  selected_echo_range = selected_echo_range.values.tolist()
@@ -0,0 +1,9 @@
1
+ from .s3 import process_survey_data_with_progress, list_raw_files_from_bucket, convert_survey_data_from_bucket, \
2
+ download_file_from_bucket
3
+
4
+ __all__ = [
5
+ "process_survey_data_with_progress",
6
+ "list_raw_files_from_bucket",
7
+ "convert_survey_data_from_bucket",
8
+ "download_file_from_bucket"
9
+ ]
@@ -0,0 +1,93 @@
1
+ import os
2
+ import tempfile
3
+ import botocore
4
+ import boto3
5
+ from asyncio import CancelledError
6
+ from rich import print
7
+ from rich.traceback import install, Traceback
8
+ from tqdm.auto import tqdm
9
+ from dask import delayed
10
+ from botocore.config import Config
11
+
12
+
13
+ install(show_locals=False, width=120)
14
+
15
+
16
+ def process_survey_data_with_progress(files_by_day, bucket_name, client, config_data, process_func):
17
+ try:
18
+ total_files = sum(len(files) for files in files_by_day.values())
19
+ progress_bar = tqdm(total=total_files, desc="Processing Files", unit="file", ncols=100)
20
+
21
+ def update_progress(*args):
22
+ progress_bar.update()
23
+
24
+ temp_dir = tempfile.mkdtemp()
25
+ futures, temp_dir = convert_survey_data_from_bucket(files_by_day, temp_dir, bucket_name, client, config_data,
26
+ process_func)
27
+
28
+ for future in futures:
29
+ future.add_done_callback(update_progress)
30
+
31
+ client.gather(futures) # Ensure all tasks complete
32
+
33
+ progress_bar.close()
34
+ os.rmdir(temp_dir)
35
+ except KeyboardInterrupt:
36
+ print("Closing down.")
37
+ except CancelledError:
38
+ print("Closing down.")
39
+ except Exception as e:
40
+ print(f"[bold red]An error occurred:[/bold red] {e}")
41
+ print(f"{Traceback()}\n")
42
+
43
+
44
+ def convert_survey_data_from_bucket(files_by_day, temp_dir, bucket_name, dask_client, config_data, process_func):
45
+ """Process survey data from S3."""
46
+ tasks = []
47
+
48
+ for day, files in files_by_day.items():
49
+ for file in files:
50
+ task = delayed(_process_raw_file)(file, day, temp_dir, bucket_name, config_data, process_func)
51
+ tasks.append(task)
52
+
53
+ # Execute all tasks in parallel
54
+ futures = dask_client.compute(tasks)
55
+
56
+ return futures, temp_dir
57
+
58
+
59
+ def list_raw_files_from_bucket(bucket_name, prefix):
60
+ """List files in the S3 bucket along with their metadata."""
61
+ s3_client = boto3.client('s3', config=Config(signature_version=botocore.UNSIGNED))
62
+ response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
63
+
64
+ files = [
65
+ {
66
+ 'Key': item['Key'],
67
+ 'Size': item['Size'],
68
+ 'LastModified': item['LastModified']
69
+ }
70
+ for item in response.get('Contents', []) if item['Key'].endswith('.raw')
71
+ ]
72
+
73
+ return files
74
+
75
+
76
+ def download_file_from_bucket(bucket_name, s3_key, local_dir):
77
+ """Download a file from S3."""
78
+ s3_client = boto3.client('s3', config=Config(signature_version=botocore.UNSIGNED))
79
+ local_path = os.path.join(local_dir, os.path.basename(s3_key))
80
+ s3_client.download_file(bucket_name, s3_key, local_path)
81
+
82
+ return local_path
83
+
84
+
85
+ def _process_raw_file(file, day, temp_dir, bucket_name, config_data, process_func):
86
+ """Process a single file."""
87
+ day_dir = os.path.join(temp_dir, day)
88
+ os.makedirs(day_dir, exist_ok=True)
89
+ local_path = download_file_from_bucket(bucket_name, file['Key'], day_dir)
90
+
91
+ process_func(local_path, config=config_data, base_path=temp_dir)
92
+ os.remove(local_path)
93
+
@@ -0,0 +1,8 @@
1
+ from .blob_storage import list_zarr_files
2
+ from .azure_processor import process_survey_data_with_progress, process_survey_data
3
+
4
+ __all__ = [
5
+ "list_zarr_files",
6
+ "process_survey_data",
7
+ "process_survey_data_with_progress"
8
+ ]
@@ -0,0 +1,66 @@
1
+ import re
2
+
3
+ from asyncio import CancelledError
4
+ from rich import print
5
+ from rich.traceback import install, Traceback
6
+ from tqdm.auto import tqdm
7
+ from dask import delayed
8
+ from .blob_storage import open_zarr_store
9
+
10
+ install(show_locals=False, width=120)
11
+
12
+
13
+ def process_survey_data_with_progress(files_by_day, azfs, container_name, client, config_data, process_func):
14
+ try:
15
+ total_files = sum(len(files) for files in files_by_day.values())
16
+ progress_bar = tqdm(total=total_files, desc="Processing Files", unit="file", ncols=100)
17
+
18
+ def update_progress(*args):
19
+ progress_bar.update()
20
+
21
+ futures = process_survey_data(files_by_day, azfs, container_name, client, config_data, process_func)
22
+
23
+ for future in futures:
24
+ future.add_done_callback(update_progress)
25
+
26
+ client.gather(futures) # Ensure all tasks complete
27
+
28
+ progress_bar.close()
29
+ except KeyboardInterrupt:
30
+ print("Closing down.")
31
+ except CancelledError:
32
+ print("Closing down.")
33
+ except Exception as e:
34
+ print(f"[bold red]An error occurred:[/bold red] {e}")
35
+ print(f"{Traceback()}\n")
36
+
37
+
38
+ def process_survey_data(files_by_day, azfs, container_name, dask_client, config_data, process_func):
39
+ """Process survey data from S3."""
40
+
41
+ tasks = []
42
+
43
+ for day, files in files_by_day.items():
44
+ for file in files:
45
+ task = delayed(_process_zarr_file)(file['Key'], azfs, container_name, config_data, process_func)
46
+ tasks.append(task)
47
+
48
+ # Execute all tasks in parallel
49
+ futures = dask_client.compute(tasks)
50
+
51
+ return futures
52
+
53
+
54
+ def _process_zarr_file(file, azfs, container_name, config_data, process_func=None):
55
+ """Process a single Zarr file."""
56
+ base_path = file.replace('.zarr', '')
57
+ pattern = rf"^{re.escape(container_name)}/"
58
+ base_path = re.sub(pattern, '', base_path)
59
+
60
+ echodata = open_zarr_store(azfs, file, chunks=config_data['chunks'])
61
+
62
+ process_func(echodata, config_data, base_path=base_path,
63
+ chunks=config_data.get('chunks'),
64
+ plot_echogram=config_data.get('plot_echogram', False),
65
+ waveform_mode=config_data.get('waveform_mode', "CW"),
66
+ depth_offset=config_data.get('depth_offset', 0.0))
@@ -0,0 +1,54 @@
1
+ import echopype as ep
2
+
3
+
4
+ def list_zarr_files(azfs, path):
5
+ """List all Zarr files in the Azure Blob Storage container along with their metadata."""
6
+ zarr_files = []
7
+ for blob in azfs.ls(path, detail=True):
8
+ if blob['type'] == 'directory' and not blob['name'].endswith('.zarr'):
9
+ subdir_files = list_zarr_files(azfs, blob['name'])
10
+ zarr_files.extend(subdir_files)
11
+ elif blob['name'].endswith('.zarr'):
12
+ zarr_files.append({
13
+ 'Key': blob['name'],
14
+ 'Size': blob['size'] if blob['size'] else 0,
15
+ 'LastModified': blob['last_modified'] if 'last_modified' in blob else 0
16
+ })
17
+
18
+ return zarr_files
19
+
20
+
21
+ def open_zarr_store(azfs, store_name, chunks=None):
22
+ """Open a Zarr store from Azure Blob Storage."""
23
+ mapper = azfs.get_mapper(store_name)
24
+
25
+ return ep.open_converted(mapper, chunks=chunks)
26
+
27
+
28
+ def _list_zarr_files_extended(azfs, path):
29
+ """List all Zarr files in the Azure Blob Storage container along with their metadata."""
30
+ zarr_files = []
31
+ for blob in azfs.ls(path, detail=True):
32
+ if blob['type'] == 'directory' and not blob['name'].endswith('.zarr'):
33
+ subdir_files = list_zarr_files(azfs, blob['name'])
34
+ zarr_files.extend(subdir_files)
35
+ else:
36
+ # Calculate the total size and most recent modification date for the .zarr folder
37
+ total_size = 0
38
+ last_modified = None
39
+ for sub_blob in azfs.ls(blob['name'], detail=True):
40
+ if sub_blob['type'] == 'file':
41
+ total_size += sub_blob['size']
42
+ if last_modified is None or sub_blob['last_modified'] > last_modified:
43
+ last_modified = sub_blob['last_modified']
44
+
45
+ zarr_files.append({
46
+ 'Key': blob['name'],
47
+ 'Size': total_size,
48
+ 'LastModified': last_modified
49
+ })
50
+
51
+ return zarr_files
52
+
53
+
54
+
@@ -86,4 +86,4 @@ def from_filename(file_name):
86
86
  creation_time = datetime.strptime(date_str + time_str, '%Y%m%d%H%M%S')
87
87
  return creation_time
88
88
 
89
- return None
89
+ return None
@@ -12,7 +12,8 @@ from rich.traceback import install, Traceback
12
12
  from rich.progress import Progress, TimeElapsedColumn, BarColumn, TextColumn
13
13
 
14
14
  from oceanstream.plot import plot_sv_data_with_progress, plot_sv_data
15
- from oceanstream.echodata import get_campaign_metadata
15
+ from oceanstream.echodata import get_campaign_metadata, read_file
16
+
16
17
  from .process import compute_sv, process_file_with_progress, read_file_with_progress
17
18
 
18
19
  install(show_locals=True, width=120)
@@ -37,27 +38,35 @@ def compute_Sv_to_zarr(echodata, config_data, base_path=None, chunks=None, plot_
37
38
  Returns:
38
39
  str: Path to the zarr file.
39
40
  """
40
- file_path = config_data["raw_path"]
41
41
  waveform_mode = kwargs.get("waveform_mode", "CW")
42
42
  encode_mode = waveform_mode == "CW" and "power" or "complex"
43
43
  Sv = compute_sv(echodata, encode_mode=encode_mode, **kwargs)
44
44
 
45
- if base_path:
46
- relative_path = file_path.relative_to(base_path)
45
+ if config_data.get('raw_path') is not None:
46
+ file_path = config_data["raw_path"]
47
+ file_base_name = file_path.stem
48
+ if base_path:
49
+ relative_path = file_path.relative_to(base_path)
47
50
 
48
- if relative_path.parent != ".":
49
- zarr_path = Path(relative_path.parent) / file_path.stem
51
+ if relative_path.parent != ".":
52
+ zarr_path = Path(relative_path.parent) / file_base_name
53
+ else:
54
+ zarr_path = relative_path.stem
50
55
  else:
51
- zarr_path = relative_path.stem
52
- else:
53
- zarr_path = file_path.stem
56
+ zarr_path = file_base_name
54
57
 
55
- output_path = Path(config_data["output_folder"]) / zarr_path
56
- output_path.mkdir(parents=True, exist_ok=True)
58
+ output_path = Path(config_data["output_folder"]) / zarr_path
59
+ output_path.mkdir(parents=True, exist_ok=True)
57
60
 
58
- echogram_path = zarr_path
59
- zarr_file_name = f"{file_path.stem}_Sv.zarr"
61
+ zarr_file_name = f"{file_base_name}_Sv.zarr"
62
+ else:
63
+ zarr_path = base_path
64
+ zarr_file_name = f"{zarr_path}_Sv.zarr"
65
+ output_path = None
66
+ file_base_name = None
67
+ file_path = None
60
68
 
69
+ echogram_path = zarr_path
61
70
  if chunks is not None:
62
71
  for var in Sv.data_vars:
63
72
  var_chunk_sizes = get_chunk_sizes(Sv[var].dims, chunks)
@@ -73,7 +82,7 @@ def compute_Sv_to_zarr(echodata, config_data, base_path=None, chunks=None, plot_
73
82
 
74
83
  if plot_echogram:
75
84
  try:
76
- plot_sv_data(ds_processed, file_base_name=file_path.stem, output_path=output_path,
85
+ plot_sv_data(ds_processed, file_base_name=file_base_name, output_path=output_path,
77
86
  echogram_path=echogram_path, config_data=config_data)
78
87
  except Exception as e:
79
88
  logging.exception(f"Error plotting echogram for {file_path}:")
@@ -139,7 +148,6 @@ async def process_raw_file_with_progress(config_data, plot_echogram, waveform_mo
139
148
 
140
149
  def convert_raw_file(file_path, config_data, base_path=None, progress_queue=None):
141
150
  logging.debug("Starting processing of file: %s", file_path)
142
- from oceanstream.echodata import read_file
143
151
 
144
152
  try:
145
153
  file_path_obj = Path(file_path)
@@ -151,11 +159,16 @@ def convert_raw_file(file_path, config_data, base_path=None, progress_queue=None
151
159
  else:
152
160
  relative_path = file_path_obj.name
153
161
 
154
- output_path = Path(config_data["output_folder"]) / relative_path
155
- output_path.mkdir(parents=True, exist_ok=True)
156
-
157
162
  echodata, encode_mode = read_file(file_config_data, use_swap=True, skip_integrity_check=True)
158
- echodata.to_zarr(save_path=output_path, overwrite=True, parallel=False)
163
+
164
+ if 'cloud_storage' in config_data:
165
+ file_name = file_path_obj.stem + ".zarr"
166
+ store = get_chunk_store(config_data['cloud_storage'], Path(relative_path) / file_name)
167
+ echodata.to_zarr(save_path=store, overwrite=True, parallel=False)
168
+ else:
169
+ output_path = Path(config_data["output_folder"]) / relative_path
170
+ output_path.mkdir(parents=True, exist_ok=True)
171
+ echodata.to_zarr(save_path=output_path, overwrite=True, parallel=False)
159
172
 
160
173
  if progress_queue:
161
174
  progress_queue.put(file_path)
@@ -164,17 +177,6 @@ def convert_raw_file(file_path, config_data, base_path=None, progress_queue=None
164
177
  print(Traceback())
165
178
 
166
179
 
167
- def get_chunk_store(storage_config, path):
168
- if storage_config['storage_type'] == 'azure':
169
- from adlfs import AzureBlobFileSystem
170
- azfs = AzureBlobFileSystem(**storage_config['storage_options'])
171
-
172
- return azfs.get_mapper(f"{storage_config['container_name']}/{path}")
173
-
174
- else:
175
- raise ValueError(f"Unsupported storage type: {storage_config['storage_type']}")
176
-
177
-
178
180
  def compute_single_file(config_data, **kwargs):
179
181
  file_path = config_data["raw_path"]
180
182
  start_time = time.time()
@@ -191,3 +193,14 @@ def compute_single_file(config_data, **kwargs):
191
193
  end_time = time.time()
192
194
  total_time = end_time - start_time
193
195
  print(f"Total time taken: {total_time:.2f} seconds")
196
+
197
+
198
+ def get_chunk_store(storage_config, path):
199
+ if storage_config['storage_type'] == 'azure':
200
+ from adlfs import AzureBlobFileSystem
201
+ azfs = AzureBlobFileSystem(**storage_config['storage_options'])
202
+
203
+ return azfs.get_mapper(f"{storage_config['container_name']}/{path}")
204
+
205
+ else:
206
+ raise ValueError(f"Unsupported storage type: {storage_config['storage_type']}")
@@ -0,0 +1,159 @@
1
+ from pathlib import Path
2
+ from rich.console import Console
3
+ from rich.table import Table
4
+ from rich import box
5
+ import pandas as pd
6
+ from IPython.display import display, HTML
7
+ import re
8
+ from datetime import datetime, timedelta
9
+
10
+
11
+ split_by_day_pattern = r'D(\d{8})-T(\d{6})'
12
+
13
+
14
+ def extract_timestamp_from_filename(file_name):
15
+ """Extract creation time from the file name if it follows a specific pattern."""
16
+ pattern = r'(\d{4}[A-Z])?-D(\d{8})-T(\d{6})'
17
+ match = re.search(pattern, file_name)
18
+ if match:
19
+ date_str = match.group(2)
20
+ time_str = match.group(3)
21
+ creation_time = datetime.strptime(date_str + time_str, '%Y%m%d%H%M%S')
22
+ return creation_time
23
+
24
+ return None
25
+
26
+
27
+ def extract_timestamp_str(filename):
28
+ """Extract timestamp from the filename."""
29
+ match = re.search(r'D(\d{8})-T(\d{6})', filename)
30
+ if match:
31
+ date_str, time_str = match.groups()
32
+ return f"{date_str}T{time_str}"
33
+ return None
34
+
35
+
36
+ def parse_filename(filename):
37
+ """Parse the date and time from the filename."""
38
+ match = re.search(r'D(\d{8})-T(\d{6})', filename)
39
+ if match:
40
+ date_str, time_str = match.groups()
41
+ return datetime.strptime(f"{date_str} {time_str}", "%Y%m%d %H%M%S")
42
+ return None
43
+
44
+
45
+ def sort_raw_files(raw_files):
46
+ return sorted(raw_files, key=lambda x: extract_timestamp_str(Path(x).stem))
47
+
48
+
49
+ def group_files_by_day(files, extract_filename_from_object=lambda x: x['Key']):
50
+ """Group files by day based on the parsed date from the filename."""
51
+ files_by_day = {}
52
+ for file in files:
53
+ date_str, _ = re.search(split_by_day_pattern, extract_filename_from_object(file)).groups()
54
+ day = date_str
55
+ if day not in files_by_day:
56
+ files_by_day[day] = []
57
+ files_by_day[day].append(file)
58
+
59
+ for day in files_by_day:
60
+ files_by_day[day] = sorted(files_by_day[day], key=lambda x: parse_filename(x['Key']))
61
+
62
+ return files_by_day
63
+
64
+
65
+ def _print_survey_summary(files_by_day):
66
+ """Print summary of files by day."""
67
+ print("Summary of files by day:")
68
+ for day, files in files_by_day.items():
69
+ print(f"Day: {day}, Number of files: {len(files)}")
70
+ print(f"Total number of days: {len(files_by_day)}")
71
+
72
+
73
+ def print_survey_summary(files_by_day, show_in_terminal=True, show_in_notebook=False, show_file_size=False):
74
+ """Print summary of files by day in terminal and Jupyter notebook."""
75
+ # Calculate statistics
76
+ statistics = calculate_survey_statistics(files_by_day)
77
+
78
+ if show_in_terminal:
79
+ # Create a summary using rich for terminal display
80
+ console = Console()
81
+ table = Table(title="Survey Summary", box=box.ROUNDED, show_header=True, header_style="bold magenta")
82
+
83
+ # Add columns to the table
84
+ table.add_column("Day", justify="center")
85
+ table.add_column("Number of Files", justify="center")
86
+ table.add_column("Total Recorded Time (HH:MM:SS)", justify="center")
87
+ if show_file_size:
88
+ table.add_column("Average File Size (MB)", justify="center")
89
+
90
+ # Add rows to the table
91
+ for day, stats in statistics.items():
92
+ row = [
93
+ datetime.strptime(day, "%Y%m%d").strftime("%b %d, %Y"),
94
+ str(stats["Number of Files"]),
95
+ stats["Total Recorded Time"]
96
+ ]
97
+ if show_file_size:
98
+ row.append(str(stats["Average File Size (MB)"]))
99
+ table.add_row(*row)
100
+
101
+ # Display the table in the terminal
102
+ console.print(table)
103
+
104
+ if show_in_notebook:
105
+ # Create a summary using pandas for Jupyter notebook display
106
+ df = pd.DataFrame.from_dict(statistics, orient='index')
107
+ df.index = pd.to_datetime(df.index, format='%Y%m%d').strftime('%b %d, %Y')
108
+
109
+ if not show_file_size:
110
+ df = df.drop(columns=["Average File Size (MB)"])
111
+
112
+ display(HTML("<h2>Survey Summary</h2>"))
113
+ display(df.style.set_table_styles(
114
+ [{
115
+ 'selector': 'thead th',
116
+ 'props': [('background-color', 'magenta'), ('color', 'white'), ('font-weight', 'bold')]
117
+ }, {
118
+ 'selector': 'tbody td',
119
+ 'props': [('border', '1px solid black')]
120
+ }, {
121
+ 'selector': 'table',
122
+ 'props': [('border-collapse', 'collapse'), ('width', '100%')]
123
+ }]
124
+ ).set_properties(**{'text-align': 'center'}))
125
+
126
+
127
+ def calculate_survey_statistics(files_by_day):
128
+ """Calculate the total recorded time and average file size per day."""
129
+ statistics = {}
130
+ for day, files in files_by_day.items():
131
+ total_time = timedelta()
132
+ previous_time = None
133
+ total_size = sum(file['Size'] for file in files)
134
+ avg_size = total_size / len(files) if files else 0
135
+
136
+ for file in files:
137
+ current_time = parse_filename(file['Key'])
138
+ if previous_time:
139
+ time_diff = current_time - previous_time
140
+ if time_diff > timedelta(minutes=10): # More than 10 minutes indicates a stop
141
+ total_time += timedelta(minutes=10)
142
+ else:
143
+ total_time += time_diff
144
+ previous_time = current_time
145
+
146
+ statistics[day] = {
147
+ "Number of Files": len(files),
148
+ "Total Recorded Time": format_duration(total_time),
149
+ "Average File Size (MB)": round(avg_size / (1024 * 1024), 2)
150
+ }
151
+ return statistics
152
+
153
+
154
+ def format_duration(duration):
155
+ """Format a timedelta duration as HH:MM:SS."""
156
+ total_seconds = int(duration.total_seconds())
157
+ hours, remainder = divmod(total_seconds, 3600)
158
+ minutes, seconds = divmod(remainder, 60)
159
+ return f"{hours:02}:{minutes:02}:{seconds:02}"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oceanstream
3
- Version: 0.0.10.dev0
3
+ Version: 0.0.12.dev0
4
4
  Summary: OceanStream: process raw sonar data at scale
5
5
  Home-page: https://oceanstream.io
6
6
  Author: Pineview Labs
@@ -62,6 +62,12 @@ oceanstream/process/folder_processor.py
62
62
  oceanstream/process/process.py
63
63
  oceanstream/process/processed_data_io.py
64
64
  oceanstream/process/requirements.txt
65
+ oceanstream/process/utils.py
66
+ oceanstream/process/aws/__init__.py
67
+ oceanstream/process/aws/s3.py
68
+ oceanstream/process/azure/__init__.py
69
+ oceanstream/process/azure/azure_processor.py
70
+ oceanstream/process/azure/blob_storage.py
65
71
  oceanstream/settings/__init__.py
66
72
  oceanstream/settings/config.py
67
73
  oceanstream/settings/defaults.config.json
@@ -1,22 +1,21 @@
1
1
  dask[array,distributed]==2024.5.2
2
2
  xarray
3
3
  numpy
4
+ pydantic
5
+ rich
6
+ tqdm
4
7
 
5
8
  [cli]
6
9
  typer
7
- rich
8
10
 
9
11
  [complete]
10
12
  typer
11
- rich
12
- pydantic
13
13
  matplotlib
14
14
  cmocean
15
15
 
16
16
  [denoise]
17
17
 
18
18
  [echodata]
19
- pydantic
20
19
 
21
20
  [exports]
22
21
 
@@ -1,4 +1,7 @@
1
1
  dask[array,distributed]==2024.5.2
2
2
  xarray
3
3
  numpy
4
+ pydantic
5
+ rich
6
+ tqdm
4
7
  git+https://github.com/OceanStreamIO/echopype-dev.git@oceanstream#egg=echopype
@@ -1,6 +1,6 @@
1
1
  [metadata]
2
2
  name = oceanstream
3
- version = 0.0.10-dev
3
+ version = 0.0.12-dev
4
4
  description = OceanStream: process raw sonar data at scale
5
5
  long_description = file: README.md
6
6
  long_description_content_type = text/markdown
@@ -29,9 +29,7 @@ setup_requires =
29
29
  [options.extras_require]
30
30
  cli =
31
31
  typer
32
- rich
33
32
  echodata =
34
- pydantic
35
33
  plot =
36
34
  matplotlib
37
35
  cmocean
@@ -1,3 +0,0 @@
1
- from .core import compute_sv, combine, convert, process_raw_file
2
-
3
- __all__ = ["compute_sv", "combine", "convert", "process_raw_file"]
@@ -1,2 +0,0 @@
1
- typer
2
- rich