water-column-sonar-processing 25.1.7__py3-none-any.whl → 25.3.0__py3-none-any.whl

water_column_sonar_processing/processing/batch_downloader.py

@@ -1,10 +1,13 @@
-import xarray as xr
+from typing import Optional
+
 import numpy as np
 import pandas as pd
+import xarray as xr
 import xbatcher
-from typing import Optional
+
 # s3fs.core.setup_logging("DEBUG")
 
+
 class BatchDownloader:
     """
     Uses the xbatcher XbatchDownloader to download data from an xarray dataset. Connection
@@ -12,13 +15,13 @@ class BatchDownloader:
     """
 
     def __init__(
-            self,
-            bucket_name: Optional[str] = "noaa-wcsd-zarr-pds",
-            ship_name: Optional[str] = "Henry_B._Bigelow",
-            cruise_name: Optional[str] = "HB0707",
-            sensor_name: Optional[str] = "EK60",
-            patch_dims: Optional[int] = 64, # TODO: change to 64
-            # input_steps: Optional[int] = 3,
+        self,
+        bucket_name: Optional[str] = "noaa-wcsd-zarr-pds",
+        ship_name: Optional[str] = "Henry_B._Bigelow",
+        cruise_name: Optional[str] = "HB0707",
+        sensor_name: Optional[str] = "EK60",
+        patch_dims: Optional[int] = 64,  # TODO: change to 64
+        # input_steps: Optional[int] = 3,
     ):
         self.bucket_name = bucket_name
         self.ship_name = ship_name
@@ -28,7 +31,7 @@ class BatchDownloader:
 
     # TODO: move this to the s3fs module
     def get_s3_zarr_store(self) -> xr.Dataset:
-        """ Returns an Xarray Dataset """
+        """Returns an Xarray Dataset"""
         s3_zarr_store_path = f"{self.bucket_name}/level_2/{self.ship_name}/{self.cruise_name}/{self.sensor_name}/{self.cruise_name}.zarr"
         # Info about the HB0707 cruise:
         #   Time: ["2007-07-11T18:20:33.657573888", "2007-07-11T18:20:53.657573888", "2007-07-13T00:55:17.454448896"]
@@ -40,7 +43,9 @@ class BatchDownloader:
         # store = s3fs.S3Map(root=s3_zarr_store_path, s3=s3_file_system, check=False)
 
         # return xr.open_zarr(store=f"s3://{s3_zarr_store_path}", consolidated=True, storage_options={'anon': True})
-        return xr.open_dataset(f"s3://{s3_zarr_store_path}", engine="zarr", storage_options={'anon': True})
+        return xr.open_dataset(
+            f"s3://{s3_zarr_store_path}", engine="zarr", storage_options={"anon": True}
+        )
         # return xr.open_zarr(store, consolidated=True)
 
     def get_toy_batch_generator(self) -> xbatcher.BatchGenerator:
@@ -48,14 +53,12 @@ class BatchDownloader:
         Returns a BatchGenerator with subsets of Sv data
         Note: this is synthetic data, for a smaller toy example
         """
-        depth = np.arange(1, 21) # N meters
-        time = pd.date_range(start="2025-01-01", end="2025-01-31", freq='D') # N days
-        frequency = [1_000, 2_000, 3_000] # N frequencies
-        Sv = np.random.rand(len(depth), len(time), len(frequency)) # synthetic data
+        depth = np.arange(1, 21)  # N meters
+        time = pd.date_range(start="2025-01-01", end="2025-01-31", freq="D")  # N days
+        frequency = [1_000, 2_000, 3_000]  # N frequencies
+        Sv = np.random.rand(len(depth), len(time), len(frequency))  # synthetic data
         cruise = xr.Dataset(
-            data_vars={
-                "Sv": (["depth", "time", "frequency"], Sv)
-            },
+            data_vars={"Sv": (["depth", "time", "frequency"], Sv)},
             coords={
                 "depth": depth,
                 "time": time,
@@ -66,28 +69,45 @@ class BatchDownloader:
         batch_generator = xbatcher.BatchGenerator(
             ds=cruise,
             # get samples that are shaped 10x10x3
-            input_dims={ 'depth': 10, 'time': 10, 'frequency': cruise.frequency.shape[0] }, # A dictionary specifying the size of the inputs in each dimension, e.g. ``{'lat': 30, 'lon': 30}`` These are the dimensions the ML library will see. All other dimensions will be stacked into one dimension called ``sample``.
+            input_dims={
+                "depth": 10,
+                "time": 10,
+                "frequency": cruise.frequency.shape[0],
+            },  # A dictionary specifying the size of the inputs in each dimension, e.g. ``{'lat': 30, 'lon': 30}`` These are the dimensions the ML library will see. All other dimensions will be stacked into one dimension called ``sample``.
             # no overlap between samples
-            input_overlap={ 'depth': 0, 'time': 0, 'frequency': 0 }, # Zero means no overlap. A dictionary specifying the overlap along each dimension
+            input_overlap={
+                "depth": 0,
+                "time": 0,
+                "frequency": 0,
+            },  # Zero means no overlap. A dictionary specifying the overlap along each dimension
         )
         return batch_generator
 
     def get_s3_batch_generator(self) -> xbatcher.BatchGenerator:
-        """ Returns a BatchGenerator with subsets of Sv data from s3 Zarr store """
+        """Returns a BatchGenerator with subsets of Sv data from s3 Zarr store"""
         cruise = self.get_s3_zarr_store()
 
         # TODO: temporarily limits to a smaller slice of the data
-        cruise_select = (cruise
-            .where(cruise.depth < 100., drop=True)
-            .sel(time=slice("2007-07-11T18:20:33", "2007-07-11T18:20:53"))
+        cruise_select = (
+            cruise.where(cruise.depth < 100.0, drop=True).sel(
+                time=slice("2007-07-11T18:20:33", "2007-07-11T18:20:53")
+            )
             # .sel(time=slice("2007-07-11T18:20:00", "2007-07-11T19:20:00"))
         )
-        print(cruise_select.Sv.shape) # (526 depth, 21 time, 4 freq)
+        print(cruise_select.Sv.shape)  # (526 depth, 21 time, 4 freq)
 
         batch_generator = xbatcher.BatchGenerator(
             ds=cruise_select,
-            input_dims={ 'depth': 10, 'time': 10, 'frequency': cruise.frequency.shape[0] }, # A dictionary specifying the size of the inputs in each dimension, e.g. ``{'lat': 30, 'lon': 30}`` These are the dimensions the ML library will see. All other dimensions will be stacked into one dimension called ``sample``.
-            input_overlap={ 'depth': 0, 'time': 0, 'frequency': 0 }, # Zero means no overlap. A dictionary specifying the overlap along each dimension
+            input_dims={
+                "depth": 10,
+                "time": 10,
+                "frequency": cruise.frequency.shape[0],
+            },  # A dictionary specifying the size of the inputs in each dimension, e.g. ``{'lat': 30, 'lon': 30}`` These are the dimensions the ML library will see. All other dimensions will be stacked into one dimension called ``sample``.
+            input_overlap={
+                "depth": 0,
+                "time": 0,
+                "frequency": 0,
+            },  # Zero means no overlap. A dictionary specifying the overlap along each dimension
             preload_batch=False,
         )
 
@@ -104,15 +124,22 @@ class BatchDownloader:
         cruise = self.get_s3_zarr_store()
 
         # TODO: temporarily limits to a smaller slice of the data
-        cruise_select = (cruise
-            .where(cruise.depth < 100., drop=True)
-            .sel(time=slice("2007-07-11T18:20:33", "2007-07-11T18:20:53"))
+        cruise_select = cruise.where(cruise.depth < 100.0, drop=True).sel(
+            time=slice("2007-07-11T18:20:33", "2007-07-11T18:20:53")
         )
-        print(cruise_select.Sv.shape) # (526 depth, 21 time, 4 freq)
+        print(cruise_select.Sv.shape)  # (526 depth, 21 time, 4 freq)
         batch_generator = xbatcher.BatchGenerator(
             ds=cruise_select,
-            input_dims={ 'depth': 10, 'time': 10, 'frequency': cruise.frequency.shape[0] }, # A dictionary specifying the size of the inputs in each dimension, e.g. ``{'lat': 30, 'lon': 30}`` These are the dimensions the ML library will see. All other dimensions will be stacked into one dimension called ``sample``.
-            input_overlap={ 'depth': 0, 'time': 0, 'frequency': 0 }, # Zero means no overlap. A dictionary specifying the overlap along each dimension
+            input_dims={
+                "depth": 10,
+                "time": 10,
+                "frequency": cruise.frequency.shape[0],
+            },  # A dictionary specifying the size of the inputs in each dimension, e.g. ``{'lat': 30, 'lon': 30}`` These are the dimensions the ML library will see. All other dimensions will be stacked into one dimension called ``sample``.
+            input_overlap={
+                "depth": 0,
+                "time": 0,
+                "frequency": 0,
+            },  # Zero means no overlap. A dictionary specifying the overlap along each dimension
             preload_batch=True,
         )
 
@@ -121,12 +148,10 @@ class BatchDownloader:
         return batch_generator
         # https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_generator
 
-"""
-(105, 21, 4)
 
-depth-start: 0.1899999976158142, depth-end: 1.899999976158142
-time-start: 2007-07-11T18:20:33.657573888, time-end: 2007-07-11T18:20:42.657573888
-frequency-start: 18000.0, frequency-end: 200000.0
-(10, 10, 4)
-np.nanmean: -53.70000076293945
-"""
+# (105, 21, 4)
+# depth-start: 0.1899999976158142, depth-end: 1.899999976158142
+# time-start: 2007-07-11T18:20:33.657573888, time-end: 2007-07-11T18:20:42.657573888
+# frequency-start: 18000.0, frequency-end: 200000.0
+# (10, 10, 4)
+# np.nanmean: -53.70000076293945

water_column_sonar_processing/processing/raw_to_zarr.py

@@ -1,25 +1,26 @@
 import gc
 import os
+from datetime import datetime
+from pathlib import Path  # , PurePath
+
 import echopype as ep
 import numcodecs
 import numpy as np
 from numcodecs import Blosc
-from datetime import datetime
-from pathlib import Path # , PurePath
 
 from water_column_sonar_processing.aws import DynamoDBManager, S3Manager
 from water_column_sonar_processing.geometry import GeometryManager
-from water_column_sonar_processing.utility import Cleaner, PipelineStatus
+from water_column_sonar_processing.utility import Cleaner
 
 
 # This code is getting copied from echofish-aws-raw-to-zarr-lambda
 class RawToZarr:
     #######################################################
     def __init__(
-            self,
-            # output_bucket_access_key,
-            # output_bucket_secret_access_key,
-            # # overwrite_existing_zarr_store,
+        self,
+        # output_bucket_access_key,
+        # output_bucket_secret_access_key,
+        # # overwrite_existing_zarr_store,
     ):
         # TODO: revert to Blosc.BITSHUFFLE, troubleshooting misc error
         self.__compressor = Blosc(cname="zstd", clevel=2)  # shuffle=Blosc.NOSHUFFLE
@@ -33,45 +34,47 @@ class RawToZarr:
     ############################################################################
     ############################################################################
     def __zarr_info_to_table(
-            self,
-            output_bucket_name,
-            table_name,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            file_name,
-            zarr_path,
-            min_echo_range,
-            max_echo_range,
-            num_ping_time_dropna,
-            start_time,
-            end_time,
-            frequencies,
-            channels
+        self,
+        # output_bucket_name,
+        table_name,
+        ship_name,
+        cruise_name,
+        sensor_name,
+        file_name,
+        # zarr_path,
+        min_echo_range,
+        max_echo_range,
+        num_ping_time_dropna,
+        start_time,
+        end_time,
+        frequencies,
+        channels,
+        water_level,
     ):
-        print('Writing Zarr information to DynamoDB table.')
+        print("Writing Zarr information to DynamoDB table.")
         dynamodb_manager = DynamoDBManager()
         dynamodb_manager.update_item(
             table_name=table_name,
             key={
-                'FILE_NAME': {'S': file_name},  # Partition Key
-                'CRUISE_NAME': {'S': cruise_name},  # Sort Key
+                "FILE_NAME": {"S": file_name},  # Partition Key
+                "CRUISE_NAME": {"S": cruise_name},  # Sort Key
             },
             expression_attribute_names={
-                '#CH': 'CHANNELS',
-                '#ET': 'END_TIME',
+                "#CH": "CHANNELS",
+                "#ET": "END_TIME",
                 # "#ED": "ERROR_DETAIL",
-                '#FR': 'FREQUENCIES',
-                '#MA': 'MAX_ECHO_RANGE',
-                '#MI': 'MIN_ECHO_RANGE',
-                '#ND': 'NUM_PING_TIME_DROPNA',
-                "#PS": "PIPELINE_STATUS",
+                "#FR": "FREQUENCIES",
+                "#MA": "MAX_ECHO_RANGE",
+                "#MI": "MIN_ECHO_RANGE",
+                "#ND": "NUM_PING_TIME_DROPNA",
+                # "#PS": "PIPELINE_STATUS",
                 "#PT": "PIPELINE_TIME",
                 "#SE": "SENSOR_NAME",
                 "#SH": "SHIP_NAME",
-                '#ST': 'START_TIME',
-                '#ZB': 'ZARR_BUCKET',
-                '#ZP': 'ZARR_PATH',
+                "#ST": "START_TIME",
+                # "#ZB": "ZARR_BUCKET",
+                # "#ZP": "ZARR_PATH",
+                "#WL": "WATER_LEVEL",
             },
             expression_attribute_values={
                 ":ch": {"L": [{"S": i} for i in channels]},
@@ -82,13 +85,14 @@ class RawToZarr:
                 ":mi": {"N": str(np.round(min_echo_range, 4))},
                 ":nd": {"N": str(num_ping_time_dropna)},
                 # ":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
-                ":ps": {"S": PipelineStatus.LEVEL_1_PROCESSING.name},
+                # ":ps": {"S": PipelineStatus.LEVEL_1_PROCESSING.name},
                 ":pt": {"S": datetime.now().isoformat(timespec="seconds") + "Z"},
                 ":se": {"S": sensor_name},
                 ":sh": {"S": ship_name},
                 ":st": {"S": start_time},
-                ":zb": {"S": output_bucket_name},
-                ":zp": { "S": zarr_path },
+                ":wl": {"N": str(np.round(water_level, 2))},
+                # ":zb": {"S": output_bucket_name},
+                # ":zp": {"S": zarr_path},
             },
             update_expression=(
                 "SET "
@@ -99,30 +103,31 @@ class RawToZarr:
                 "#MA = :ma, "
                 "#MI = :mi, "
                 "#ND = :nd, "
-                "#PS = :ps, "
+                # "#PS = :ps, "
                 "#PT = :pt, "
                 "#SE = :se, "
                 "#SH = :sh, "
                 "#ST = :st, "
-                "#ZB = :zb, "
-                "#ZP = :zp"
+                "#WL = :wl"
+                # "#ZB = :zb, "
+                # "#ZP = :zp"
             ),
         )
-        print('Done writing Zarr information to DynamoDB table.')
+        print("Done writing Zarr information to DynamoDB table.")
 
     ############################################################################
     ############################################################################
     ############################################################################
     def __upload_files_to_output_bucket(
-            self,
-            output_bucket_name,
-            local_directory,
-            object_prefix,
-            endpoint_url,
+        self,
+        output_bucket_name,
+        local_directory,
+        object_prefix,
+        endpoint_url,
     ):
         # Note: this will be passed credentials if using NODD
         s3_manager = S3Manager(endpoint_url=endpoint_url)
-        print('Uploading files using thread pool executor.')
+        print("Uploading files using thread pool executor.")
         all_files = []
         for subdir, dirs, files in os.walk(local_directory):
             for file in files:
@@ -138,38 +143,50 @@ class RawToZarr:
 
     ############################################################################
     def raw_to_zarr(
-            self,
-            table_name,
-            input_bucket_name,
-            output_bucket_name,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            raw_file_name,
-            endpoint_url=None,
-            include_bot=True,
+        self,
+        table_name,
+        input_bucket_name,
+        output_bucket_name,
+        ship_name,
+        cruise_name,
+        sensor_name,
+        raw_file_name,
+        endpoint_url=None,
+        include_bot=True,
     ):
         """
         Downloads the raw files, processes them with echopype, writes geojson, and uploads files
         to the nodd bucket.
         """
-        print(f'Opening raw: {raw_file_name} and creating zarr store.')
+        print(f"Opening raw: {raw_file_name} and creating zarr store.")
         geometry_manager = GeometryManager()
         cleaner = Cleaner()
-        cleaner.delete_local_files(file_types=["*.zarr", "*.json"]) # TODO: include bot and raw?
+        cleaner.delete_local_files(
+            file_types=["*.zarr", "*.json"]
+        )  # TODO: include bot and raw?
 
         s3_manager = S3Manager(endpoint_url=endpoint_url)
-        s3_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{raw_file_name}"
+        s3_file_path = (
+            f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{raw_file_name}"
+        )
         bottom_file_name = f"{Path(raw_file_name).stem}.bot"
-        s3_bottom_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{bottom_file_name}"
-        s3_manager.download_file(bucket_name=input_bucket_name, key=s3_file_path, file_name=raw_file_name)
+        s3_bottom_file_path = (
+            f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{bottom_file_name}"
+        )
+        s3_manager.download_file(
+            bucket_name=input_bucket_name, key=s3_file_path, file_name=raw_file_name
+        )
         # TODO: add the bottom file
         if include_bot:
-            s3_manager.download_file(bucket_name=input_bucket_name, key=s3_bottom_file_path, file_name=bottom_file_name)
+            s3_manager.download_file(
+                bucket_name=input_bucket_name,
+                key=s3_bottom_file_path,
+                file_name=bottom_file_name,
+            )
 
         try:
             gc.collect()
-            print('Opening raw file with echopype.')
+            print("Opening raw file with echopype.")
             # s3_file_path = f"s3://{bucket_name}/data/raw/{ship_name}/{cruise_name}/{sensor_name}/{file_name}"
             # s3_file_path = Path(f"s3://noaa-wcsd-pds/data/raw/{ship_name}/{cruise_name}/{sensor_name}/{file_name}")
             echodata = ep.open_raw(
@@ -180,14 +197,20 @@ class RawToZarr:
                 # max_chunk_size=300,
                 # storage_options={'anon': True } # 'endpoint_url': self.endpoint_url} # this was creating problems
             )
-            print('Compute volume backscattering strength (Sv) from raw data.')
+            print("Compute volume backscattering strength (Sv) from raw data.")
             ds_sv = ep.calibrate.compute_Sv(echodata)
+            ds_sv = ep.consolidate.add_depth(
+                ds_sv, echodata
+            )  # TODO: consolidate with other depth values
+            water_level = ds_sv["water_level"].values
             gc.collect()
-            print('Done computing volume backscatter strength (Sv) from raw data.')
+            print("Done computing volume backscatter strength (Sv) from raw data.")
             # Note: detected_seafloor_depth is located at echodata.vendor.detected_seafloor_depth
             # but is not written out with ds_sv
             if "detected_seafloor_depth" in list(echodata.vendor.variables):
-                ds_sv["detected_seafloor_depth"] = echodata.vendor.detected_seafloor_depth
+                ds_sv["detected_seafloor_depth"] = (
+                    echodata.vendor.detected_seafloor_depth
+                )
             #
             frequencies = echodata.environment.frequency_nominal.values
             #################################################################
@@ -200,10 +223,12 @@ class RawToZarr:
                 sensor_name=sensor_name,
                 file_name=raw_file_name,
                 endpoint_url=endpoint_url,
-                write_geojson=True
+                write_geojson=True,
             )
             ds_sv = ep.consolidate.add_location(ds_sv, echodata)
-            ds_sv.latitude.values = lat # overwriting echopype gps values to include missing values
+            ds_sv.latitude.values = (
+                lat  # overwriting echopype gps values to include missing values
+            )
             ds_sv.longitude.values = lon
             # gps_data, lat, lon = self.__get_gps_data(echodata=echodata)
             #################################################################
@@ -216,8 +241,12 @@ class RawToZarr:
             # This is the number of missing values found throughout the lat/lon
             num_ping_time_dropna = lat[~np.isnan(lat)].shape[0]  # symmetric to lon
             #
-            start_time = np.datetime_as_string(ds_sv.ping_time.values[0], unit='ms') + "Z"
-            end_time = np.datetime_as_string(ds_sv.ping_time.values[-1], unit='ms') + "Z"
+            start_time = (
+                np.datetime_as_string(ds_sv.ping_time.values[0], unit="ms") + "Z"
+            )
+            end_time = (
+                np.datetime_as_string(ds_sv.ping_time.values[-1], unit="ms") + "Z"
+            )
             channels = list(ds_sv.channel.values)
             #
             #################################################################
@@ -225,7 +254,9 @@ class RawToZarr:
             store_name = f"{Path(raw_file_name).stem}.zarr"
             # Sv = ds_sv.Sv
             # ds_sv['Sv'] = Sv.astype('int32', copy=False)
-            ds_sv.to_zarr(store=store_name) # ds_sv.Sv.sel(channel=ds_sv.channel.values[0]).shape
+            ds_sv.to_zarr(
+                store=store_name
+            )  # ds_sv.Sv.sel(channel=ds_sv.channel.values[0]).shape
             gc.collect()
             #################################################################
             output_zarr_prefix = f"level_1/{ship_name}/{cruise_name}/{sensor_name}/"
@@ -237,7 +268,9 @@ class RawToZarr:
                 sub_prefix=f"level_1/{ship_name}/{cruise_name}/{sensor_name}/{Path(raw_file_name).stem}.zarr",
             )
             if len(child_objects) > 0:
-                print('Zarr store data already exists in s3, deleting existing and continuing.')
+                print(
+                    "Zarr store data already exists in s3, deleting existing and continuing."
+                )
                 s3_manager.delete_nodd_objects(
                     bucket_name=output_bucket_name,
                     objects=child_objects,
@@ -247,37 +280,42 @@ class RawToZarr:
                 output_bucket_name=output_bucket_name,
                 local_directory=store_name,
                 object_prefix=output_zarr_prefix,
-                endpoint_url=endpoint_url
+                endpoint_url=endpoint_url,
             )
             #################################################################
             self.__zarr_info_to_table(
-                output_bucket_name=output_bucket_name,
+                # output_bucket_name=output_bucket_name,
                 table_name=table_name,
                 ship_name=ship_name,
                 cruise_name=cruise_name,
                 sensor_name=sensor_name,
                 file_name=raw_file_name,
-                zarr_path=os.path.join(output_zarr_prefix, store_name),
+                # zarr_path=os.path.join(output_zarr_prefix, store_name),
                 min_echo_range=min_echo_range,
                 max_echo_range=max_echo_range,
                 num_ping_time_dropna=num_ping_time_dropna,
                 start_time=start_time,
                 end_time=end_time,
                 frequencies=frequencies,
-                channels=channels
+                channels=channels,
+                water_level=water_level,
             )
             #######################################################################
             # TODO: verify count of objects matches, publish message, update status
             #######################################################################
-            print('Finished raw-to-zarr conversion.')
+            print("Finished raw-to-zarr conversion.")
         except Exception as err:
-            print(f'Exception encountered creating local Zarr store with echopype: {err}')
+            print(
+                f"Exception encountered creating local Zarr store with echopype: {err}"
+            )
             raise RuntimeError(f"Problem creating local Zarr store, {err}")
         finally:
             gc.collect()
             print("Finally.")
-            cleaner.delete_local_files(file_types=["*.raw", "*.bot", "*.zarr", "*.json"])
-        print('Done creating local zarr store.')
+            cleaner.delete_local_files(
+                file_types=["*.raw", "*.bot", "*.zarr", "*.json"]
+            )
+        print("Done creating local zarr store.")
 
     ############################################################################
     # TODO: does this get called?
@@ -365,5 +403,6 @@ class RawToZarr:
 
     ############################################################################
 
+
 ################################################################################
 ############################################################################

water_column_sonar_processing/utility/constants.py

@@ -9,7 +9,16 @@ class Constants(Flag):
     # chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon data
     # NOTE: larger value here will speed up the TurfJS download of data in the UI
     # Problem interpolating the data: cannot reshape array of size 65536 into shape...
-    SPATIOTEMPORAL_CHUNK_SIZE = 16384 # e.g. int(2**14)
+    SPATIOTEMPORAL_CHUNK_SIZE = int(2**16) - 1024  # e.g. int(2**14)
+    # TODO: create test for SPATIOTEMPORAL_CHUNK_SIZE with requirement!
+
+    LEVEL_0 = "raw"
+    LEVEL_1 = "level_1"
+    LEVEL_2 = "level_2"
+    LEVEL_3 = "level_3"
+
+    EK60 = "EK60"  # TODO: use for "instrument"
+    EK80 = "EK80"
 
 
 class Coordinates(Enum):

water_column_sonar_processing/utility/pipeline_status.py

@@ -107,19 +107,15 @@ class PipelineStatus(Flag):
 # Status.LEVEL_1_PROCESSING.value < Status.LEVEL_2_PROCESSING.value
 
 # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cloudformation-stack.html
-"""
-CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
-ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
-DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
-UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
-UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
-UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
-REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
-IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
 
-failure - noun - 
-failed - verb - "verbs should be avoided"
-
-success - noun
-
-"""
+# CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
+# ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
+# DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
+# UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
+# UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
+# UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
+# REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
+# IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
+# failure - noun -
+# failed - verb - "verbs should be avoided"
+# success - noun