water-column-sonar-processing 0.0.9__py3-none-any.whl → 26.1.9__py3-none-any.whl

water_column_sonar_processing/model/zarr_manager.py

@@ -1,276 +1,633 @@
 import os
-import numcodecs
+from importlib import metadata
+from typing import Optional
+
 import numpy as np
 import xarray as xr
 import zarr
-from numcodecs import Blosc
-
-from water_column_sonar_processing.aws import S3FSManager
-from water_column_sonar_processing.utility import Constants
-from water_column_sonar_processing.utility import Timestamp
-from water_column_sonar_processing.utility import Coordinates
+from zarr.codecs import BloscCodec, BloscShuffle
+from zarr.core.group import Group
 
-numcodecs.blosc.use_threads = False
-numcodecs.blosc.set_nthreads(1)
+from water_column_sonar_processing.utility import Constants, Coordinates, Timestamp
 
+# https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html
+compressors = BloscCodec(
+    cname="zstd",
+    clevel=9,
+    shuffle=BloscShuffle.bitshuffle,
+)
 
-# TODO: when ready switch to version 3 of model spec
-# ZARR_V3_EXPERIMENTAL_API = 1
 
-
-# creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
+# creates the latlon dataset: foo = ep.consolidate.add_location(ds_Sv, echodata)
 class ZarrManager:
     #######################################################
     def __init__(
         self,
+        # endpoint_url: Optional[str] = None,
     ):
-        # TODO: revert to Blosc.BITSHUFFLE, troubleshooting misc error
-        self.__compressor = Blosc(cname="zstd", clevel=2)  # shuffle=Blosc.NOSHUFFLE
         self.__overwrite = True
-        self.__num_threads = numcodecs.blosc.get_nthreads()
-        self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
-        self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
+        self.key = os.environ.get("OUTPUT_BUCKET_ACCESS_KEY")
+        self.secret = os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY")
 
     #######################################################
+    @staticmethod
     def get_depth_values(
-        self,
-        min_echo_range: float = 1.0,  # minimum depth measured (zero non-inclusive) from whole cruise
-        max_echo_range: float = 100.0,  # maximum depth measured from whole cruise
-    ):
+        max_echo_range: float,  # maximum depth measured from whole cruise
+        cruise_min_epsilon: float = 0.20,  # delta subsequent measurements
+    ) -> np.ndarray[tuple]:
         # Gets the set of depth values that will be used when resampling and
-        # regridding the data to a cruise level model store.
-        # Note: returned values do not start at zero.
-        print("Getting depth values.")
-        all_cruise_depth_values = np.linspace(
-            start=min_echo_range,
-            stop=max_echo_range,
-            num=int(max_echo_range / min_echo_range) + 1,
+        # regridding the dataset to a cruise level model store.
+        # Note: returned values start at zero!
+        # For more info see here: https://echopype.readthedocs.io/en/stable/data-proc-additional.html
+        all_cruise_depth_values = np.linspace(  # TODO: PROBLEM HERE
+            start=0,  # start it at zero
+            stop=np.ceil(max_echo_range),  # round up
+            num=int(np.ceil(max_echo_range) / cruise_min_epsilon) + 1,
             endpoint=True,
         )
 
-        print("Done getting depth values.")
+        if np.any(np.isnan(all_cruise_depth_values)):
+            raise Exception("Problem depth values returned were NaN.")
+
         return all_cruise_depth_values.round(decimals=2)
 
     #######################################################
     def create_zarr_store(
         self,
-        path: str,
+        path: str,  # 'level_2/Henry_B._Bigelow/HB0707/EK60/HB0707.model/tmp/HB0707.zarr/.zattrs'
         ship_name: str,
         cruise_name: str,
         sensor_name: str,
-        frequencies: list,  # units in Hz
-        width: int,  # TODO: needs better name... "ping_time"
-        min_echo_range: float,  # smallest resolution in meters
+        frequencies: list,  # units in Hz, type(frequencies) == np.ndarray
+        width: int,
         max_echo_range: float,
         calibration_status: bool = False,  # Assume uncalibrated
     ) -> str:
-        print(
-            f"Creating local zarr_manager store at {cruise_name}.zarr for ship {ship_name}"
-        )
-
-        # There should be no repeated frequencies
-        assert len(frequencies) == len(set(frequencies))
-        # TODO: eventually switch coordinate to "channel"
-
-        print(f"Debugging number of threads: {self.__num_threads}")
-
-        zarr_path = f"{path}/{cruise_name}.zarr"
-        store = zarr.DirectoryStore(path=zarr_path, normalize_keys=False)
-        root = zarr.group(store=store, overwrite=self.__overwrite, cache_attrs=True)
-
-        #####################################################################
-        # --- Coordinate: Time --- #
-        # https://zarr.readthedocs.io/en/stable/spec/v2.html#data-type-encoding
-        root.create_dataset(
-            name=Coordinates.TIME.value,
-            data=np.repeat(0.0, width),
-            shape=width,
-            chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-                # Constants.TILE_SIZE.value,
-            #),  # TODO: the chunking scheme doesn't seem to be working here
-            dtype=np.dtype(Coordinates.TIME_DTYPE.value),
-            compressor=self.__compressor,
-            fill_value=np.nan,  # TODO: do i want nan's?
-            overwrite=self.__overwrite,
-        )
-
-        root.time.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
-
-        root.time.attrs["calendar"] = Coordinates.TIME_CALENDAR.value
-        root.time.attrs["units"] = Coordinates.TIME_UNITS.value
-        root.time.attrs["long_name"] = Coordinates.TIME_LONG_NAME.value
-        root.time.attrs["standard_name"] = Coordinates.TIME_STANDARD_NAME.value
-
-        #####################################################################
-        # --- Coordinate: Depth --- #
-        depth_values = self.get_depth_values(
-            min_echo_range=min_echo_range, max_echo_range=max_echo_range
-        )
-
-        root.create_dataset(
-            name=Coordinates.DEPTH.value,
-            # TODO: verify that these values are correct
-            data=depth_values,
-            shape=len(depth_values),
-            chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-            dtype=np.dtype(
-                Coordinates.DEPTH_DTYPE.value
-            ),  # float16 == 2 significant digits would be ideal
-            compressor=self.__compressor,
-            fill_value=np.nan,
-            overwrite=self.__overwrite,
-        )
-        # TODO: change to exception
-        assert not np.any(np.isnan(depth_values))
-
-        root.depth.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.DEPTH.value]
-
-        root.depth.attrs["long_name"] = Coordinates.DEPTH_LONG_NAME.value
-        root.depth.attrs["units"] = Coordinates.DEPTH_UNITS.value
-
-        #####################################################################
-        # --- Coordinate: Latitude --- #
-        root.create_dataset(
-            name=Coordinates.LATITUDE.value,
-            # data=np.repeat(0.0, width),
-            shape=width,
-            chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-            dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
-            compressor=self.__compressor,
-            fill_value=np.nan,
-            overwrite=self.__overwrite,
-        )
-
-        # Note: LATITUDE is indexed by TIME
-        root.latitude.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
-
-        root.latitude.attrs["long_name"] = Coordinates.LATITUDE_LONG_NAME.value
-        root.latitude.attrs["units"] = Coordinates.LATITUDE_UNITS.value
-
-        #####################################################################
-        # --- Coordinate: Longitude --- #
-        root.create_dataset(
-            name=Coordinates.LONGITUDE.value,
-            # data=np.repeat(0.0, width),  # root.longitude[:] = np.nan
-            shape=width,
-            chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-            dtype=np.dtype(Coordinates.LONGITUDE_DTYPE.value),
-            compressor=self.__compressor,
-            fill_value=np.nan,
-            overwrite=self.__overwrite,
-        )
-
-        # Note: LONGITUDE is indexed by TIME
-        root.longitude.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
-
-        root.longitude.attrs["long_name"] = Coordinates.LONGITUDE_LONG_NAME.value
-        root.longitude.attrs["units"] = Coordinates.LONGITUDE_UNITS.value
-
-        #####################################################################
-        # TODO: verify adding this variable for where the bottom was detected
-        # --- Coordinate: Bottom --- #
-        root.create_dataset(
-            name=Coordinates.BOTTOM.value,
-            data=np.repeat(0.0, width),  # root.longitude[:] = np.nan
-            shape=width,
-            chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-            dtype=np.dtype(Coordinates.BOTTOM_DTYPE.value),
-            compressor=self.__compressor,
-            fill_value=0.0,
-            overwrite=self.__overwrite,
-        )
-
-        # BOTTOM is indexed by TIME
-        root.bottom.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
-
-        root.bottom.attrs["long_name"] = Coordinates.BOTTOM_LONG_NAME.value
-        root.bottom.attrs["units"] = Coordinates.BOTTOM_UNITS.value
-
-        #####################################################################
-        # --- Coordinate: Frequency --- #
-        root.create_dataset(
-            name=Coordinates.FREQUENCY.value,
-            data=frequencies,
-            shape=len(frequencies),
-            chunks=len(frequencies),
-            dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value),
-            compressor=self.__compressor,
-            fill_value=0.0,
-            overwrite=self.__overwrite,
-        )
-
-        # TODO: best coordinate would be channel with str type
-        root.frequency.attrs["_ARRAY_DIMENSIONS"] = [
-            Coordinates.FREQUENCY.value
-        ]  # TODO: is this correct
-
-        root.frequency.attrs["long_name"] = Coordinates.FREQUENCY_LONG_NAME.value
-        root.frequency.attrs["standard_name"] = (
-            Coordinates.FREQUENCY_STANDARD_NAME.value
-        )
-        root.frequency.attrs["units"] = Coordinates.FREQUENCY_UNITS.value
-
-        #####################################################################
-        # --- Sv Data --- #
-        root.create_dataset(
-            name=Coordinates.SV.value,
-            shape=(len(depth_values), width, len(frequencies)),
-            chunks=(Constants.TILE_SIZE.value, Constants.TILE_SIZE.value, len(frequencies)),
-            dtype=np.dtype(
-                Coordinates.SV_DTYPE.value
-            ),  # TODO: try to experiment with 'float16'
-            compressor=self.__compressor,
-            fill_value=np.nan,
-            overwrite=self.__overwrite,
-        )
-
-        root.Sv.attrs["_ARRAY_DIMENSIONS"] = [
-            Coordinates.DEPTH.value,
-            Coordinates.TIME.value,
-            Coordinates.FREQUENCY.value,
-        ]
-
-        root.Sv.attrs["long_name"] = Coordinates.SV_LONG_NAME.value
-        root.Sv.attrs["units"] = Coordinates.SV_UNITS.value
-        root.Sv.attrs["tile_size"] = Constants.TILE_SIZE.value
-
-        #####################################################################
-        # --- Metadata --- #
-        root.attrs["ship_name"] = ship_name
-        root.attrs["cruise_name"] = cruise_name
-        root.attrs["sensor_name"] = sensor_name
-        #
-        root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
-        root.attrs["processing_software_version"] = (
-            "0.0.6"  # TODO: get programmatically
-        )
-        root.attrs["processing_software_time"] = Timestamp.get_timestamp()
-        #
-        root.attrs["calibration_status"] = calibration_status
-
-        zarr.consolidate_metadata(store)
-        #####################################################################
         """
-        # zzz = zarr.open('https://echofish-dev-master-118234403147-echofish-zarr-store.s3.us-west-2.amazonaws.com/GU1002_resample.zarr')
-        # zzz.time[0] = 1274979445.423
-        # Initialize all to origin time, will be overwritten late
+        Creates a new zarr store in a local temporary directory(?)
         """
-        return zarr_path
+        try:
+            print(f"Creating local zarr store, {cruise_name}.zarr for ship {ship_name}")
+            if len(frequencies) != len(set(frequencies)):
+                raise Exception(
+                    "Number of frequencies does not match number of channels"
+                )
+
+            zarr_path = f"{path}/{cruise_name}.zarr"
+            #####################################################################
+            frequencies = np.array(
+                frequencies, dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value)
+            )
+            #####################################################################
+            # Define the chunk sizes and the encoding
+            depth_chunk_shape = (Constants.TILE_SIZE.value,)
+            time_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            frequency_chunk_shape = (len(frequencies),)
+            latitude_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            longitude_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            bottom_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            speed_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            distance_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            sv_chunk_shape = (Constants.TILE_SIZE.value, Constants.TILE_SIZE.value, 1)
+            #####################################################################
+            root = zarr.create_group(store=zarr_path, zarr_format=3, overwrite=True)
+            #####################################################################
+            # --- Coordinate: Time --- #
+            # https://zarr.readthedocs.io/en/stable/spec/v2.html#data-type-encoding
+            #  "data_type": "int64", "fill_value": 0, "units": "nanoseconds since 1970-01-01", "calendar": "proleptic_gregorian"
+            #
+            time_values = np.repeat(0.0, width)
+            time_values.astype(np.dtype(Coordinates.TIME_DTYPE.value))
+            root.create_array(
+                name=Coordinates.TIME.value,
+                # shape=width_indices,
+                # dtype=np.dtype(Coordinates.TIME_DTYPE.value),
+                data=time_values,
+                chunks=time_chunk_shape,
+                compressor=compressors,
+                fill_value=np.nan,
+                attributes=dict(
+                    calendar=Coordinates.TIME_CALENDAR.value,
+                    units=Coordinates.TIME_UNITS.value,
+                    long_name=Coordinates.TIME_LONG_NAME.value,
+                    standard_name=Coordinates.TIME_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],
+                overwrite=True,
+            )
+            #####################################################################
+            #####################################################################
+            # # --- Coordinate: Depth --- #
+            depth_data_values = self.get_depth_values(
+                max_echo_range=max_echo_range,
+            )
+            depth_data = np.array(
+                depth_data_values, dtype=Coordinates.DEPTH_DTYPE.value
+            )
+            root.create_array(
+                name=Coordinates.DEPTH.value,
+                # shape=depth_indices,
+                # dtype=np.dtype(Coordinates.DEPTH_DTYPE.value),
+                data=depth_data,
+                chunks=depth_chunk_shape,
+                compressor=compressors,
+                # fill_value=np.nan,
+                attributes=dict(
+                    units=Coordinates.DEPTH_UNITS.value,
+                    long_name=Coordinates.DEPTH_LONG_NAME.value,
+                    standard_name=Coordinates.DEPTH_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.DEPTH.value],  # TODO: is this right
+                overwrite=True,
+            )
+            # #####################################################################
+            # # --- Coordinate: Latitude --- #
+            # latitude_values = np.rep(np.nan, width_indices)
+            # latitude_values.astype(np.dtype(Coordinates.LATITUDE_DTYPE.value))
+            root.create_array(
+                name=Coordinates.LATITUDE.value,
+                shape=width,
+                dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
+                # data=latitude_values,
+                chunks=latitude_chunk_shape,
+                compressor=compressors,
+                fill_value=np.nan,
+                attributes=dict(
+                    units=Coordinates.LATITUDE_UNITS.value,
+                    long_name=Coordinates.LATITUDE_LONG_NAME.value,
+                    standard_name=Coordinates.LATITUDE_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],
+                overwrite=True,
+            )
+            # #####################################################################
+            # # --- Coordinate: Longitude --- #
+            # longitude_values = np.arange(0, width_indices)
+            # longitude_values.astype(np.dtype(Coordinates.LONGITUDE_DTYPE.value))
+            root.create_array(
+                name=Coordinates.LONGITUDE.value,
+                shape=width,
+                dtype=np.dtype(Coordinates.LONGITUDE_DTYPE.value),
+                # data=longitude_values,
+                chunks=longitude_chunk_shape,
+                compressor=compressors,
+                fill_value=np.nan,
+                attributes=dict(
+                    units=Coordinates.LONGITUDE_UNITS.value,
+                    long_name=Coordinates.LONGITUDE_LONG_NAME.value,
+                    standard_name=Coordinates.LONGITUDE_STANDARD_NAME.value,
+                ),
+                dimension_names=[
+                    Coordinates.TIME.value
+                ],  # Note: LONGITUDE is indexed by TIME
+                overwrite=True,
+            )
+            # #####################################################################
+            # # --- Coordinate: Bottom --- #
+            # bottom_values = np.repeat(12.34, width_indices)
+            # bottom_values.astype(np.dtype(Coordinates.BOTTOM_DTYPE.value))
+            root.create_array(
+                name=Coordinates.BOTTOM.value,
+                shape=width,
+                dtype=np.dtype(Coordinates.BOTTOM_DTYPE.value),
+                # data=bottom_values,
+                chunks=bottom_chunk_shape,
+                compressor=compressors,
+                fill_value=np.nan,
+                attributes=dict(
+                    units=Coordinates.BOTTOM_UNITS.value,
+                    long_name=Coordinates.BOTTOM_LONG_NAME.value,
+                    standard_name=Coordinates.BOTTOM_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],  # Note: _ is indexed by TIME
+                overwrite=True,
+            )
+            # #####################################################################
+            # # --- Coordinate: Speed --- #
+            # speed_values = np.repeat(5.67, width_indices)
+            # speed_values.astype(np.dtype(Coordinates.SPEED_DTYPE.value))
+            root.create_array(
+                name=Coordinates.SPEED.value,
+                shape=width,
+                dtype=np.dtype(Coordinates.SPEED_DTYPE.value),
+                # data=speed_values,
+                chunks=speed_chunk_shape,
+                compressor=compressors,
+                fill_value=np.nan,
+                attributes=dict(
+                    units=Coordinates.SPEED_UNITS.value,
+                    long_name=Coordinates.SPEED_LONG_NAME.value,
+                    standard_name=Coordinates.SPEED_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],  # Note: _ is indexed by TIME
+                overwrite=True,
+            )
+            # #####################################################################
+            # # --- Coordinate: Distance --- #
+            # distance_values = np.repeat(8.90, width_indices)
+            # distance_values.astype(np.dtype(Coordinates.DISTANCE_DTYPE.value))
+            root.create_array(
+                name=Coordinates.DISTANCE.value,
+                shape=width,
+                dtype=np.dtype(Coordinates.DISTANCE_DTYPE.value),
+                # data=distance_values,
+                chunks=distance_chunk_shape,
+                compressor=compressors,
+                fill_value=np.nan,
+                attributes=dict(
+                    units=Coordinates.DISTANCE_UNITS.value,
+                    long_name=Coordinates.DISTANCE_LONG_NAME.value,
+                    standard_name=Coordinates.DISTANCE_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],  # Note: _ is indexed by TIME
+                overwrite=True,
+            )
+            # #####################################################################
+            # # --- Coordinate: Frequency --- #
+            root.create_array(
+                name=Coordinates.FREQUENCY.value,
+                # shape=frequency_indices,
+                # dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value),
+                data=frequencies,
+                # chunks=(Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,),
+                chunks=frequency_chunk_shape,
+                compressor=compressors,
+                # fill_value=0,
+                attributes=dict(
+                    units=Coordinates.FREQUENCY_UNITS.value,
+                    long_name=Coordinates.FREQUENCY_LONG_NAME.value,
+                    standard_name=Coordinates.FREQUENCY_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.FREQUENCY.value],
+                overwrite=True,
+            )
+            # #####################################################################
+            # # --- Sv Data --- #
+            root.create_array(
+                name=Coordinates.SV.value,
+                shape=(len(depth_data), width, len(frequencies)),
+                dtype=np.dtype(Coordinates.SV_DTYPE.value),
+                # data=,
+                chunks=sv_chunk_shape,
+                compressor=compressors,
+                fill_value=np.nan,
+                attributes=dict(
+                    units=Coordinates.SV_UNITS.value,
+                    long_name=Coordinates.SV_LONG_NAME.value,
+                    standard_name=Coordinates.SV_STANDARD_NAME.value,
+                ),
+                dimension_names=[
+                    Coordinates.DEPTH.value,
+                    Coordinates.TIME.value,
+                    Coordinates.FREQUENCY.value,
+                ],
+                overwrite=True,
+            )
+            #####################################################################
+            # # --- Metadata --- #
+            root.attrs["ship_name"] = ship_name
+            root.attrs["cruise_name"] = cruise_name
+            root.attrs["sensor_name"] = sensor_name
+            #
+            root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
+            # NOTE: for the version to be parsable you need to build the python package locally first.
+            root.attrs["processing_software_version"] = metadata.version(
+                "water-column-sonar-processing"
+            )
+            root.attrs["processing_software_time"] = Timestamp.get_timestamp()
+            #
+            root.attrs["calibration_status"] = calibration_status
+            root.attrs["tile_size"] = Constants.TILE_SIZE.value
+            #
+            return zarr_path
+        except Exception as err:
+            raise RuntimeError(f"Problem trying to create zarr store, {err}")
 
-    ############################################################################
-    # def update_zarr_store(
+    # #######################################################
+    # def create_zarr_store_old(
     #         self,
-    #         path: str,
+    #         path: str,  # 'level_2/Henry_B._Bigelow/HB0707/EK60/HB0707.model/tmp/HB0707.zarr/.zattrs'
     #         ship_name: str,
-    #         cruise_name: str,  # TODO: just pass stem
+    #         cruise_name: str,
     #         sensor_name: str,
-    # ) -> None:
+    #         frequencies: list,  # units in Hz
+    #         width: int,
+    #         max_echo_range: float,
+    #         # cruise_min_epsilon: float,  # smallest resolution in meters
+    #         calibration_status: bool = False,  # Assume uncalibrated
+    # ) -> str:
     #     """
-    #     Opens an existing Zarr store living in a s3 bucket for the purpose
-    #     of updating just a subset of the cruise-level Zarr store associated
-    #     with a file-level Zarr store.
+    #     Creates a new zarr store in a local temporary directory(?)
     #     """
-    #     pass
+    #     try:
+    #         print(f"Creating local zarr store, {cruise_name}.zarr for ship {ship_name}")
+    #         if len(frequencies) != len(set(frequencies)):
+    #             raise Exception(
+    #                 "Number of frequencies does not match number of channels"
+    #             )
+    #
+    #         zarr_path = f"{path}/{cruise_name}.zarr"
+    #         #####################################################################
+    #         # Define the chunk sizes and the encoding
+    #         # 1_000_000 data points for quickest download
+    #         spatiotemporal_chunk_size = int(1e6)
+    #         depth_chunk_shape = (512,)
+    #         time_chunk_shape = (spatiotemporal_chunk_size,)
+    #         frequency_chunk_shape = (len(frequencies),)
+    #         latitude_chunk_shape = (spatiotemporal_chunk_size,)
+    #         longitude_chunk_shape = (spatiotemporal_chunk_size,)
+    #         bottom_chunk_shape = (spatiotemporal_chunk_size,)
+    #         speed_chunk_shape = (spatiotemporal_chunk_size,)
+    #         distance_chunk_shape = (spatiotemporal_chunk_size,)
+    #         sv_chunk_shape = (512, 512, 1)  # TODO: move to constants
+    #
+    #         #####################################################################
+    #         ##### Depth #####
+    #         depth_data_values = self.get_depth_values(
+    #             max_echo_range=max_echo_range,
+    #         )
+    #
+    #         depth_data = np.array(
+    #             depth_data_values, dtype=Coordinates.DEPTH_DTYPE.value
+    #         )
+    #         depth_da = xr.DataArray(
+    #             data=depth_data,
+    #             dims=Coordinates.DEPTH.value,
+    #             name=Coordinates.DEPTH.value,
+    #             attrs=dict(
+    #                 units=Coordinates.DEPTH_UNITS.value,
+    #                 long_name=Coordinates.DEPTH_LONG_NAME.value,
+    #                 standard_name=Coordinates.DEPTH_STANDARD_NAME.value,
+    #             ),
+    #         )
+    #
+    #         ##### Time #####
+    #         # https://zarr.readthedocs.io/en/stable/spec/v2.html#data-type-encoding
+    #         time_data = np.array(
+    #             np.repeat(np.datetime64(0, "ns"), width),
+    #             dtype="datetime64[ns]",
+    #         )
+    #         time_da = xr.DataArray(
+    #             data=time_data,
+    #             dims=Coordinates.TIME.value,
+    #             name=Coordinates.TIME.value,
+    #             attrs=dict(
+    #                 # Note: cal & units are written automatically by xarray
+    #                 # calendar="proleptic_gregorian",
+    #                 # units="seconds since 1970-01-01 00:00:00",
+    #                 long_name=Coordinates.TIME_LONG_NAME.value,
+    #                 standard_name=Coordinates.TIME_STANDARD_NAME.value,
+    #             ),
+    #         )
+    #
+    #         ##### Frequency #####
+    #         frequency_data = np.array(
+    #             frequencies,
+    #             dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value),
+    #         )
+    #         frequency_da = xr.DataArray(
+    #             data=frequency_data,
+    #             dims=Coordinates.FREQUENCY.value,
+    #             name=Coordinates.FREQUENCY.value,
+    #             attrs=dict(
+    #                 units=Coordinates.FREQUENCY_UNITS.value,
+    #                 long_name=Coordinates.FREQUENCY_LONG_NAME.value,
+    #                 standard_name=Coordinates.FREQUENCY_STANDARD_NAME.value,
+    #             ),
+    #         )
+    #
+    #         ##### Latitude #####
+    #         gps_data = np.array(
+    #             np.repeat(np.nan, width),
+    #             dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
+    #         )
+    #         latitude_da = xr.DataArray(
+    #             data=gps_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
+    #             name=Coordinates.LATITUDE.value,
+    #             attrs=dict(
+    #                 units=Coordinates.LATITUDE_UNITS.value,
+    #                 long_name=Coordinates.LATITUDE_LONG_NAME.value,
+    #                 standard_name=Coordinates.LATITUDE_STANDARD_NAME.value,
+    #             ),
+    #         )  # Note: LATITUDE is indexed by TIME
+    #
+    #         ##### Longitude #####
+    #         longitude_da = xr.DataArray(
+    #             data=gps_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
+    #             name=Coordinates.LONGITUDE.value,
+    #             attrs=dict(
+    #                 units=Coordinates.LONGITUDE_UNITS.value,
+    #                 long_name=Coordinates.LONGITUDE_LONG_NAME.value,
+    #                 standard_name=Coordinates.LONGITUDE_STANDARD_NAME.value,
+    #             ),
+    #         )  # Note: LONGITUDE is indexed by TIME
+    #
+    #         ##### Bottom #####
+    #         bottom_data = np.array(
+    #             np.repeat(np.nan, width), dtype=np.dtype(Coordinates.BOTTOM_DTYPE.value)
+    #         )
+    #         bottom_da = xr.DataArray(
+    #             data=bottom_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
+    #             name=Coordinates.BOTTOM.value,
+    #             attrs=dict(
+    #                 units=Coordinates.BOTTOM_UNITS.value,
+    #                 long_name=Coordinates.BOTTOM_LONG_NAME.value,
+    #                 standard_name=Coordinates.BOTTOM_STANDARD_NAME.value,
+    #             ),
+    #         )
+    #
+    #         ##### Speed #####
+    #         speed_data = np.array(
+    #             np.repeat(np.nan, width), dtype=np.dtype(Coordinates.SPEED_DTYPE.value)
+    #         )
+    #         speed_da = xr.DataArray(
+    #             data=speed_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
+    #             name=Coordinates.SPEED.value,
+    #             attrs=dict(
+    #                 units=Coordinates.SPEED_UNITS.value,
+    #                 long_name=Coordinates.SPEED_LONG_NAME.value,
+    #                 standard_name=Coordinates.SPEED_STANDARD_NAME.value,
+    #             ),
+    #         )
+    #
+    #         ##### Distance #####
+    #         distance_data = np.array(
+    #             np.repeat(np.nan, width),
+    #             dtype=np.dtype(Coordinates.DISTANCE_DTYPE.value),
+    #         )
+    #         distance_da = xr.DataArray(
+    #             data=distance_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
+    #             name=Coordinates.DISTANCE.value,
+    #             attrs=dict(
+    #                 units=Coordinates.DISTANCE_UNITS.value,
+    #                 long_name=Coordinates.DISTANCE_LONG_NAME.value,
+    #                 standard_name=Coordinates.DISTANCE_STANDARD_NAME.value,
+    #             ),
+    #         )
+    #
+    #         ##### Sv #####
+    #         gc.collect()
+    #         # sv_data = np.empty(
+    #         #     (len(depth_data), width, len(frequencies)),
+    #         #     # (2501, 4_100_782, 4), # large cruise used for testing
+    #         #     dtype=np.dtype(Coordinates.SV_DTYPE.value),
+    #         # )
+    #         sv_data = np.full(
+    #             (len(depth_data), width, len(frequencies)),
+    #             np.nan,
+    #             dtype=np.dtype(Coordinates.SV_DTYPE.value),
+    #         )
+    #         print(f"one: {sys.getsizeof(sv_data)}")
+    #         # sv_data[:] = np.nan  # initialize all
+    #
+    #         sv_da = xr.DataArray(
+    #             data=sv_data,
+    #             coords=dict(
+    #                 depth=depth_da,
+    #                 time=time_da,
+    #                 frequency=frequency_da,
+    #                 #
+    #                 latitude=latitude_da,
+    #                 longitude=longitude_da,
+    #                 bottom=bottom_da,
+    #                 speed=speed_da,
+    #                 distance=distance_da,
+    #             ),
+    #             dims=(  # Depth * Time * Frequency
+    #                 Coordinates.DEPTH.value,
+    #                 Coordinates.TIME.value,
+    #                 Coordinates.FREQUENCY.value,
+    #             ),
+    #             name=Coordinates.SV.value,
+    #             attrs=dict(
+    #                 units=Coordinates.SV_UNITS.value,
+    #                 long_name=Coordinates.SV_LONG_NAME.value,
+    #                 standard_name=Coordinates.SV_STANDARD_NAME.value,
+    #                 tiles_size=Constants.TILE_SIZE.value,
+    #                 _FillValue=np.nan,
+    #             ),
+    #         )
+    #         print(f"two: {sys.getsizeof(sv_data)}")  # getting to at least here
+    #         del sv_data
+    #         sv_da.encoding = {"compressors": [compressor], "chunks": sv_chunk_shape}
+    #         # sv_da = sv_da.astype(np.float32)  # was crashing here
+    #         gc.collect()
+    #         #####################################################################
+    #         ### Now create the xarray.Dataset
+    #         ds = xr.Dataset(
+    #             data_vars=dict(
+    #                 Sv=sv_da,
+    #                 #
+    #                 bottom=bottom_da,
+    #                 speed=speed_da,
+    #                 distance=distance_da,
+    #             ),
+    #             coords=dict(
+    #                 depth=depth_da,
+    #                 time=time_da,
+    #                 frequency=frequency_da,
+    #                 #
+    #                 latitude=latitude_da,
+    #                 longitude=longitude_da,
+    #             ),
+    #             attrs=dict(
+    #                 # --- Metadata --- #
+    #                 ship_name=ship_name,
+    #                 cruise_name=cruise_name,
+    #                 sensor_name=sensor_name,
+    #                 processing_software_name=Coordinates.PROJECT_NAME.value,
+    #                 # NOTE: for the version to be parsable you need to build the python package
+    #                 #  locally first.
+    #                 processing_software_version=importlib.metadata.version(
+    #                     "water-column-sonar-processing"
+    #                 ),
+    #                 processing_software_time=Timestamp.get_timestamp(),
+    #                 calibration_status=calibration_status,
+    #                 tile_size=Constants.TILE_SIZE.value,
+    #             ),
+    #         )
+    #         del sv_da
+    #         gc.collect()
+    #         print(f"three: {sys.getsizeof(ds)}")
+    #         #####################################################################
+    #         encodings = dict(
+    #             depth={
+    #                 "compressors": [compressor],
+    #                 "chunks": depth_chunk_shape,
+    #             },
+    #             time={
+    #                 "compressors": [compressor],
+    #                 "chunks": time_chunk_shape,
+    #                 "units": Coordinates.TIME_UNITS.value,
+    #             },
+    #             frequency={
+    #                 "compressors": [compressor],
+    #                 "chunks": frequency_chunk_shape,
+    #             },
+    #             latitude={
+    #                 "compressors": [compressor],
+    #                 "chunks": latitude_chunk_shape,
+    #             },
+    #             longitude={
+    #                 "compressors": [compressor],
+    #                 "chunks": longitude_chunk_shape,
+    #             },
+    #             bottom={
+    #                 "compressors": [compressor],
+    #                 "chunks": bottom_chunk_shape,
+    #             },
+    #             speed={
+    #                 "compressors": [compressor],
+    #                 "chunks": speed_chunk_shape,
+    #             },
+    #             distance={
+    #                 "compressors": [compressor],
+    #                 "chunks": distance_chunk_shape,
+    #             },
+    #             Sv={
+    #                 "compressors": [compressor],
+    #                 "chunks": sv_chunk_shape,
+    #             },
+    #         )
+    #         gc.collect()
+    #         ds.to_zarr(
+    #             store=zarr_path,
+    #             mode="w",  # “w” means create (overwrite if exists)
+    #             encoding=encodings,
+    #             consolidated=False,
+    #             safe_chunks=False,
+    #             align_chunks=True,
+    #             zarr_format=3,
+    #             write_empty_chunks=False,  # Might need to change this
+    #         )
+    #         #####################################################################
+    #         return zarr_path
+    #     except Exception as err:
+    #         raise RuntimeError(f"Problem trying to create zarr store, {err}")
+    #     # finally:
+    #     #     cleaner = Cleaner()
+    #     #     cleaner.delete_local_files()
+    #     # TODO: should delete zarr store in temp directory too?
 
     ############################################################################
     def open_s3_zarr_store_with_zarr(
@@ -278,60 +635,106 @@ class ZarrManager:
         ship_name: str,
         cruise_name: str,
         sensor_name: str,
-        # zarr_synchronizer: Union[str, None] = None,
-    ):
+        output_bucket_name: str,
+        endpoint_url: Optional[str] = None,
+    ) -> Group:
         # Mounts a Zarr store using pythons Zarr implementation. The mounted store
         #  will have read/write privileges so that store can be updated.
-        print("Opening Zarr store with Zarr.")
+        print("Opening L2 Zarr store with Zarr for writing.")
         try:
-            s3fs_manager = S3FSManager()
-            root = f"{self.output_bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr"
-            store = s3fs_manager.s3_map(s3_zarr_store_path=root)
-            # synchronizer = model.ProcessSynchronizer(f"/tmp/{ship_name}_{cruise_name}.sync")
-            cruise_zarr = zarr.open(store=store, mode="r+")
+            level = str(Constants.LEVEL_2.value)
+            store = f"s3://{output_bucket_name}/{level}/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr"
+            print(f"endpoint url: {endpoint_url}")
+            cruise_zarr = zarr.open(
+                store=store,
+                mode="r+",
+                zarr_format=3,
+                storage_options={
+                    "endpoint_url": endpoint_url,
+                    "key": self.key,
+                    "secret": self.secret,
+                },
+            )
+            print("Done opening store with Zarr.")
+            return cruise_zarr
         except Exception as err:  # Failure
-            print(f"Exception encountered opening Zarr store with Zarr.: {err}")
-            raise
-        print("Done opening Zarr store with Zarr.")
-        return cruise_zarr
+            raise RuntimeError(f"Exception encountered opening store with Zarr, {err}")
 
-    ############################################################################
+    ###########################################################################
+    @staticmethod
     def open_s3_zarr_store_with_xarray(
-        self,
         ship_name: str,
         cruise_name: str,
         sensor_name: str,
         file_name_stem: str,
+        bucket_name: str,
+        # level: str, # TODO: add level
+        endpoint_url: Optional[str] = None,  # needed for moto testing
     ) -> xr.Dataset:
-        print("Opening Zarr store in S3 as Xarray.")
+        print("Opening L1 Zarr store in S3 with Xarray.")
         try:
-            zarr_path = f"s3://{self.output_bucket_name}/level_1/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.zarr"
-            s3fs_manager = S3FSManager()
-            store_s3_map = s3fs_manager.s3_map(s3_zarr_store_path=zarr_path)
-            ds = xr.open_zarr(
-                store=store_s3_map, consolidated=None
-            )  # synchronizer=SYNCHRONIZER
+            zarr_path = f"s3://{bucket_name}/level_1/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.zarr"
+            kwargs = {"consolidated": False}
+            ds = xr.open_dataset(
+                filename_or_obj=zarr_path,
+                engine="zarr",
+                backend_kwargs={
+                    "storage_options": {
+                        "endpoint_url": endpoint_url,
+                        "anon": True,
+                    },
+                },
+                **kwargs,
+            )
+            return ds
         except Exception as err:
-            print("Problem opening Zarr store in S3 as Xarray.")
-            raise err
-        print("Done opening Zarr store in S3 as Xarray.")
-        return ds
+            raise RuntimeError(f"Problem opening Zarr store in S3 as Xarray, {err}")
 
-    ############################################################################
+    ###########################################################################
+    # TODO: can this be consolidated with above
+    @staticmethod
+    def open_l2_zarr_store_with_xarray(
+        ship_name: str,
+        cruise_name: str,
+        sensor_name: str,
+        bucket_name: str,
+        endpoint_url: Optional[str] = None,  # needed for moto testing
+    ) -> xr.Dataset:
+        print("Opening L2 Zarr store in S3 with Xarray.")
+        try:
+            level = str(Constants.LEVEL_2.value)
+            zarr_path = f"s3://{bucket_name}/{level}/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr"
+            kwargs = {"consolidated": False}
+            ds = xr.open_dataset(
+                filename_or_obj=zarr_path,
+                engine="zarr",
+                backend_kwargs={
+                    "storage_options": {
+                        "endpoint_url": endpoint_url,
+                        "anon": True,
+                    }
+                },
+                **kwargs,
+            )
+            return ds
+        except Exception as err:
+            raise RuntimeError(f"Problem opening Zarr store in S3 as Xarray, {err}")
 
-    #######################################################
+    ###########################################################################
+
+    ###########################################################################
     # def create_process_synchronizer(self):
     #     # TODO: explore aws redis options
     #     pass
 
-    #######################################################
+    ###########################################################################
     # def verify_cruise_store_data(self):
     #     # TODO: run a check on a finished model store to ensure that
     #     #   none of the time, latitude, longitude, or depth values
     #     #   are NaN.
     #     pass
 
-    #######################################################
+    ###########################################################################
 
 
 ###########################################################