water-column-sonar-processing 25.11.1__py3-none-any.whl → 26.1.14__py3-none-any.whl

water_column_sonar_processing/model/zarr_manager.py

@@ -1,18 +1,21 @@
-import importlib.metadata
+import os
+from importlib import metadata
+from typing import Optional
 
 import numpy as np
 import xarray as xr
 import zarr
 from zarr.codecs import BloscCodec, BloscShuffle
-from zarr.storage import LocalStore
+from zarr.core.group import Group
 
-from water_column_sonar_processing.aws import S3FSManager
 from water_column_sonar_processing.utility import Constants, Coordinates, Timestamp
 
-# TODO: change clevel to 9?!
-compressor = BloscCodec(cname="zstd", clevel=9, shuffle=BloscShuffle.shuffle)
-
-# TODO: when ready switch to version 3 of model spec
+# https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html
+compressors = BloscCodec(
+    cname="zstd",
+    clevel=9,
+    shuffle=BloscShuffle.bitshuffle,
+)
 
 
 # creates the latlon dataset: foo = ep.consolidate.add_location(ds_Sv, echodata)
@@ -20,33 +23,32 @@ class ZarrManager:
     #######################################################
     def __init__(
         self,
+        # endpoint_url: Optional[str] = None,
     ):
         self.__overwrite = True
+        self.key = os.environ.get("OUTPUT_BUCKET_ACCESS_KEY")
+        self.secret = os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY")
 
     #######################################################
+    @staticmethod
     def get_depth_values(
-        self,
-        # min_echo_range: float,  # minimum depth measured (zero non-inclusive) from whole cruise
         max_echo_range: float,  # maximum depth measured from whole cruise
-        cruise_min_epsilon: float = 0.25,  # resolution between subsequent measurements
-    ):  # TODO: define return type
+        cruise_min_epsilon: float = 0.20,  # delta subsequent measurements
+    ) -> np.ndarray[tuple]:
         # Gets the set of depth values that will be used when resampling and
         # regridding the dataset to a cruise level model store.
         # Note: returned values start at zero!
         # For more info see here: https://echopype.readthedocs.io/en/stable/data-proc-additional.html
-        print("Computing depth values.")
         all_cruise_depth_values = np.linspace(  # TODO: PROBLEM HERE
-            start=0,  # just start it at zero
-            stop=max_echo_range,
-            num=int(max_echo_range / cruise_min_epsilon)
-            + 1,  # int(np.ceil(max_echo_range / cruise_min_epsilon))?
+            start=0,  # start it at zero
+            stop=np.ceil(max_echo_range),  # round up
+            num=int(np.ceil(max_echo_range) / cruise_min_epsilon) + 1,
             endpoint=True,
-        )  # np.arange(min_echo_range, max_echo_range, step=min_echo_range) # this is worse
+        )
 
         if np.any(np.isnan(all_cruise_depth_values)):
             raise Exception("Problem depth values returned were NaN.")
 
-        print("Done computing depth values.")
         return all_cruise_depth_values.round(decimals=2)
 
     #######################################################
@@ -56,667 +58,684 @@ class ZarrManager:
         ship_name: str,
         cruise_name: str,
         sensor_name: str,
-        frequencies: list,  # units in Hz
-        width: int,  # TODO: needs better name... "ping_time"
-        # min_echo_range: float,
+        frequencies: list,  # units in Hz, type(frequencies) == np.ndarray
+        width: int,
         max_echo_range: float,
-        cruise_min_epsilon: float,  # smallest resolution in meters
         calibration_status: bool = False,  # Assume uncalibrated
     ) -> str:
+        """
+        Creates a new zarr store in a local temporary directory(?)
+        This includes the water_level on top of the max_echo_range already, nothing extra needs to be done.
+        """
         try:
-            # TODO: problem throwing exceptions here
-            print(
-                f"Creating local zarr_manager store at {cruise_name}.zarr for ship {ship_name}"
-            )
-            # There can not currently be repeated frequencies
-            # TODO: eventually switch coordinate to "channel" because frequencies can repeat
+            print(f"Creating local zarr store, {cruise_name}.zarr for ship {ship_name}")
             if len(frequencies) != len(set(frequencies)):
                 raise Exception(
                     "Number of frequencies does not match number of channels"
                 )
 
             zarr_path = f"{path}/{cruise_name}.zarr"
-            # store = zarr.DirectoryStore(path=zarr_path, normalize_keys=False)
-            ### https://zarr.readthedocs.io/en/latest/user-guide/groups/ ###
-            # store = zarr.group(path=zarr_path)
-            store = LocalStore(root=zarr_path)
-            root = zarr.group(
-                store=store,  # zarr_path,
-                overwrite=self.__overwrite,  # cache_attrs=True
-                zarr_format=3,
+            #####################################################################
+            frequencies = np.array(
+                frequencies, dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value)
             )
-
+            #####################################################################
+            # Define the chunk sizes and the encoding
+            depth_chunk_shape = (Constants.TILE_SIZE.value,)
+            time_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            frequency_chunk_shape = (len(frequencies),)
+            latitude_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            longitude_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            bottom_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            speed_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            distance_chunk_shape = (Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,)
+            sv_chunk_shape = (Constants.TILE_SIZE.value, Constants.TILE_SIZE.value, 1)
+            #####################################################################
+            root = zarr.create_group(store=zarr_path, zarr_format=3, overwrite=True)
             #####################################################################
             # --- Coordinate: Time --- #
             # https://zarr.readthedocs.io/en/stable/spec/v2.html#data-type-encoding
-            time_data = np.repeat(0.0, width)
-            time_data.astype(np.dtype(Coordinates.TIME_DTYPE.value), copy=False)
-
-            time = root.create_array(  # deprecated: Use Group.create_array instead.
+            #  "data_type": "int64", "fill_value": 0, "units": "nanoseconds since 1970-01-01", "calendar": "proleptic_gregorian"
+            #
+            time_values = np.repeat(0.0, width)
+            time_values.astype(np.dtype(Coordinates.TIME_DTYPE.value))
+            root.create_array(
                 name=Coordinates.TIME.value,
-                data=time_data,
-                # shape=width,
-                chunks=(Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,),
+                # shape=width_indices,
                 # dtype=np.dtype(Coordinates.TIME_DTYPE.value),
-                compressors=compressor,
+                data=time_values,
+                chunks=time_chunk_shape,
+                compressors=compressors,
                 fill_value=np.nan,
-                overwrite=self.__overwrite,
-                dimension_names=(Coordinates.TIME.value,),
+                attributes=dict(
+                    calendar=Coordinates.TIME_CALENDAR.value,
+                    units=Coordinates.TIME_UNITS.value,
+                    long_name=Coordinates.TIME_LONG_NAME.value,
+                    standard_name=Coordinates.TIME_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],
+                overwrite=True,
             )
-
-            # time.metadata.dimension_names = (Coordinates.TIME.value,)
-
-            time.attrs["calendar"] = Coordinates.TIME_CALENDAR.value
-            time.attrs["units"] = Coordinates.TIME_UNITS.value
-            time.attrs["long_name"] = Coordinates.TIME_LONG_NAME.value
-            time.attrs["standard_name"] = Coordinates.TIME_STANDARD_NAME.value
-
             #####################################################################
-            # --- Coordinate: Depth --- #
-            depth_data = self.get_depth_values(
-                # min_echo_range=min_echo_range,
+            #####################################################################
+            # # --- Coordinate: Depth --- #
+            depth_data_values = self.get_depth_values(
                 max_echo_range=max_echo_range,
-                cruise_min_epsilon=cruise_min_epsilon,
             )
             depth_data = np.array(
-                depth_data, dtype=np.dtype(Coordinates.DEPTH_DTYPE.value)
+                depth_data_values, dtype=Coordinates.DEPTH_DTYPE.value
             )
-
-            depth = root.create_array(
+            root.create_array(
                 name=Coordinates.DEPTH.value,
-                # TODO: verify that these values are correct
+                # shape=depth_indices,
+                # dtype=np.dtype(Coordinates.DEPTH_DTYPE.value),
                 data=depth_data,
-                # shape=len(depth_values),
-                chunks=(Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,),
-                # dtype=np.dtype(
-                #     Coordinates.DEPTH_DTYPE.value
-                # ),  # float16 == 2 significant digits would be ideal
-                compressors=compressor,
-                fill_value=np.nan,
-                overwrite=self.__overwrite,
-                dimension_names=(Coordinates.DEPTH.value,),
-            )
-
-            if np.any(np.isnan(depth_data)):
-                raise Exception("Some depth values returned were NaN.")
-
-            # depth.metadata.dimension_names = (Coordinates.DEPTH.value,)
-
-            depth.attrs["units"] = Coordinates.DEPTH_UNITS.value
-            depth.attrs["long_name"] = Coordinates.DEPTH_LONG_NAME.value
-            depth.attrs["standard_name"] = Coordinates.DEPTH_STANDARD_NAME.value
-
-            #####################################################################
-            # --- Coordinate: Latitude --- #
-            gps_data = np.array(
-                np.repeat(np.nan, width),
-                dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
+                chunks=depth_chunk_shape,
+                compressors=compressors,
+                # fill_value=np.nan,
+                attributes=dict(
+                    units=Coordinates.DEPTH_UNITS.value,
+                    long_name=Coordinates.DEPTH_LONG_NAME.value,
+                    standard_name=Coordinates.DEPTH_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.DEPTH.value],  # TODO: is this right
+                overwrite=True,
             )
-
-            latitude = root.create_array(
+            # #####################################################################
+            # # --- Coordinate: Latitude --- #
+            # latitude_values = np.rep(np.nan, width_indices)
+            # latitude_values.astype(np.dtype(Coordinates.LATITUDE_DTYPE.value))
+            root.create_array(
                 name=Coordinates.LATITUDE.value,
-                # dataset=np.repeat(0.0, width),  # root.longitude[:] = np.nan
-                data=gps_data,
-                # shape=width,
-                chunks=(Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,),
-                # dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
-                compressors=compressor,
+                shape=width,
+                dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
+                # data=latitude_values,
+                chunks=latitude_chunk_shape,
+                compressors=compressors,
                 fill_value=np.nan,
-                overwrite=self.__overwrite,
-                dimension_names=(Coordinates.TIME.value,),
+                attributes=dict(
+                    units=Coordinates.LATITUDE_UNITS.value,
+                    long_name=Coordinates.LATITUDE_LONG_NAME.value,
+                    standard_name=Coordinates.LATITUDE_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],
+                overwrite=True,
             )
-
-            # Note: LATITUDE is indexed by TIME
-            # latitude.metadata.dimension_names = (Coordinates.TIME.value,)
-
-            latitude.attrs["units"] = Coordinates.LATITUDE_UNITS.value
-            latitude.attrs["long_name"] = Coordinates.LATITUDE_LONG_NAME.value
-            latitude.attrs["standard_name"] = Coordinates.LATITUDE_STANDARD_NAME.value
-
-            #####################################################################
-            # --- Coordinate: Longitude --- #
-            longitude = root.create_array(
+            # #####################################################################
+            # # --- Coordinate: Longitude --- #
+            # longitude_values = np.arange(0, width_indices)
+            # longitude_values.astype(np.dtype(Coordinates.LONGITUDE_DTYPE.value))
+            root.create_array(
                 name=Coordinates.LONGITUDE.value,
-                # dataset=np.repeat(0.0, width),  # root.longitude[:] = np.nan
-                data=gps_data,
-                # shape=width,
-                chunks=(Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,),
-                # dtype=np.dtype(Coordinates.LONGITUDE_DTYPE.value),
-                compressors=compressor,
+                shape=width,
+                dtype=np.dtype(Coordinates.LONGITUDE_DTYPE.value),
+                # data=longitude_values,
+                chunks=longitude_chunk_shape,
+                compressors=compressors,
                 fill_value=np.nan,
-                overwrite=self.__overwrite,
-                dimension_names=(Coordinates.TIME.value,),
-            )
-
-            # Note: LONGITUDE is indexed by TIME
-            # longitude.metadata.dimension_names = (Coordinates.TIME.value,)
-
-            longitude.attrs["units"] = Coordinates.LONGITUDE_UNITS.value
-            longitude.attrs["long_name"] = Coordinates.LONGITUDE_LONG_NAME.value
-            longitude.attrs["standard_name"] = Coordinates.LONGITUDE_STANDARD_NAME.value
-
-            #####################################################################
-            # TODO: verify adding this variable for where the bottom was detected
-            # --- Coordinate: Bottom --- #
-            bottom_data = np.array(
-                np.repeat(np.nan, width), dtype=np.dtype(Coordinates.BOTTOM_DTYPE.value)
+                attributes=dict(
+                    units=Coordinates.LONGITUDE_UNITS.value,
+                    long_name=Coordinates.LONGITUDE_LONG_NAME.value,
+                    standard_name=Coordinates.LONGITUDE_STANDARD_NAME.value,
+                ),
+                dimension_names=[
+                    Coordinates.TIME.value
+                ],  # Note: LONGITUDE is indexed by TIME
+                overwrite=True,
             )
-
-            bottom = root.create_array(
+            # #####################################################################
+            # # --- Coordinate: Bottom --- #
+            # bottom_values = np.repeat(12.34, width_indices)
+            # bottom_values.astype(np.dtype(Coordinates.BOTTOM_DTYPE.value))
+            root.create_array(
                 name=Coordinates.BOTTOM.value,
-                data=bottom_data,
-                # shape=width,
-                chunks=(Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,),
-                # dtype=np.dtype(Coordinates.BOTTOM_DTYPE.value),
-                compressors=compressor,
+                shape=width,
+                dtype=np.dtype(Coordinates.BOTTOM_DTYPE.value),
+                # data=bottom_values,
+                chunks=bottom_chunk_shape,
+                compressors=compressors,
                 fill_value=np.nan,
-                overwrite=self.__overwrite,
-                dimension_names=(Coordinates.TIME.value,),
+                attributes=dict(
+                    units=Coordinates.BOTTOM_UNITS.value,
+                    long_name=Coordinates.BOTTOM_LONG_NAME.value,
+                    standard_name=Coordinates.BOTTOM_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],  # Note: _ is indexed by TIME
+                overwrite=True,
             )
-
-            # BOTTOM is indexed by TIME
-            # bottom.metadata.dimension_names = (Coordinates.TIME.value,)
-
-            bottom.attrs["units"] = Coordinates.BOTTOM_UNITS.value
-            bottom.attrs["long_name"] = Coordinates.BOTTOM_LONG_NAME.value
-            bottom.attrs["standard_name"] = Coordinates.BOTTOM_STANDARD_NAME.value
-
-            #####################################################################
-            # TODO: verify adding this variable with test
-            # --- Coordinate: Speed --- #
-            speed_data = np.repeat(np.nan, width)
-            speed_data.astype(np.dtype(Coordinates.SPEED_DTYPE.value), copy=False)
-
-            speed = root.create_array(
+            # #####################################################################
+            # # --- Coordinate: Speed --- #
+            # speed_values = np.repeat(5.67, width_indices)
+            # speed_values.astype(np.dtype(Coordinates.SPEED_DTYPE.value))
+            root.create_array(
                 name=Coordinates.SPEED.value,
-                data=np.repeat(np.nan, width),  # root.longitude[:] = np.nan
-                # shape=width,
-                chunks=(Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,),
-                # dtype=np.dtype(Coordinates.SPEED_DTYPE.value),
-                compressors=compressor,
+                shape=width,
+                dtype=np.dtype(Coordinates.SPEED_DTYPE.value),
+                # data=speed_values,
+                chunks=speed_chunk_shape,
+                compressors=compressors,
                 fill_value=np.nan,
-                overwrite=self.__overwrite,
-                dimension_names=(Coordinates.TIME.value,),  # NOTE: 'TIME'
+                attributes=dict(
+                    units=Coordinates.SPEED_UNITS.value,
+                    long_name=Coordinates.SPEED_LONG_NAME.value,
+                    standard_name=Coordinates.SPEED_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],  # Note: _ is indexed by TIME
+                overwrite=True,
             )
-
-            # SPEED is indexed by TIME
-            # speed.metadata.dimension_names = (Coordinates.TIME.value,)
-
-            speed.attrs["units"] = Coordinates.SPEED_UNITS.value
-            speed.attrs["long_name"] = Coordinates.SPEED_LONG_NAME.value
-            speed.attrs["standard_name"] = Coordinates.SPEED_STANDARD_NAME.value
-
-            #####################################################################
-            # TODO: verify adding this variable with test
-            # --- Coordinate: Speed --- #
-            distance_data = np.repeat(np.nan, width)
-            distance_data.astype(np.dtype(Coordinates.DISTANCE_DTYPE.value), copy=False)
-
-            distance = root.create_array(
+            # #####################################################################
+            # # --- Coordinate: Distance --- #
+            # distance_values = np.repeat(8.90, width_indices)
+            # distance_values.astype(np.dtype(Coordinates.DISTANCE_DTYPE.value))
+            root.create_array(
                 name=Coordinates.DISTANCE.value,
-                data=np.repeat(np.nan, width),  # root.longitude[:] = np.nan
-                # shape=width,
-                chunks=(Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,),
-                # dtype=np.dtype(Coordinates.SPEED_DTYPE.value),
-                compressors=compressor,
+                shape=width,
+                dtype=np.dtype(Coordinates.DISTANCE_DTYPE.value),
+                # data=distance_values,
+                chunks=distance_chunk_shape,
+                compressors=compressors,
                 fill_value=np.nan,
-                overwrite=self.__overwrite,
-                dimension_names=(Coordinates.TIME.value,),  # NOTE: 'TIME'
-            )
-
-            # DISTANCE is indexed by TIME
-            # distance.metadata.dimension_names = (Coordinates.TIME.value,)
-
-            distance.attrs["units"] = Coordinates.DISTANCE_UNITS.value
-            distance.attrs["long_name"] = Coordinates.DISTANCE_LONG_NAME.value
-            distance.attrs["standard_name"] = Coordinates.DISTANCE_STANDARD_NAME.value
-
-            #####################################################################
-            # --- Coordinate: Frequency --- #
-            frequency_data = np.array(
-                frequencies, dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value)
+                attributes=dict(
+                    units=Coordinates.DISTANCE_UNITS.value,
+                    long_name=Coordinates.DISTANCE_LONG_NAME.value,
+                    standard_name=Coordinates.DISTANCE_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.TIME.value],  # Note: _ is indexed by TIME
+                overwrite=True,
             )
-            # frequency_data.astype(np.dtype(Coordinates.FREQUENCY_DTYPE.value), copy=False)
-
-            frequency = root.create_array(
+            # #####################################################################
+            # # --- Coordinate: Frequency --- #
+            root.create_array(
                 name=Coordinates.FREQUENCY.value,
-                data=frequency_data,
-                # shape=len(frequencies),
-                chunks=(len(frequencies),),
+                # shape=frequency_indices,
                 # dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value),
-                compressors=compressor,
-                fill_value=0.0,
-                overwrite=self.__overwrite,
-                dimension_names=(Coordinates.FREQUENCY.value,),
+                data=frequencies,
+                # chunks=(Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,),
+                chunks=frequency_chunk_shape,
+                compressors=compressors,
+                # fill_value=0,
+                attributes=dict(
+                    units=Coordinates.FREQUENCY_UNITS.value,
+                    long_name=Coordinates.FREQUENCY_LONG_NAME.value,
+                    standard_name=Coordinates.FREQUENCY_STANDARD_NAME.value,
+                ),
+                dimension_names=[Coordinates.FREQUENCY.value],
+                overwrite=True,
             )
-
-            # TODO: best coordinate would be channel with str type
-            # frequency.metadata.dimension_names = (Coordinates.FREQUENCY.value,)
-
-            frequency.attrs["units"] = Coordinates.FREQUENCY_UNITS.value
-            frequency.attrs["long_name"] = Coordinates.FREQUENCY_LONG_NAME.value
-            frequency.attrs["standard_name"] = Coordinates.FREQUENCY_STANDARD_NAME.value
-
-            #####################################################################
-            # --- Sv Data --- #
-            sv = root.create_array(
+            # #####################################################################
+            # # --- Sv Data --- #
+            root.create_array(
                 name=Coordinates.SV.value,
                 shape=(len(depth_data), width, len(frequencies)),
-                chunks=(
-                    Constants.TILE_SIZE.value,
-                    Constants.TILE_SIZE.value,
-                    1,
-                ),
                 dtype=np.dtype(Coordinates.SV_DTYPE.value),
-                compressors=compressor,
+                # data=,
+                chunks=sv_chunk_shape,
+                compressors=compressors,
                 fill_value=np.nan,
-                overwrite=self.__overwrite,
-                dimension_names=(
+                attributes=dict(
+                    units=Coordinates.SV_UNITS.value,
+                    long_name=Coordinates.SV_LONG_NAME.value,
+                    standard_name=Coordinates.SV_STANDARD_NAME.value,
+                ),
+                dimension_names=[
                     Coordinates.DEPTH.value,
                     Coordinates.TIME.value,
                     Coordinates.FREQUENCY.value,
-                ),
+                ],
+                overwrite=True,
             )
-            # sv.metadata.dimension_names = (
-            #     Coordinates.DEPTH.value,
-            #     Coordinates.TIME.value,
-            #     Coordinates.FREQUENCY.value,
-            # )
-            # sv.attrs["_ARRAY_DIMENSIONS"] = [
-            #     Coordinates.DEPTH.value,
-            #     Coordinates.TIME.value,
-            #     Coordinates.FREQUENCY.value,
-            # ]
-
-            sv.attrs["units"] = Coordinates.SV_UNITS.value
-            sv.attrs["long_name"] = Coordinates.SV_LONG_NAME.value
-            sv.attrs["tile_size"] = Constants.TILE_SIZE.value
-
             #####################################################################
-            # --- Metadata --- #
+            # # --- Metadata --- #
             root.attrs["ship_name"] = ship_name
             root.attrs["cruise_name"] = cruise_name
             root.attrs["sensor_name"] = sensor_name
             #
             root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
-
-            # NOTE: for the version to be parsable you need to build the python package
-            #  locally first.
-            current_project_version = importlib.metadata.version(
+            # NOTE: for the version to be parsable you need to build the python package locally first.
+            root.attrs["processing_software_version"] = metadata.version(
                 "water-column-sonar-processing"
             )
-            root.attrs["processing_software_version"] = current_project_version
             root.attrs["processing_software_time"] = Timestamp.get_timestamp()
             #
             root.attrs["calibration_status"] = calibration_status
             root.attrs["tile_size"] = Constants.TILE_SIZE.value
-
-            # TODO: ZarrUserWarning: Consolidated metadata is currently not part in the Zarr format 3 specification. It may not be supported by other zarr implementations and may change in the future.
-            # zarr.consolidate_metadata(zarr_path)
-            #####################################################################
-            """
-            # zzz = zarr.open('https://echofish-dev-master-118234403147-echofish-zarr-store.s3.us-west-2.amazonaws.com/GU1002_resample.zarr')
-            # zzz.time[0] = 1274979445.423
-            # Initialize all to origin time, will be overwritten late
-            """
+            #
             return zarr_path
         except Exception as err:
             raise RuntimeError(f"Problem trying to create zarr store, {err}")
-        # finally:
-        #     cleaner = Cleaner()
-        #     cleaner.delete_local_files()
-        # TODO: should delete zarr store in temp directory too?
 
-    #######################################################
-    #
-    # LEVEL 3 - LEVEL 3 - LEVEL 3 - LEVEL 3 # TODO: move to separate project for zarr 3?
-    #
-    # def create_zarr_store_level_3(
-    #     self,
-    #     path: str,  # 'level_2/Henry_B._Bigelow/HB0707/EK60/HB0707.model/tmp/HB0707.zarr/.zattrs'
-    #     ship_name: str,
-    #     cruise_name: str,
-    #     sensor_name: str,
-    #     frequencies: list,  # units in Hz
-    #     width: int,  # TODO: needs better name... "ping_time"
-    #     min_echo_range: float,  # smallest resolution in meters --> 1.0 meters
-    #     max_echo_range: float,
-    #     cruise_min_epsilon: float,
-    #     calibration_status: bool = False,  # Assume uncalibrated
+    # #######################################################
+    # def create_zarr_store_old(
+    #         self,
+    #         path: str,  # 'level_2/Henry_B._Bigelow/HB0707/EK60/HB0707.model/tmp/HB0707.zarr/.zattrs'
+    #         ship_name: str,
+    #         cruise_name: str,
+    #         sensor_name: str,
+    #         frequencies: list,  # units in Hz
+    #         width: int,
+    #         max_echo_range: float,
+    #         # cruise_min_epsilon: float,  # smallest resolution in meters
+    #         calibration_status: bool = False,  # Assume uncalibrated
     # ) -> str:
-    #     compressor = Blosc(cname="zstd", clevel=9, shuffle=1)
-    #     TILE_SIZE = 1024
+    #     """
+    #     Creates a new zarr store in a local temporary directory(?)
+    #     """
     #     try:
-    #         # TODO: problem throwing exceptions here
-    #         print(
-    #             f"Creating level 3 local zarr_manager store at {cruise_name}.zarr for ship {ship_name}"
-    #         )
+    #         print(f"Creating local zarr store, {cruise_name}.zarr for ship {ship_name}")
     #         if len(frequencies) != len(set(frequencies)):
     #             raise Exception(
     #                 "Number of frequencies does not match number of channels"
     #             )
     #
-    #         # print(f"Debugging number of threads: {self.__num_threads}")
-    #
     #         zarr_path = f"{path}/{cruise_name}.zarr"
-    #         store = zarr.DirectoryStore(path=zarr_path, normalize_keys=False)
-    #         root = zarr.group(store=store, overwrite=self.__overwrite, cache_attrs=True)
-    #
     #         #####################################################################
-    #         # --- Coordinate: Time --- #
-    #         # https://zarr.readthedocs.io/en/stable/spec/v2.html#data-type-encoding
-    #         time = root.create_array(
-    #             name=Coordinates.TIME.value,
-    #             data=np.repeat(0.0, width),
-    #             shape=width,
-    #             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-    #             dtype=np.dtype(Coordinates.TIME_DTYPE.value),
-    #             compressor=compressor,
-    #             # fill_value=np.nan,
-    #             overwrite=self.__overwrite,
-    #         )
-    #
-    #         time.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
-    #         time.attrs["calendar"] = Coordinates.TIME_CALENDAR.value
-    #         time.attrs["units"] = Coordinates.TIME_UNITS.value
-    #         time.attrs["long_name"] = Coordinates.TIME_LONG_NAME.value
-    #         time.attrs["standard_name"] = Coordinates.TIME_STANDARD_NAME.value
+    #         # Define the chunk sizes and the encoding
+    #         # 1_000_000 data points for quickest download
+    #         spatiotemporal_chunk_size = int(1e6)
+    #         depth_chunk_shape = (512,)
+    #         time_chunk_shape = (spatiotemporal_chunk_size,)
+    #         frequency_chunk_shape = (len(frequencies),)
+    #         latitude_chunk_shape = (spatiotemporal_chunk_size,)
+    #         longitude_chunk_shape = (spatiotemporal_chunk_size,)
+    #         bottom_chunk_shape = (spatiotemporal_chunk_size,)
+    #         speed_chunk_shape = (spatiotemporal_chunk_size,)
+    #         distance_chunk_shape = (spatiotemporal_chunk_size,)
+    #         sv_chunk_shape = (512, 512, 1)  # TODO: move to constants
     #
     #         #####################################################################
-    #         # --- Coordinate: Depth --- #
-    #         depth_values = self.get_depth_values(
-    #             # min_echo_range=min_echo_range,
+    #         ##### Depth #####
+    #         depth_data_values = self.get_depth_values(
     #             max_echo_range=max_echo_range,
-    #             cruise_min_epsilon=cruise_min_epsilon,
     #         )
     #
-    #         root.create_dataset(
+    #         depth_data = np.array(
+    #             depth_data_values, dtype=Coordinates.DEPTH_DTYPE.value
+    #         )
+    #         depth_da = xr.DataArray(
+    #             data=depth_data,
+    #             dims=Coordinates.DEPTH.value,
     #             name=Coordinates.DEPTH.value,
-    #             # TODO: verify that these values are correct
-    #             data=depth_values,
-    #             shape=len(depth_values),
-    #             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-    #             dtype=np.dtype(
-    #                 Coordinates.DEPTH_DTYPE.value  # TODO: convert to integers and only get whole number depths
-    #             ),  # float16 == 2 significant digits would be ideal
-    #             compressor=compressor,
-    #             # fill_value=np.nan,
-    #             overwrite=self.__overwrite,
+    #             attrs=dict(
+    #                 units=Coordinates.DEPTH_UNITS.value,
+    #                 long_name=Coordinates.DEPTH_LONG_NAME.value,
+    #                 standard_name=Coordinates.DEPTH_STANDARD_NAME.value,
+    #             ),
     #         )
     #
-    #         if np.any(np.isnan(depth_values)):
-    #             raise Exception("Some depth values returned were NaN.")
-    #
-    #         root.depth.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.DEPTH.value]
-    #         root.depth.attrs["units"] = Coordinates.DEPTH_UNITS.value
-    #         root.depth.attrs["long_name"] = Coordinates.DEPTH_LONG_NAME.value
-    #         root.depth.attrs["standard_name"] = Coordinates.DEPTH_STANDARD_NAME.value
+    #         ##### Time #####
+    #         # https://zarr.readthedocs.io/en/stable/spec/v2.html#data-type-encoding
+    #         time_data = np.array(
+    #             np.repeat(np.datetime64(0, "ns"), width),
+    #             dtype="datetime64[ns]",
+    #         )
+    #         time_da = xr.DataArray(
+    #             data=time_data,
+    #             dims=Coordinates.TIME.value,
+    #             name=Coordinates.TIME.value,
+    #             attrs=dict(
+    #                 # Note: cal & units are written automatically by xarray
+    #                 # calendar="proleptic_gregorian",
+    #                 # units="seconds since 1970-01-01 00:00:00",
+    #                 long_name=Coordinates.TIME_LONG_NAME.value,
+    #                 standard_name=Coordinates.TIME_STANDARD_NAME.value,
+    #             ),
+    #         )
     #
-    #         #####################################################################
-    #         # --- Coordinate: Latitude --- #
-    #         root.create_dataset(
-    #             name=Coordinates.LATITUDE.value,
-    #             # dataset=np.repeat(0.0, width),  # root.longitude[:] = np.nan
-    #             data=np.repeat(np.nan, width),
-    #             shape=width,
-    #             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-    #             dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
-    #             compressor=compressor,
-    #             fill_value=np.nan,  # needs to be nan to validate if any missing
-    #             overwrite=self.__overwrite,
+    #         ##### Frequency #####
+    #         frequency_data = np.array(
+    #             frequencies,
+    #             dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value),
+    #         )
+    #         frequency_da = xr.DataArray(
+    #             data=frequency_data,
+    #             dims=Coordinates.FREQUENCY.value,
+    #             name=Coordinates.FREQUENCY.value,
+    #             attrs=dict(
+    #                 units=Coordinates.FREQUENCY_UNITS.value,
+    #                 long_name=Coordinates.FREQUENCY_LONG_NAME.value,
+    #                 standard_name=Coordinates.FREQUENCY_STANDARD_NAME.value,
+    #             ),
     #         )
     #
-    #         # Note: LATITUDE is indexed by TIME
-    #         root.latitude.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
-    #         root.latitude.attrs["units"] = Coordinates.LATITUDE_UNITS.value
-    #         root.latitude.attrs["long_name"] = Coordinates.LATITUDE_LONG_NAME.value
-    #         root.latitude.attrs["standard_name"] = (
-    #             Coordinates.LATITUDE_STANDARD_NAME.value
+    #         ##### Latitude #####
+    #         gps_data = np.array(
+    #             np.repeat(np.nan, width),
+    #             dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
     #         )
+    #         latitude_da = xr.DataArray(
+    #             data=gps_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
+    #             name=Coordinates.LATITUDE.value,
+    #             attrs=dict(
+    #                 units=Coordinates.LATITUDE_UNITS.value,
+    #                 long_name=Coordinates.LATITUDE_LONG_NAME.value,
+    #                 standard_name=Coordinates.LATITUDE_STANDARD_NAME.value,
+    #             ),
+    #         )  # Note: LATITUDE is indexed by TIME
     #
-    #         #####################################################################
-    #         # --- Coordinate: Longitude --- #
-    #         root.create_dataset(
+    #         ##### Longitude #####
+    #         longitude_da = xr.DataArray(
+    #             data=gps_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
     #             name=Coordinates.LONGITUDE.value,
-    #             # dataset=np.repeat(0.0, width),  # root.longitude[:] = np.nan
-    #             data=np.repeat(np.nan, width),
-    #             shape=width,
-    #             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-    #             dtype=np.dtype(Coordinates.LONGITUDE_DTYPE.value),
-    #             compressor=compressor,
-    #             fill_value=np.nan,
-    #             overwrite=self.__overwrite,
-    #         )
+    #             attrs=dict(
+    #                 units=Coordinates.LONGITUDE_UNITS.value,
+    #                 long_name=Coordinates.LONGITUDE_LONG_NAME.value,
+    #                 standard_name=Coordinates.LONGITUDE_STANDARD_NAME.value,
+    #             ),
+    #         )  # Note: LONGITUDE is indexed by TIME
     #
-    #         # Note: LONGITUDE is indexed by TIME
-    #         root.longitude.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
-    #         root.longitude.attrs["units"] = Coordinates.LONGITUDE_UNITS.value
-    #         root.longitude.attrs["long_name"] = Coordinates.LONGITUDE_LONG_NAME.value
-    #         root.longitude.attrs["standard_name"] = (
-    #             Coordinates.LONGITUDE_STANDARD_NAME.value
+    #         ##### Bottom #####
+    #         bottom_data = np.array(
+    #             np.repeat(np.nan, width), dtype=np.dtype(Coordinates.BOTTOM_DTYPE.value)
     #         )
-    #
-    #         #####################################################################
-    #         # TODO: verify adding this variable for where the bottom was detected
-    #         # --- Coordinate: Bottom --- #
-    #         root.create_dataset(
+    #         bottom_da = xr.DataArray(
+    #             data=bottom_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
     #             name=Coordinates.BOTTOM.value,
-    #             data=np.repeat(0.0, width),  # root.longitude[:] = np.nan
-    #             shape=width,
-    #             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-    #             dtype=np.dtype(
-    #                 Coordinates.BOTTOM_DTYPE.value
-    #             ),  # TODO: should also only be integers
-    #             compressor=compressor,
-    #             fill_value=0.0,
-    #             overwrite=self.__overwrite,
+    #             attrs=dict(
+    #                 units=Coordinates.BOTTOM_UNITS.value,
+    #                 long_name=Coordinates.BOTTOM_LONG_NAME.value,
+    #                 standard_name=Coordinates.BOTTOM_STANDARD_NAME.value,
+    #             ),
     #         )
     #
-    #         # BOTTOM is indexed by TIME
-    #         root.bottom.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
-    #         root.bottom.attrs["units"] = Coordinates.BOTTOM_UNITS.value
-    #         root.bottom.attrs["long_name"] = Coordinates.BOTTOM_LONG_NAME.value
-    #         root.bottom.attrs["standard_name"] = Coordinates.BOTTOM_STANDARD_NAME.value
-    #
-    #         #####################################################################
-    #         # TODO: verify adding this variable with test
-    #         # --- Coordinate: Speed --- #
-    #         root.create_dataset(
+    #         ##### Speed #####
+    #         speed_data = np.array(
+    #             np.repeat(np.nan, width), dtype=np.dtype(Coordinates.SPEED_DTYPE.value)
+    #         )
+    #         speed_da = xr.DataArray(
+    #             data=speed_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
     #             name=Coordinates.SPEED.value,
-    #             data=np.repeat(np.nan, width),  # root.longitude[:] = np.nan
-    #             shape=width,
-    #             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-    #             dtype=np.dtype(Coordinates.SPEED_DTYPE.value),  # TODO: also round?
-    #             compressor=compressor,
-    #             fill_value=np.nan,
-    #             overwrite=self.__overwrite,
+    #             attrs=dict(
+    #                 units=Coordinates.SPEED_UNITS.value,
+    #                 long_name=Coordinates.SPEED_LONG_NAME.value,
+    #                 standard_name=Coordinates.SPEED_STANDARD_NAME.value,
+    #             ),
     #         )
     #
-    #         # SPEED is indexed by TIME
-    #         root.speed.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
-    #         root.speed.attrs["units"] = Coordinates.SPEED_UNITS.value
-    #         root.speed.attrs["long_name"] = Coordinates.SPEED_LONG_NAME.value
-    #         root.speed.attrs["standard_name"] = Coordinates.SPEED_STANDARD_NAME.value
-    #
-    #         #####################################################################
-    #         # --- Coordinate: Frequency --- #
-    #         root.create_dataset(
-    #             name=Coordinates.FREQUENCY.value,
-    #             data=frequencies,
-    #             shape=len(frequencies),
-    #             chunks=len(frequencies),
-    #             dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value),
-    #             compressor=compressor,
-    #             fill_value=0.0,
-    #             overwrite=self.__overwrite,
+    #         ##### Distance #####
+    #         distance_data = np.array(
+    #             np.repeat(np.nan, width),
+    #             dtype=np.dtype(Coordinates.DISTANCE_DTYPE.value),
+    #         )
+    #         distance_da = xr.DataArray(
+    #             data=distance_data,
+    #             coords=dict(
+    #                 time=time_da,
+    #             ),
+    #             dims=Coordinates.TIME.value,  # Note: "TIME"
+    #             name=Coordinates.DISTANCE.value,
+    #             attrs=dict(
+    #                 units=Coordinates.DISTANCE_UNITS.value,
+    #                 long_name=Coordinates.DISTANCE_LONG_NAME.value,
+    #                 standard_name=Coordinates.DISTANCE_STANDARD_NAME.value,
+    #             ),
     #         )
     #
-    #         # TODO: best coordinate would be channel with str type
-    #         root.frequency.attrs["_ARRAY_DIMENSIONS"] = [
-    #             Coordinates.FREQUENCY.value
-    #         ]  # TODO: is this correct
-    #         root.frequency.attrs["units"] = Coordinates.FREQUENCY_UNITS.value
-    #         root.frequency.attrs["long_name"] = Coordinates.FREQUENCY_LONG_NAME.value
-    #         root.frequency.attrs["standard_name"] = (
-    #             Coordinates.FREQUENCY_STANDARD_NAME.value
+    #         ##### Sv #####
+    #         gc.collect()
+    #         # sv_data = np.empty(
+    #         #     (len(depth_data), width, len(frequencies)),
+    #         #     # (2501, 4_100_782, 4), # large cruise used for testing
+    #         #     dtype=np.dtype(Coordinates.SV_DTYPE.value),
+    #         # )
+    #         sv_data = np.full(
+    #             (len(depth_data), width, len(frequencies)),
+    #             np.nan,
+    #             dtype=np.dtype(Coordinates.SV_DTYPE.value),
     #         )
+    #         print(f"one: {sys.getsizeof(sv_data)}")
+    #         # sv_data[:] = np.nan  # initialize all
     #
-    #         #####################################################################
-    #         # --- Sv Data --- #
-    #         root.create_dataset(
+    #         sv_da = xr.DataArray(
+    #             data=sv_data,
+    #             coords=dict(
+    #                 depth=depth_da,
+    #                 time=time_da,
+    #                 frequency=frequency_da,
+    #                 #
+    #                 latitude=latitude_da,
+    #                 longitude=longitude_da,
+    #                 bottom=bottom_da,
+    #                 speed=speed_da,
+    #                 distance=distance_da,
+    #             ),
+    #             dims=(  # Depth * Time * Frequency
+    #                 Coordinates.DEPTH.value,
+    #                 Coordinates.TIME.value,
+    #                 Coordinates.FREQUENCY.value,
+    #             ),
     #             name=Coordinates.SV.value,
-    #             shape=(len(depth_values), width, len(frequencies)),
-    #             chunks=(
-    #                 TILE_SIZE,
-    #                 TILE_SIZE,
-    #                 len(frequencies),
+    #             attrs=dict(
+    #                 units=Coordinates.SV_UNITS.value,
+    #                 long_name=Coordinates.SV_LONG_NAME.value,
+    #                 standard_name=Coordinates.SV_STANDARD_NAME.value,
+    #                 tiles_size=Constants.TILE_SIZE.value,
+    #                 _FillValue=np.nan,
     #             ),
-    #             dtype=np.dtype("int8"),  # Coordinates.SV_DTYPE.value
-    #             compressor=compressor,  # TODO: get compression working?!
-    #             # fill_value=np.nan,
-    #             overwrite=self.__overwrite,
     #         )
-    #
-    #         root.Sv.attrs["_ARRAY_DIMENSIONS"] = [
-    #             Coordinates.DEPTH.value,
-    #             Coordinates.TIME.value,
-    #             Coordinates.FREQUENCY.value,
-    #         ]
-    #         root.Sv.attrs["units"] = Coordinates.SV_UNITS.value
-    #         root.Sv.attrs["long_name"] = Coordinates.SV_LONG_NAME.value
-    #         root.Sv.attrs["tile_size"] = TILE_SIZE
-    #
+    #         print(f"two: {sys.getsizeof(sv_data)}")  # getting to at least here
+    #         del sv_data
+    #         sv_da.encoding = {"compressors": [compressor], "chunks": sv_chunk_shape}
+    #         # sv_da = sv_da.astype(np.float32)  # was crashing here
+    #         gc.collect()
     #         #####################################################################
-    #         # --- Metadata --- #
-    #         root.attrs["ship_name"] = ship_name
-    #         root.attrs["cruise_name"] = cruise_name
-    #         root.attrs["sensor_name"] = sensor_name
-    #         #
-    #         root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
-    #
-    #         current_project_version = importlib.metadata.version(
-    #             "water_column_sonar_processing"
+    #         ### Now create the xarray.Dataset
+    #         ds = xr.Dataset(
+    #             data_vars=dict(
+    #                 Sv=sv_da,
+    #                 #
+    #                 bottom=bottom_da,
+    #                 speed=speed_da,
+    #                 distance=distance_da,
+    #             ),
+    #             coords=dict(
+    #                 depth=depth_da,
+    #                 time=time_da,
+    #                 frequency=frequency_da,
+    #                 #
+    #                 latitude=latitude_da,
+    #                 longitude=longitude_da,
+    #             ),
+    #             attrs=dict(
+    #                 # --- Metadata --- #
+    #                 ship_name=ship_name,
+    #                 cruise_name=cruise_name,
+    #                 sensor_name=sensor_name,
+    #                 processing_software_name=Coordinates.PROJECT_NAME.value,
+    #                 # NOTE: for the version to be parsable you need to build the python package
+    #                 #  locally first.
+    #                 processing_software_version=importlib.metadata.version(
+    #                     "water-column-sonar-processing"
+    #                 ),
+    #                 processing_software_time=Timestamp.get_timestamp(),
+    #                 calibration_status=calibration_status,
+    #                 tile_size=Constants.TILE_SIZE.value,
+    #             ),
+    #         )
+    #         del sv_da
+    #         gc.collect()
+    #         print(f"three: {sys.getsizeof(ds)}")
+    #         #####################################################################
+    #         encodings = dict(
+    #             depth={
+    #                 "compressors": [compressor],
+    #                 "chunks": depth_chunk_shape,
+    #             },
+    #             time={
+    #                 "compressors": [compressor],
+    #                 "chunks": time_chunk_shape,
+    #                 "units": Coordinates.TIME_UNITS.value,
+    #             },
+    #             frequency={
+    #                 "compressors": [compressor],
+    #                 "chunks": frequency_chunk_shape,
+    #             },
+    #             latitude={
+    #                 "compressors": [compressor],
+    #                 "chunks": latitude_chunk_shape,
+    #             },
+    #             longitude={
+    #                 "compressors": [compressor],
+    #                 "chunks": longitude_chunk_shape,
+    #             },
+    #             bottom={
+    #                 "compressors": [compressor],
+    #                 "chunks": bottom_chunk_shape,
+    #             },
+    #             speed={
+    #                 "compressors": [compressor],
+    #                 "chunks": speed_chunk_shape,
+    #             },
+    #             distance={
+    #                 "compressors": [compressor],
+    #                 "chunks": distance_chunk_shape,
+    #             },
+    #             Sv={
+    #                 "compressors": [compressor],
+    #                 "chunks": sv_chunk_shape,
+    #             },
+    #         )
+    #         gc.collect()
+    #         ds.to_zarr(
+    #             store=zarr_path,
+    #             mode="w",  # “w” means create (overwrite if exists)
+    #             encoding=encodings,
+    #             consolidated=False,
+    #             safe_chunks=False,
+    #             align_chunks=True,
+    #             zarr_format=3,
+    #             write_empty_chunks=False,  # Might need to change this
     #         )
-    #         root.attrs["processing_software_version"] = current_project_version
-    #         root.attrs["processing_software_time"] = Timestamp.get_timestamp()
-    #         #
-    #         # TODO: add level somewhere?
-    #         #
-    #         root.attrs["calibration_status"] = calibration_status
-    #         root.attrs["tile_size"] = TILE_SIZE
-    #
-    #         zarr.consolidate_metadata(store)
     #         #####################################################################
     #         return zarr_path
     #     except Exception as err:
-    #         raise RuntimeError(f"Problem trying to create level 3 zarr store, {err}")
+    #         raise RuntimeError(f"Problem trying to create zarr store, {err}")
     #     # finally:
     #     #     cleaner = Cleaner()
     #     #     cleaner.delete_local_files()
     #     # TODO: should delete zarr store in temp directory too?
 
-    ############################################################################
-    # def update_zarr_store(
-    #         self,
-    #         path: str,
-    #         ship_name: str,
-    #         cruise_name: str,  # TODO: just pass stem
-    #         sensor_name: str,
-    # ) -> None:
-    #     """
-    #     Opens an existing Zarr store living in a s3 bucket for the purpose
-    #     of updating just a subset of the cruise-level Zarr store associated
-    #     with a file-level Zarr store.
-    #     """
-    #     pass
-
     ############################################################################
     def open_s3_zarr_store_with_zarr(
         self,
         ship_name: str,
         cruise_name: str,
         sensor_name: str,
-        # zarr_synchronizer: Union[str, None] = None, # TODO:
         output_bucket_name: str,
-        endpoint_url=None,
-    ):  #  -> zarr.hierarchy.Group:
+        endpoint_url: Optional[str] = None,
+    ) -> Group:
         # Mounts a Zarr store using pythons Zarr implementation. The mounted store
         #  will have read/write privileges so that store can be updated.
         print("Opening L2 Zarr store with Zarr for writing.")
         try:
-            s3fs_manager = S3FSManager(endpoint_url=endpoint_url)
-            root = f"{output_bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr"
-            store = s3fs_manager.s3_map(s3_zarr_store_path=root)
-            # synchronizer = model.ProcessSynchronizer(f"/tmp/{ship_name}_{cruise_name}.sync")
-            cruise_zarr = zarr.open(store=store, mode="r+")
-        except Exception as err:  # Failure
-            raise RuntimeError(
-                f"Exception encountered opening Zarr store with Zarr, {err}"
+            level = str(Constants.LEVEL_2.value)
+            store = f"s3://{output_bucket_name}/{level}/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr"
+            print(f"endpoint url: {endpoint_url}")
+            cruise_zarr = zarr.open(
+                store=store,
+                mode="r+",
+                zarr_format=3,
+                storage_options={
+                    "endpoint_url": endpoint_url,
+                    "key": self.key,
+                    "secret": self.secret,
+                },
             )
-        print("Done opening Zarr store with Zarr.")
-        return cruise_zarr
+            print("Done opening store with Zarr.")
+            return cruise_zarr
+        except Exception as err:  # Failure
+            raise RuntimeError(f"Exception encountered opening store with Zarr, {err}")
 
-    ############################################################################
+    ###########################################################################
+    @staticmethod
     def open_s3_zarr_store_with_xarray(
-        self,
         ship_name: str,
         cruise_name: str,
         sensor_name: str,
         file_name_stem: str,
-        input_bucket_name: str,
-        endpoint_url=None,
+        bucket_name: str,
+        # level: str, # TODO: add level
+        endpoint_url: Optional[str] = None,  # needed for moto testing
     ) -> xr.Dataset:
-        print(
-            "Opening L1 Zarr store in S3 with Xarray."
-        )  # TODO: Is this only used for reading from?
+        print("Opening L1 Zarr store in S3 with Xarray.")
         try:
-            zarr_path = f"s3://{input_bucket_name}/level_1/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.zarr"
-            s3fs_manager = S3FSManager(endpoint_url=endpoint_url)
-            store_s3_map = s3fs_manager.s3_map(s3_zarr_store_path=zarr_path)
-            ds = xr.open_dataset(filename_or_obj=store_s3_map, engine="zarr", chunks={})
+            zarr_path = f"s3://{bucket_name}/level_1/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.zarr"
+            kwargs = {"consolidated": False}
+            ds = xr.open_dataset(
+                filename_or_obj=zarr_path,
+                engine="zarr",
+                backend_kwargs={
+                    "storage_options": {
+                        "endpoint_url": endpoint_url,
+                        "anon": True,
+                    },
+                },
+                **kwargs,
+            )
             return ds
         except Exception as err:
             raise RuntimeError(f"Problem opening Zarr store in S3 as Xarray, {err}")
-        finally:
-            print("Exiting opening Zarr store in S3 as Xarray.")
 
+    ###########################################################################
+    # TODO: can this be consolidated with above
+    @staticmethod
     def open_l2_zarr_store_with_xarray(
-        self,
         ship_name: str,
         cruise_name: str,
         sensor_name: str,
         bucket_name: str,
-        endpoint_url=None,
+        endpoint_url: Optional[str] = None,  # needed for moto testing
     ) -> xr.Dataset:
         print("Opening L2 Zarr store in S3 with Xarray.")
         try:
-            zarr_path = f"s3://{bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr"
-            s3fs_manager = S3FSManager(endpoint_url=endpoint_url)
-            store_s3_map = s3fs_manager.s3_map(s3_zarr_store_path=zarr_path)
+            level = str(Constants.LEVEL_2.value)
+            zarr_path = f"s3://{bucket_name}/{level}/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr"
+            kwargs = {"consolidated": False}
             ds = xr.open_dataset(
-                filename_or_obj=store_s3_map,
+                filename_or_obj=zarr_path,
                 engine="zarr",
+                backend_kwargs={
+                    "storage_options": {
+                        "endpoint_url": endpoint_url,
+                        "anon": True,
+                    }
+                },
+                **kwargs,
             )
+            return ds
         except Exception as err:
             raise RuntimeError(f"Problem opening Zarr store in S3 as Xarray, {err}")
-        print("Done opening Zarr store in S3 as Xarray.")
-        return ds
 
-    ############################################################################
+    ###########################################################################
 
-    #######################################################
+    ###########################################################################
     # def create_process_synchronizer(self):
     #     # TODO: explore aws redis options
     #     pass
 
-    #######################################################
+    ###########################################################################
     # def verify_cruise_store_data(self):
     #     # TODO: run a check on a finished model store to ensure that
     #     #   none of the time, latitude, longitude, or depth values
     #     #   are NaN.
     #     pass
 
-    #######################################################
+    ###########################################################################
 
 
 ###########################################################