water-column-sonar-processing 0.0.9__py3-none-any.whl → 0.0.11__py3-none-any.whl

water_column_sonar_processing/aws/dynamodb_manager.py

@@ -111,17 +111,21 @@ class DynamoDBManager:
         expression_attribute_names,
         expression_attribute_values,
         update_expression,
-    ):
-        response = self.__dynamodb_client.update_item(
-            TableName=table_name,
-            Key=key,
-            ExpressionAttributeNames=expression_attribute_names,
-            ExpressionAttributeValues=expression_attribute_values,
-            UpdateExpression=update_expression,
-        )
-        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
-        assert response['ConsumedCapacity']['TableName'] == table_name
-        assert status_code == 200, "Problem, unable to update dynamodb table."
+    ): # TODO: convert to boolean
+        try:
+            response = self.__dynamodb_client.update_item(
+                TableName=table_name,
+                Key=key,
+                ExpressionAttributeNames=expression_attribute_names,
+                ExpressionAttributeValues=expression_attribute_values,
+                UpdateExpression=update_expression,
+            )
+            status_code = response["ResponseMetadata"]["HTTPStatusCode"]
+            print(f"HTTPStatusCode: {status_code}")
+            # assert status_code == 200, "Problem, unable to update dynamodb table."
+            # assert response['ConsumedCapacity']['TableName'] == table_name
+        except Exception as err:
+            print(f"Problem was encountered while updating item: {err}")
 
     #####################################################################
     # TODO: change to "get_cruise_as_df"

water_column_sonar_processing/aws/s3_manager.py

@@ -3,6 +3,8 @@ import os
 import boto3
 from collections.abc import Generator
 from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import botocore
 from boto3.s3.transfer import TransferConfig
 from botocore.config import Config
 from botocore.exceptions import ClientError
@@ -14,7 +16,10 @@ GB = 1024**3
 
 
 #########################################################################
-def chunked(ll: list, n: int) -> Generator:
+def chunked(
+    ll: list,
+    n: int
+) -> Generator:
     # Yields successively n-sized chunks from ll.
     for i in range(0, len(ll), n):
         yield ll[i : i + n]
@@ -24,16 +29,9 @@ class S3Manager:
     #####################################################################
     def __init__(
         self,
-        # input_endpoint_url: str,
-        # output_endpoint_url: str,
-        # endpoint_url
-        # TODO: Need to allow passing in of credentials when writing to protected bucket
     ):
         self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
         self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
-        # self.endpoint_url = endpoint_url
-        # self.input_endpoint_url = input_endpoint_url
-        # self.output_endpoint_url = output_endpoint_url
         self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.s3_client_config = Config(max_pool_connections=MAX_POOL_CONNECTIONS)
         self.s3_transfer_config = TransferConfig(
@@ -51,14 +49,12 @@ class S3Manager:
             service_name="s3",
             config=self.s3_client_config,
             region_name=self.s3_region,
-            # endpoint_url=endpoint_url, # TODO: temporary
         )
         self.s3_resource = boto3.resource(
             service_name="s3",
             config=self.s3_client_config,
             region_name=self.s3_region,
         )
-        # self.paginator = self.s3_client.get_paginator(operation_name='list_objects_v2')
         self.s3_session_noaa_wcsd_zarr_pds = boto3.Session(
             aws_access_key_id=os.environ.get("OUTPUT_BUCKET_ACCESS_KEY"),
             aws_secret_access_key=os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY"),
@@ -68,7 +64,6 @@ class S3Manager:
             service_name="s3",
             config=self.s3_client_config,
             region_name=self.s3_region,
-            # endpoint_url=endpoint_url, # TODO: temporary
         )
         self.s3_resource_noaa_wcsd_zarr_pds = self.s3_session_noaa_wcsd_zarr_pds.resource(
             service_name="s3",
@@ -78,12 +73,12 @@ class S3Manager:
         self.paginator = self.s3_client.get_paginator('list_objects_v2')
         self.paginator_noaa_wcsd_zarr_pds = self.s3_client_noaa_wcsd_zarr_pds.get_paginator('list_objects_v2')
 
-    def get_client(self): # TODO: do i need this?
-        return self.s3_session.client(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
+    # def get_client(self): # TODO: do i need this?
+    #     return self.s3_session.client(
+    #         service_name="s3",
+    #         config=self.s3_client_config,
+    #         region_name=self.s3_region,
+    #     )
 
     #####################################################################
     def create_bucket(
@@ -146,18 +141,6 @@ class S3Manager:
         return all_uploads
 
     #####################################################################
-    # def upload_nodd_file2(
-    #         self,
-    #         body: str,
-    #         bucket: str,
-    #         key: str,
-    # ):
-    #     self.s3_client_noaa_wcsd_zarr_pds.put_object(
-    #         Body=body,
-    #         Bucket=bucket,
-    #         Key=key,
-    #     )
-
     # TODO: this uses resource, try to use client
     def upload_file(
             self,
@@ -190,11 +173,36 @@ class S3Manager:
                 all_files.append([local_path, s3_key])
 
         all_uploads = self.upload_files_with_thread_pool_executor(
+            output_bucket_name=self.output_bucket_name,
             all_files=all_files,
         )
         print("Done uploading files to output bucket.")
         return all_uploads
 
+    #####################################################################
+    def check_if_object_exists(
+            self,
+            bucket_name,
+            key_name
+    ) -> bool:
+        s3_manager2 = S3Manager()
+        s3_manager2.list_objects(bucket_name=bucket_name, prefix=key_name)
+        s3_client_noaa_wcsd_zarr_pds = self.s3_client_noaa_wcsd_zarr_pds
+        try:
+            # response = s3_resource_noaa_wcsd_zarr_pds.Object(bucket_name, key_name).load()
+            s3_client_noaa_wcsd_zarr_pds.head_object(Bucket=bucket_name, Key=key_name)
+        except botocore.exceptions.ClientError as e:
+            if e.response['Error']['Code'] == "404":
+                # The object does not exist.
+                return False
+            elif e.response['Error']['Code'] == 403:
+                # Unauthorized, including invalid bucket
+                return False
+            else:
+                # Something else has gone wrong.
+                raise
+        return True
+
     #####################################################################
     # used: raw-to-zarr
     def list_objects(  # noaa-wcsd-pds and noaa-wcsd-zarr-pds
@@ -202,6 +210,7 @@ class S3Manager:
         bucket_name,
         prefix
     ):
+        # TODO: this isn't working for geojson detecting objects!!!!!!!
         # analog to "find_children_objects"
         # Returns a list of key strings for each object in bucket defined by prefix
         # s3_client = self.s3_client
@@ -227,7 +236,11 @@ class S3Manager:
 
     #####################################################################
     # TODO: change name to "directory"
-    def folder_exists_and_not_empty(self, bucket_name: str, path: str) -> bool:
+    def folder_exists_and_not_empty(
+        self,
+        bucket_name: str,
+        path: str
+    ) -> bool:
         if not path.endswith("/"):
             path = path + "/"
         s3_client = self.s3_client
@@ -319,23 +332,15 @@ class S3Manager:
         print("downloaded file")
 
     #####################################################################
-    # not used
-    # def delete_nodd_object(  # noaa-wcsd-model-pds
-    #         self,
-    #         bucket_name,
-    #         key
-    # ):  # -> dict:
-    #     #return self.__s3_client.delete_object(Bucket=bucket_name, Key=key)
-    #     self.s3_client.delete_object(Bucket=bucket_name, Key=key)
-
-    #####################################################################
+    # TODO: need to test this!!!
     def delete_nodd_objects(  # nodd-bucket
         self,
+        bucket_name,
         objects: list,
     ):
         try:
             print(
-                f"Deleting {len(objects)} objects in {self.output_bucket_name} in batches."
+                f"Deleting {len(objects)} objects in {bucket_name} in batches."
             )
             objects_to_delete = []
             for obj in objects:
@@ -343,12 +348,28 @@ class S3Manager:
             # Note: request can contain a list of up to 1000 keys
             for batch in chunked(ll=objects_to_delete, n=1000):
                 self.s3_client_noaa_wcsd_zarr_pds.delete_objects(
-                    Bucket=self.output_bucket_name, Delete={"Objects": batch}
+                    Bucket=bucket_name, Delete={"Objects": batch}
                 )
             print(f"Deleted files.")
         except Exception as err:
             print(f"Problem was encountered while deleting objects: {err}")
 
+    #####################################################################
+    # TODO: need to test this!!!
+    def delete_nodd_object(
+            self,
+            bucket_name,
+            key_name,
+    ):
+        try:
+            print(
+                f"Deleting {key_name} objects in {bucket_name}."
+            )
+            self.s3_client_noaa_wcsd_zarr_pds.delete_object(Bucket=bucket_name, Key=key_name)
+            print(f"Deleted file.")
+        except Exception as err:
+            print(f"Problem was encountered while deleting objects: {err}")
+
     #####################################################################
     # not used TODO: remove
     def put(self, bucket_name, key, body):  # noaa-wcsd-model-pds

water_column_sonar_processing/cruise/create_empty_zarr_store.py

@@ -47,7 +47,7 @@ class CreateEmptyZarrStore:
         ):
             for file in files:
                 local_path = os.path.join(subdir, file)
-                # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.model/.zattrs'
+                # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.model/..zattrs'
                 s3_key = f'{object_prefix}/{cruise_name}.model{local_path.split(f"{cruise_name}.model")[-1]}'
                 all_files.append([local_path, s3_key])
         #

water_column_sonar_processing/geometry/geometry_manager.py

@@ -138,15 +138,13 @@ class GeometryManager:
 
                 print("Checking s3 and deleting any existing GeoJSON file.")
                 s3_manager = S3Manager()
-                s3_objects = s3_manager.list_objects(
+                geojson_object_exists = s3_manager.check_if_object_exists(
                     bucket_name=output_bucket_name,
-                    prefix=f"{geo_json_prefix}/{geo_json_name}"
+                    key_name=f"{geo_json_prefix}/{geo_json_name}"
                 )
-                if len(s3_objects) > 0:
-                    print(
-                        "GeoJSON already exists in s3, deleting existing and continuing."
-                    )
-                    s3_manager.delete_nodd_objects(objects=s3_objects)
+                if geojson_object_exists:
+                    print("GeoJSON already exists in s3, deleting existing and continuing.")
+                    s3_manager.delete_nodd_object(bucket_name=output_bucket_name, key_name=f"{geo_json_prefix}/{geo_json_name}")
 
                 print("Upload GeoJSON to s3.")
                 s3_manager.upload_nodd_file(

water_column_sonar_processing/model/zarr_manager.py

@@ -86,8 +86,6 @@ class ZarrManager:
             data=np.repeat(0.0, width),
             shape=width,
             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-                # Constants.TILE_SIZE.value,
-            #),  # TODO: the chunking scheme doesn't seem to be working here
             dtype=np.dtype(Coordinates.TIME_DTYPE.value),
             compressor=self.__compressor,
             fill_value=np.nan,  # TODO: do i want nan's?
@@ -125,14 +123,16 @@ class ZarrManager:
 
         root.depth.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.DEPTH.value]
 
-        root.depth.attrs["long_name"] = Coordinates.DEPTH_LONG_NAME.value
         root.depth.attrs["units"] = Coordinates.DEPTH_UNITS.value
+        root.depth.attrs["long_name"] = Coordinates.DEPTH_LONG_NAME.value
+        root.depth.attrs["standard_name"] = Coordinates.DEPTH_STANDARD_NAME.value
 
         #####################################################################
         # --- Coordinate: Latitude --- #
         root.create_dataset(
             name=Coordinates.LATITUDE.value,
-            # data=np.repeat(0.0, width),
+            # data=np.repeat(0.0, width),  # root.longitude[:] = np.nan
+            data=np.repeat(np.nan, width),
             shape=width,
             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
             dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
@@ -144,14 +144,16 @@ class ZarrManager:
         # Note: LATITUDE is indexed by TIME
         root.latitude.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
 
-        root.latitude.attrs["long_name"] = Coordinates.LATITUDE_LONG_NAME.value
         root.latitude.attrs["units"] = Coordinates.LATITUDE_UNITS.value
+        root.latitude.attrs["long_name"] = Coordinates.LATITUDE_LONG_NAME.value
+        root.latitude.attrs["standard_name"] = Coordinates.LATITUDE_STANDARD_NAME.value
 
         #####################################################################
         # --- Coordinate: Longitude --- #
         root.create_dataset(
             name=Coordinates.LONGITUDE.value,
             # data=np.repeat(0.0, width),  # root.longitude[:] = np.nan
+            data=np.repeat(np.nan, width),
             shape=width,
             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
             dtype=np.dtype(Coordinates.LONGITUDE_DTYPE.value),
@@ -163,8 +165,9 @@ class ZarrManager:
         # Note: LONGITUDE is indexed by TIME
         root.longitude.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
 
-        root.longitude.attrs["long_name"] = Coordinates.LONGITUDE_LONG_NAME.value
         root.longitude.attrs["units"] = Coordinates.LONGITUDE_UNITS.value
+        root.longitude.attrs["long_name"] = Coordinates.LONGITUDE_LONG_NAME.value
+        root.longitude.attrs["standard_name"] = Coordinates.LONGITUDE_STANDARD_NAME.value
 
         #####################################################################
         # TODO: verify adding this variable for where the bottom was detected
@@ -183,8 +186,9 @@ class ZarrManager:
         # BOTTOM is indexed by TIME
         root.bottom.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
 
-        root.bottom.attrs["long_name"] = Coordinates.BOTTOM_LONG_NAME.value
         root.bottom.attrs["units"] = Coordinates.BOTTOM_UNITS.value
+        root.bottom.attrs["long_name"] = Coordinates.BOTTOM_LONG_NAME.value
+        root.bottom.attrs["standard_name"] = Coordinates.BOTTOM_STANDARD_NAME.value
 
         #####################################################################
         # --- Coordinate: Frequency --- #
@@ -204,11 +208,11 @@ class ZarrManager:
             Coordinates.FREQUENCY.value
         ]  # TODO: is this correct
 
+        root.frequency.attrs["units"] = Coordinates.FREQUENCY_UNITS.value
         root.frequency.attrs["long_name"] = Coordinates.FREQUENCY_LONG_NAME.value
         root.frequency.attrs["standard_name"] = (
             Coordinates.FREQUENCY_STANDARD_NAME.value
         )
-        root.frequency.attrs["units"] = Coordinates.FREQUENCY_UNITS.value
 
         #####################################################################
         # --- Sv Data --- #
@@ -230,8 +234,8 @@ class ZarrManager:
             Coordinates.FREQUENCY.value,
         ]
 
-        root.Sv.attrs["long_name"] = Coordinates.SV_LONG_NAME.value
         root.Sv.attrs["units"] = Coordinates.SV_UNITS.value
+        root.Sv.attrs["long_name"] = Coordinates.SV_LONG_NAME.value
         root.Sv.attrs["tile_size"] = Constants.TILE_SIZE.value
 
         #####################################################################
@@ -242,7 +246,7 @@ class ZarrManager:
         #
         root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
         root.attrs["processing_software_version"] = (
-            "0.0.6"  # TODO: get programmatically
+            "0.0.9"  # TODO: get programmatically, echopype>utils>prov.py
         )
         root.attrs["processing_software_time"] = Timestamp.get_timestamp()
         #

water_column_sonar_processing/processing/raw_to_zarr.py

@@ -9,7 +9,7 @@ from pathlib import Path # , PurePath
 
 from water_column_sonar_processing.aws import DynamoDBManager, S3Manager
 from water_column_sonar_processing.geometry import GeometryManager
-from water_column_sonar_processing.utility import Cleaner
+from water_column_sonar_processing.utility import Cleaner, PipelineStatus
 
 TEMPDIR = "/tmp"
 
@@ -53,10 +53,6 @@ class RawToZarr:
     ):
         print('Writing Zarr information to DynamoDB table.')
         dynamodb_manager = DynamoDBManager()
-
-        # The problem is that these values were never populated
-        # and so when the query looks for values that aren't there
-        # they fail
         dynamodb_manager.update_item(
             table_name=table_name,
             key={
@@ -87,7 +83,8 @@ class RawToZarr:
                 ":ma": {"N": str(np.round(max_echo_range, 4))},
                 ":mi": {"N": str(np.round(min_echo_range, 4))},
                 ":nd": {"N": str(num_ping_time_dropna)},
-                ":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
+                # ":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
+                ":ps": {"S": PipelineStatus.LEVEL_1_PROCESSING.name},
                 ":pt": {"S": datetime.now().isoformat(timespec="seconds") + "Z"},
                 ":se": {"S": sensor_name},
                 ":sh": {"S": ship_name},
@@ -113,6 +110,7 @@ class RawToZarr:
                 "#ZP = :zp"
             ),
         )
+        print('Done writing Zarr information to DynamoDB table.')
 
     ############################################################################
     ############################################################################
@@ -143,16 +141,29 @@ class RawToZarr:
     def raw_to_zarr(
             self,
             table_name,
+            input_bucket_name,
             output_bucket_name,
             ship_name,
             cruise_name,
             sensor_name,
             raw_file_name,
     ):
+        """
+        Downloads the raw files, processes them with echopype, writes geojson, and uploads files
+        to the nodd bucket.
+        """
         print(f'Opening raw: {raw_file_name} and creating zarr store.')
         geometry_manager = GeometryManager()
         cleaner = Cleaner()
-        cleaner.delete_local_files(file_types=["*.zarr", "*.json"]) # TODO: include bot and raw?ß
+        cleaner.delete_local_files(file_types=["*.zarr", "*.json"]) # TODO: include bot and raw?
+
+        s3_manager = S3Manager()
+        s3_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{raw_file_name}"
+        bottom_file_name = f"{Path(raw_file_name).stem}.bot"
+        s3_bottom_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{bottom_file_name}"
+        s3_manager.download_file(bucket_name=input_bucket_name, key=s3_file_path, file_name=raw_file_name)
+        s3_manager.download_file(bucket_name=input_bucket_name, key=s3_bottom_file_path, file_name=bottom_file_name)
+
         try:
             gc.collect()
             print('Opening raw file with echopype.')
@@ -168,7 +179,12 @@ class RawToZarr:
             )
             print('Compute volume backscattering strength (Sv) from raw data.')
             ds_sv = ep.calibrate.compute_Sv(echodata)
-            print('Done computing volume backscattering strength (Sv) from raw data.')
+            print('Done computing volume backscatter strength (Sv) from raw data.')
+            # Note: detected_seafloor_depth is located at echodata.vendor.detected_seafloor_depth
+            # but is not written out with ds_sv
+            if "detected_seafloor_depth" in list(echodata.vendor.variables):
+                ds_sv["detected_seafloor_depth"] = echodata.vendor.detected_seafloor_depth
+            #
             frequencies = echodata.environment.frequency_nominal.values
             #################################################################
             # Get GPS coordinates
@@ -187,12 +203,9 @@ class RawToZarr:
             # TODO: this var name is supposed to represent minimum resolution of depth measurements
             # TODO revert this so that smaller diffs can be used
             # The most minimum the resolution can be is as small as 0.25 meters
-            min_echo_range = np.maximum(
-                0.25,
-                np.nanmin(np.diff(ds_sv.echo_range.values))
-            )
+            min_echo_range = np.round(np.nanmin(np.diff(ds_sv.echo_range.values)), 2)
             max_echo_range = float(np.nanmax(ds_sv.echo_range))
-            #
+            # This is the number of missing values found throughout the lat/lon
             num_ping_time_dropna = lat[~np.isnan(lat)].shape[0]  # symmetric to lon
             #
             start_time = np.datetime_as_string(ds_sv.ping_time.values[0], unit='ms') + "Z"
@@ -204,14 +217,27 @@ class RawToZarr:
             store_name = f"{Path(raw_file_name).stem}.zarr"
             ds_sv.to_zarr(store=store_name)
             #################################################################
-            # TODO: do i still need this?
-            # print('Note: Adding GeoJSON inside Zarr store')
-            # self.__write_geojson_to_file( # Was trying to write geojson to the L1 zarr store
-            #     store_name=store_name,
-            #     data=gps_data
-            # )
-            #################################################################
             output_zarr_prefix = f"level_1/{ship_name}/{cruise_name}/{sensor_name}/"
+            #################################################################
+            # If zarr store already exists then delete
+            s3_manager = S3Manager()
+            child_objects = s3_manager.get_child_objects(
+                bucket_name=output_bucket_name,
+                sub_prefix=f"level_1/{ship_name}/{cruise_name}/{sensor_name}/{Path(raw_file_name).stem}.zarr",
+            )
+            if len(child_objects) > 0:
+                print('Zarr store data already exists in s3, deleting existing and continuing.')
+                s3_manager.delete_nodd_objects(
+                    bucket_name=output_bucket_name,
+                    objects=child_objects,
+                )
+            #################################################################
+            self.__upload_files_to_output_bucket(
+                output_bucket_name=output_bucket_name,
+                local_directory=store_name,
+                object_prefix=output_zarr_prefix
+            )
+            #################################################################
             self.__zarr_info_to_table(
                 output_bucket_name=output_bucket_name,
                 table_name=table_name,
@@ -228,34 +254,15 @@ class RawToZarr:
                 frequencies=frequencies,
                 channels=channels
             )
-            ###################################################################
-            #######################################################################
-            self.__upload_files_to_output_bucket(
-                output_bucket_name=output_bucket_name,
-                local_directory=store_name,
-                object_prefix=output_zarr_prefix
-            )
-            #######################################################################
-            # # TODO: verify count of objects matches
-            # s3_objects = self.__s3.list_objects(
-            #     bucket_name=self.__output_bucket,
-            #     prefix=f"{zarr_prefix}/{os.path.splitext(input_file_name)[0]}.zarr/",
-            #     access_key_id=self.__output_bucket_access_key,
-            #     secret_access_key=self.__output_bucket_secret_access_key
-            # )
             #######################################################################
-            # self.__update_processing_status(
-            #     file_name=input_file_name,
-            #     cruise_name=cruise_name,
-            #     pipeline_status='SUCCESS_RAW_TO_ZARR'
-            # )
+            # TODO: verify count of objects matches, publish message, update status
             #######################################################################
-            # self.__publish_done_message(input_message)
-            print('here')
+            print('Finished raw-to-zarr conversion.')
         except Exception as err:
             print(f'Exception encountered creating local Zarr store with echopype: {err}')
             raise RuntimeError(f"Problem creating local Zarr store, {err}")
         finally:
+            print("Finally.")
             cleaner.delete_local_files(file_types=["*.raw", "*.bot", "*.zarr", "*.json"])
         print('Done creating local zarr store.')
 

{water_column_sonar_processing-0.0.9.dist-info → water_column_sonar_processing-0.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: water_column_sonar_processing
-Version: 0.0.9
+Version: 0.0.11
 Summary: A processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
@@ -24,7 +24,7 @@ Requires-Dist: numcodecs==0.13.1
 Requires-Dist: numpy==1.26.4
 Requires-Dist: pandas==2.2.3
 Requires-Dist: pyarrow==18.1.0
-Requires-Dist: python-dotenv==1.0.0
+Requires-Dist: python-dotenv==1.0.1
 Requires-Dist: requests==2.32.3
 Requires-Dist: s3fs==2023.12.1
 Requires-Dist: scipy==1.14.1
@@ -114,6 +114,7 @@ python -m twine upload --repository pypi dist/*
 ```
 
 # Pre Commit Hook
+see here for installation: https://pre-commit.com/
 https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
 ```
 pre-commit install --allow-missing-config
@@ -132,3 +133,8 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
 20241125
 5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
 3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
+
+
+# TODO:
+add https://pypi.org/project/setuptools-scm/
+for extracting the version

{water_column_sonar_processing-0.0.9.dist-info → water_column_sonar_processing-0.0.11.dist-info}/RECORD

@@ -1,32 +1,32 @@
 water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
 water_column_sonar_processing/process.py,sha256=-yQtK3rnZq6lGAr3q02zLDe1NuMH9c0PiUOxKzG_r18,5386
 water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
-water_column_sonar_processing/aws/dynamodb_manager.py,sha256=sZHn-hgCt3K3w0x5BcXfF5jLMt_F11dAtQHJToij9nU,10008
-water_column_sonar_processing/aws/s3_manager.py,sha256=ctNWMkgqMlwbwmXHgwKEV8otLwIjr-dHX6bQ2rOw1ug,14718
+water_column_sonar_processing/aws/dynamodb_manager.py,sha256=gMDAXLE_p_nKmNZYICKA9T56PYDqtXBySlysSOVnWrI,10250
+water_column_sonar_processing/aws/s3_manager.py,sha256=kS48Vu_jE_fOKbwKOhCLWKDSqHzOGVEdZ_Lc4MaMCfA,15291
 water_column_sonar_processing/aws/s3fs_manager.py,sha256=thVJPQKhbvF1g-Ue3BYgwazFOFDYOICIEJx4zkXBQ1E,2381
 water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
 water_column_sonar_processing/aws/sqs_manager.py,sha256=NSUrWmnSC8h8Gf7gT0U8zFaQQ-yX89h0Q0mDLKGqp2Y,1597
 water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
-water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=uLwHZazndSy4puXrS-2PrGhicV-umsCCiXoqt2MMpkM,7317
+water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=uQiZoKm16jD0SUuXmhuPryxdE-6bUc6BlCi2UtmzUpw,7318
 water_column_sonar_processing/cruise/resample_regrid.py,sha256=4Tw6Ro9mQZOr0uIph6foz6a1OeFAZW0SMUT_asIwvKw,12309
 water_column_sonar_processing/geometry/__init__.py,sha256=_ol5nI8AL30pYXeAh5rtP7YmQggitPC6LA_kuTfPJ0Q,231
-water_column_sonar_processing/geometry/geometry_manager.py,sha256=7WZ1UerY_h3uOKc3mcaOpvhgZ1yV3gD-CUnhZJl1BOQ,10550
+water_column_sonar_processing/geometry/geometry_manager.py,sha256=0Q9IRiBr6XvxUg5M2vCPtUhbnYnwa5pJI1ayfWXMgMs,10587
 water_column_sonar_processing/geometry/geometry_simplification.py,sha256=im1HG9nfYIerQv3w-PUHzphw2B7aGgnsA3Zcdy2oTmA,3016
 water_column_sonar_processing/geometry/pmtile_generation.py,sha256=7Lm08Jr6YaM4nYmexClxbIMOqSV1teo9wMm6dfjFuNA,12384
 water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
 water_column_sonar_processing/index/index_manager.py,sha256=YS6y_THfGAZpjfBZOj5n8O1aY_BnBYS781eNHfhpip0,11239
 water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
-water_column_sonar_processing/model/zarr_manager.py,sha256=fpRkk6Qg1_LVdLg1M_X0J9Lchp2OJygTxfekEk1Mi88,13641
+water_column_sonar_processing/model/zarr_manager.py,sha256=TbcVux-GWfX4XJ7UT20E7dI_h_islrKsGtjx_VwSsLg,14003
 water_column_sonar_processing/processing/__init__.py,sha256=UwdB3BnoUxy4q3k9-ZjBF6KzmCWVDcqbcArTeHgmvGA,118
 water_column_sonar_processing/processing/cruise_sampler.py,sha256=hadPrnH5nz7_oG_4pND7YbMFH6NMR9d6p3xAXedtKU8,15927
-water_column_sonar_processing/processing/raw_to_zarr.py,sha256=QBz58P-hYTZwg6hhf4u_kNg710lrfxPwnvIr7UmK30I,16125
+water_column_sonar_processing/processing/raw_to_zarr.py,sha256=OPu4CoIlHQFW38iY4DLe5A5Ttrdz4NXtjYThrB-FuPs,16874
 water_column_sonar_processing/utility/__init__.py,sha256=yDObMOL0_OxKWet5wffK2-XVJgoE9iwiY2q04GZrtBQ,234
 water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
 water_column_sonar_processing/utility/constants.py,sha256=EbzsorvYKadsPjuutRjQKKByGibhFm0Gw6D-Sp2ZD3I,2143
 water_column_sonar_processing/utility/pipeline_status.py,sha256=O-0SySqdRGJ6bs3zQe1NV9vkOpmsRM7zj5QoHgzYioY,4395
 water_column_sonar_processing/utility/timestamp.py,sha256=bO0oir7KxxoEHPGRkz9FCBfOligkocUyRiWRzAq8fnU,361
-water_column_sonar_processing-0.0.9.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
-water_column_sonar_processing-0.0.9.dist-info/METADATA,sha256=5mtfW5UU7dpl2oKGUi7GKTbjsrcMo13ackWFpNVkjr4,4432
-water_column_sonar_processing-0.0.9.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-water_column_sonar_processing-0.0.9.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
-water_column_sonar_processing-0.0.9.dist-info/RECORD,,
+water_column_sonar_processing-0.0.11.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
+water_column_sonar_processing-0.0.11.dist-info/METADATA,sha256=KFkI1367kV7L7pl8SIK4UFwUVJvUCHkRTPwBCqpnxWA,4566
+water_column_sonar_processing-0.0.11.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+water_column_sonar_processing-0.0.11.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
+water_column_sonar_processing-0.0.11.dist-info/RECORD,,