water-column-sonar-processing 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

water_column_sonar_processing/__init__.py

@@ -1,8 +1,6 @@
 from __future__ import absolute_import
 
 from . import aws, cruise, geometry, index, model, processing, utility
-# from .model import ZarrManager
-# from .process import Process
 
 __all__ = [
     "aws",

water_column_sonar_processing/aws/__init__.py

@@ -1,7 +1,7 @@
 from .dynamodb_manager import DynamoDBManager
-from .s3_manager import S3Manager
+from .s3_manager import S3Manager, chunked
 from .s3fs_manager import S3FSManager
 from .sns_manager import SNSManager
 from .sqs_manager import SQSManager
 
-__all__ = ["DynamoDBManager", "S3Manager", "S3FSManager", "SNSManager", "SQSManager"]
+__all__ = ["DynamoDBManager", "S3Manager", "chunked", "S3FSManager", "SNSManager", "SQSManager"]

water_column_sonar_processing/aws/s3_manager.py

@@ -3,6 +3,8 @@ import os
 import boto3
 from collections.abc import Generator
 from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import botocore
 from boto3.s3.transfer import TransferConfig
 from botocore.config import Config
 from botocore.exceptions import ClientError
@@ -14,7 +16,10 @@ GB = 1024**3
 
 
 #########################################################################
-def chunked(ll: list, n: int) -> Generator:
+def chunked(
+    ll: list,
+    n: int
+) -> Generator:
     # Yields successively n-sized chunks from ll.
     for i in range(0, len(ll), n):
         yield ll[i : i + n]
@@ -24,16 +29,9 @@ class S3Manager:
     #####################################################################
     def __init__(
         self,
-        # input_endpoint_url: str,
-        # output_endpoint_url: str,
-        # endpoint_url
-        # TODO: Need to allow passing in of credentials when writing to protected bucket
     ):
         self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
         self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
-        # self.endpoint_url = endpoint_url
-        # self.input_endpoint_url = input_endpoint_url
-        # self.output_endpoint_url = output_endpoint_url
         self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.s3_client_config = Config(max_pool_connections=MAX_POOL_CONNECTIONS)
         self.s3_transfer_config = TransferConfig(
@@ -51,14 +49,12 @@ class S3Manager:
             service_name="s3",
             config=self.s3_client_config,
             region_name=self.s3_region,
-            # endpoint_url=endpoint_url, # TODO: temporary
         )
         self.s3_resource = boto3.resource(
             service_name="s3",
             config=self.s3_client_config,
             region_name=self.s3_region,
         )
-        # self.paginator = self.s3_client.get_paginator(operation_name='list_objects_v2')
         self.s3_session_noaa_wcsd_zarr_pds = boto3.Session(
             aws_access_key_id=os.environ.get("OUTPUT_BUCKET_ACCESS_KEY"),
             aws_secret_access_key=os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY"),
@@ -68,7 +64,6 @@ class S3Manager:
             service_name="s3",
             config=self.s3_client_config,
             region_name=self.s3_region,
-            # endpoint_url=endpoint_url, # TODO: temporary
         )
         self.s3_resource_noaa_wcsd_zarr_pds = self.s3_session_noaa_wcsd_zarr_pds.resource(
             service_name="s3",
@@ -78,12 +73,12 @@ class S3Manager:
         self.paginator = self.s3_client.get_paginator('list_objects_v2')
         self.paginator_noaa_wcsd_zarr_pds = self.s3_client_noaa_wcsd_zarr_pds.get_paginator('list_objects_v2')
 
-    def get_client(self): # TODO: do i need this?
-        return self.s3_session.client(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
+    # def get_client(self): # TODO: do i need this?
+    #     return self.s3_session.client(
+    #         service_name="s3",
+    #         config=self.s3_client_config,
+    #         region_name=self.s3_region,
+    #     )
 
     #####################################################################
     def create_bucket(
@@ -146,18 +141,6 @@ class S3Manager:
         return all_uploads
 
     #####################################################################
-    # def upload_nodd_file2(
-    #         self,
-    #         body: str,
-    #         bucket: str,
-    #         key: str,
-    # ):
-    #     self.s3_client_noaa_wcsd_zarr_pds.put_object(
-    #         Body=body,
-    #         Bucket=bucket,
-    #         Key=key,
-    #     )
-
     # TODO: this uses resource, try to use client
     def upload_file(
             self,
@@ -190,11 +173,36 @@ class S3Manager:
                 all_files.append([local_path, s3_key])
 
         all_uploads = self.upload_files_with_thread_pool_executor(
+            output_bucket_name=self.output_bucket_name,
             all_files=all_files,
         )
         print("Done uploading files to output bucket.")
         return all_uploads
 
+    #####################################################################
+    def check_if_object_exists(
+            self,
+            bucket_name,
+            key_name
+    ) -> bool:
+        s3_manager2 = S3Manager()
+        s3_manager2.list_objects(bucket_name=bucket_name, prefix=key_name)
+        s3_client_noaa_wcsd_zarr_pds = self.s3_client_noaa_wcsd_zarr_pds
+        try:
+            # response = s3_resource_noaa_wcsd_zarr_pds.Object(bucket_name, key_name).load()
+            s3_client_noaa_wcsd_zarr_pds.head_object(Bucket=bucket_name, Key=key_name)
+        except botocore.exceptions.ClientError as e:
+            if e.response['Error']['Code'] == "404":
+                # The object does not exist.
+                return False
+            elif e.response['Error']['Code'] == 403:
+                # Unauthorized, including invalid bucket
+                return False
+            else:
+                # Something else has gone wrong.
+                raise
+        return True
+
     #####################################################################
     # used: raw-to-zarr
     def list_objects(  # noaa-wcsd-pds and noaa-wcsd-zarr-pds
@@ -202,6 +210,7 @@ class S3Manager:
         bucket_name,
         prefix
     ):
+        # TODO: this isn't working for geojson detecting objects!!!!!!!
         # analog to "find_children_objects"
         # Returns a list of key strings for each object in bucket defined by prefix
         # s3_client = self.s3_client
@@ -227,7 +236,11 @@ class S3Manager:
 
     #####################################################################
     # TODO: change name to "directory"
-    def folder_exists_and_not_empty(self, bucket_name: str, path: str) -> bool:
+    def folder_exists_and_not_empty(
+        self,
+        bucket_name: str,
+        path: str
+    ) -> bool:
         if not path.endswith("/"):
             path = path + "/"
         s3_client = self.s3_client
@@ -319,23 +332,15 @@ class S3Manager:
         print("downloaded file")
 
     #####################################################################
-    # not used
-    # def delete_nodd_object(  # noaa-wcsd-model-pds
-    #         self,
-    #         bucket_name,
-    #         key
-    # ):  # -> dict:
-    #     #return self.__s3_client.delete_object(Bucket=bucket_name, Key=key)
-    #     self.s3_client.delete_object(Bucket=bucket_name, Key=key)
-
-    #####################################################################
+    # TODO: need to test this!!!
     def delete_nodd_objects(  # nodd-bucket
         self,
+        bucket_name,
         objects: list,
     ):
         try:
             print(
-                f"Deleting {len(objects)} objects in {self.output_bucket_name} in batches."
+                f"Deleting {len(objects)} objects in {bucket_name} in batches."
             )
             objects_to_delete = []
             for obj in objects:
@@ -343,12 +348,28 @@ class S3Manager:
             # Note: request can contain a list of up to 1000 keys
             for batch in chunked(ll=objects_to_delete, n=1000):
                 self.s3_client_noaa_wcsd_zarr_pds.delete_objects(
-                    Bucket=self.output_bucket_name, Delete={"Objects": batch}
+                    Bucket=bucket_name, Delete={"Objects": batch}
                 )
             print(f"Deleted files.")
         except Exception as err:
             print(f"Problem was encountered while deleting objects: {err}")
 
+    #####################################################################
+    # TODO: need to test this!!!
+    def delete_nodd_object(
+            self,
+            bucket_name,
+            key_name,
+    ):
+        try:
+            print(
+                f"Deleting {key_name} objects in {bucket_name}."
+            )
+            self.s3_client_noaa_wcsd_zarr_pds.delete_object(Bucket=bucket_name, Key=key_name)
+            print(f"Deleted file.")
+        except Exception as err:
+            print(f"Problem was encountered while deleting objects: {err}")
+
     #####################################################################
     # not used TODO: remove
     def put(self, bucket_name, key, body):  # noaa-wcsd-model-pds

water_column_sonar_processing/cruise/create_empty_zarr_store.py

@@ -3,10 +3,10 @@ import os
 import numcodecs
 import numpy as np
 
-from src.water_column_sonar_processing.aws.dynamodb_manager import DynamoDBManager
-from src.water_column_sonar_processing.aws.s3_manager import S3Manager
-from src.water_column_sonar_processing.model.zarr_manager import ZarrManager
-from src.water_column_sonar_processing.utility.cleaner import Cleaner
+from water_column_sonar_processing.aws import DynamoDBManager
+from water_column_sonar_processing.aws import S3Manager
+from water_column_sonar_processing.model import ZarrManager
+from water_column_sonar_processing.utility import Cleaner
 
 numcodecs.blosc.use_threads = False
 numcodecs.blosc.set_nthreads(1)
@@ -47,7 +47,7 @@ class CreateEmptyZarrStore:
         ):
             for file in files:
                 local_path = os.path.join(subdir, file)
-                # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.model/.zattrs'
+                # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.model/..zattrs'
                 s3_key = f'{object_prefix}/{cruise_name}.model{local_path.split(f"{cruise_name}.model")[-1]}'
                 all_files.append([local_path, s3_key])
         #

water_column_sonar_processing/cruise/resample_regrid.py

@@ -7,9 +7,9 @@ import numpy as np
 import pandas as pd
 import xarray as xr
 
-from src.water_column_sonar_processing.aws.dynamodb_manager import DynamoDBManager
-from src.water_column_sonar_processing.geometry.geometry_manager import GeometryManager
-from src.water_column_sonar_processing.model.zarr_manager import ZarrManager
+from water_column_sonar_processing.aws import DynamoDBManager
+from water_column_sonar_processing.geometry import GeometryManager
+from water_column_sonar_processing.model import ZarrManager
 
 numcodecs.blosc.use_threads = False
 numcodecs.blosc.set_nthreads(1)

water_column_sonar_processing/geometry/geometry_manager.py

@@ -5,8 +5,8 @@ import geopandas
 import numpy as np
 import pandas as pd
 
-from src.water_column_sonar_processing.aws.s3_manager import S3Manager
-from src.water_column_sonar_processing.utility.cleaner import Cleaner
+from water_column_sonar_processing.aws import S3Manager
+from water_column_sonar_processing.utility import Cleaner
 
 """
 //  [Decimal / Places / Degrees	/ Object that can be recognized at scale / N/S or E/W at equator, E/W at 23N/S, E/W at 45N/S, E/W at 67N/S]
@@ -138,15 +138,13 @@ class GeometryManager:
 
                 print("Checking s3 and deleting any existing GeoJSON file.")
                 s3_manager = S3Manager()
-                s3_objects = s3_manager.list_objects(
+                geojson_object_exists = s3_manager.check_if_object_exists(
                     bucket_name=output_bucket_name,
-                    prefix=f"{geo_json_prefix}/{geo_json_name}"
+                    key_name=f"{geo_json_prefix}/{geo_json_name}"
                 )
-                if len(s3_objects) > 0:
-                    print(
-                        "GeoJSON already exists in s3, deleting existing and continuing."
-                    )
-                    s3_manager.delete_nodd_objects(objects=s3_objects)
+                if geojson_object_exists:
+                    print("GeoJSON already exists in s3, deleting existing and continuing.")
+                    s3_manager.delete_nodd_object(bucket_name=output_bucket_name, key_name=f"{geo_json_prefix}/{geo_json_name}")
 
                 print("Upload GeoJSON to s3.")
                 s3_manager.upload_nodd_file(

water_column_sonar_processing/geometry/pmtile_generation.py

@@ -12,8 +12,6 @@ import pyogrio
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from shapely.geometry import LineString
 
-from src.water_column_sonar_processing.aws import S3Manager, S3FSManager
-
 MAX_POOL_CONNECTIONS = 64
 MAX_CONCURRENCY = 64
 MAX_WORKERS = 64

water_column_sonar_processing/index/index_manager.py

@@ -4,7 +4,7 @@ import pandas as pd
 from datetime import datetime
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import as_completed
-from src.water_column_sonar_processing.aws.s3_manager import S3Manager
+from water_column_sonar_processing.aws import S3Manager
 
 
 class IndexManager:

water_column_sonar_processing/model/zarr_manager.py

@@ -5,9 +5,10 @@ import xarray as xr
 import zarr
 from numcodecs import Blosc
 
-from src.water_column_sonar_processing.aws.s3fs_manager import S3FSManager
-from src.water_column_sonar_processing.utility.constants import Constants, Coordinates
-from src.water_column_sonar_processing.utility.timestamp import Timestamp
+from water_column_sonar_processing.aws import S3FSManager
+from water_column_sonar_processing.utility import Constants
+from water_column_sonar_processing.utility import Timestamp
+from water_column_sonar_processing.utility import Coordinates
 
 numcodecs.blosc.use_threads = False
 numcodecs.blosc.set_nthreads(1)
@@ -85,8 +86,6 @@ class ZarrManager:
             data=np.repeat(0.0, width),
             shape=width,
             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
-                # Constants.TILE_SIZE.value,
-            #),  # TODO: the chunking scheme doesn't seem to be working here
             dtype=np.dtype(Coordinates.TIME_DTYPE.value),
             compressor=self.__compressor,
             fill_value=np.nan,  # TODO: do i want nan's?
@@ -124,14 +123,16 @@ class ZarrManager:
 
         root.depth.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.DEPTH.value]
 
-        root.depth.attrs["long_name"] = Coordinates.DEPTH_LONG_NAME.value
         root.depth.attrs["units"] = Coordinates.DEPTH_UNITS.value
+        root.depth.attrs["long_name"] = Coordinates.DEPTH_LONG_NAME.value
+        root.depth.attrs["standard_name"] = Coordinates.DEPTH_STANDARD_NAME.value
 
         #####################################################################
         # --- Coordinate: Latitude --- #
         root.create_dataset(
             name=Coordinates.LATITUDE.value,
-            # data=np.repeat(0.0, width),
+            # data=np.repeat(0.0, width),  # root.longitude[:] = np.nan
+            data=np.repeat(np.nan, width),
             shape=width,
             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
             dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
@@ -143,14 +144,16 @@ class ZarrManager:
         # Note: LATITUDE is indexed by TIME
         root.latitude.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
 
-        root.latitude.attrs["long_name"] = Coordinates.LATITUDE_LONG_NAME.value
         root.latitude.attrs["units"] = Coordinates.LATITUDE_UNITS.value
+        root.latitude.attrs["long_name"] = Coordinates.LATITUDE_LONG_NAME.value
+        root.latitude.attrs["standard_name"] = Coordinates.LATITUDE_STANDARD_NAME.value
 
         #####################################################################
         # --- Coordinate: Longitude --- #
         root.create_dataset(
             name=Coordinates.LONGITUDE.value,
             # data=np.repeat(0.0, width),  # root.longitude[:] = np.nan
+            data=np.repeat(np.nan, width),
             shape=width,
             chunks=Constants.SPATIOTEMPORAL_CHUNK_SIZE.value,
             dtype=np.dtype(Coordinates.LONGITUDE_DTYPE.value),
@@ -162,8 +165,9 @@ class ZarrManager:
         # Note: LONGITUDE is indexed by TIME
         root.longitude.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
 
-        root.longitude.attrs["long_name"] = Coordinates.LONGITUDE_LONG_NAME.value
         root.longitude.attrs["units"] = Coordinates.LONGITUDE_UNITS.value
+        root.longitude.attrs["long_name"] = Coordinates.LONGITUDE_LONG_NAME.value
+        root.longitude.attrs["standard_name"] = Coordinates.LONGITUDE_STANDARD_NAME.value
 
         #####################################################################
         # TODO: verify adding this variable for where the bottom was detected
@@ -182,8 +186,9 @@ class ZarrManager:
         # BOTTOM is indexed by TIME
         root.bottom.attrs["_ARRAY_DIMENSIONS"] = [Coordinates.TIME.value]
 
-        root.bottom.attrs["long_name"] = Coordinates.BOTTOM_LONG_NAME.value
         root.bottom.attrs["units"] = Coordinates.BOTTOM_UNITS.value
+        root.bottom.attrs["long_name"] = Coordinates.BOTTOM_LONG_NAME.value
+        root.bottom.attrs["standard_name"] = Coordinates.BOTTOM_STANDARD_NAME.value
 
         #####################################################################
         # --- Coordinate: Frequency --- #
@@ -203,11 +208,11 @@ class ZarrManager:
             Coordinates.FREQUENCY.value
         ]  # TODO: is this correct
 
+        root.frequency.attrs["units"] = Coordinates.FREQUENCY_UNITS.value
         root.frequency.attrs["long_name"] = Coordinates.FREQUENCY_LONG_NAME.value
         root.frequency.attrs["standard_name"] = (
             Coordinates.FREQUENCY_STANDARD_NAME.value
         )
-        root.frequency.attrs["units"] = Coordinates.FREQUENCY_UNITS.value
 
         #####################################################################
         # --- Sv Data --- #
@@ -229,8 +234,8 @@ class ZarrManager:
             Coordinates.FREQUENCY.value,
         ]
 
-        root.Sv.attrs["long_name"] = Coordinates.SV_LONG_NAME.value
         root.Sv.attrs["units"] = Coordinates.SV_UNITS.value
+        root.Sv.attrs["long_name"] = Coordinates.SV_LONG_NAME.value
         root.Sv.attrs["tile_size"] = Constants.TILE_SIZE.value
 
         #####################################################################
@@ -241,7 +246,7 @@ class ZarrManager:
         #
         root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
         root.attrs["processing_software_version"] = (
-            "0.0.6"  # TODO: get programmatically
+            "0.0.9"  # TODO: get programmatically, echopype>utils>prov.py
         )
         root.attrs["processing_software_time"] = Timestamp.get_timestamp()
         #

water_column_sonar_processing/process.py

@@ -3,10 +3,10 @@ import os
 
 import numpy as np
 
-from src.water_column_sonar_processing.aws.dynamodb_manager import DynamoDBManager
-from src.water_column_sonar_processing.aws.s3_manager import S3Manager
-from src.water_column_sonar_processing.aws.s3fs_manager import S3FSManager
-from src.water_column_sonar_processing.aws.sns_manager import SNSManager
+from water_column_sonar_processing.aws import DynamoDBManager
+from water_column_sonar_processing.aws import S3Manager
+from water_column_sonar_processing.aws import S3FSManager
+from water_column_sonar_processing.aws import SNSManager
 
 
 ###########################################################

water_column_sonar_processing/processing/cruise_sampler.py

@@ -4,7 +4,7 @@ import echopype as ep
 import numpy as np
 from numcodecs import Blosc
 
-from src.water_column_sonar_processing.utility import Cleaner
+from water_column_sonar_processing.utility import Cleaner
 
 TEMPDIR = "/tmp"
 

water_column_sonar_processing/processing/raw_to_zarr.py

@@ -7,9 +7,9 @@ from numcodecs import Blosc
 from datetime import datetime
 from pathlib import Path # , PurePath
 
-from src.water_column_sonar_processing.aws import DynamoDBManager, S3Manager
-from src.water_column_sonar_processing.geometry.geometry_manager import GeometryManager
-from src.water_column_sonar_processing.utility import Cleaner
+from water_column_sonar_processing.aws import DynamoDBManager, S3Manager
+from water_column_sonar_processing.geometry import GeometryManager
+from water_column_sonar_processing.utility import Cleaner
 
 TEMPDIR = "/tmp"
 
@@ -149,10 +149,14 @@ class RawToZarr:
             sensor_name,
             raw_file_name,
     ):
+        """
+        Downloads the raw files, processes them with echopype, writes geojson, and uploads files
+        to the nodd bucket.
+        """
         print(f'Opening raw: {raw_file_name} and creating zarr store.')
         geometry_manager = GeometryManager()
         cleaner = Cleaner()
-        cleaner.delete_local_files(file_types=["*.zarr", "*.json"]) # TODO: include bot and raw?ß
+        cleaner.delete_local_files(file_types=["*.zarr", "*.json"]) # TODO: include bot and raw?
         try:
             gc.collect()
             print('Opening raw file with echopype.')
@@ -204,14 +208,27 @@ class RawToZarr:
             store_name = f"{Path(raw_file_name).stem}.zarr"
             ds_sv.to_zarr(store=store_name)
             #################################################################
-            # TODO: do i still need this?
-            # print('Note: Adding GeoJSON inside Zarr store')
-            # self.__write_geojson_to_file( # Was trying to write geojson to the L1 zarr store
-            #     store_name=store_name,
-            #     data=gps_data
-            # )
-            #################################################################
             output_zarr_prefix = f"level_1/{ship_name}/{cruise_name}/{sensor_name}/"
+            #################################################################
+            # If zarr store already exists then delete
+            s3_manager = S3Manager()
+            child_objects = s3_manager.get_child_objects(
+                bucket_name=output_bucket_name,
+                sub_prefix=f"level_1/{ship_name}/{cruise_name}/{sensor_name}/{Path(raw_file_name).stem}.zarr",
+            )
+            if len(child_objects) > 0:
+                print('Zarr store data already exists in s3, deleting existing and continuing.')
+                s3_manager.delete_nodd_objects(
+                    bucket_name=output_bucket_name,
+                    objects=child_objects,
+                )
+            #################################################################
+            self.__upload_files_to_output_bucket(
+                output_bucket_name=output_bucket_name,
+                local_directory=store_name,
+                object_prefix=output_zarr_prefix
+            )
+            #################################################################
             self.__zarr_info_to_table(
                 output_bucket_name=output_bucket_name,
                 table_name=table_name,
@@ -228,29 +245,9 @@ class RawToZarr:
                 frequencies=frequencies,
                 channels=channels
             )
-            ###################################################################
-            #######################################################################
-            self.__upload_files_to_output_bucket(
-                output_bucket_name=output_bucket_name,
-                local_directory=store_name,
-                object_prefix=output_zarr_prefix
-            )
-            #######################################################################
-            # # TODO: verify count of objects matches
-            # s3_objects = self.__s3.list_objects(
-            #     bucket_name=self.__output_bucket,
-            #     prefix=f"{zarr_prefix}/{os.path.splitext(input_file_name)[0]}.zarr/",
-            #     access_key_id=self.__output_bucket_access_key,
-            #     secret_access_key=self.__output_bucket_secret_access_key
-            # )
             #######################################################################
-            # self.__update_processing_status(
-            #     file_name=input_file_name,
-            #     cruise_name=cruise_name,
-            #     pipeline_status='SUCCESS_RAW_TO_ZARR'
-            # )
+            # TODO: verify count of objects matches, publish message, update status
             #######################################################################
-            # self.__publish_done_message(input_message)
             print('here')
         except Exception as err:
             print(f'Exception encountered creating local Zarr store with echopype: {err}')

water_column_sonar_processing/utility/__init__.py

@@ -1,6 +1,6 @@
 from .cleaner import Cleaner
-from .constants import Constants
+from .constants import Constants, Coordinates
 from .pipeline_status import PipelineStatus
 from .timestamp import Timestamp
 
-__all__ = ["Cleaner", "Constants", "PipelineStatus", "Timestamp"]
+__all__ = ["Cleaner", "Constants", "Coordinates", "PipelineStatus", "Timestamp"]

{water_column_sonar_processing-0.0.8.dist-info → water_column_sonar_processing-0.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: water_column_sonar_processing
-Version: 0.0.8
+Version: 0.0.10
 Summary: A processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
@@ -24,7 +24,7 @@ Requires-Dist: numcodecs==0.13.1
 Requires-Dist: numpy==1.26.4
 Requires-Dist: pandas==2.2.3
 Requires-Dist: pyarrow==18.1.0
-Requires-Dist: python-dotenv==1.0.0
+Requires-Dist: python-dotenv==1.0.1
 Requires-Dist: requests==2.32.3
 Requires-Dist: s3fs==2023.12.1
 Requires-Dist: scipy==1.14.1
@@ -114,6 +114,7 @@ python -m twine upload --repository pypi dist/*
 ```
 
 # Pre Commit Hook
+see here for installation: https://pre-commit.com/
 https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
 ```
 pre-commit install --allow-missing-config
@@ -132,3 +133,8 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
 20241125
 5 failed, 35 passed, 3 skipped, 1 warning in 9.71s
 3 failed, 38 passed, 3 skipped, 1 warning in 7.24s
+
+
+# TODO:
+add https://pypi.org/project/setuptools-scm/
+for extracting the version

{water_column_sonar_processing-0.0.8.dist-info → water_column_sonar_processing-0.0.10.dist-info}/RECORD

@@ -1,32 +1,32 @@
-water_column_sonar_processing/__init__.py,sha256=Ipl74g4btRHNYqgJ4Ro957HQC4YXBoIrGhgMmBPP0Uw,290
-water_column_sonar_processing/process.py,sha256=mb8_UpcOTy7RAAMlv-nF5hZG5zFoMsFxlT4OyvrpObk,5455
-water_column_sonar_processing/aws/__init__.py,sha256=u5J-TOEVgAQsMdc5LMo1igUESRclzV8gf-b0jUaJ9Gg,277
+water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
+water_column_sonar_processing/process.py,sha256=-yQtK3rnZq6lGAr3q02zLDe1NuMH9c0PiUOxKzG_r18,5386
+water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
 water_column_sonar_processing/aws/dynamodb_manager.py,sha256=sZHn-hgCt3K3w0x5BcXfF5jLMt_F11dAtQHJToij9nU,10008
-water_column_sonar_processing/aws/s3_manager.py,sha256=ctNWMkgqMlwbwmXHgwKEV8otLwIjr-dHX6bQ2rOw1ug,14718
+water_column_sonar_processing/aws/s3_manager.py,sha256=kS48Vu_jE_fOKbwKOhCLWKDSqHzOGVEdZ_Lc4MaMCfA,15291
 water_column_sonar_processing/aws/s3fs_manager.py,sha256=thVJPQKhbvF1g-Ue3BYgwazFOFDYOICIEJx4zkXBQ1E,2381
 water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
 water_column_sonar_processing/aws/sqs_manager.py,sha256=NSUrWmnSC8h8Gf7gT0U8zFaQQ-yX89h0Q0mDLKGqp2Y,1597
 water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
-water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=ev5jMZiwLyY1zrIAQhAw9X55eD3rxMrW5PpotjlOxDE,7382
-water_column_sonar_processing/cruise/resample_regrid.py,sha256=aCbY84wtQma7GK9QooMqXzrJuIpC8IPUp1cRwN8ZOoA,12368
+water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=uQiZoKm16jD0SUuXmhuPryxdE-6bUc6BlCi2UtmzUpw,7318
+water_column_sonar_processing/cruise/resample_regrid.py,sha256=4Tw6Ro9mQZOr0uIph6foz6a1OeFAZW0SMUT_asIwvKw,12309
 water_column_sonar_processing/geometry/__init__.py,sha256=_ol5nI8AL30pYXeAh5rtP7YmQggitPC6LA_kuTfPJ0Q,231
-water_column_sonar_processing/geometry/geometry_manager.py,sha256=xN1zyD_4Apry69BNcm5GtBKPpbHaPc_TYRYYFXjtwMc,10577
+water_column_sonar_processing/geometry/geometry_manager.py,sha256=0Q9IRiBr6XvxUg5M2vCPtUhbnYnwa5pJI1ayfWXMgMs,10587
 water_column_sonar_processing/geometry/geometry_simplification.py,sha256=im1HG9nfYIerQv3w-PUHzphw2B7aGgnsA3Zcdy2oTmA,3016
-water_column_sonar_processing/geometry/pmtile_generation.py,sha256=Hh7UP8_mSC4_vw0F-LEv0DCT0FBYzbuNWZ0bITdg3gI,12458
+water_column_sonar_processing/geometry/pmtile_generation.py,sha256=7Lm08Jr6YaM4nYmexClxbIMOqSV1teo9wMm6dfjFuNA,12384
 water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
-water_column_sonar_processing/index/index_manager.py,sha256=k_OOpU8FNQ5pz4HZkr1xGLakLbuyeBm8LZcFVqjU2cQ,11254
+water_column_sonar_processing/index/index_manager.py,sha256=YS6y_THfGAZpjfBZOj5n8O1aY_BnBYS781eNHfhpip0,11239
 water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
-water_column_sonar_processing/model/zarr_manager.py,sha256=hKikypzqqxGPUpjkSrf_RweHihaBuAxIQyOichf4Vhs,13637
+water_column_sonar_processing/model/zarr_manager.py,sha256=TbcVux-GWfX4XJ7UT20E7dI_h_islrKsGtjx_VwSsLg,14003
 water_column_sonar_processing/processing/__init__.py,sha256=UwdB3BnoUxy4q3k9-ZjBF6KzmCWVDcqbcArTeHgmvGA,118
-water_column_sonar_processing/processing/cruise_sampler.py,sha256=d5_rwk9ucCGxNnL4gjDoY12fN6t9jOwJJenftGV4dGE,15931
-water_column_sonar_processing/processing/raw_to_zarr.py,sha256=W5BswuiHSnHj6jhVY_31qHo6OQHbYEvP2cqBliIqVZQ,16154
-water_column_sonar_processing/utility/__init__.py,sha256=nyqPobcvwftr6T4MNxNtQtfbWzW9Kgpbp6JO7Gr5IZI,206
+water_column_sonar_processing/processing/cruise_sampler.py,sha256=hadPrnH5nz7_oG_4pND7YbMFH6NMR9d6p3xAXedtKU8,15927
+water_column_sonar_processing/processing/raw_to_zarr.py,sha256=7vvoNe0jlB34R5mBPceQjL9N_5X0GTWs9xpCqvRK1nQ,15931
+water_column_sonar_processing/utility/__init__.py,sha256=yDObMOL0_OxKWet5wffK2-XVJgoE9iwiY2q04GZrtBQ,234
 water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
 water_column_sonar_processing/utility/constants.py,sha256=EbzsorvYKadsPjuutRjQKKByGibhFm0Gw6D-Sp2ZD3I,2143
 water_column_sonar_processing/utility/pipeline_status.py,sha256=O-0SySqdRGJ6bs3zQe1NV9vkOpmsRM7zj5QoHgzYioY,4395
 water_column_sonar_processing/utility/timestamp.py,sha256=bO0oir7KxxoEHPGRkz9FCBfOligkocUyRiWRzAq8fnU,361
-water_column_sonar_processing-0.0.8.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
-water_column_sonar_processing-0.0.8.dist-info/METADATA,sha256=N2RBgvC6W-KnKzS4CoijfHeQ5PVIVhhBNOZkIRH4Kuc,4432
-water_column_sonar_processing-0.0.8.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-water_column_sonar_processing-0.0.8.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
-water_column_sonar_processing-0.0.8.dist-info/RECORD,,
+water_column_sonar_processing-0.0.10.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
+water_column_sonar_processing-0.0.10.dist-info/METADATA,sha256=qFNeJ3GduRHKfcJRYShO9LamuMREk66qm18IUUXsMg8,4566
+water_column_sonar_processing-0.0.10.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+water_column_sonar_processing-0.0.10.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
+water_column_sonar_processing-0.0.10.dist-info/RECORD,,