water-column-sonar-processing 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl

water_column_sonar_processing/aws/s3fs_manager.py

@@ -1,6 +1,6 @@
 import os
-import s3fs
 
+import s3fs
 
 # TODO: S3FS_LOGGING_LEVEL=DEBUG
 
@@ -8,12 +8,12 @@ import s3fs
 class S3FSManager:
     #####################################################################
     def __init__(
-            self,
+        self,
     ):
         self.__s3_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.s3fs = s3fs.S3FileSystem(
-            key=os.environ.get('OUTPUT_BUCKET_ACCESS_KEY'),
-            secret=os.environ.get('OUTPUT_BUCKET_SECRET_ACCESS_KEY'),
+            key=os.environ.get("OUTPUT_BUCKET_ACCESS_KEY"),
+            secret=os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY"),
             # asynchronous=True
             # use_ssl=False,
             # skip_instance_cache=True,
@@ -24,10 +24,7 @@ class S3FSManager:
         )
 
     #####################################################################
-    def add_file(
-            self,
-            filename
-    ):
+    def add_file(self, filename):
         full_path = f"{os.getenv('OUTPUT_BUCKET_NAME')}/testing/{filename}"
         print(full_path)
 
@@ -37,12 +34,7 @@ class S3FSManager:
         print(ff)
 
     #####################################################################
-    def upload_data(
-            self,
-            bucket_name,
-            file_path,
-            prefix
-    ):
+    def upload_data(self, bucket_name, file_path, prefix):
         # TODO: this works in theory but use boto3 to upload files
         s3_path = f"s3://{bucket_name}/{prefix}/"
         s3_file_system = self.s3fs
@@ -50,18 +42,20 @@ class S3FSManager:
 
     #####################################################################
     def s3_map(
-            self,
-            s3_zarr_store_path,  # f's3://{bucket}/{input_zarr_path}'
+        self,
+        s3_zarr_store_path,  # f's3://{bucket}/{input_zarr_path}'
     ):
         # The "s3_zarr_store_path" is defined as f's3://{bucket}/{input_zarr_path}'
         # create=False, not false because will be writing
         # return s3fs.S3Map(root=s3_zarr_store_path, s3=self.s3fs, check=True)
-        return s3fs.S3Map(root=s3_zarr_store_path, s3=self.s3fs)  # create=False, not false because will be writing
+        return s3fs.S3Map(
+            root=s3_zarr_store_path, s3=self.s3fs
+        )  # create=False, not false because will be writing
 
     #####################################################################
     def exists(
-            self,
-            geo_json_s3_path,
+        self,
+        geo_json_s3_path,
     ):
         s3_file_system = self.s3fs
         return s3_file_system.exists(path=geo_json_s3_path)

water_column_sonar_processing/aws/sns_manager.py

@@ -1,5 +1,5 @@
 import os
-# import json
+
 import boto3
 
 
@@ -7,32 +7,22 @@ import boto3
 class SNSManager:
     #######################################################
     def __init__(
-            self,
+        self,
     ):
         self.__sns_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.__sns_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-            region_name=self.__sns_region
+            aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
+            aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
+            region_name=self.__sns_region,
         )
         self.__sns_resource = self.__sns_session.resource(
-            service_name="sns",
-            region_name=self.__sns_region
+            service_name="sns", region_name=self.__sns_region
         )
         self.__sns_client = self.__sns_session.client(
-            service_name="sns",
-            region_name=self.__sns_region
+            service_name="sns", region_name=self.__sns_region
         )
 
     #######################################################
-    # TODO: pick one
-    # def publish_message(self, topic_arn, message):
-    #     response = self.__sns_client.publish(
-    #         TopicArn=topic_arn,
-    #         Message=message
-    #     )
-    #     print(f"Topic Response: {topic_arn} : '{message}' => {response}")
-
     # TODO: pick one
     def publish(self, topic_arn, message):
         response = self.__sns_client.publish(
@@ -55,13 +45,12 @@ class SNSManager:
     #######################################################
     def subscribe(self, topic_arn, endpoint):
         self.__sns_client.subscribe(
-            TopicArn=topic_arn,
-            Protocol='sqs',
-            Endpoint=endpoint
+            TopicArn=topic_arn, Protocol="sqs", Endpoint=endpoint
         )
 
     #######################################################
     def list_topics(self):
         print(self.__sns_client.list_topics())
 
-###########################################################
+
+###########################################################

water_column_sonar_processing/aws/sqs_manager.py

@@ -1,34 +1,29 @@
 import os
+
 import boto3
-# import time
 
 
 ###########################################################
 class SQSManager:
     #######################################################
     def __init__(
-            self,
+        self,
     ):
         self.__sqs_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.__sqs_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-            region_name=self.__sqs_region
+            aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
+            aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
+            region_name=self.__sqs_region,
         )
         self.__sqs_resource = self.__sqs_session.resource(
-            service_name="sqs",
-            region_name=self.__sqs_region
+            service_name="sqs", region_name=self.__sqs_region
         )
         self.__sqs_client = self.__sqs_session.client(
-            service_name="sqs",
-            region_name=self.__sqs_region
+            service_name="sqs", region_name=self.__sqs_region
         )
 
     #######################################################
-    def create_queue(
-            self,
-            queue_name
-    ):
+    def create_queue(self, queue_name):
         response = self.__sqs_client.create_queue(QueueName=queue_name)
         return response
 
@@ -38,13 +33,10 @@ class SQSManager:
         return sqs_queue
 
     #######################################################
-    def list_queues(
-            self,
-            queue_name_prefix
-    ):
+    def list_queues(self, queue_name_prefix):
         # Note: SQS control plane is eventually consistent, meaning that it
         # takes a while to propagate the data accross the systems.
         response = self.__sqs_client.list_queues(QueueNamePrefix=queue_name_prefix)
         print(response)
 
-    #######################################################
+    #######################################################

water_column_sonar_processing/cruise/__init__.py

@@ -0,0 +1,4 @@
+from .create_empty_zarr_store import CreateEmptyZarrStore
+from .resample_regrid import ResampleRegrid
+
+__all__ = ["CreateEmptyZarrStore", "ResampleRegrid"]

water_column_sonar_processing/cruise/create_empty_zarr_store.py

@@ -1,46 +1,48 @@
 import os
+
 import numcodecs
 import numpy as np
 
-from water_column_sonar_processing.utility.cleaner import Cleaner
 from water_column_sonar_processing.aws.dynamodb_manager import DynamoDBManager
 from water_column_sonar_processing.aws.s3_manager import S3Manager
 from water_column_sonar_processing.model.zarr_manager import ZarrManager
+from water_column_sonar_processing.utility.cleaner import Cleaner
 
 numcodecs.blosc.use_threads = False
 numcodecs.blosc.set_nthreads(1)
 
-TEMPDIR = "/tmp"
-
+# TEMPDIR = "/tmp"
 # TODO: when ready switch to version 3 of model spec
 # ZARR_V3_EXPERIMENTAL_API = 1
 # creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
 
+
 class CreateEmptyZarrStore:
     #######################################################
     def __init__(
-            self,
+        self,
     ):
         self.__overwrite = True
-        # TODO: create output_bucket and input_bucket variables here?
         self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
         self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
 
     #######################################################
 
     def upload_zarr_store_to_s3(
-            self,
-            local_directory: str,
-            object_prefix: str,
-            cruise_name: str,
+        self,
+        local_directory: str,
+        object_prefix: str,
+        cruise_name: str,
     ) -> None:
-        print('uploading model store to s3')
+        print("uploading model store to s3")
         s3_manager = S3Manager()
         #
-        print('Starting upload with thread pool executor.')
+        print("Starting upload with thread pool executor.")
         # # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
         all_files = []
-        for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.zarr_manager"):
+        for subdir, dirs, files in os.walk(
+            f"{local_directory}/{cruise_name}.zarr_manager"
+        ):
             for file in files:
                 local_path = os.path.join(subdir, file)
                 # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.model/.zattrs'
@@ -51,48 +53,62 @@ class CreateEmptyZarrStore:
         s3_manager.upload_files_with_thread_pool_executor(
             all_files=all_files,
         )
-        print('Done uploading with thread pool executor.')
+        print("Done uploading with thread pool executor.")
         # TODO: move to common place
 
     #######################################################
     def create_cruise_level_zarr_store(
-            self,
-            ship_name: str,
-            cruise_name: str,
-            sensor_name: str,
-            table_name: str
+        self,
+        ship_name: str,
+        cruise_name: str,
+        sensor_name: str,
+        table_name: str,
+        tempdir: str,
     ) -> None:
         try:
             # HB0806 - 123, HB0903 - 220
             dynamo_db_manager = DynamoDBManager()
+            s3_manager = S3Manager()
 
             df = dynamo_db_manager.get_table_as_df(
                 table_name=table_name,
                 ship_name=ship_name,
                 cruise_name=cruise_name,
-                sensor_name=sensor_name
+                sensor_name=sensor_name,
             )
 
-            # filter the dataframe just for enums >= LEVEL_1_PROCESSING
+            # TODO: filter the dataframe just for enums >= LEVEL_1_PROCESSING
             # df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
 
             # TODO: VERIFY GEOJSON EXISTS as prerequisite!!!
 
             print(f"DataFrame shape: {df.shape}")
-            cruise_channels = list(set([i for sublist in df['CHANNELS'].dropna() for i in sublist]))
+            cruise_channels = list(
+                set([i for sublist in df["CHANNELS"].dropna() for i in sublist])
+            )
             cruise_channels.sort()
 
-            consolidated_zarr_width = np.sum(df['NUM_PING_TIME_DROPNA'].dropna().astype(int))
+            consolidated_zarr_width = np.sum(
+                df["NUM_PING_TIME_DROPNA"].dropna().astype(int)
+            )
 
             # [3] calculate the max/min measurement resolutions for the whole cruise
-            cruise_min_echo_range = float(np.min(df['MIN_ECHO_RANGE'].dropna().astype(float)))
+            cruise_min_echo_range = float(
+                np.min(df["MIN_ECHO_RANGE"].dropna().astype(float))
+            )
 
             # [4] calculate the maximum of the max depth values
-            cruise_max_echo_range = float(np.max(df['MAX_ECHO_RANGE'].dropna().astype(float)))
-            print(f"cruise_min_echo_range: {cruise_min_echo_range}, cruise_max_echo_range: {cruise_max_echo_range}")
+            cruise_max_echo_range = float(
+                np.max(df["MAX_ECHO_RANGE"].dropna().astype(float))
+            )
+            print(
+                f"cruise_min_echo_range: {cruise_min_echo_range}, cruise_max_echo_range: {cruise_max_echo_range}"
+            )
 
             # [5] get number of channels
-            cruise_frequencies = [float(i) for i in df['FREQUENCIES'].dropna().values.flatten()[0]]
+            cruise_frequencies = [
+                float(i) for i in df["FREQUENCIES"].dropna().values.flatten()[0]
+            ]
             print(cruise_frequencies)
 
             new_width = int(consolidated_zarr_width)
@@ -102,7 +118,6 @@ class CreateEmptyZarrStore:
             print(store_name)
             ################################################################
             # Delete existing model store if it exists
-            s3_manager = S3Manager()
             zarr_prefix = os.path.join("level_2", ship_name, cruise_name, sensor_name)
             child_objects = s3_manager.get_child_objects(
                 bucket_name=self.output_bucket_name,
@@ -115,14 +130,16 @@ class CreateEmptyZarrStore:
             ################################################################
             # Create new model store
             zarr_manager = ZarrManager()
-            new_height = len(zarr_manager.get_depth_values(
-                min_echo_range=cruise_min_echo_range,
-                max_echo_range=cruise_max_echo_range
-            ))
+            new_height = len(
+                zarr_manager.get_depth_values(
+                    min_echo_range=cruise_min_echo_range,
+                    max_echo_range=cruise_max_echo_range,
+                )
+            )
             print(f"new_height: {new_height}")
 
             zarr_manager.create_zarr_store(
-                path=TEMPDIR,
+                path=tempdir,
                 ship_name=ship_name,
                 cruise_name=cruise_name,
                 sensor_name=sensor_name,
@@ -134,7 +151,7 @@ class CreateEmptyZarrStore:
             )
             #################################################################
             self.upload_zarr_store_to_s3(
-                local_directory=TEMPDIR,
+                local_directory=tempdir,
                 object_prefix=zarr_prefix,
                 cruise_name=cruise_name,
             )
@@ -161,6 +178,7 @@ class CreateEmptyZarrStore:
         finally:
             cleaner = Cleaner()
             cleaner.delete_local_files()
+            # TODO: should delete zarr store in temp directory too?
         print("Done creating cruise level model store")